Scrape NYSE

#!/bin/sh -x
 
PATH=$PATH:/usr/local/bin
BASEDIR=/home/stockh/xyz.com/api/scripts/
DATAFILE=${BASEDIR}../data/nyse_index_symbols.txt
LOGFILE=${BASEDIR}../logs/scraping
 
echo running $0 on `date` >> $LOGFILE
lynx -dump http://www.csidata.com/factsheets/nyse.txt > /tmp/nyse_index.txt 2>> $LOGFILE
 
if [ $? = 0 ]; then
 
  mv ${BASEDIR}../files/nyse_index.txt ${BASEDIR}../files/nyse_index.txt.bak 2>> $LOGFILE
  mv /tmp/nyse_index.txt ${BASEDIR}../files/nyse_index.txt 2>> $LOGFILE
 
  #cat ${BASEDIR}../files/nyse_index.txt | perl -ne 'if (s/^(.*)[\s\s]+(\S+)[\s\s]+(\S+)\s(\S+)/\1\2\3\4/) { print }'
  # get symbols and do + to -P translation for yahoo
  cat ${BASEDIR}../files/nyse_index.txt | perl -ne 'if (s/^(.*)[\s\s]+(\S+)[\s\s]+(\S+)\s(\S+)/\2/) { $_ =~ s/\+/-P/; print }' > /tmp/nyse_index_symbols.txt 2>> $LOGFILE
 
  mv ${BASEDIR}../data/nyse_index_symbols.txt ${DATAFILE}.bak 2>> $LOGFILE
 
  # removed windows newlines
  tr -d "\015" < /tmp/nyse_index_symbols.txt > $DATAFILE 2>> $LOGFILE
 
fi
 
echo `diff $DATAFILE ${DATFILE}.bak` >> $LOGFILE
echo `wc -l $DATAFILE ` lines in new datafile >> $LOGFILE
echo `wc -l ${DATAFILE}.bak ` lines in old datafile >> $LOGFILE