#!/usr/bin/ksh
#
# wget_lrv_gmag.ksh
#
# Author: Lydia Philpott (lphilpott@igpp.ucla.edu)
#
#
# This script uses wget to download data from magnetometer site LRV
# 10 second preliminary data is available for recent days
# 1 minute definitive data is available for the previous month
# Both sorts of data are downloaded here.
#
# $1 = year

## Set environment variables
#
  if [[ -z $THMSOC ]]
  then
    THMSOC=/disks/socware/thmsoc_dp_current
    export THMSOC
  fi
	. $THMSOC/src/config/soc_it_to_me.config
	. $THMSOC/src/config/wget_lrv_gmag.config

#
## Handle input arguments
#
year=${1}
site=lrv

#
## Set logfiles
#
	processdate=$(date '+%y%m%d')
	LOGFILE=${LOGDIR}/webdownloads/lrv_gmag_log_${processdate}
	WGETLOG=${LOGDIR}/webdownloads/wget_output_lrv_gmag_${processdate}_${site}_$$


	echo ":$$:lrv_gmag_log:" >> $LOGFILE
	echo ":$$:lrv_gmag_log:Starting script wget_lrv_gmag at $(date)" >> $LOGFILE

#
## Run wget to download data 10 second preliminary data for LRV
#

        yeartwodigit=$(echo $year |awk '{print substr($1,3,2)}')
	wgetTime=$(date '+%Y-%m-%d %T')
	url=${WEBSITE}/
	filelist="*.sec.gz"
	/usr/bin/wget -r --level=1 -nH --cut-dirs=2 -N -o $WGETLOG -P $LOCALWEBMIRROR/${site}/${year} -A $filelist $url

	echo ":$$:lrv_gmag_log: Download Complete: 10 second data" >> $LOGFILE

echo $WGETLOG >> $LOGFILE
cp $WGETLOG /home/thmsoc/clrussells_stuff/lrv/${year}${doy}test.tmp

#
## Parse through wget output to see what was downloaded
#
	grep saved $WGETLOG | grep -v listing | awk '{print $6 " " $8}' > /tmp/wget_lrv_gmag$$

	while read line
	do
		path=$(echo $line |awk '{print $1}')
		size=$(echo $line |awk '{print $2}')

		fileName=$(basename $path)
		fileName=${fileName%*\'}

		# Parse date information from filename
		year=$(echo $fileName |awk '{print "20" substr($1,1,2)}')
		doy=$(echo $fileName |awk '{print substr($1,3,3)}')
                date=$(doy2date $year $doy)
		mon=$(echo $date |awk -F/ '{print $2}')
		day=$(echo $date |awk -F/ '{print $3}')
		fileTime=$(echo $year $mon $day | awk '{printf("%s-%02s-%02s 00:00:00", $1,$2,$3)}')

echo $path >> $LOGFILE
echo $size >> $LOGFILE
echo $fileName >> $LOGFILE
echo $year >> $LOGFILE
echo $doy >> $LOGFILE
echo $date >> $LOGFILE
echo $mon >> $LOGFILE
echo $day >> $LOGFILE
echo $fileTime >> $LOGFILE

		# Verify file exists
		if [[ ! -a ${LOCALWEBMIRROR}/${site}/${year}/${fileName} ]]
		then
		    echo "$$:lrv_gmag_log:" >> $LOGFILE
		    echo "$$:lrv_gmag_log: ${LOCALWEBMIRROR}/${site}/${year}/${fileName} not found. " >> $LOGFILE
		    echo "$$:lrv_gmag_log: continuing..................." >> $LOGFILE

			#printf "%s\n" "script: $0" \
			#"Warning: ${LOCALWEBMIRROR}/${site}/${year}/${fileName} not found." \
			#"Check $LOGFILE" |
			#mailx -s "LRV Download Discrepancy" $SOC_EMAIL
			msg="Warning: LRV Download Discrepancy - ${LOCALWEBMIRROR}/${site}/${year}/${fileName} not found. Check $LOGFILE"
			thmsoc_dblog.php $(basename $0) 1 "$msg"		### added 2012-06-26 - bsadeghi
		fi

		fileSize=${size%\]}
		fileSize=${fileSize#\[}

		# Update database with stats
		${PHP_SCRIPTS}/wget_lrv_gmag.php $fileName $wgetTime $fileTime $fileSize
		#echo "$fileName $wgetTime $fileTime $fileSize"
		# Create a file used for CDF processing
		cp ${LOCALWEBMIRROR}/${site}/${year}/${fileName} ${GMAGWORKDIR}/$fileName

	done < /tmp/wget_lrv_gmag$$

#
## Cleanup
#
	rm -f /tmp/wget_lrv_gmag$$

#
## Now run wget to download data 1 minute definitive data for the current month
#
	wgetTime=$(date '+%Y-%m-%d %T')
	url=${WEBSITE}/$year/
	filelist="$site*.min"

echo "Starting one minute download" >> $LOGFILE
echo $wgetTime >> $LOGFILE
echo $url >> $LOGFILE
echo $filelist >> $LOGFILE

	/usr/bin/wget -r --level=1 -nH --cut-dirs=3 -N -a $WGETLOG -P $LOCALWEBMIRROR/${site}/${year} -A $filelist $url

	echo ":$$:lrv_gmag_log: Download Complete: 1 minute data" >> $LOGFILE
#
## Parse through wget output to see what was downloaded
#

echo $WGETLOG >> $LOGFILE
cp $WGETLOG /home/thmsoc/clrussells_stuff/lrv/${year}${doy}test1.tmp

	grep saved $WGETLOG | grep -v listing | awk '{print $6 " " $8}' > /tmp/wget_lrv_gmag$$

	while read line
	do
		path=$(echo $line |awk '{print $1}')
		size=$(echo $line |awk '{print $2}')

		fileName=$(basename $path)
		fileName=${fileName%*\'}

echo $path >> $LOGFILE
echo $size >> $LOGFILE
echo $fileName >> $LOGFILE

		# Parse date information from filename
		year=$(echo $fileName |awk '{print "20" substr($1,4,2)}')

		#year=$(echo $fileName |awk '{print "20" substr($1,1,2)}')
		month=$(echo $fileName |awk '{print substr($1,6,3)}')
		# the filetime will show the beginning date rather than the end date
		# but this is only for recording in the datebase anyway
		date=$(date --date="1-$month-$year" '+%Y-%m-%d')
		fileTime="${date} 00:00:00"

echo $year >> $LOGFILE
echo "year"
echo $year
echo $doy >> $LOGFILE
echo $date >> $LOGFILE
echo $mon >> $LOGFILE
echo $day >> $LOGFILE
echo $filetime >> $LOGFILE

		# Verify file exists
		if [[ ! -a ${LOCALWEBMIRROR}/${site}/${year}/${fileName} ]]
		then
		    echo "$$:lrv_gmag_log:" >> $LOGFILE
		    echo "$$:lrv_gmag_log: ${LOCALWEBMIRROR}/${site}/${year}/${fileName} not found. " >> $LOGFILE
		    echo "$$:lrv_gmag_log: continuing..................." >> $LOGFILE

			#printf "%s\n" "script: $0" \
			#"Warning: ${LOCALWEBMIRROR}/${site}/${year}/${fileName} not found." \
			#"Check $LOGFILE" |
			#mailx -s "LRV Download Discrepancy" $SOC_EMAIL
			msg="Warning: LRV Download Discrepancy - ${LOCALWEBMIRROR}/${site}/${year}/${fileName} not found. Check $LOGFILE"
			thmsoc_dblog.php $(basename $0) 1 "$msg"		### added 2012-06-26 - bsadeghi
		fi

		fileSize=${size%\]}
		fileSize=${fileSize#\[}

		# Update database with stats
		${PHP_SCRIPTS}/wget_lrv_gmag.php $fileName $wgetTime $fileTime $fileSize
		#echo "$fileName $wgetTime $fileTime $fileSize"
		# Create a file used for CDF processing - Minute files will not be processed on to cdf
		#cp ${LOCALWEBMIRROR}/${site}/${year}/${fileName} ${GMAGWORKDIR}/$fileName

	done < /tmp/wget_lrv_gmag$$

#
## Cleanup
#
	rm -f /tmp/wget_lrv_gmag$$

exit 0