cd cd #!/usr/bin/ksh
#
# wget_usgs_gmag.ksh
#
# Author: Lydia Philpott (lphilpott@igpp.ucla.edu)
#
# Based heavily on wget_intermagnet_gmag.ksh
#
# This script uses wget to download data from USGS magnetometer sites
#
# This script is initially being set up to download data from BRW (Barrow). With the intention 
# that it be extendable to other sites in the future.
#
# $1 = network
# $2 = site
# $3 = year
# $4 = month

#
## Handle input arguments
#
network=${1}
site=${2}
year=${3}
month=${4}

#
## Set environment variables
#
  if [[ -z $THMSOC ]]
  then
    THMSOC=/disks/socware/thmsoc_dp_current
    export THMSOC
  fi
	. $THMSOC/src/config/soc_it_to_me.config
	. $THMSOC/src/config/wget_$network_gmag.config


#
## Set logfiles
#
	processdate=$(date '+%y%m%d')
	LOGFILE=${LOGDIR}/webdownloads/${network}_gmag_log_${processdate}
	WGETLOG=${LOGDIR}/webdownloads/wget_output_${network}_gmag_${processdate}_${site}_$$


	echo ":$$:${network}_gmag_log:" >> $LOGFILE
	echo ":$$:${network}_gmag_log:Starting script wget_${network}_gmag at $(date)" >> $LOGFILE

#
## Initialize wget parameters for each network
#

    wgetTime=$(date '+%Y-%m-%d %T')
    OPTIONS=' -r -nH --cut-dirs=5 -N -o ${WGETLOG} '	
    url==${WEBSITE}/${site}/${year}

    if [[ ${NETWORK} = "usgs" ]]
    then 
	if [[ $os = "SunOS" ]]
        then
  	    siteupper=$(echo $site | nawk '{print toupper($1)}')
 	else
            siteupper=$(echo $site | awk '{print toupper($1)}')
        fi
        url=${WEBSITE}/$siteupper/OneSecond/
        sitequery="$site$year$month*"

    elif [[ ${NETWORK} = "maccs" ]]
    then 
        sitequery='*.sec'

    elif [[ ${NETWORK} = "maccs" ]]
    then 
        sitequery='*.sec'

    elif [[ ${NETWORK} = "lrv" ]]
    then 
        url=${WEBSITE}/
        sitequery='*.sec.gz'

    elif [[ ${NETWORK} = "intermagnet" ]]
    then 
        
        url=${WEBSITE}/second/variation/IAGA2002/${year}/${month}
        sitequery="${site}*"
        password='--user=imagthemis --password=data4th3M '
        OPTIONS="${OPTIONS}${password}"

    else [[ ${NETWORK} = "antarctic" ]]       
        sitequery=""
        password='--user=THEMIS --password=Angelopoulos '
        OPTIONS="${OPTIONS}${password}"
    fi

#
## Run wget to download data from usgs for given site
#
    /usr/bin/wget ${OPTIONS} -P $LOCALWEBMIRROR/${site}/${year}/${month} -A $sitequery -np $url

    echo ":$$:${network}_gmag_log: Download Complete: $site" >> $LOGFILE

#
## Parse through wget output to see what was downloaded
#
    grep saved $WGETLOG | grep -v 'index\|robot' | awk '{print $6 " " $8}' > ${TMPDIR}/wget_${network}_gmag$$

	while read line
	do
		path=$(echo $line |awk '{print $1}')
		size=$(echo $line |awk '{print $2}')

		fileName=$(basename $path)
		fileName=${fileName%*\'}

		# Parse date information from filename
		year=$(echo $fileName |awk -F_ '{print substr($1,4,4)}')
		mon=$(echo $fileName |awk -F_ '{print substr($1,8,2)}')
		day=$(echo $fileName |awk -F_ '{print substr($1,10,2)}')
		fileTime="${year}-${mon}-${day} 00:00:00"

		# Verify file exists
		if [[ ! -a ${LOCALWEBMIRROR}/${site}/${year}/${month}/${fileName} ]]
		then
		    echo "$$:${network}_gmag_log:" >> $LOGFILE
		    echo "$$:${network}_gmag_log: ${LOCALWEBMIRROR}/${site}/${year}/${month}/${fileName} not found. " >> $LOGFILE
		    echo "$$:${network}_gmag_log: continuing..................." >> $LOGFILE
		fi

		fileSize=${size%\]}
		fileSize=${fileSize#\[}

		# Update database with stats
		${PHP_SCRIPTS}/wget_${network}_gmag.php $fileName $wgetTime $fileTime $fileSize

		# Create a file used for CDF processing
		cp ${LOCALWEBMIRROR}/${site}/${year}/${month}/${fileName} ${GMAGWORKDIR}/$fileName

	done < ${TMPDIR}/wget_${network}_gmag$$

#
## Cleanup
#
	rm -f ${TMPDIR}/wget_${network}_gmag$$


exit 0