#!/usr/bin/ksh
#
# NO, IT DOESN'T WORK ON GAIA!(now just using this to run in parallel with others)
#
# epo_magprocess_date.ksh -  Combine the XYZ and BDH magnetometer measurement
# plots from the EPO GMAGS, for one particular date.
#
# epo_magprocess_date_test_gaia.ksh - can be run in parallel with epo_magprocess_date_test.ksh
# if lots of processing needed, testing to see if I can run this on gaia
#
#		#Input argument:	$1 = date in the form mmddyyyy
# Changed to read dates from a file (in form mmddyyyy)
# Set the school name in the file, or uncomment bits to loop through all
#

## Set the environment variables
#

  if [[ -z $THMSOC ]]
  then
    THMSOC=/disks/socware/thmsoc_dp_current
    export THMSOC
  fi
  
#. $THMSOC/src/config/thmsoc_epo.config
. /home/thmsoc/lphilpotts_stuff/thmsoc_epo_test.config

## Check for lock file
#
#  if [[ -a $LOCKFILE ]] ; then
#    echo "$$:" >> $LOGFILE
#    echo "$$: A Append Process is currently active " >> $LOGFILE
#    echo "$$: Quittin..................." >> $LOGFILE
#    exit 0
#  fi
#  echo $$ > $LOCKFILE

## Function Definitions
#

function no_merge 	# Print warning if file merge cannot occur
{
		echo "$$:" >> $LOGFILE
		echo "$$:   WARNING: One or more of the following files were not found:" >> $LOGFILE
		echo "$$:	$1" >> $LOGFILE
		echo "$$:	$2" >> $LOGFILE
		if [[ -z $4 ]]
		then
			echo "$$:   Cannot create file = $3" >> $LOGFILE
		else
			echo "$$:	$3" >> $LOGFILE
			echo "$$:   Cannot create file = $4" >> $LOGFILE
		fi
}

## Move to the directory containg the EPO data
#
cd $EPO_HOME

## Set a counter (n) and then set a variable to the
## first school name in the SCHOOL array which was
## defined in the config file above
#
set -A SCHOOL HOTS LOYS
#n=0
#school=${SCHOOL[$n]}
#school=PINE
#change date list!
# CHANGING so that dates are YYYYMMDD (same as date_list.txt)
for date in $(cat /home/thmsoc/lphilpotts_stuff/missing_jpg_date_list_test.txt)
do
  inputdate=$(echo $date | awk '{print substr($1,5,2) substr($1,7,2) substr($1,1,4)}')
  inputyear=$(echo $inputdate | awk '{print substr($1,5,4)}')
  echo $inputyear
  inputmonth=$(echo $inputdate | awk '{print substr($1,1,2)}')
  echo $inputmonth
  inputday=$(echo $inputdate | awk '{print substr($1,3,2)}')
  echo $inputday

  echo "$$:" >> $LOGFILE
  echo "$$:----------- Starting epo_magfile_append at $(date) ---------" >> $LOGFILE

n=0
school=${SCHOOL[$n]}
# Now lets start looping through the schools
#
while [[ -n $school ]]
do
	echo "$$: Starting processing for school = $school" >> $LOGFILE 
	school_dir=$(echo $school | sed 'y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/')
	chkStatus=""

  ##############################################
	## Search for x, y, and z time series plots ##
  ##############################################
	#
	echo "About to find TS1..."
	find ./${school_dir}/${inputyear}/${inputmonth}/${inputday} -name ${school}_TS1_${inputdate}* > /tmp/ts1_files_reprocess3
	xyzTSstatus=""
	echo "TS1 find done..."
	if [[ ! -s /tmp/ts1_files_reprocess3 ]]
	then
		xyzTSstatus=1
		chkStatus=1
	fi

	## .....and merge plots
	#
	echo "$$:" >> $LOGFILE
	echo "$$:......processing XYZ Time Series plots" >> $LOGFILE
	#while [[ -s /tmp/ts1_files ]]
	while read ts1_line
        do
		#xmag_file_name=$(head -1 /tmp/ts1_files)
		xmag_file_name=$ts1_line
	        date_string=$(echo $xmag_file_name | awk  -F_ '{print $3 "_" $4}')
		path_string=$(echo $xmag_file_name | awk  -F_ '{print $1}')
		ymag_file_name=$(echo "${path_string}_TS2_${date_string}")
		zmag_file_name=$(echo "${path_string}_TS3_${date_string}")
		b_file_name=$(echo "${path_string}_TStamp_${date_string}")
		xyzmag_file_name=$(echo $xmag_file_name |awk -F_ '{print $1 "_TS123_" $3 "_" $4}')

		if [[ ! -a $ymag_file_name || ! -a $zmag_file_name || ! -a $b_file_name ]]
		then
			no_merge $ymag_file_name $zmag_file_name $b_file_name $xyzmag_file_name
		else
			echo "$$:" >> $LOGFILE
			echo "$$: 	Merging files:" >> $LOGFILE
			echo "$$:	$xmag_file_name" >> $LOGFILE
			echo "$$:	$ymag_file_name" >> $LOGFILE
			echo "$$:	$zmag_file_name" >> $LOGFILE
			echo "$$:	$b_file_name" >> $LOGFILE
			echo "$$:	New filename = $xyzmag_file_name" >> $LOGFILE
			#echo "About to merge files TS123..."

			/usr/local/bin/convert $xmag_file_name $ymag_file_name $zmag_file_name $b_file_name -append $xyzmag_file_name

			#echo "Merge files done..."
			mysql_table=${school}_TS123
			mysql_date=$(echo $date_string | awk -F_ '{print substr($1,5,4) "-" substr($1,1,2) "-" substr($1,3,2)}')
			mysql_time=$(echo $date_string | awk -F_ '{print substr($2,1,2) ":" substr($2,3,2) ":" substr($2,5,2)}')
			echo "$$:  Updating MySQL database table $mysql_table......." >> $LOGFILE
			#echo "Update database..."
			/usr/local/bin/php /home/thmsoc/lphilpotts_stuff/epo_cmdline_input_test.php $mysql_table $mysql_date $mysql_time $xyzmag_file_name >> $LOGFILE
			echo "$$:  ...........update complete. " >> $LOGFILE
			#echo "Database updated..."
		fi
		#sed '1d' /tmp/ts1_files > /tmp/TS1_FILES
		#mv /tmp/TS1_FILES /tmp/ts1_files
	done < /tmp/ts1_files_reprocess3
	rm /tmp/ts1_files_reprocess3

  ##############################################
	## Search for B, D, and H time series plots ##
  ##############################################
	#
	find ./${school_dir}/${inputyear}/${inputmonth}/${inputday} -name ${school}_TSB_${inputdate}\* > /tmp/tsb_files_reprocess3
	bdhTSstatus=""
	if [[ ! -s /tmp/tsb_files_reprocess3 ]]
	then
		bdhTSstatus=1
		chkStatus=1
	fi

	## .....and merge plots
	#
	echo "$$:" >> $LOGFILE
	echo "$$:......processing BDH Time Series plots" >> $LOGFILE
	echo "Processing TSBDH..."
	#while [[ -s /tmp/tsb_files ]]
	while read tsb_line
        do
		#bmag_file_name=$(head -1 /tmp/tsb_files)
		bmag_file_name=$tsb_line
	        date_string=$(echo $bmag_file_name | awk  -F_ '{print $3 "_" $4}')
		path_string=$(echo $bmag_file_name | awk  -F_ '{print $1}')
		dmag_file_name=$(echo "${path_string}_TSD_${date_string}")
		hmag_file_name=$(echo "${path_string}_TSH_${date_string}")
		b_file_name=$(echo "${path_string}_TStamp_${date_string}")
		bdhmag_file_name=$(echo $bmag_file_name |awk -F_ '{print $1 "_TSBDH_" $3 "_" $4}')
		if [[ ! -a $dmag_file_name || ! -a $hmag_file_name || ! -a $b_file_name ]]
		then 
			no_merge $dmag_file_name $hmag_file_name $b_file_name $bdhmag_file_name
		else
			echo "$$:" >> $LOGFILE
			echo "$$: 	Merging files:" >> $LOGFILE
			echo "$$:	$bmag_file_name" >> $LOGFILE
			echo "$$:	$dmag_file_name" >> $LOGFILE
			echo "$$:	$hmag_file_name" >> $LOGFILE
			echo "$$:	$b_file_name" >> $LOGFILE
			echo "$$:	New filename = $bdhmag_file_name" >> $LOGFILE

                        #Changing order of plots in EPO dataset to HDB, pcruce, 2009-12-07
			#/usr/local/bin/convert $bmag_file_name $dmag_file_name $hmag_file_name $b_file_name -append $bdhmag_file_name
			/usr/local/bin/convert $hmag_file_name $dmag_file_name $bmag_file_name $b_file_name -append $bdhmag_file_name

			mysql_table=${school}_TSBDH
			mysql_date=$(echo $date_string | awk -F_ '{print substr($1,5,4) "-" substr($1,1,2) "-" substr($1,3,2)}')
			mysql_time=$(echo $date_string | awk -F_ '{print substr($2,1,2) ":" substr($2,3,2) ":" substr($2,5,2)}')
			echo "$$:  Updating MySQL database table $mysql_table......." >> $LOGFILE
			/usr/local/bin/php /home/thmsoc/lphilpotts_stuff/epo_cmdline_input_test.php $mysql_table $mysql_date $mysql_time $bdhmag_file_name >> $LOGFILE
			echo "$$:  ...........update complete. " >> $LOGFILE
		fi
		#	sed '1d' /tmp/tsb_files > /tmp/TSB_FILES
		#	mv /tmp/TSB_FILES /tmp/tsb_files
	done < /tmp/tsb_files_reprocess3
	rm /tmp/tsb_files_reprocess3 

	#############################################
	## Search for x, y, and z day series plots ##
	#############################################
	#
	find ./${school_dir}/${inputyear}/${inputmonth}/${inputday} -name ${school}_DS1_${inputdate}\* > /tmp/ds1_files_reprocess3
	xyzDSstatus=""
	if [[ ! -s /tmp/ds1_files_reprocess3 ]]
	then
		xyzDSstatus=1
		chkStatus=1
	fi

	## .....and merge plots
	#
	echo "$$:" >> $LOGFILE
	echo "$$:......processing XYZ Day Series plots" >> $LOGFILE
	echo "Processing DS123..."
	#while [[ -s /tmp/ds1_files ]]
	while read ds1_line
        do
		#xmag_file_name=$(head -1 /tmp/ds1_files)
		xmag_file_name=$ds1_line
	        date_string=$(echo $xmag_file_name | awk  -F_ '{print $3 "_" $4}')
		path_string=$(echo $xmag_file_name | awk  -F_ '{print $1}')
		ymag_file_name=$(echo "${path_string}_DS2_${date_string}")
		zmag_file_name=$(echo "${path_string}_DS3_${date_string}")
	    b_file_name=$(echo "${path_string}_DStamp_${date_string}")
		xyzmag_file_name=$(echo $xmag_file_name |awk -F_ '{print $1 "_DS123_" $3 "_" $4}')

		if [[ ! -a $ymag_file_name || ! -a $zmag_file_name || ! -a $b_file_name ]]
		then
			no_merge $ymag_file_name $zmag_file_name $b_file_name $xyzmag_file_name 
		else
			echo "$$:" >> $LOGFILE
			echo "$$: 	Merging files:" >> $LOGFILE
			echo "$$:	$xmag_file_name" >> $LOGFILE
			echo "$$:	$ymag_file_name" >> $LOGFILE
			echo "$$:	$zmag_file_name" >> $LOGFILE
			echo "$$:	$b_file_name" >> $LOGFILE
			echo "$$:	New filename = $xyzmag_file_name" >> $LOGFILE

			/usr/local/bin/convert $xmag_file_name $ymag_file_name $zmag_file_name $b_file_name -append $xyzmag_file_name

			mysql_table=${school}_DS123
			mysql_date=$(echo $date_string | awk -F_ '{print substr($1,5,4) "-" substr($1,1,2) "-" substr($1,3,2)}')
			mysql_time=$(echo $date_string | awk -F_ '{print substr($2,1,2) ":" substr($2,3,2) ":00" }')
			echo "$$:  Updating MySQL database table $mysql_table......." >> $LOGFILE
			/usr/local/bin/php /home/thmsoc/lphilpotts_stuff/epo_cmdline_input_test.php $mysql_table $mysql_date $mysql_time $xyzmag_file_name >> $LOGFILE
			echo "$$:  ...........update complete. " >> $LOGFILE
		fi
		#sed '1d' /tmp/ds1_files > /tmp/DS1_FILES
		#mv /tmp/DS1_FILES /tmp/ds1_files
	done < /tmp/ds1_files_reprocess3
	rm /tmp/ds1_files_reprocess3

	#############################################
	## Search for B, D, and H day series plots ##
	#############################################
	#
	find ./${school_dir}/${inputyear}/${inputmonth}/${inputday} -name ${school}_DSB_${inputdate}\* > /tmp/dsb_files_reprocess3
	bdhDSstatus=""
	if [[ ! -s /tmp/dsb_files_reprocess3 ]]
	then
		bdhDSstatus=1
		chkStatus=1
	fi

	## .....and merge plots
	#
	echo "$$:" >> $LOGFILE
	echo "$$:......processing BDH Day Series plots" >> $LOGFILE
	echo "Processing DSBDH..."
	#while [[ -s /tmp/dsb_files ]]
	while read dsb_line
        do
		#bmag_file_name=$(head -1 /tmp/dsb_files)
		bmag_file_name=$dsb_line
	        date_string=$(echo $bmag_file_name | awk  -F_ '{print $3 "_" $4}')
		path_string=$(echo $bmag_file_name | awk  -F_ '{print $1}')
		dmag_file_name=$(echo "${path_string}_DSD_${date_string}")
		hmag_file_name=$(echo "${path_string}_DSH_${date_string}")
		b_file_name=$(echo "${path_string}_DStamp_${date_string}")
		bdhmag_file_name=$(echo $bmag_file_name |awk -F_ '{print $1 "_DSBDH_" $3 "_" $4}')

		if [[ ! -a $dmag_file_name || ! -a $hmag_file_name || ! -a $b_file_name ]]
		then 
			no_merge $dmag_file_name $hmag_file_name $b_file_name $bdhmag_file_name 
		else
			echo "$$:" >> $LOGFILE
			echo "$$: 	Merging files:" >> $LOGFILE
			echo "$$:	$bmag_file_name" >> $LOGFILE
			echo "$$:	$dmag_file_name" >> $LOGFILE
			echo "$$:	$hmag_file_name" >> $LOGFILE
			echo "$$:	$b_file_name" >> $LOGFILE
			echo "$$:	New filename = $bdhmag_file_name" >> $LOGFILE
                        
                        #Changing order of plots in EPO dataset to HDB, pcruce, 2009-12-07
			#/usr/local/bin/convert $bmag_file_name $dmag_file_name $hmag_file_name $b_file_name -append $bdhmag_file_name
			/usr/local/bin/convert $hmag_file_name $dmag_file_name $bmag_file_name $b_file_name -append $bdhmag_file_name

			echo "$$:" >> $LOGFILE
			echo "$$: Updating MySQL database................" >> $LOGFILE
			mysql_table=${school}_DSBDH
			mysql_date=$(echo $date_string | awk -F_ '{print substr($1,5,4) "-" substr($1,1,2) "-" substr($1,3,2)}')
			mysql_time=$(echo $date_string | awk -F_ '{print substr($2,1,2) ":" substr($2,3,2) ":00" }')
			echo "$$:  Updating MySQL database table $mysql_table......." >> $LOGFILE
			/usr/local/bin/php /home/thmsoc/lphilpotts_stuff/epo_cmdline_input_test.php $mysql_table $mysql_date $mysql_time $bdhmag_file_name >> $LOGFILE
			echo "$$:  ...........update complete. " >> $LOGFILE
		fi
		#sed '1d' /tmp/dsb_files > /tmp/DSB_FILES
		#mv /tmp/DSB_FILES /tmp/dsb_files
	done < /tmp/dsb_files_reprocess3
	rm /tmp/dsb_files_reprocess3 

	####################################
	## Search for Time Series Spectra ##
	####################################
	#
	find ./${school_dir}/${inputyear}/${inputmonth}/${inputday} -name ${school}_TSpectra_${inputdate}\* > /tmp/ts1_files_reprocess3
	specTSstatus=""
	if [[ ! -s /tmp/ts1_files_reprocess3 ]]
	then
		specTSstatus=1
		chkStatus=1
	fi
	echo "$$:" >> $LOGFILE
	echo "$$:......processing Spectra Time Series plots" >> $LOGFILE
	echo "Processing TSpectra..."
	#while [[ -s /tmp/ts1_files ]]
	while read ts1_line
	do
		#tspectra_file_name=$(head -1 /tmp/ts1_files)
		tspectra_file_name=$ts1_line
	        date_string=$(echo $tspectra_file_name | awk  -F_ '{print $3 "_" $4}')

			echo "$$:" >> $LOGFILE
			echo "$$:" >> $LOGFILE
			echo "$$:  Found file: $tspectra_file_name" >> $LOGFILE
			mysql_table=${school}_TSpectra
			mysql_date=$(echo $date_string | awk -F_ '{print substr($1,5,4) "-" substr($1,1,2) "-" substr($1,3,2)}')
			mysql_time=$(echo $date_string | awk -F_ '{print substr($2,1,2) ":" substr($2,3,2) ":" substr($2,5,2)}')
			echo "$$:  Updating MySQL database table $mysql_table......." >> $LOGFILE
			/usr/local/bin/php /home/thmsoc/lphilpotts_stuff/epo_cmdline_input_test.php $mysql_table $mysql_date $mysql_time $tspectra_file_name >> $LOGFILE
			echo "$$:  ...........update complete. " >> $LOGFILE

		#sed '1d' /tmp/ts1_files > /tmp/TS1_FILES
		#mv /tmp/TS1_FILES /tmp/ts1_files
	done < /tmp/ts1_files_reprocess3
	rm /tmp/ts1_files_reprocess3 

	####################################
	## Search for Day Series Spectra  ##
	####################################
	#
	find ./${school_dir}/${inputyear}/${inputmonth}/${inputday} -name ${school}_DSpectra_${inputdate}\* > /tmp/ds1_files_reprocess3
	specDSstatus=""
	if [[ ! -s /tmp/ds1_files_reprocess3 ]]
	then
		specDSstatus=1
		chkStatus=1
	fi
	echo "$$:" >> $LOGFILE
	echo "$$:......processing Spectra Time Series plots" >> $LOGFILE
	echo "Processing DSpectra..."
	#while [[ -s /tmp/ds1_files ]]
	while read ds1_line
        do
		#dspectra_file_name=$(head -1 /tmp/ds1_files)
		dspectra_file_name=$ds1_line
	        date_string=$(echo $dspectra_file_name | awk  -F_ '{print $3 "_" $4}')

			echo "$$:" >> $LOGFILE
			echo "$$:" >> $LOGFILE
			echo "$$:  Found file: $dspectra_file_name" >> $LOGFILE
			mysql_table=${school}_DSpectra
			mysql_date=$(echo $date_string | awk -F_ '{print substr($1,5,4) "-" substr($1,1,2) "-" substr($1,3,2)}')
			mysql_time=$(echo $date_string | awk -F_ '{print substr($2,1,2) ":" substr($2,3,2) ":00" }')
			echo "$$:  Updating MySQL database table $mysql_table......." >> $LOGFILE
			/usr/local/bin/php /home/thmsoc/lphilpotts_stuff/epo_cmdline_input_test.php $mysql_table $mysql_date $mysql_time $dspectra_file_name >> $LOGFILE
			echo "$$:  ...........update complete. " >> $LOGFILE
		
		#sed '1d' /tmp/ds1_files > /tmp/DS1_FILES
		#mv /tmp/DS1_FILES /tmp/ds1_files
	done < /tmp/ds1_files_reprocess3
	rm /tmp/ds1_files_reprocess3
	 
	if [[ -n $chkStatus ]]
	then
	    #Moving from email to logfile notification
		#printf "%s\n%s\n" "THEMIS EPO-GMAG Recovery Warning" \
		#"Files not received for $FIND_MTIME days. " \
		#"Check the following:" \
		#"School = $school" \
		#"xyzTSstatus = $xyzTSstatus" \
		#"bdhTSstatus = $bdhTSstatus" \
		#"xyzDSstatus = $xyzDSstatus" \
		#"bdhDSstatus = $bdhDSstatus" \
		#"specTSstatus = $specTSstatus" \
		#"specDSstatus = $specDSstatus" | mail $CONTACT_LIST
		
		echo "THEMIS EPO-GMAG Recovery Warning" >> $LOGFILE
	    echo "Files not received for $FIND_MTIME days. " >> $LOGFILE
		echo "Check the following:" >> $LOGFILE
		echo "School = $school" >> $LOGFILE
		echo "xyzTSstatus = $xyzTSstatus" >> $LOGFILE
		echo "bdhTSstatus = $bdhTSstatus" >> $LOGFILE
		echo "xyzDSstatus = $xyzDSstatus" >> $LOGFILE
		echo "bdhDSstatus = $bdhDSstatus" >> $LOGFILE
		echo "specTSstatus = $specTSstatus" >> $LOGFILE
		echo "specDSstatus = $specDSstatus" >> $LOGFILE
	fi			
	n=$(( n + 1 ))
	school=${SCHOOL[$n]}

done
done #dates in file

#rm $LOCKFILE

echo "$$:" >> $LOGFILE
echo "$$:----------- Ending epo_magfile_append at $(date) ---------" >> $LOGFILE

exit 0