#!/usr/bin/ksh
#
# ucla_rmd2all.ksh 
#
#	When UCLA Ground Magnetometer (GMAG) raw data (RMD) have been
# rsync-mirrored from the UCLA or UCalgary server, this script 
# will initiate the sequence of file format conversions:
#
#			RMD->FF->Merged_FF->CDF->ASCII
#
#			Author: Tim Quinn
# 		Created: September 2006
#
#			Notes: This scipt calls code written by UCLA which will
#			only comile on SunOS.
#
#			Updates:
#
#				11/28/06 Removed stack plot production, which will run
#								 independently.

#
## Set the environment variables
#
  if [[ -z $THMSOC ]]
  then
    THMSOC=/disks/socware/thmsoc_dp_current
    export THMSOC
  fi
	. $THMSOC/src/config/soc_it_to_me.config
	. $THMSOC/src/config/ucla_rmd2all.config

	echo ":$$:ucla_rmd2all:" >> $LOGFILE
	echo ":$$:ucla_rmd2all:Starting ucla_rmd2all at $(date)" >> $LOGFILE

#
## Script MUST run on SunOS. Check and quit otherwise
#
	os=$(uname -s)
	if [[ $os != "SunOS" ]]
	then
		echo Script MUST run on SunOS. Quitting......
		exit 0
	fi

#
## Process any RMD directories that were produced as part of the 
## rsync-mirror process
#
site_dir=VLDR
year = 2007
month = 01
#echo $(ls -d ${ATHA_MIRROR_HOME}/VLDR/2007/01/*)
#MIR_DIR = ${ATHA_MIRROR_HOME}/${site}/${year}/${month}/*
#echo $MIR_DIR
	for dir_file in $(ls -d ${ATHA_MIRROR_HOME}/VLDR/2007/01/* 2>/dev/null )
	do
#	for dir_file in $(ls -d $GMAGMIRROR/*_RMDdir* 2>/dev/null )
#	do
		echo ":$$:ucla_rmd2all:" >> $LOGFILE
		echo ":$$:ucla_rmd2all: Checking content of file $dir_file" >> $LOGFILE
		gbo_or_epo=$(echo $dir_file | awk -F/ '{print substr($6,1,3)}')
		lastDir=NaN
		for dir in $(cat $dir_file)
		do
				
			if [[ $dir = $lastDir ]]
			then
				continue
            fi
            
			if [[ $gbo_or_epo = "EPO" ]]
			then
				filename=$(basename $(ls ${EPO_MIRROR_HOME}/${dir}/*RMD |head -1))
			elif [[ $gbo_or_epo = "ATH" ]]
			then
				site=$(echo $dir | awk '{print substr($1,1,4)}')
				year=$(echo $dir | awk '{print "20" substr($1,5,2)}')
				month=$(echo $dir | awk '{print substr($1,7,2)}')
				day=$(echo $dir | awk '{print substr($1,9,2)}')
				dir="${site}/${year}/${month}/${day}/${site}"
				filename=$(basename $(ls ${ATHA_MIRROR_HOME}/${dir}*RMD |head -1))
			else
				site=$(echo $dir | awk '{print substr($1,1,4)}')
				year=$(echo $dir | awk '{print "20" substr($1,5,2)}')
				month=$(echo $dir | awk '{print substr($1,7,2)}')
				day=$(echo $dir | awk '{print substr($1,9,2)}')
				dir="/${year}/${month}/${day}/${site}"
				filename=$(basename $(ls ${GBO_MIRROR_HOME}/${dir}*RMD |head -1))
		    fi
			
			fileroot=$(echo $filename | awk -F. '{print $1}')
			site=$(echo $fileroot |awk '{print substr($1,1,4)}')

			echo ":$$:ucla_rmd2all:" >> $LOGFILE
			echo ":$$:ucla_rmd2all: Looking at $gbo_or_epo files in $dir" >> $LOGFILE
	
	        #skip if empty site string
	        if [[ -z $site ]]
	        then
	          lastDir=$dir
	          continue
	        fi
	
			## First: Check if calibration file exists
			#
			if [[ ! -e ${TABLE_FILES_DIR}/${site}.txt ]]
			then
				# Calibration file does not exist. Go to next dir
				echo ":$$:ucla_rmd2all:" >> $LOGFILE
				echo ":$$:ucla_rmd2all:WARNING: Cal file ${site}.txt does not exist." >> $LOGFILE
				echo ":$$:ucla_rmd2all:.........trying next directory." >> $LOGFILE
#				printf "%s\n" "Subject: THEMIS GMAG Processing Problem" \
#				"script: $0" \
#				"Warning: Cal file ${site}.txt does not exist." |
#				mailx -s "THEMIS GMAG Processing Problem" $SOC_EMAIL
				msg="THEMIS GMAG Processing Problem - Warning: Cal file ${site}.txt does not exist."
				thmsoc_dblog.php $(basename $0) 1 "$msg"		### added 2012-06-25 - bsadeghi
				lastDir = $dir #stop repetitious mailing
				continue
			fi

			## create a directory name that includes siteyyyymmdd_rmd in 
			## garage subdir of epo ff storage dir
			#

			tmpdir=$(echo $dir | awk -F/ '{print $1$2$3$4}')

			mkdir -p ${UCB_FLAT_HOME}/process_rmd2ff/${tmpdir}
			
			## Copy all rmd files from RMD dir to temp dir
			#
			if [[ $gbo_or_epo = "EPO" ]]
			then
				cp -r ${EPO_MIRROR_HOME}/${dir}/*RMD ${UCB_FLAT_HOME}/process_rmd2ff/${tmpdir}
			elif [[ $gbo_or_epo = "ATH" ]]
			then
			    cp -r ${ATHA_MIRROR_HOME}/${dir}*RMD ${UCB_FLAT_HOME}/process_rmd2ff/${tmpdir}
			else
				cp -r ${GBO_MIRROR_HOME}/${dir}*RMD ${UCB_FLAT_HOME}/process_rmd2ff/${tmpdir}
			fi

			## Change directories to temp dir and run rmd2ff on each RMD file
			#
			cd ${UCB_FLAT_HOME}/process_rmd2ff/${tmpdir}
			echo ":$$:ucla_rmd2all:" >> $LOGFILE
			echo ":$$:ucla_rmd2all:	Running rmd2ff on files in $(pwd)......." >> $LOGFILE
			for file in $(ls *RMD)
			do
				fileroot=$(echo $file | awk -F. '{print $1}')
				site=$(echo $fileroot |awk '{print substr($1,1,4)}')
				rm -f $LOGDIR/rmd2ff_${fileroot}.log
				##${RMD2FFPATH}/RMD2FF -batch -in_file $file -table ${TABLE_FILES_DIR}/${site}.txt -outfile $fileroot >$LOGDIR/rmd2ff_${fileroot}.log
				${RMD2FFPATH}/RMD2FF -ctime -batch -in_file $file -table ${TABLE_FILES_DIR}/${site}.txt -outfile $fileroot 1>/dev/null 2>${LOGDIR}/RMD2FF_BAD$$
				if [[ -s ${LOGDIR}/RMD2FF_BAD$$ ]]
				then
					echo ":$$:ucla_rmd2all:">> $LOGFILE
					echo ":$$:ucla_rmd2all: Problem in RMD2FF run. Check ${LOGDIR}/RMD2FF_BAD$$">> $LOGFILE
					printf "%s\n" "Subject: THEMIS SOC Processing Problem" \
					"script: $0" \
					"Warning: RMD2FF Problem with $file" |
					mailx -s "THEMIS RMD2FF Processing Message" $SOC_EMAIL
					msg="THEMIS RMD2FF Processing Message - Warning: RMD2FF Problem with $file"
					thmsoc_dblog.php $(basename $0) 1 "$msg"		### added 2012-06-25 - bsadeghi
				else
					rm -f ${LOGDIR}/RMD2FF_BAD$$
				fi
			done
			echo ":$$:ucla_rmd2all:	rmd2ff complete. ">> $LOGFILE
				

			## Now merge the flat files into one 24 hour file
			#
			for file in $(ls *ffd)
			do
				fileroot=$(echo $file | awk -F. '{print $1}')
				echo $fileroot >> filelist.txt
			done
			newfilename=$(echo $fileroot |awk '{print substr($1,1,10)}')
			echo filelist.txt >> ffmerge.input
			echo "2 3 4" >> ffmerge.input
			echo x >> ffmerge.input
			echo first >> ffmerge.input
			echo last >> ffmerge.input
			echo $newfilename >> ffmerge.input
			echo n >> ffmerge.input
			echo y >> ffmerge.input
			rm -f $LOGDIR/ffmerge_${newfilename}
			echo ":$$:ucla_rmd2all:" >> $LOGFILE
			echo ":$$:ucla_rmd2all:	Running ffmerge on files ......" >> $LOGFILE
			#${FFMERGEPATH}/ffmergesungnucmp < ffmerge.input > $LOGDIR/ffmerge_${newfilename} 
			${FFMERGEPATH}/ffmergesungnucmp < ffmerge.input 1> /dev/null
			#echo ":$$:ucla_rmd2all:	ffmerge complete. Check log ffmerge_${newfilename}." >> $LOGFILE
			echo ":$$:ucla_rmd2all:	ffmerge complete. ">> $LOGFILE
	
			# Now a single flat file exists for that day in the form
			# schlyymmdd.ffd, schlyymmdd.ffh and must be changed to
			# Byydoy_schl
			#
			fullyear=$(echo $newfilename |awk '{print "20" substr($1,5,2)}')
			partyear=$(echo $newfilename |awk '{print  substr($1,5,2)}')
			month=$(echo $newfilename |awk '{print substr($1,7,2)}')
			day=$(echo $newfilename |awk '{print substr($1,9,2)}')
			sitedir=$(echo $site | nawk '{print tolower($1)}')
			doy=$(${KSHSCRIPTS}/date2doy ${fullyear}-${month}-${day} | awk '{printf("%03d",$1)}')
			finalfileroot=B${partyear}${doy}_${site}
			cdftxtgifroot=thg_l2_mag_${sitedir}_${fullyear}${month}${day}_v01
			echo ":$$:ucla_rmd2all:	New files: ${newfilename}.ffd and ${newfilename}.ffh." >> $LOGFILE
	
			mkdir -p ${UCB_FLAT_HOME}/${sitedir}/${fullyear}
			mv ${newfilename}.ffd ${UCB_FLAT_HOME}/${sitedir}/${fullyear}/${finalfileroot}.ffd
			mv ${newfilename}.ffh ${UCB_FLAT_HOME}/${sitedir}/${fullyear}/${finalfileroot}.ffh
			echo ":$$:ucla_rmd2all:	New 24 hour flat file = ${finalfileroot}" >> $LOGFILE
	
			# TBD: Update database with merged FF stats 
			# TBD: insert php script that was used to update 
			#      db as part of rsync process
	
			## Make CDF file
			# TBD: Update database with merged CDF stats 
			ucla_flat2cdf.ksh $finalfileroot $gbo_or_epo 1>> $LOGFILE 2>/dev/null

	
			## Make ASCII file
			# TBD: Update database with merged ASCII stats 
			ucla_flat2ascii.ksh $finalfileroot $gbo_or_epo 1>> $LOGFILE 2>/dev/null
	
			## Delete temp dir
			#
			cd ${UCB_FLAT_HOME}
			rm -rf ${UCB_FLAT_HOME}/process_rmd2ff/${tmpdir}

			lastDir=$dir
			
		done
	
		rm -f $dir_file
	
	done

#
## Cleanup
#
	exit 0