#!/usr/bin/ksh # # ucla_rmd2ff.ksh # # Convert UCLA Ground Mag raw (RMD) data files into Flat File # format. # # RMD->FF->Merged_FF # ## Set the environment variables # . $THMSOC/src/config/ucla_rmd2all.config . $THMSOC/src/config/soc_it_to_me.config ## Check for lock file # if [[ -a /var/tmp/ucla_rmd2ff_lock ]] ; then echo ":$$:ucla_rmd2ff" >> $LOGFILE echo ":$$:ucla_rmd2ff: A ucla_rmd2ff Process is currently active " >> $LOGFILE echo ":$$:ucla_rmd2ff: Quittin..................." >> $LOGFILE exit 0 fi echo $$ > /var/tmp/ucla_rmd2ff_lock echo ":$$:ucla_rmd2ff:" >> $LOGFILE echo ":$$:ucla_rmd2ff:Starting ucla_rmd2all at $(date)" >> $LOGFILE ## Process any RMD directories that were produced as part of the ## rsync-mirror process # for dir_file in $(ls -d $WORKDIR/*_RMDdir*) do echo ":$$:ucla_rmd2ff:" >> $LOGFILE echo ":$$:ucla_rmd2ff: Checking content of file $dir_file" >> $LOGFILE gbo_or_epo=$(echo $dir_file | awk -F/ '{print substr($6,1,3)}') for dir in $(cat $dir_file) do if [[ $gbo_or_epo = "EPO" ]] then filename=$(basename $(ls ${EPO_MIRROR_HOME}/${dir}/*RMD |head -1)) else site=$(echo $dir | awk '{print substr($1,1,4)}') year=$(echo $dir | awk '{print "20" substr($1,5,2)}') month=$(echo $dir | awk '{print substr($1,7,2)}') day=$(echo $dir | awk '{print substr($1,9,2)}') dir="/${year}/${month}/${day}/${site}" filename=$(basename $(ls ${GBO_MIRROR_HOME}/${dir}*RMD |head -1)) fi fileroot=$(echo $filename | awk -F. '{print $1}') site=$(echo $fileroot |awk '{print substr($1,1,4)}') echo ":$$:ucla_rmd2ff:" >> $LOGFILE echo ":$$:ucla_rmd2ff: Looking at $gbo_or_epo files in $dir" >> $LOGFILE ## First: Check if calibration file exists # if [[ ! -e ${TABLE_FILES_DIR}/${site}.txt ]] then # Calibration file does not exist. Go to next dir echo ":$$:ucla_rmd2ff:" >> $LOGFILE echo ":$$:ucla_rmd2ff:WARNING: Cal file ${site}.txt does not exist." >> $LOGFILE echo ":$$:ucla_rmd2ff:.........trying next directory." >> $LOGFILE continue fi ## create a directory name that includes siteyyyymmdd_rmd in ## garage subdir of epo ff storage dir # tmpdir=$(echo $dir | awk -F/ '{print $1$2$3$4}') mkdir -p ${UCB_FLAT_HOME}/process_rmd2ff/${tmpdir} ## Copy all rmd files from RMD dir to temp dir # if [[ $gbo_or_epo = "EPO" ]] then cp -r ${EPO_MIRROR_HOME}/${dir}/*RMD ${UCB_FLAT_HOME}/process_rmd2ff/${tmpdir} else cp -r ${GBO_MIRROR_HOME}/${dir}*RMD ${UCB_FLAT_HOME}/process_rmd2ff/${tmpdir} fi ## Change directories to temp dir and run rmd2ff on each RMD file # cd ${UCB_FLAT_HOME}/process_rmd2ff/${tmpdir} echo ":$$:ucla_rmd2ff:" >> $LOGFILE echo ":$$:ucla_rmd2ff: Running rmd2ff on files in $(pwd)......." >> $LOGFILE for file in $(ls *RMD) do fileroot=$(echo $file | awk -F. '{print $1}') site=$(echo $fileroot |awk '{print substr($1,1,4)}') rm -f $LOGDIR/rmd2ff_${fileroot}.log ##${RMD2FFPATH}/RMD2FF -batch -in_file $file -table ${TABLE_FILES_DIR}/${site}.txt -outfile $fileroot >$LOGDIR/rmd2ff_${fileroot}.log ${RMD2FFPATH}/RMD2FF -batch -in_file $file -table ${TABLE_FILES_DIR}/${site}.txt -outfile $fileroot 1>/dev/null #echo ":$$:ucla_rmd2ff: rmd2ff complete. Check logfile rmd2ff_${fileroot}.log." >> $LOGFILE done echo ":$$:ucla_rmd2ff: rmd2ff complete. ">> $LOGFILE ## Now merge the flat files into one 24 hour file # for file in $(ls *ffd) do fileroot=$(echo $file | awk -F. '{print $1}') echo $fileroot >> filelist.txt done newfilename=$(echo $fileroot |awk '{print substr($1,1,10)}') echo filelist.txt >> ffmerge.input echo "2 3 4" >> ffmerge.input echo x >> ffmerge.input echo first >> ffmerge.input echo last >> ffmerge.input echo $newfilename >> ffmerge.input echo n >> ffmerge.input echo y >> ffmerge.input rm -f $LOGDIR/ffmerge_${newfilename} echo ":$$:ucla_rmd2ff:" >> $LOGFILE echo ":$$:ucla_rmd2ff: Running ffmerge on files ......" >> $LOGFILE #${FFMERGEPATH}/ffmergesungnucmp < ffmerge.input > $LOGDIR/ffmerge_${newfilename} ${FFMERGEPATH}/ffmergesungnucmp < ffmerge.input 1> /dev/null #echo ":$$:ucla_rmd2ff: ffmerge complete. Check log ffmerge_${newfilename}." >> $LOGFILE echo ":$$:ucla_rmd2ff: ffmerge complete. ">> $LOGFILE # Now a single flat file exists for that day in the form # schlyymmdd.ffd, schlyymmdd.ffh and must be changed to # Byydoy_schl # fullyear=$(echo $newfilename |awk '{print "20" substr($1,5,2)}') partyear=$(echo $newfilename |awk '{print substr($1,5,2)}') month=$(echo $newfilename |awk '{print substr($1,7,2)}') day=$(echo $newfilename |awk '{print substr($1,9,2)}') sitedir=$(echo $site | nawk '{print tolower($1)}') doy=$(${KSHSCRIPTS}/date2doy ${fullyear}-${month}-${day} | awk '{printf("%03d",$1)}') finalfileroot=B${partyear}${doy}_${site} cdftxtgifroot=thg_l2_mag_${sitedir}_${fullyear}${month}${day}_v01 echo ":$$:ucla_rmd2ff: New files: ${newfilename}.ffd and ${newfilename}.ffh." >> $LOGFILE mkdir -p ${UCB_FLAT_HOME}/${sitedir}/${fullyear} mv ${newfilename}.ffd ${UCB_FLAT_HOME}/${sitedir}/${fullyear}/${finalfileroot}.ffd mv ${newfilename}.ffh ${UCB_FLAT_HOME}/${sitedir}/${fullyear}/${finalfileroot}.ffh echo ":$$:ucla_rmd2ff: New 24 hour flat file = ${finalfileroot}" >> $LOGFILE # TBD: Update database with merged FF stats # TBD: insert php script that was used to update db as part of rsync process # TBD: delete temp dir cd ${UCB_FLAT_HOME} rm -rf ${UCB_FLAT_HOME}/process_rmd2ff/${tmpdir} done rm -f $dir_file done rm -f /var/tmp/ucla_rmd2ff_lock exit 0