;+ ;NAME: ; thm_mk_data_dir ;PURPOSE: ; Given the root directory for themis data, and a themis data ; filename, create a directory structure. ;CALLING SEQUENCE: ; path = thm_mk_data_dir(time, data_dir=data_dir) ;INPUT: ; time = an input time array, and tplottable format ;OUTPUT: ; path = the string for the data path ;KEYWORDS: ; data_dir= the root directory, the default is '/disks/data/themis' ; name_only= only return the name, don't create directories ;HISTORY: ; 14-nov-2006, jmm, jimm@ssl.berkeley.edu ; 14-may-2007, jmm, Resurrected. No need to care about slashes, and ; IDL 6.3 handles '/' correctly, Also this now takes an ; input time array, and not a filename ;- Function thm_mk_data_dir, time0, data_dir = data_dir, $ name_only = name_only, _extra = _extra otp = -1 slash = '/' nxxx = n_elements(time0) xxx = strmid(time_string(time0), 0, 10) dirs = strarr(nxxx) For j = 0, nxxx-1 Do Begin dj = strsplit(xxx[j], '-', /extract) dirs[j] = dj[0]+'/'+dj[1]+'/'+dj[2] Endfor If(keyword_set(data_dir)) Then thdd = data_dir $ Else thdd = getenv('THEMIS_DATA_DIR') otp = thdd+slash+dirs If(Not keyword_set(name_only)) Then Begin ;build up the directory structure, piece by piece pp = file_search(thdd) If(is_string(pp) Eq 0) Then Begin message, /info, 'Creating: '+thdd file_mkdir, thdd Endif For j = 0, nxxx-1 Do file_mkdir, dirs[j] Endif Return, otp End ;+ ;NAME: ; thm_asi_v2 ;PURPOSE: ; The idea is to create high-resolution ASI data ; CDF files and keogram data. ;PROCEDURE: ;Jim, ;please find attached a "cook-book" how to reprocess the THEMIS ASI data. ;Please read it and let me know if there are any questions. For a test ;you could just pretend that 2007-01-01 needs to be reprocessed because ;data from a new station "fykn" appeared. ;Harald ;The automatic reprocessing of THEMIS-ASI data relates to the two datatypes "ask" and ;"asf". Both datatypes are created from full resolution all-sky images which are ;stored in ;/disks/themisdata/thg/mirrors/asi/hi_res_copy/ ;in subdirectories ;yyyy/mm/dd/ssss_themis??/*/*/ ;where ssss stands for the 20 different THEMIS GBO stations. Whenever new files ;appear below that directory structure the ask- and asf-cdf-files should be ;reprocessed. A test should be performed once a week. ;The reprocessing is all IDL-based. A simple way to make sure that all the required ;programs can be found would be to change the working directory to ;/disks/sprite/disk1/hfrey/idl/themis-gbo/. After the IDL session was started the ;following programs should be compiled: ;@compile_cdaweb ;@compile_IDLmakecdf ;@/disks/sprite/disk1/hfrey/idl/themis-gbo/thg_asi_start_sw ;.r /disks/sprite/disk1/hfrey/idl/themis-gbo/cdf/IDLmakecdf ;.r /disks/sprite/disk1/hfrey/idl/themis-gbo/thm_asi_create_ask_cdf ;.r /disks/sprite/disk1/hfrey/idl/themis-gbo/thm_asi_create_asf_cdf ;There are then two different levels of reprocessing. Whenever new files appear below ;yyyy/mm/dd/ and in any subdirectory below that directory the ask-cdf-file has to be ;reprocessed because it holds data from all potential 20 GBO stations. In IDL the ;following call needs to be issued: ;thm_asi_create_ask_cdf,'yyyy-mm-dd',corners=1 ;where yyyy, mm, and dd are the year, month, and day for which the reprocessing ;should be done. For a normal day this will run for about 20 minutes and a cdf-file ;will be created and moved to the appropriate location in /disks/themisdata/. ;The other level of reprocessing is related to the asf-cdf-files which hold the data ;for one station only. If either a completely new subdirectory appears below ;yyyy/mm/dd/ with the name ssss_themis??/ or when below such an existing directory ;new files appear, then the date and the name of that station have to be found and ;passed to and IDL process ;for hour=0,23 do thm_asi_create_asf_cdf,'yyyy-mm-dd','ssss',hour ;Several cdf-files will be created (one for every hour with data) and moved to their ;appropriate location on /disks/themisdata/. Pro thm_asi_shell, data_start_date = data_start_date, $ search_start_date = search_start_date, $ keograms = keograms, _extra = _extra ;put in a catch here, to allow multiple files to be processed common thm_asi_shell_private, load_position ;-------------------------------- catch, error_status if error_status ne 0 then begin print, 'Error Message:' help, /last_message, output = err_msg For kerr = 0, n_elements(err_msg)-1 Do print, err_msg[kerr] case load_position of 'create_asf': begin print, '***************ASF SKIPPED****************' goto, skip_hour end 'create_ask': begin print, '***************ASK SKIPPED****************' goto, skip_day end else: begin print, 'Entire process bombed, Sorry' return end endcase endif ;-------------------------------- load_position = 'init' set_plot, 'z' ;just in case hr_dir = '/disks/themisdata/thg/mirrors/asi/hi_res_copy/' hr_dir0 = '/disks/themisdata/thg/mirrors/asi/hi_res_copy' ;First search the data directory for new files, do this for the ;last 7 days, unless data_start_date is set one_day = 86400.0d0 txd = systime(/seconds) ;current day is needed txd0 = time_double(strmid(time_string(txd), 0, 10)+'/00:00:00') If(keyword_set(data_start_date)) Then Begin ndays = ceil((txd0-time_double(data_start_date))/one_day) Endif Else ndays = 7 stime70 = txd0-one_day*dindgen(ndays) ;filetimes are in these units days = time_string(stime70) days00 = strmid(days, 0, 10) ;create the day directory for each day since the start. dirs = thm_mk_data_dir(days, /name_only, data_dir = hr_dir0) ;check for files less than 1 week old, unless search_start_date is set If(keyword_set(data_start_date)) Then Begin nsdays = ceil((txd0-time_double(search_start_date))/one_day) Endif Else nsdays = 7 ttest = txd0-one_day*nsdays ;get days and stations to process days_to_process = '' stations_to_process = '' For j = 0, ndays-1 Do Begin ;find all of the files ppp = file_search(dirs[j], '*') temp_stations = '' If(is_string(ppp)) Then Begin ;Find files that have dates finfo = file_info(ppp) ;use modified time, in case there is a file changed ccc = where(finfo.mtime Ge ttest, nccc) ;If there are files, save the day and directory for processing If(nccc Gt 0) Then Begin f0 = ppp[ccc] temp_stations = strarr(nccc) ok_ccc = bytarr(nccc) For k = 0, nccc-1 Do Begin ;the directory structure is predictable, at least after 'hi_res_copy' ttt = strsplit(f0[k], '/', /extract) copyd = where(ttt Eq 'hi_res_copy') If(copyd[0] Ne -1) And (n_elements(ttt) Ge copyd[0]+4) Then Begin temp_stations[k] = strmid(ttt[copyd[0]+4], 0, 4) ok_ccc[k] = 1b Endif Endfor ok = where(ok_ccc) If(ok[0] Ne -1) Then Begin temp_stations = temp_stations[ok] ssu = uniq(temp_stations) temp_stations = temp_stations[ssu] Endif Endif Endif ; If(days00[j] Eq '2007-02-28') Then stop If(is_string(temp_stations)) Then Begin days_to_process = [temporary(days_to_process), $ replicate(days00[j], n_elements(temp_stations))] stations_to_process = [temporary(stations_to_process), $ temporary(temp_stations)] Endif Endfor ;Now strip the first element If(n_elements(stations_to_process) Gt 1) Then Begin stations_to_process = stations_to_process[1:*] days_to_process = days_to_process[1:*] ;Now do the processing np = n_elements(stations_to_process) openw, 1, 'thm_asi_v2_test.otp' For j = 0, np-1 Do Begin printf, 1, days_to_process[j], ' ', stations_to_process[j] Endfor close, 1 For j = 0, np-1 Do Begin load_position = 'create_ask' message, /info, 'Processing: '+days_to_process[j] thm_asi_create_ask_cdf, days_to_process[j], corners = 1 If(Not keyword_set(keograms)) Then Begin For hour = 0, 23 Do Begin load_position = 'create_asf' thm_asi_create_asf_cdf, days_to_process[j], $ stations_to_process[j], hour skip_hour: Endfor Endif skip_day: Endfor message, /info, 'FINISHED SUCCESSFULLY' Endif Else Begin message, /info, 'No Data to Process' message, /info, 'FINISHED SUCCESSFULLY' Endelse Return End