;+ ;Procedure: ; mms_get_local_files ; ;Purpose: ; Search for local MMS files in case a list cannot be retrieved from the ; remote server. Returns a sorted list of file paths. ; ;Calling Sequence: ; ; files = mms_get_local_file_info( probe=probe, instrument=instrument, $ ; data_rate=data_rate, level=level, datatype=datatype, trange=trange) ; ;Input: ; probe: (string) Full spacecraft designation, e.g. 'mms1' ; instrument: (string) Instrument designation, e.g. 'hpca' ; data_rate: (string) Data collection mode? e.g. 'srvy' ; level: (string) Data processing level, e.g. 'l1b' ; trange: (string/double) Two element time range, e.g. ['2015-06-22','2015-06-23'] ; datatype: (string) Optional datatype specification, e.g. 'moments' ; ;Output: ; return value: Sorted string array of file paths, if successful; 0 otherwise ; ;Notes: ; -Input strings should not contain wildcards (datatype may be '*') ; ; ;$LastChangedBy: egrimes $ ;$LastChangedDate: 2019-02-12 11:20:23 -0800 (Tue, 12 Feb 2019) $ ;$LastChangedRevision: 26613 $ ;$URL: svn+ssh://thmsvn@ambrosia.ssl.berkeley.edu/repos/spdsoft/tags/spedas_4_0/projects/mms/common/load_data/mms_get_local_files.pro $ ;- function mms_get_local_files, probe = probe, instrument = instrument, data_rate = data_rate, $ level = level, datatype = datatype, trange = trange_in, cdf_version = cdf_version, $ latest_version = latest_version, min_version = min_version, major_version=major_version, mirror = mirror compile_opt idl2, hidden ;return value in case of error error = 0 ;verify all inputs are present if undefined(probe) || $ undefined(instrument) || $ undefined(data_rate) || $ undefined(level) || $ undefined(trange_in) then begin dprint, dlevel=0, 'Missing required input to search for local files' return, error endif trange = time_double(trange_in) ;---------------------------------------------------------------- ;Get list of files by probe and type of data ;---------------------------------------------------------------- ;path & filename separators s = path_sep() f = '_' ;inputs common to all file paths and folder names basic_inputs = [probe, instrument, data_rate, level] ;if datatype is a wildcard then allow any match ;empty strings should be fine if undefined(datatype) || datatype eq '*' then begin dir_datatype = '[^'+s+']+' file_datatype = '[^'+f+']+' endif else begin dir_datatype = datatype file_datatype = datatype endelse ;directory and file name search patterns ; -assume directories are of the form: ; (srvy, SITL): spacecraft/instrument/rate/level[/datatype]/year/month/ ; (brst): spacecraft/instrument/rate/level[/datatype]/year/month/day/ ; -assume file names are of the form: ; spacecraft_instrument_rate_level[_datatype]_YYYYMMDD[hhmmss]_version.cdf if data_rate eq 'brst' then begin dir_pattern = strjoin( basic_inputs, s) + '('+s+dir_datatype+')?' +s+ '[0-9]{4}' +s+ '[0-9]{2}' + s + '[0-9]{2}' + s endif else begin dir_pattern = strjoin( basic_inputs, s) + '('+s+dir_datatype+')?' +s+ '[0-9]{4}' +s+ '[0-9]{2}' + s endelse file_pattern = strjoin( basic_inputs, f) + '('+f+file_datatype+')?' +f+ '([0-9]{8,14})' ;escape backslash in case of Windows search_pattern = escape_string(dir_pattern + file_pattern, list='\') ;get list of all .cdf files in local directory ;all_files = file_search(!mms.local_data_dir,'*.cdf') ; Updated with performance enhancement from Naritoshi Kitamura, 11/17/2015, ; to be more specific on which directory to look into. This can significantly speed up searching for local files if keyword_set(mirror) then begin instr_data_dir = filepath('', ROOT_DIR=!mms.mirror_data_dir, $ SUBDIRECTORY=[probe, instrument, data_rate, level]) endif else begin instr_data_dir = filepath('', ROOT_DIR=!mms.local_data_dir, $ SUBDIRECTORY=[probe, instrument, data_rate, level]) endelse all_files = file_search(instr_data_dir,'*.cdf') ;perform search idx = where( stregex( all_files, search_pattern, /bool, /fold_case), n_files) if n_files eq 0 then begin ; suppress redundant error message ; dprint, dlevel=2, 'No local files found for: '+strjoin(basic_inputs,' ') + ' ' +$ ; (undefined(datatype) ? '':datatype) return, error endif files = all_files[idx] ;---------------------------------------------------------------- ;Restrict list to files within the time range ;---------------------------------------------------------------- ;extract file info from file names ; [file name sans version, data type, time] file_strings = stregex( files, file_pattern, /subexpr, /extract, /fold_case) ;get file start times time_strings = file_strings[2,*] tformat = 'YYYYMMDD' + (strlen(time_strings[0]) gt 8 ? 'hhmmss':'') times = time_double(time_strings, tformat=tformat) ;determine which files are within the requested time range ; TODO: This check is inadequate as it cannot determine if ; files whose (start) times precede the time range will ; intersect it. Possible solutions: ; -ascertain time range of each file by type (unlikely) ; -guess file cadence based on file times within requested ; time range (inconsistent/untrustworthy) ; -sort files by time and always load the file preceding the ; first in-range file, then allow time_clip to remove ; unwanted data (kludgy) time_idx = where( times ge trange[0] and times lt trange[1], n_times) if n_times eq 0 then begin ; suppress redundant error message ;dprint, dlevel=2, 'No local files found between '+time_string(trange[0])+' and '+time_string(trange[1]) return, error endif ;restrict list of files to those in the time range files = files[time_idx] file_strings = file_strings[*,time_idx] ;---------------------------------------------------------------- ;Extract the latest version of each file ;---------------------------------------------------------------- files_out = unh_mms_file_filter(files, /no_time, version=cdf_version, min_version=min_version, latest_version=latest_version, major_version=major_version) if keyword_set(mirror) then begin mirror_dir = !mms.mirror_data_dir local_dir = !mms.local_data_dir ; need to spawn to handle shortcuts in the directories, e.g., ~/mirror_data -> /Users/username/mirror_data spawn, 'echo ' + mirror_dir, mirror_dir spawn, 'echo ' + local_dir, local_dir for fi=0, n_elements(files_out)-1 do begin mirror_file = files_out[fi] ; for windows machines, need forward slashes mirror_file = strjoin(strsplit(mirror_file, '\', /extract), '/') str_replace, mirror_file, mirror_dir, local_dir append_array, local_files, mirror_file ; make the local data directory, if needed directory = file_dirname(local_files[fi]) if file_test(directory,/dir) eq 0 then begin file_mkdir2, directory endif endfor file_copy, files_out, local_files, /overwrite, /allow_same files_out = local_files endif ;ensure files are in chronological order, just in case (see note in mms_load_data) files_out = files_out[bsort(files_out)] return, files_out end