;+
;Procedure: THM_LOAD_XXX
;
;Purpose: Generic THEMIS Data File Loading routine, meant to be called by
; type specific thm_load procedures.
;
;keywords:
; post_process_proc: name of procedure to call after cdf2tplot is called
; will be called w/ keywords sname, dt (datatype), lvl,
; and _extra.
; relpath_funct: name of routine to call in place of file_dailynames
; may simply be a wrapper.
; will be called w/ keywords sname, dt (datatype), lvl,
; and _extra.
; cdf_to_tplot: user-supplied procedure to override cdf2tplot
; sname = site or probe name. The default is 'all',
; type_sname = string, set to 'probe' or 'site'
; /all_sites_in_one: set this if all sites are contained in a single file.
; vsnames = space-separated list of valid probes/sites
; datatype = Can be any datatype from the list of valid datatypes
; or 'all'
; vdatatypes = space-separated list of valid data types
; file_vdatatypes = space-separated list of file types corresponding to each
; valid data type. If there is a one-to-one correpspondence
; between filetype and datatype, vfiletypes may be left undefined.
; If all datatypes are in a single file, then file_vdatatypes may
; contain a single name, rather than a list.
; TRANGE= (Optional) Time range of interest (2 element array), if
; this is not set, the default is to prompt the user. Note
; that if the input time range is not a full day, a full
; day's data is loaded
; level = the level of the data, the default is 'l1', or Level 1
; data. A string (e.g., 'l2') or an integer can be used.
; vlevels=A space-separated list of valid levels, e.g. 'l1 l2'
; proc_type = the type of data, i.e. 'raw' or 'calibrated'. This is
; for validating the 'type' keyword to thm_load procs.
; vtypes =A space-separated list of valid types, e.g. 'raw calibrated'
; vL2datatypes= space-separated list of datatypes valid for L2 data
; vL2coord= space-separated list of coordinates valid for L2 data
; file_vL2datatypes=same as file_vdatatypes, but for L2 data. Defaults to
; value of file_vdatatypes.
; coord = coordinate system of data to be loaded. For L2, may be an array or
; space-separated list, which will checked against vL2coord.
; For L1, no checking: passed on to post_process_proc.
; CDF_DATA: named variable in which to return cdf data structure: only works
; for a single spacecraft and datafile name.
; VARNAMES: names of variables to load from cdf: default is all.
; /GET_SUPPORT_DATA: load support_data variables as well as data variables
; into tplot variables.
; /DOWNLOADONLY: download file but don't read it.
; /NO_UPDATE: prevent contact to server if local file already exists.
; /valid_names, if set, then this routine will return the valid probe, datatype
; and/or level options in named variables supplied as
; arguments to the corresponding keywords.
; files named varible for output of pathnames of local files.
; /VERBOSE set to output some useful info
; suffix suffix to add to names of tplot variables loaded from CDF
; Note that the suffix is *not* applied to support_data.
; SCM_CAL: structure that contains calibration paramters
; msg_out: A named variable to output any useful error messages,
; messages can be printed to the console later when they
; will be more visible to the user.
;
;Notes:
; This routine is (should be) platform independent.
;
; $LastChangedBy: pcruce $
; $LastChangedDate: 2014-01-08 16:26:27 -0800 (Wed, 08 Jan 2014) $
; $LastChangedRevision: 13823 $
; $URL: svn+ssh://thmsvn@ambrosia.ssl.berkeley.edu/repos/spdsoft/tags/spedas_1_00/projects/themis/common/thm_load_xxx.pro $
;-
pro thm_load_xxx, sname = sname, datatype = datatype, trange = trange, $
level = level, coord = coord, proc_type = type, version = version, $
verbose = verbose, downloadonly = downloadonly, $
cdf_data = cdf_data, get_cdf_data = get_cdf_data, $
get_support_data = get_support_data, varformat = varformat, $
tplotnames = tplotnames, valid_names = valid_names, $
files = files, $
type_sname = type_sname, $
vsnames = vsnames, vdatatypes = vdatatypes, $
file_vdatatypes = file_vdatatypes, $
vtypes = vtypes, $
vlevels = vlevels, deflevel = deflevel, $
vL2datatypes = vL2datatypes, vL2coord = vL2coord, $
file_vL2datatypes = file_vL2datatypes, $
relpath_funct = relpath_funct, $
cdf_to_tplot = cdf_to_tplot, $
post_process_proc = post_process_proc, $
addmaster = addmaster, midfix = midfix, $
no_download = no_download, relpathnames_all = relpathnames_all, $
all_sites_in_one = all_sites_in_one, suffix = suffix, $
progobj = progobj, $ ;jmm, 15-may-2007
scm_cal = scm_cal, $
no_update = no_update, $
alternate_load_params=alternate_load_params, $
no_time_clip = no_time_clip, $
no_implicit_wildcard=no_implicit_wildcard, $
use_eclipse_corrections=use_eclipse_corrections, $
msg_out=msg_out, $
_ref_extra = _extra
compile_opt idl2
if keyword_set(alternate_load_params) then begin
dprint, 'Using alternate load parameters'
load_params=alternate_load_params
endif else begin
thm_init
load_params=!themis
endelse
thm_load_proc_arg, sname = sname, datatype = datatype, $
level = level, coord = coord, proc_type = type, $
verbose = verbose, $
varformat = varformat, valid_names = valid_names, $
type_sname = type_sname, $
vsnames = vsnames, vdatatypes = vdatatypes, $
file_vdatatypes = file_vdatatypes, $
vtypes = vtypes, $
vlevels = vlevels, deflevel = deflevel, $
vL2datatypes = vL2datatypes, vL2coord = vL2coord, $
file_vL2datatypes = file_vL2datatypes, $
no_download = no_download, $
progobj = progobj, $
osname = snames, odt = dts, olvl = lvls, my_themis = my_themis, $
oft = fts, ofdt = fdts, $
load_params=load_params, $
use_eclipse_corrections=use_eclipse_corrections, $
no_update = no_update, $
msg_out = msg_out
vb = size(verbose, /type) ne 0 ? verbose : load_params.verbose
nlvls = n_elements(lvls)
ndts = n_elements(dts)
nfts = n_elements(fts)
nsnames = n_elements(snames)
if nlvls*ndts*nfts*nsnames le 0 then return
if keyword_set(valid_names) then return
if get_cdf_data && nlvls*nfts*nsnames gt 1 then begin
dprint, 'can only get cdf_data for a single file type'
return
endif
if keyword_set(all_sites_in_one) then begin
;; site name is not included in pathname of data file
nsnames = 1 ;for ASK
endif
;get file names, loop over all snames, levels and datatypes
files_ptrarr = ptrarr(nsnames, nfts, nlvls)
for k = 0, nlvls-1 do $
for j = 0, nfts-1 do $
for i = 0, nsnames-1 do begin
snamei = snames[i]
ftj = fts[j]
lvlk = lvls[k]
if keyword_set(relpath_funct) then begin
;; call a datatype specific pathname function, because
;; we don't want to write one-box-fits-all heuristics
relpathnames = call_function(relpath_funct, sname = snamei, $
filetype = ftj, level = lvlk, $
version = version, trange = trange, $
addmaster = addmaster, _extra = _extra)
endif else begin
;; use standard heuristics to determine pathname
;; if these don't work, please consider using your own relpath_funct
;; before adding code which may affect the other instrument load routines
if strcmp(strlowcase(type_sname), 'probe') then begin
relpath = 'th'+snamei+'/'+lvlk+'/'+ ftj+'/'
prefix = 'th'+snamei+'_'+lvlk+'_'+ftj+'_'
dir = 'YYYY/'
endif else if strcmp(strlowcase(type_sname), 'site') then begin
relpath = 'thg/'+lvlk+'/'+ftj+'/'+snamei + '/'
prefix = 'thg_'+lvlk+'_'+ftj+'_'+snamei + '_'
dir = 'YYYY/'
endif
If(version Eq '') Then Begin
ending = '.cdf'
Endif Else ending = '_'+version+'.cdf'
relpathnames = file_dailynames(relpath, prefix, ending, dir = dir, $
trange = trange, addmaster = addmaster)
endelse
;get the full path name, save it for later for reading the cdf.
my_themis.no_download = 1
files_ptrarr[i, j, k] = $
ptr_new(file_retrieve(relpathnames, _extra = my_themis))
;build an array with all relpathnames, so all files can be downloaded
;with one call to file_retrieve.
if i+j+k eq 0 then relpathnames_all = relpathnames else $
relpathnames_all = [relpathnames_all, relpathnames]
if vb ge 7 then dprint, 'files', *files_ptrarr[i, j, k]
endfor ;end of loop over all snames, levels and datatypes.
;download files for all snames, levels, and datatypes
if ~load_params.no_download && ~keyword_set(no_download) then begin
if vb ge 7 then dprint, 'relpathnames', relpathnames_all
my_themis.no_download = 0
files = file_retrieve(relpathnames_all, _extra = my_themis)
endif
if keyword_set(downloadonly) then begin
ptr_free, files_ptrarr
return
endif
;load data into tplot variables loop over all snames, levels and datatypes
for k = 0, nlvls-1 do $
for j = 0, nfts-1 do $
for i = 0, nsnames-1 do begin
if keyword_set(all_sites_in_one) then begin
snamei = snames ; this is for ASK
endif else snamei = snames[i]
ftj = fts[j]
fdtj = fdts[j]
lvlk = lvls[k]
files = file_search(*files_ptrarr[i, j, k], count = filecount)
If(filecount Eq 0) Then Begin
dprint, 'Files Not Found:'
If(ptr_valid(files_ptrarr[i, j, k])) Then Begin
dprint, *files_ptrarr[i, j, k]
Endif Else dprint, 'Invalid files pointer'
Continue ;no file found, go to the next
Endif
; note the trailing '*' is necessary to get, for example,
; both tha_fgh and tha_fgh_hed, given datatype of fgh
star = keyword_set(get_support_data) ? '*' : ''
; 2010-01-26 JWL: For thm_load_state, we need to disable the implied
; trailing wildcard that enables support variables like tha_fgh_hed
; to be autoloaded, even if only "fgh" was supplied as an input
; datatype to thm_load_fgm.
;
; But if thm_load_state requests datatype "pos", we don't necessarily
; want to load "pos_gse" and "pos_gsm" even if /get_support_data
; is specified. So in thm_load_state, we'll call thm_load_xxx
; with /no_implicit_wildcard, and the other load routines will
; not be affected.
if keyword_set(no_implicit_wildcard) then begin
star = ''
endif
fdtj_arr = strsplit(fdtj, ' ', /extract)
if n_elements(fdtj_arr) eq 1 then fdtj_arr = fdtj_arr[0] ; for ASK
if keyword_set(varformat) then begin
varformatj = varformat
endif else if strcmp(strlowcase(type_sname), 'probe') then begin
varformatj = strjoin('*'+fdtj_arr+star, ' ')
endif else begin
varformatj = strjoin('*'+fdtj_arr+'_'+snamei+star, ' ')
endelse
if vb ge 6 then printdat, varformatj ;;;;
if get_cdf_data then begin
if not keyword_set(varformat) then varformat = varformatj
cdf_data = cdf_load_vars(*files_ptrarr[0, 0, 0], varnames = varnames, $
verbose = vb, /all, varformat = varformat)
ptr_free, files_ptrarr
return
endif
if keyword_set(cdf_to_tplot) then begin
call_procedure, cdf_to_tplot, file = files, $
all = all, verbose = vb, tplotnames = tplotnames, $
varformat = varformatj, midfix = midfix, midpos = 4, $
suffix = suffix, varnames = varnames, _extra = _extra
endif else begin
if keyword_set(verbose) then dprint, transpose(['Loading...', files])
cdf2tplot, file = files, all = all, verbose = vb, $
tplotnames = tplotnames, suffix = suffix, $
midfix = midfix, midpos = 4, varformat = varformatj
endelse
tn_pre_proc = tnames()
; clip data to requested trange, if trange exists, jmm, 2009-08-11
If (keyword_set(trange) && n_elements(trange) Eq 2) $
Then tr = timerange(trange) $
else tr = timerange()
;explicit check for existence of tplotnames to avoid later crash, jmm, 27-aug-2009
If(is_string(tplotnames)) Then Begin
for ivar = 0, n_elements(tplotnames)-1 do begin ;change index variable to ivar, jmm, 28-jul-2009
if tnames(tplotnames[ivar]) eq '' then continue
; check special case from efi loading
if strmid(tplotnames[ivar], 16, 17, /reverse_offset) eq '_thm_cal_efi_priv' then continue
; check special case from state loading
if strmid(tplotnames[ivar], 10, 11, /reverse_offset) eq '_state_temp' then continue
; check for scm data, skip cut until after calibration
if strmid(tplotnames[ivar], 3, 3) eq '_sc' then continue
if ~keyword_set(no_time_clip) then begin
time_clip, tplotnames[ivar], min(tr), max(tr), /replace, error = tr_err
if tr_err then del_data, tplotnames[ivar]
endif
endfor
tn_post_clip = tnames()
; make ssl_set_intersection doesn't get scalar inputs
if n_elements(tn_pre_proc) eq 1 then tn_pre_proc = [tn_pre_proc]
if n_elements(tn_post_clip) eq 1 then tn_post_clip = [tn_post_clip]
if n_elements(tplotnames) eq 1 then tplotnames = [tplotnames]
tn_for_cal = ssl_set_intersection(tplotnames, tn_post_clip)
if keyword_set(post_process_proc) then begin
if keyword_set(scm_cal) then Begin
call_procedure, post_process_proc, sname = snamei, filetype = ftj, $
datatype = fdtj, suffix = suffix, coord = coord, $
level = lvlk, verbose = vb, tplotnames = tplotnames, $
progobj = progobj, proc_type = type, trange = trange, $
scm_cal = scm_cal, get_support_data = get_support_data, $
use_eclipse_corrections = use_eclipse_corrections, _extra = _extra
endif else Begin
if ~array_equal(tn_for_cal, -1L,/no_type) then $
call_procedure, post_process_proc, sname = snamei, filetype = ftj, $
datatype = fdtj, suffix = suffix, coord = coord, $
level = lvlk, verbose = vb, tplotnames = tn_for_cal, $
progobj = progobj, files = files, proc_type = type, $
get_support_data = get_support_data, trange = trange, $
use_eclipse_corrections = use_eclipse_corrections, _extra = _extra
endelse
endif
; make sure tplot_vars created in post_procs get added to list
tn_post_proc = tnames()
if n_elements(tn_post_proc) eq 1 then tn_post_proc = [tn_post_proc]
post_proc_names = ssl_set_complement(tn_pre_proc, tn_post_proc)
if size(post_proc_names, /type) eq 7 then tplotnames = [tplotnames, post_proc_names]
; clip data to requested trange, if trange exists, jmm, 2009-08-11
for ivar = 0, n_elements(tplotnames)-1 do begin
if tnames(tplotnames[ivar]) eq '' then continue
; check special case from efi loading
if strmid(tplotnames[ivar], 16, 17, /reverse_offset) eq '_thm_cal_efi_priv' then continue
; check special case from state loading
if strmid(tplotnames[ivar], 10, 11, /reverse_offset) eq '_state_temp' then continue
if ~keyword_set(no_time_clip) then begin
time_clip, tplotnames[ivar], min(tr), max(tr), /replace, error = tr_err
if tr_err then del_data, tplotnames[ivar]
endif
endfor
Endif ;end for If block checking for tplotnames
endfor ;end of loop over all snames, levels and datatypes.
ptr_free, files_ptrarr
end