;+
;Procedure: THM_LOAD_EFI
;
;Purpose:  Loads THEMIS EFI data
;
;keywords:
;  probe = Probe name. The default is 'all', i.e., load all available probes.
;          This can be an array of strings, e.g., ['a', 'b'] or a
;          single string delimited by spaces, e.g., 'a b'
;  datatype = The type of data to be loaded, can be an array of strings
;          or single string separate by spaces.  The default is 'all'
;  TRANGE= (Optional) Time range of interest  (2 element array), if
;          this is not set, the default is to prompt the user. Note
;          that if the input time range is not a full day, a full
;          day's data is loaded
;  level = the level of the data, the default is 'l1', or level-1
;          data. A string (e.g., 'l2') or an integer can be used. 'all'
;          can be passed in also, to get all levels.
;  CDF_DATA: named variable in which to return cdf data structure: only works
;          for a single spacecraft and datafile name.
;  VARNAMES: names of variables to load from cdf: default is all.
;  /GET_SUPPORT_DATA: load support_data variables as well as data variables
;                      into tplot variables.
;  /DOWNLOADONLY: download file but don't read it.
;  /valid_names, if set, then this routine will return the valid probe, datatype
;          and/or level options in named variables supplied as
;          arguments to the corresponding keywords.
;  files   named varible for output of pathnames of local files.
;  /VERBOSE  set to output some useful info
;  /no_download: use only files which are online locally.
;  relpathnames_all: named variable in which to return all files that are
;          required for specified timespan, probe, datatype, and level.
;          If present, no files will be downloaded, and no data will be loaded.
;  type: 'calibrated' or 'raw'
;  coord: what coordinate system you would like your data in.
;  TEST: Disables selected /CONTINUE to MESSAGE.  For QA testing only.
;
;Example:
;   thg_load_efi,/get_suppport_data,probe=['a', 'b']
;Notes:
;
;Modifications:
;  Added TEST kw to disable certain /CONTINUE to MESSAGE, and passed TEST
;    through to THM_CAL_EFI.PRO, W.M.Feuerstein, 4/7/2008 (M).
;  Fixed crash on passing an argument for RELPATHNAMES_ALL, WMF, 4/9/2008 (Tu).
;
; $LastChangedBy: jimmpc $
; $LastChangedDate: 2009-05-29 15:10:52 -0700 (Fri, 29 May 2009) $
; $LastChangedRevision: 6004 $
; $URL: svn+ssh://thmsvn@ambrosia.ssl.berkeley.edu/repos/thmsoc/tags/tdas_5_1/idl/themis/spacecraft/fields/thm_load_efi.pro $
;-

pro thm_load_efi,probe=probe, datatype=datatype, trange=trange, $
                 level=level, verbose=verbose, downloadonly=downloadonly, $
                 no_download=no_download,type=type, coord=coord,varformat=varformat,$
                 cdf_data=cdf_data,get_support_data=get_support_data, $
                 varnames=varnames, valid_names = valid_names, files=files, $
                 relpathnames_all = relpathnames_all,$
                 suffix=suffix, progobj=progobj, test=test

  thm_init
; If verbose keyword is defined, override !themis.verbose
  vb = size(verbose, /type) ne 0 ? verbose : !themis.verbose
  if not keyword_set(suffix) then suffix=''
  if arg_present(relpathnames_all) then begin
     downloadonly=1
     no_download=1
  end

  thm_load_proc_arg, sname=probe, datatype=datatype, proc_type=type, $
                     level=level, verbose=verbose, no_download=no_download, $
                     valid_names = valid_names, $
                     vsnames = 'a b c d e', $
                     type_sname = 'probe', $
                     vdatatypes =    'vaf vap vaw vbf vbp vbw eff efp efw eff_0 efp_0 efw_0 eff_dot0 efp_dot0 efw_dot0', $
                     file_vdatatypes='vaf vap vaw vbf vbp vbw eff efp efw eff   efp   efw   eff      efp      efw', $
                     vL2datatypes= 'vaf vap vaw vbf vbp vbw eff efp efw eff_0 efp_0 efw_0 efs efs_0 efs_dot0', $
                     file_vL2datatypes= 'efi', $
                     vtypes='raw calibrated', deftype = 'calibrated', $
                     vlevels = 'l1 l2', $
                     deflevel = 'l1', $
                     osname=probes, odt=dts, olvl=lvls, $
                     oft=fts, ofdt=fdts, otyp = typ, $
                     load_params=!themis,$
                     my_themis=my_themis

  nlvls = n_elements(lvls)
  ndts = n_elements(dts)
  nfts = n_elements(fts)
  nprobes = n_elements(probes)

  if keyword_set(valid_names) then return

  if nlvls*ndts*nfts*nprobes le 0 then return

  if arg_present(cdf_data) && nlvls*ndts*nsnames gt 1 then begin
     print, 'can only get cdf_data for a single datatype'
     return
  endif

;  help, typ ;;;;
  if lvls[0] eq 'l1' then begin
     ;; default action for loading level 1 is to calibrate
     if strmatch(typ, 'calibrated') then begin
        ;; we're calibrating, so make sure we get support data
        if not keyword_set(get_support_data) then begin
           get_support_data = 1
           delete_support_data = 1
        endif
     endif
  endif


;get file names, loop over all snames, levels and datatypes
  for k = 0, nlvls-1 do $
     for j = 0, nfts-1 do $
        for i = 0, nprobes-1 do begin
     probei = probes[i]
     ftj = fts[j]
     lvlk = lvls[k]

     relpath = 'th'+probei+'/'+lvlk+'/'+ ftj+'/'
     prefix = 'th'+probei+'_'+lvlk+'_'+ftj+'_'
     dir = 'YYYY/'
     ending = '_v01.cdf'

     relpathnames = file_dailynames(relpath, prefix, ending, dir=dir, trange = trange,addmaster=addmaster)

     if vb ge 7 then print, 'relpathnames : ', relpathnames

     if arg_present(relpathnames_all) then begin
        if i+j+k eq 0 then relpathnames_all = relpathnames else relpathnames_all = [relpathnames_all, relpathnames]
     endif

     ;;download files for this probe, level, and datatype

     ;; my_themis is a copy of !themis, which may have no_download set
     files = file_retrieve(relpathnames, _extra=my_themis, progobj=progobj)

     if keyword_set(downloadonly) then continue

     if arg_present(cdf_data) then begin
        cdf_data = cdf_load_vars(files,varnames=varnames, verbose=vb,/all)
        return
     endif

     if  ~keyword_set(varformat) then begin
        if keyword_set(get_support_data) then $
           ;varformat='th?_??? th?_???_hed' $
           varformat='th?_??? th?_???_hed th?_???_hed_ac' else varformat='th?_???'
     endif

     if keyword_set(vb) then print,transpose(['Loading...',files])

     cdf2tplot, file=files, all=all, verbose=vb, varformat=varformat, tplotnames=tplotnames

     ;;Reset 'tha_hed_ac' to simulate boundary (for testing only!):
     ;;============================================================
     ;get_data,'th'+probe+'_'+'eff'+'_hed_ac',data=hed_ac
     ;;hed_ac.y[0:115] = 1b
     ;;hed_ac.y[115:224] = 1b
     ;hed_ac.y[115:224] = 255b    ;This line tests hed_ac=255.
     ;store_data,'th'+probe+'_'+'eff'+'_hed_ac',data=hed_ac


     if not keyword_set(suffix) then suffix = ''
     ;; add '_raw' (or whatever) suffix to tplot var name
     ;; and add DLIMIT tags to data quantities
     for l=0, n_elements(tplotnames)-1 do begin
        tplot_var = tplotnames[l]
        get_data, tplot_var, data=d_str, limit=l_str, dlimit=dl_str
        if data_type(dl_str) eq 8 && dl_str.cdf.vatt.var_type eq 'data' then begin
           tplot_var_raw = tplot_var+suffix
           data_att = { data_type:'raw', coord_sys:'efi_sensor',units:'ADC'}
           str_element, dl_str, 'data_att', data_att, /add


           if strmatch( strlowcase( tplot_var), 'th'+probei+'_v??') then begin
                 colors = [ 1, 2, 3, 4, 5, 6]
                 labels = [ 'V1', 'V2', 'V3', 'V4', 'V5', 'V6']
              endif
           if strmatch( strlowcase( tplot_var), 'th'+probei+'_e??') then begin
                 colors = [ 2, 4, 6]
                 labels = [ 'e12', 'e34', 'e56']
              endif

           str_element, dl_str, 'colors', colors, /add
           str_element, dl_str, 'labels', labels, /add
           str_element, dl_str, 'labflag', 1, /add
           str_element, dl_str, 'ytitle', string( tplot_var_raw, 'ADC', format='(A,"!C!C[",A,"]")'), /add


           store_data, delete=tplot_var
;           store_data, tplot_var_raw, data=d_str, limit=l_str, dlimit=dl_str
           store_data, 'veryunusualprefixtempfoo_'+tplot_var_raw, data=d_str, limit=l_str, dlimit=dl_str

           etype=strmid(tplot_var,4)

        endif

     endfor

  endfor                    ;end of loop over all probes, levels and datatypes.

  if keyword_set(typ) && typ eq 'calibrated' && ~arg_present(relpathnames_all) then begin

    thm_cal_efi, probe=probes, datatype=dts, coord=coord,in_suf=suffix, out_suf=suffix, test=test, stored_tnames = stored_tnames

  endif

  ;Don't care about history, so unique:
  ;
  if ~(~size(stored_tnames, /type)) then stored_tnames = stored_tnames[uniq(stored_tnames, sort(stored_tnames))] else stored_tnames = ''

  ;Convert temporary variables (that are requested as permanent) to permanent names (remove prefix):
  ;*************************************************************************************************
  tempnames= tnames('veryunusualprefixtempfoo_*')
  w=where(strlen(dts) eq 3)
  if w[0] ne -1 then primary_dts= dts[w] else primary_dts = ''
  if tempnames[0] ne '' && primary_dts[0] ne '' then begin
    for i= 0, n_elements(tempnames)-1 do begin
      ;
      ;If requested and not produced by THM_CAL_EFI, then rename to the permanent TPLOT variable name, else delete:
      ;************************************************************************************************************
      if (where(strmid(tempnames[i], 2+strlen(suffix), 3, /reverse_offset) eq primary_dts))[0] ne -1 then begin     ;Requested?
        case 1 of
;          tnames(strmid(tempnames[i], 25)) eq '': store_data, tempnames[i], newname= strmid(tempnames[i], 25)
          (where( strmid(tempnames[i], 25) eq stored_tnames ))[0] eq -1: begin
            if tnames(strmid(tempnames[i], 25)) ne '' then store_data, strmid(tempnames[i], 25), /del
            store_data, tempnames[i], newname= strmid(tempnames[i], 25)
          end
          else: store_data, tempnames[i], /delete
        endcase
      endif
    endfor
  endif
  ;
  ;Remove remaining temp variables:
  ;********************************
  ;
  tempnames= tnames('veryunusualprefixtempfoo_*')
  if tempnames[0] ne '' then store_data, tempnames, /delete
      

  if vb ge 8 && arg_present(relpathnames_all) then print, 'relpathnames_all: ', relpathnames_all

  if keyword_set(delete_support_data) then begin
     if not size(dts, /n_dim) gt 0 then dt = strsplit(dt, ' ', /extract)

     for i = 0, n_elements(dt)-1L do begin
        if tnames('th'+probe+'_'+dts[i]+'_hed') ne '' then del_data, 'th'+probe+'_'+dts[i]+'_hed'
     endfor
  endif

end