;+
;Procedure: THM_LOAD_XXX
;
;Purpose:  Generic THEMIS Data File Loading routine, meant to be called by
;   type specific thm_load procedures.
;
;keywords:
;  post_process_proc: name of procedure to call after cdf2tplot is called
;                     will be called w/ keywords sname, dt (datatype), lvl,
;                     and _extra.
;  relpath_funct: name of routine to call in place of file_dailynames
;                 may simply be a wrapper.
;                 will be called w/ keywords sname, dt (datatype), lvl,
;                 and _extra.
;  cdf_to_tplot: user-supplied procedure to override cdf2tplot
;  sname  = site or probe name. The default is 'all',
;  type_sname = string, set to 'probe' or 'site'
;  /all_sites_in_one: set this if all sites are contained in a single file.
;  vsnames = space-separated list of valid probes/sites
;  datatype = The type of data to be loaded, for this case, there is only
;          one option, the default value of 'fgm', so this is a
;          placeholder should there be more that one data type. 'all'
;          can be passed in also, to get all variables.
;  vdatatypes = space-separated list of valid data types
;  file_vdatatypes = space-separated list of file types corresponding to each
;          valid data type.  If there is a one-to-one correpspondence
;          between filetype and datatype, vfiletypes may be left undefined.
;          If all datatypes are in a single file, then vfiltypes may contain 
;          a single name, rather than a list.
;  TRANGE= (Optional) Time range of interest  (2 element array), if
;          this is not set, the default is to prompt the user. Note
;          that if the input time range is not a full day, a full
;          day's data is loaded
;  level = the level of the data, the default is 'l1', or Level 1
;          data. A string (e.g., 'l2') or an integer can be used. 
;  vlevels=A space-separated list of valid levels, e.g. 'l1 l2'
;  vL2datatypes= space-separated list of datatypes valid for L2 data
;  vL2coord= space-separated list of coordinates valid for L2 data 
;  file_vL2datatypes=same as file_vdatatypes, but for L2 data.  Defaults to 
;          value of file_vdatatypes.
;  coord = coordinate system of data to  be loaded.  For L2, may be an array or
;          space-separated list, which will checked against vL2coord.  
;          For L1, no checking: passed on to post_process_proc.
;  CDF_DATA: named variable in which to return cdf data structure: only works
;          for a single spacecraft and datafile name.
;  VARNAMES: names of variables to load from cdf: default is all.
;  /GET_SUPPORT_DATA: load support_data variables as well as data variables
;                      into tplot variables.
;  /DOWNLOADONLY: download file but don't read it.
;  /valid_names, if set, then this routine will return the valid probe, datatype
;          and/or level options in named variables supplied as
;          arguments to the corresponding keywords.
;  files   named varible for output of pathnames of local files.
;  /VERBOSE  set to output some useful info
;  suffix  suffix to add to names of tplot variables loaded from CDF
;
;Notes:
;  This routine is (should be) platform independent.
;
; $LastChangedBy: jimm $
; $LastChangedDate: 2007-07-20 13:04:44 -0700 (Fri, 20 Jul 2007) $
; $LastChangedRevision: 1194 $
; $URL: svn+ssh://thmsvn@ambrosia.ssl.berkeley.edu/repos/thmsoc/tags/tdas_2_02/idl/themis/common/thm_load_xxx.pro $
;-


pro thm_load_xxx,sname=sname, datatype=datatype, trange=trange, $
                 level=level, verbose=verbose, downloadonly=downloadonly, $
                 cdf_data=cdf_data,get_cdf_data=get_cdf_data, $
                 get_support_data=get_support_data, varformat=varformat, $
                 tplotnames=tplotnames, valid_names = valid_names, $
                 files=files, $
                 type_sname=type_sname, $
                 vsnames=vsnames, vdatatypes=vdatatypes, $
                 file_vdatatypes=file_vdatatypes, $
                 vlevels=vlevels, $
                 deflevel=deflevel, version=version, $
                 coord=coord, vL2datatypes=vL2datatypes, vL2coord=vL2coord, $
                 file_vL2datatypes=file_vL2datatypes, $
                 relpath_funct=relpath_funct, $
                 cdf_to_tplot=cdf_to_tplot, $
                 post_process_proc=post_process_proc, $
                 addmaster=addmaster, midfix=midfix, $
                 no_download=no_download, relpathnames_all=relpathnames_all, $
                 all_sites_in_one=all_sites_in_one, suffix=suffix, $
                 progobj = progobj, $ ;jmm, 15-may-2007
                 _ref_extra = _extra

  thm_init
; If verbose keyword is defined, override !themis.verbose
  vb = size(verbose, /type) ne 0 ? verbose : !themis.verbose

; Valid sname names
  vsnames = strsplit(vsnames, ' ', /extract)
; Valid data names
  vdatatypes = strsplit(vdatatypes, ' ', /extract)
;valid data levels
  vlevels = strsplit(vlevels, ' ', /extract)

; parse out data level
  if keyword_set(deflevel) then lvl = deflevel else lvl = 'l1'
  if n_elements(level) gt 0 then begin
    if size(level, /type) Eq 7 then begin
      If(level[0] Ne '') Then lvl = strcompress(strlowcase(level), /remove_all)
    endif else lvl = 'l'+strcompress(string(fix(level)), /remove_all)
  endif
  lvls = thm_check_valid_name(strlowcase(lvl), vlevels)
  if not keyword_set(lvls) then return
  if n_elements(lvls) gt 1 then begin
     message, /info, level = -1, $
              'only one value may be specified for level'
     return
  endif
  if keyword_set(vb) then printdat, lvls, /value, varname='Level'

;valid coordinate systems for L2
  if keyword_set(vL2coord) then $
     vL2coord = strsplit(vL2coord, ' ', /extract)

  if lvls[0] eq 'l2' && keyword_set(vL2datatypes) then $
     vdatatypes = strsplit(vL2datatypes, ' ', /extract)


;If valid_names is set, return the options
  if keyword_set(valid_names) then begin
    sname = vsnames
    message, /info, level = -1, $
             string(strjoin(sname, ','), $
                    format = '( "Valid '+type_sname+'s:",X,A,".")')

    datatype = vdatatypes
    message, /info, level = -1, $
             string(strjoin(datatype, ','), $
                    format = '( "Valid '+lvls[0]+' datatypes:",X,A,".")')

    if lvls[0] eq 'l2' && keyword_set(vL2coord) then begin
       coord = vL2coord
       message, /info, level = -1, $
                string(strjoin(coord, ','), $
                       format = '( "Valid '+lvls[0]+' coords:",X,A,".")')
    endif

    level = vlevels
    message, /info, level = -1, $
             string(strjoin(level, ','), format = '( "Valid levels:",X,A,".")')
    return
  endif
;parse out snames
  if not keyword_set(sname) then snames = vsnames $
  else snames = thm_check_valid_name(strlowcase(sname), vsnames, /include_all)
  if not keyword_set(snames) then return
  if keyword_set(vb) then printdat, snames, /value, varname=type_sname+'s'
;datatype
  if not keyword_set(datatype) then dts = vdatatypes $
  else dts = thm_check_valid_name(strlowcase(datatype), vdatatypes, /include_all)
  if not keyword_set(dts) then return
  if keyword_set(vb) then printdat, dts, /value, varname='Datatypes'
  
; Level2 probe data : set varformat to get each possible combination of 
; datatype and coord
  if lvls[0] eq 'l2' && strlowcase(type_sname) eq 'probe' $
     && ~keyword_set(varformat) && keyword_set(vL2coord) then begin
     if not keyword_set(coord) then crds = vL2coord $
     else crds = thm_check_valid_name(strlowcase(coord), vL2coord, /include_all)
     if not keyword_set(crds) then return
     if keyword_set(vb) then printdat, crds, /value, varname='Coord'
     cross_dt_coord = array_cross(dts, crds)
     varformat = 'th?_' + strjoin(cross_dt_coord, '_')
  endif

  if vb ge 7 then printdat,!themis

  nlvls = n_elements(lvls)
  ndts = n_elements(dts)
  nsnames = n_elements(snames)

  if lvls[0] eq 'l2' && keyword_set(file_vL2datatypes) then $
     file_vdatatypes = file_vL2datatypes

  if keyword_set(file_vdatatypes) then begin
     fts=strarr(ndts)
     file_vdatatypes = strsplit(file_vdatatypes, ' ', /extract)
     
     if n_elements(file_vdatatypes) eq 1 then begin
        ; fill out file_vdatatypes array to same dimension as vdatatypes
        file_vdatatypes = strarr(n_elements(vdatatypes)) + file_vdatatypes[0]
     endif

     for j = 0, ndts-1 do begin
        fts[j] = file_vdatatypes[where(vdatatypes eq dts[j])]
     endfor
     fts = fts[UNIQ(fts, SORT(fts))]
     nfts = n_elements(fts)
     ;; for each filetype, get a list of requested datatypes
     fdts=strarr(nfts)
     for j = 0, nfts-1 do begin
        file_datatypes = vdatatypes[where(file_vdatatypes eq fts[j])]
        fdts[j] = strjoin(strfilter(file_datatypes,dts), ' ')
     endfor
     if vb ge 7 then printdat, fts  ;;;;
     if vb ge 7 then printdat, fdts                 ;;;;
  endif else begin
     fts = dts
     nfts = n_elements(fts)
     fdts = dts
  endelse
     
  if get_cdf_data && nlvls*nfts*nsnames gt 1 then begin
     print, 'can only get cdf_data for a single file type'
     return
  endif

  my_themis = !themis
  my_themis.no_download = 1
  my_themis.verbose=vb

  if keyword_set(all_sites_in_one) then begin
     ;; site name is not included in pathname of data file
     nsnames = 1 ;for ASK
  endif 
;get file names, loop over all snames, levels and datatypes
  files_ptrarr = ptrarr(nsnames,nfts,nlvls)
  for k = 0, nlvls-1 do $
     for j = 0, nfts-1 do $
        for i = 0, nsnames-1 do begin

     snamei = snames[i]
     ftj = fts[j]
     lvlk = lvls[k]

     if keyword_set(relpath_funct) then begin
        ;; call a datatype specific pathname function, because
        ;; we don't want to write one-box-fits-all heuristics 
        relpathnames = call_function(relpath_funct, sname=snamei, $
                                     filetype=ftj, level=lvlk, $
                                     version=version, trange=trange, $
                                     addmaster=addmaster, _extra = _extra)
     endif else begin
        ;; use standard heuristics to determine pathname
        ;; if these don't work, plese consider using your own relpath_funct
        ;; before adding code which may affect the other instrument load routines
        if strcmp(strlowcase(type_sname), 'probe') then begin
           relpath = 'th'+snamei+'/'+lvlk+'/'+ ftj+'/'
           prefix = 'th'+snamei+'_'+lvlk+'_'+ftj+'_'
           dir = 'YYYY/'
        endif else if strcmp(strlowcase(type_sname), 'site') then begin
           relpath = 'thg/'+lvlk+'/'+ftj+'/'+snamei + '/'
           prefix = 'thg_'+lvlk+'_'+ftj+'_'+snamei + '_'
           dir = 'YYYY/'
        endif

        ending = '_'+version+'.cdf'

        relpathnames = file_dailynames(relpath, prefix, ending, dir=dir, $
                                       trange = trange,addmaster=addmaster)
     endelse

; get the full path name, save it for later for reading the cdf.
     files_ptrarr[i,j,k] = $
        ptr_new(file_retrieve(relpathnames, progobj = progobj, _extra = my_themis))

; build an array with all relpathnames, so all files can be downloaded
; with one call to file_retrieve.
     if i+j+k eq 0 then relpathnames_all = relpathnames else $
        relpathnames_all = [relpathnames_all, relpathnames]

     if vb ge 7 then print, 'files', *files_ptrarr[i,j,k]

  endfor ;end of loop over all snames, levels and datatypes.

;download files for all snames, levels, and datatypes

  if ~ !themis.no_download && ~keyword_set(no_download) then begin
     if vb ge 7 then print, 'relpathnames', relpathnames_all
     files = file_retrieve(relpathnames_all, progobj = progobj, _extra=!themis)
  endif 

  if keyword_set(downloadonly) then begin
     ptr_free, files_ptrarr
     return
  endif

  ; load data into tplot variables  loop over all snames, levels and datatypes
  for k = 0, nlvls-1 do $
     for j = 0, nfts-1 do $
        for i = 0, nsnames-1 do begin

     if keyword_set(all_sites_in_one) then begin
        snamei = snames         ; this is for ASK
     endif else snamei = snames[i]
     ftj = fts[j]
     fdtj = fdts[j]
     lvlk = lvls[k]

     files = *files_ptrarr[i,j,k]
     ; note the trailing '*' is necessary to get, for example,
     ; both tha_fgh and tha_fgh_hed, given datatype of fgh    
     star = keyword_set(get_support_data) ? '*' : ''
     fdtj_arr = strsplit(fdtj, ' ',/extract)
     if n_elements(fdtj_arr) eq 1 then fdtj_arr = fdtj_arr[0] ; for ASK
     if keyword_set(varformat) then begin
        varformatj = varformat 
     endif else if strcmp(strlowcase(type_sname), 'probe') then begin
        varformatj = strjoin('*'+fdtj_arr+star, ' ')
     endif else begin
        varformatj = strjoin('*'+fdtj_arr+'_'+snamei+star, ' ')
     endelse  
      
     if vb ge 6 then printdat, varformatj ;;;;

     if get_cdf_data then begin
        if not keyword_set(varformat) then varformat = varformatj
        cdf_data = cdf_load_vars(*files_ptrarr[0,0,0],varnames=varnames, $
                                 verbose=vb,/all, varformat=varformat)
        ptr_free, files_ptrarr
        return
     endif

     if keyword_set(cdf_to_tplot) then begin
        call_procedure, cdf_to_tplot, file=files, $
                        all=all,verbose=vb, tplotnames=tplotnames, $
;                        get_support_data=get_support_data, $
                        varformat=varformatj, midfix=midfix, midpos=4,  $
                        suffix=suffix, varnames=varnames, _extra=_extra
     endif else begin
        if keyword_set(verbose) then print,transpose(['Loading...',files])
        cdf2tplot,file=files,all=all,verbose=vb, $
;                  get_support_data=get_support_data, $
                  tplotnames=tplotnames, suffix=suffix, $
                  midfix=midfix, midpos=4, varformat=varformatj
     endelse

     if keyword_set(post_process_proc) then $
        call_procedure, post_process_proc, sname=snamei, filetype=ftj, $
                        datatype=fdtj, suffix=suffix, coord=coord, $
                        level=lvlk, verbose=vb, tplotnames=tplotnames, $
                        progobj = progobj, _extra = _extra

  endfor ;end of loop over all snames, levels and datatypes.

  ptr_free, files_ptrarr

end