;+
; FUNCTION: FILE_RETRIEVE
; Purpose:
;  FILE_RETRIEVE provides a simple, transportable interface to retrieve data files.
;  It will download files from a remote web server and copy them into a local (cache) directory 
;  maintaining the original directory structure. It returns the list of local file names. 
;  By default files are only downloaded if the remote file is more recent. 
;  This routine is specifically designed to be used with the same file system that the web server is using to serve files.
;  It can look for a MASTER_FILE that indicates the master files system is in use and it then bypasses the download process. 
;  The file system can be a mix of directories that hold the original files being served as well as a copy of files from external servers.
;  The routine correctly handles multiple users sharing (and writing to) the same directories.
;  
;  Works on LINUX, MAC, and Windows 
;  
;Usage:
; files = file_retrieve(pathnames, [keyword options])
;
;Suggested usage:
;
;First get a default structure that specifies where the files come from and where they will be stored locally
;  source = file_retrieve(/default_structure,REMOTE_DATA_DIR='http://sprg.ssl.berkeley.edu/data/',master_file='maven/.master')
;  
;   ; Retrieve a MAVEN mag 1 sec resolution file for 2014-11-18
;  files = file_retrieve( 'maven/data/sci/mag/l2/sav/1sec/2014/11/mvn_mag_l2_pl_1sec_20141118.sav' ,_extra=source)  
;  
;    ; Retrieve an array of filenames within a time range:
;  files = file_retrieve( 'maven/data/sci/mag/l2/sav/1sec/YYYY/MM/mvn_mag_l2_pl_1sec_YYYYMMDD.sav',trange=['2014-12-30','2015-1-3'] ,_extra=source)  
;
;  
;    ; Retrieve "globbed files from the SPDF:    
;    ; A typical URL at SPDF is:  'http://spdf.sci.gsfc.nasa.gov/pub/data/wind/mfi/mfi_h0/2015/wi_h0_mfi_20151008_v03.cdf
;        The source of these files is given by:
;  source = file_retrieve(/default,  REMOTE_DATA_DIR='http://spdf.sci.gsfc.nasa.gov/pub/data/'  , LOCAL_DATA_DIR = root_data_dir()+'istp/'  )
;    ; The path specifiation is given by:  
;  pathname =      'wind/mfi/mfi_h0/YYYY/wi_h0_mfi_YYYYMMDD_v??.cdf'
;  files = file_retrieve(pathname, trange=['2014-12-25','2015-1-4'],_extra=source,/last_version)
;  
;  Subsequent calls will be much faster since the files will have been downloaded.
;  
;
;Arguments:
;    pathnames: String or string array with partial path to the remote file. 
;               (will be appended to remote_data_dir)
;    [newpathnames]: (optional) String or string array with partial path to file destination.
;                   (Will be appended to local_data_dir)  (NOT RECOMMENDED TO USE THIS OPTION.
;
;Keywords:
;    REMOTE_DATA_DIR:  String defining remote data directory
;                      Pathnames will be appended to this variable.
;    LOCAL_DATA_DIR:  String or string array w/ local data directory(s)
;                     If newpathnames is set it will be appended to this variable; if not, 
;                     pathnames will be appended. 
;    MASTER_FILE:     (file pathname)   if the file: LOCAL_DATA_DIR+MASTER_FILE exists then no download or contact with the server is made. (same effect as NO_SERVER - but evaluated at run time)
;    NO_SERVER:       Set this keyword to prevent any contact with a remote server.
;    
;    TRANGE:       One or two element array indicating the time range of interest.  If set, then PATHNAMES will be expanded into an array of pathnames using the special character sequences to translate:
;         YYYY, yy, MM, DD,  hh,  mm, ss, .f, DOY, DOW, TDIFF are special characters that will be substituted with the appropriate date/time field
;         Be especially careful of extensions that begin with '.f' since these will be translated into a fractional second.
;         See "time_string"  TFORMAT keyword for more info.
;         
;    LAST_VERSION:  If set, then only the last of multiple file versions is downloaded and returned. (used in conjuction with "globbed" pathnames and version numbers.)
;    
;    USER_PASS:   Username and password for secured systems;    USER_PASS='username:password'
;    ARCHIVE_EXT:  string;   Set archiving extension. (i.e.:  ARCHIVE_EXT= '.arc'). to rename old files instead of deleting them. Prevents accidental file deletion.
;    ARCHIVE_DIR:  string;   Set archiving subdirectory. (i.e.:  ARCHIVE_DIR = '.archive/')
;
;    VALID_ONLY:  Set this keyword to return only existing files.
;
;    PRESERVE_MTIME: if set, the local file will be given a modification time that is the same as the modification time of the remote server's file modification time.
;       This keyword is ignored on (windows) machines that don't have touch installed. (No cygwin or GNU utils)  Default is 1
;
;    VERBOSE:  Set Verbosity - 0 print almost nothing ,  2 is typical,  4 and above is for debugging.
;
;    if_modified_since:    Set to 0 to force download
;  user_agent:   String - User agent text to be sent to web server.
;  file_mode:   permissions for new files.   Default is '666'o  ; 
;  dir_mode:     permissions for newly created directories.  Default is '777'o  ;
;  progobj:     Experimental option for a progress bar widget.  (please ignore for now)
;  min_age_limit:   Files younger than this age (in seconds) are assumed current (avoids the need to recheck server)  Default is 300
;  no_download:0   ,      $    ; similar to NO_SERVER keyword. Should still allow remote directory retrieval - but not files.
;  no_update:0     ,      $    ; Set to 1 to prevent contact to server if local file already exists. (this is similar to no_clobber)
;  
;History: 
;    2012-6-25:  local_data_dir and remote_data_dir accept array inputs 
;                with the same # of elements as pathnames/newpathnames   -DO NOT USE this option!
;
;$LastChangedBy: davin-mac $
;$LastChangedDate: 2025-10-07 15:45:48 -0700 (Tue, 07 Oct 2025) $
;$LastChangedRevision: 33712 $
;$URL: svn+ssh://thmsvn@ambrosia.ssl.berkeley.edu/repos/spdsoft/trunk/general/misc/file_retrieve.pro $
;-


;  The following is a crude function to determine if the internet is available. 
; returns null string if no server can be reached
function server_available,servers,verbose=verbose
   if ~keyword_set(servers) then servers=['sprg.ssl.berkeley.edu']; ,'google.com','amazon.com','ssl.berkeley.edu']
   for i=0,n_elements(servers)-1 do begin
      server = servers[i]
      port = 80
      unit = 0
      socket, unit, Server,  Port, /get_lun,/swap_if_little_endian,error=error,read_timeout=5,connect_timeout=5
      dprint,dlevel=2,verbose=verbose,server,error  ;,!error_state.msg
      if keyword_set(unit) then free_lun,unit
      if ~keyword_set(error) then return,server
   endfor
   return,''
end


pro file_retrieve_reset_internet,delay=delay
  common file_retrieve_com, no_internet_until,wait_time
  if ~keyword_set(delay) then delay =-1
  no_internet_until = systime(1) + delay
end



function file_retrieve,pathnames, newpathnames, source=source, psource=psource, structure_format=structure_format,default_structure=default_structure,  $
    use_wget=use_wget, nowait=nowait, $
    local_data_dir=local_data_dir,remote_data_dir=remote_data_dir, $
    master_file=master_file,  $
    trange=trange,monthly_res=monthly_res,daily_res=daily_res,hourly_res=hourly_res,resolution=resolution,phase_shift=phase_shift, $
    min_age_limit=min_age_limit , $
    valid_only=valid_only,   $
    file_mode = file_mode,  $   ; permissions for new files. (if non-zero)
    dir_mode = dir_mode,   $   ; permissions for newly created directories.
    recurse_limit=recurse_limit, $
    user_agent=user_agent,   $
    user_pass=user_pass, $
    preserve_mtime=preserve_mtime,  $
    restore_mtime=restore_mtime, $
    ascii_mode=ascii_mode,   $
    strict_html=strict_html, $
    no_download=no_download,no_server=no_server, $
    no_update=no_update, $
    update_after = update_after, $
    if_modified_since=if_modified_since, $
    archive_ext=archive_ext, $
    archive_dir=archive_dir, $
    last_version = last_version , $
    oldversion_dir = oldversion_dir, $
    oldversion_ext = oldversion_ext, $
    force_download=force_download,$
    no_clobber=no_clobber, ignore_filesize=ignore_filesize, $
    verbose=verbose,progress=progress,progobj=progobj,links=links
    
common file_retrieve_com, no_internet_until,wait_time
if ~keyword_set(wait_time) then wait_time = 300
if ~keyword_set(no_internet_until) then no_internet_until = systime(1)-1.


dprint,dlevel=4,verbose=verbose,'Start; $Id: file_retrieve.pro 33712 2025-10-07 22:45:48Z davin-mac $'
if size(/type, local_data_dir)  ne 7 then local_data_dir = root_data_dir()

if keyword_set(structure_format)  && structure_format eq 1 then begin    ; Old version maintained for legacy code   - don't use this any more.
;   swver = strsplit('$Id: file_retrieve.pro 33712 2025-10-07 22:45:48Z davin-mac $',/extract)
;   user_agent =  strjoin(swver[1:3],' ')+' IDL'+!version.release + ' ' + !VERSION.OS + '/' + !VERSION.ARCH+ ' (' + (getenv('USER') ? getenv('USER') : getenv('USERNAME'))+')'
   if n_elements(user_agent) eq 0 then user_agent=''
   str= {   $
      retrieve_struct,       $
      init:0,                $
      local_data_dir:local_data_dir,  $ ;getenv('ROOT_DATA_DIR'),
      remote_data_dir:'',    $
      progress: 1    ,       $    ; Currently unused keyword   (progress is printed by default)
      user_agent:user_agent, $    ; User agent text to be sent to web server.
      file_mode:'666'o  ,    $    ; permissions for new files. (if non-zero)
      dir_mode: '777'o  ,    $    ; permissions for newly created directories.
      preserve_mtime: 1 ,    $    ; Set file modification to same as on file on server  (uses file_touch executable)
      progobj: obj_new(),    $    ; Experimental option for a progress bar widget.  (please ignore for now)
      min_age_limit: 300L  , $    ;   Files younger than this age (in seconds) are assumed current (avoids the need to recheck server)
      no_server:0     ,      $    ; Set to 1 to prevent any contact with a remote server.
      no_download:0   ,      $    ; similar to NO_SERVER keyword. Should still allow remote directory retrieval - but not files.
      no_update:0     ,      $    ; Set to 1 to prevent contact to server if local file already exists. (this is similar to no_clobber)
      no_clobber:0    ,      $    ; Set to 1 to prevent existing files from being overwritten. (A warning message will be displayed if remote server has)
      archive_ext:''  ,      $    ; Set archiving extension. (i.e.:  '.arc'). to rename old files instead of deleting them. Prevents accidental file deletion.
      archive_dir:''  ,      $    ; Set archiving subdirectory. (i.e.:  'archive/') 
      ignore_filesize:0 ,    $    ; Set to 1 to ignore the remote/local file sizes when determining if updates are needed.
      ignore_filedate:0 ,    $    ; Not yet operational.
      downloadonly:0  ,      $    ; Set to 1 to only download files but not load files into memory.
      use_wget:0          ,   $   ; Experimental option (uses the routine SSL_WGET instead of file_http_copy)
      nowait:0        ,      $    ; Used with wget to download files in the background.
      verbose:2 ,             $
      force_download: 0       $   ;Allows download to be forced no matter modification time.  Useful when moving between different repositories(e.g. QA and production data)
   }
   return, str
endif

if keyword_set(default_structure)  then begin     ; pathnames not provided -  return a default source structure
   if not keyword_set(psource) then psource = {   $
     local_data_dir:  local_data_dir,  $
     remote_data_dir:  size(/type,remote_data_dir) eq 7 ? remote_data_dir : '',    $
;    verbose:2 ,             $
     no_server:0    $           ; Set to 1 to prevent any contact with a remote server.
  }
  str_element,/add,psource,'MASTER_FILE',master_file
  str_element,/add,psource,'VERBOSE',verbose
  str_element,/add,psource,'MIN_AGE_LIMIT',min_age_limit
  str_element,/add,psource,'USER_PASS',user_pass
  str_element,/add,psource,'VALID_ONLY',valid_only
  return,psource
endif

if keyword_set(source) then return, file_retrieve(pathnames,newpathnames,_extra=source,links=links)


;if keyword_set(no_download) then no_server = no_download ; Leave this line commented out.  The keyword NO_SERVER is independent of the NO_DOWNLOAD keyword 
;if not keyword_set(local_data_dir) then   local_data_dir = './'
;if not keyword_set(remote_data_dir) then   remote_data_dir = ''
vb = keyword_set(verbose) ? verbose : 0
if n_elements(progress) eq 0 then progress=1

;if keyword_set(progress) then begin
;    progobj = obj_new('progressbar')
;endif


;  This section will generate filenames based on a time range (and time resolution defaults to 1 day)
if keyword_set(trange) then begin
  filenames = ''
  for i=0,n_elements(pathnames)-1 do begin
    pathnames_expanded = time_intervals(trange=trange,monthly_res=monthly_res,daily_res=daily_res,resolution=resolution,phase_shift=phase_shift,tformat=pathnames[i])
    num_pn = n_elements(pathnames_expanded)
    dprint,dlevel=(num_pn gt 1) ? 2 : 3,verbose=verbose,strtrim(num_pn,2)+' Pathnames expanded from "'+pathnames[i]+'" using TRANGE from: '+strjoin( time_string(trange) ,'  to: ')
    fns = file_retrieve(pathnames_expanded,local_data_dir=local_data_dir,remote_data_dir=remote_data_dir, $
      use_wget=use_wget, nowait=nowait, $
      master_file=master_file,  $
      min_age_limit=min_age_limit ,  valid_only=valid_only,   $
      file_mode = file_mode,  dir_mode = dir_mode,   $
      recurse_limit=recurse_limit, $
      user_agent=user_agent,    user_pass=user_pass, $
      preserve_mtime=preserve_mtime,   restore_mtime=restore_mtime, $
      ascii_mode=ascii_mode,     strict_html=strict_html, $
      no_download=no_download,no_server=no_server, $
      no_update=no_update, if_modified_since=if_modified_since, update_after=update_after, $
      archive_ext=archive_ext,  archive_dir=archive_dir, $
      last_version = last_version , $
      oldversion_dir = oldversion_dir,  oldversion_ext = oldversion_ext, $
      force_download=force_download,   no_clobber=no_clobber, ignore_filesize=ignore_filesize, $
      verbose=verbose,progress=progress,progobj=progobj)
    if keyword_set(fns) then append_array, filenames,fns
  endfor
  return,filenames
endif






;fullnames = filepath(root_dir=local_data_dir, pathnames)
fullnames = local_data_dir + pathnames   ; trailing '/' is recommended, but not required on local_data_dir
n0 = n_elements(fullnames)

if keyword_set(use_wget) and total(/preserv,strmatch(pathnames,'*[ \* \? \[ \] ]*') ) ne 0 then begin
     use_wget=0
     dprint,dlevel=1,verbose=verbose,'Warning! WGET can not be used with wildcards!'
endif


if keyword_set(remote_data_dir) &&  ~(keyword_set(no_server) || keyword_set(no_download) || ( (size(/type,master_file) eq 7) && file_test(local_data_dir+master_file)  ) ) then begin
  
  if systime(1) gt no_internet_until then begin

    if keyword_set(use_wget) then $
      ssl_wget,serverdir=remote_data_dir,localdir=local_data_dir,pathname=pathnames,verbose=verbose ,nowait=nowait $
    else begin
      ;Set some defaults that are really essential for proper working of the system: 
      if ~keyword_set(dir_mode) then dir_mode ='777'o
      if ~keyword_set(file_mode) then file_mode ='666'o
      if n_elements(min_age_limit) eq 0 then min_age_limit=300L   ;  Wait a reasonable time (5 minutes) before trying again
      if n_elements(progress) eq 0 then progress = 1  ;    Display progress on file downloads periodically
      if n_elements(preserve_mtime) eq 0 then preserve_mtime=1  ;   Set the local file modification time to the servers modification time
      
      http0 = strmid(remote_data_dir,0,7) eq 'http://'
      If obj_valid(progobj) Then progobj -> update, 0.0, text = string(format="('Retrieving ',i0,' files from ',a)",n0,remote_data_dir) ;jmm, 15-may-2007
      for i = 0l,n0-1 do begin
        fn = fullnames[i]
        pn = pathnames[i]
        npn = keyword_set(newpathnames) ? newpathnames[i] : ''

        ;2012-6-25: these variables may be single value or array   (Who made this change?  - Might not be consistent with other options/ methods!)
        ; error checks should probably be added to check # of elements between local_data_dir and
        ; remote_data_dir (if arrays), pathnames, and newpathnames
        http = n_elements(http0) gt 1 ? http0[i]:http0
        ldd = n_elements(local_data_dir) gt 1 ? local_data_dir[i]:local_data_dir
        rdd = n_elements(remote_data_dir) gt 1 ? remote_data_dir[i]:remote_data_dir

        ;         if keyword_set(no_update) and file_test(fn,/regular) then continue
        if http then begin
          file_http_copy,pn,npn,url_info=url_info,serverdir=rdd,localdir=ldd,verbose=verbose, $
            no_clobber=no_clobber,no_update=no_update, update_after=update_after , $
            ignore_filesize=ignore_filesize,progobj=progobj, progress=progress, $
            no_download = no_download, archive_ext=archive_ext,archive_dir=archive_dir,  $
            ascii_mode=ascii_mode,  recurse_limit=recurse_limit,   if_modified_since=if_modified_since, $
            user_agent=user_agent, user_pass=user_pass, strict_html=strict_html , $
            preserve_mtime = preserve_mtime, restore_mtime=restore_mtime, $
            file_mode=file_mode,dir_mode=dir_mode,last_version=last_version, $
            min_age_limit=min_age_limit,force_download=force_download, $
            error =error,links=links
            
          if keyword_set(error) then begin   
            dprint,dlevel=1,verbose=verbose,'Network Connection Error detected- Will use local copies only. ',error
            if ~server_available() then begin   ; This is a dangerous solution - Can't distiguish between "Remote server down" and "No connection to internet"
              no_internet_until = systime(1) + wait_time  
              dprint,dlevel=0,verbose=verbose,'Disabling checks of server for '+strtrim(wait_time,2)+' seconds'  
            endif 
            break
          endif
          if url_info[0].io_error ne 0 then begin
            dprint, "File or URL i/o error detected.  See !error_state for more info"
            printdat,!error_state
            return,''
          endif
        endif  else begin
          file_copy2,serverdir=remote_data_dir,localdir=local_data_dir,pathname=pn,verbose=verbose,no_clobber=no_update
        endelse
      endfor

    endelse

  endif else begin
    dprint,'No Internet available until '+time_string(no_internet_until,/local)+ ' Unable to check: '+remote_data_dir+pathnames[0]
  endelse
  
endif

; The following bit of code should find the highest version number if globbing is used.

fullnames2 = ''
for i=0,n_elements(fullnames)-1 do begin
   ff = file_search(fullnames[i],count=c)
   case c of
   0:    begin
           dprint,dlevel=3,verbose=vb,'No matching file: "'+fullnames[i]+'"'
           if ~keyword_set(valid_only) then append_array,fullnames2,fullnames[i]
         end
   1:    begin
           dprint,dlevel=3,verbose=vb,'Found: "'+ff[0]+'"'
;           fullnames[i] = ff[0]
           append_array,fullnames2,ff[0]
         end
   else: begin
           if keyword_set(last_version) then begin
;             dprint,dlevel=2,verbose=vb,strtrim(c,2)+' matches found for: "'+fullnames[i]+'"  Using last version.'
             dprint,dlevel=3,verbose=vb,'Using last version of '+strtrim(c,2)+' matches: '+ff[c-1]
             append_array,fullnames2,ff[c-1]
             file_archive,ff[0:c-2],verbose=verbose,archive_dir=oldversion_dir,archive_ext=oldversion_ext
           endif else begin
             dprint,dlevel=2,verbose=vb,strtrim(c,2)+' matches found for: "'+fullnames[i]+'"'
             append_array,fullnames2,ff
;             fullnames = ff
           endelse
         end
   endcase
endfor

   

return,fullnames2
end

