;+ ; PROCEDURE: ; kgy_file_retrieve ; PURPOSE: ; Retrieve data and label files ; CALLING SEQUENCE: ; files = kgy_file_retrieve('pbf/LEVEL1_VER1_DB/YYYYMMDD/PBF1_C_YYYYMMDD_*_I.DAT*',/daily_names,/valid_only,/last_version) ; INPUTS: ; PUBLIC: use publicly available data ; pathnames: String or string array with partial path to the remote file. ; (will be appended to local_data_dir) ; KEYWORDS: ; DAILY_NAMES : resolution (in days) for generating file names. ; YYYY, yy, MM, DD, hh, mm, ss, .f, DOY, DOW, TDIFF are special characters that will be substituted with the appropriate date/time field ; Be especially careful of extensions that begin with '.f' since these will be translated into a fractional second. ; See "time_string" TFORMAT keyword for more info. ; TRANGE : two element vector containing start and end times (UNIX_TIME or UT string). if not present then timerange() is called to obtain the limits. ; SOURCE: alternate file source. Default is whatever is return by the function: ppi_file_source() (see "ppi_file_source" for more info) ; FILES: if provided these will be passed through as output. ; VALID_ONLY: Set to 1 to prevent non existent files from being returned. ; CREATE_DIR: Generates a filename and creates the directories needed to create the file without errors. Will not check for file on remote server. ; ; KEYWORDS Passed on to "FILE_RETRIEVE": ; LAST_VERSION : [0,1] if set then only the last matching file is returned. (Default is 1) ; VALID_ONLY: [0,1] If set then only existing files are returned. (Default is defined by source keyword) ; VERBOSE: set verbosity level (2 is typical) ; LIMITATIONS: ; Beware of file pathnames that include the character sequences: YY, MM, DD, hh, mm, ss, .f since these can be retranslated to the time ; CREATED BY: ; Yuki Harada on 2015-07-05 ; Mostly copied from 'mvn_pfp_file_retrieve' ; ; $LastChangedBy: haraday $ ; $LastChangedDate: 2017-11-21 12:02:46 -0800 (Tue, 21 Nov 2017) $ ; $LastChangedRevision: 24333 $ ; $URL: svn+ssh://thmsvn@ambrosia.ssl.berkeley.edu/repos/spdsoft/tags/spedas_4_0/projects/kaguya/general/kgy_file_retrieve.pro $ ;- function kgy_file_retrieve,pathname,trange=trange,verbose=verbose,source=src,files=files,last_version=last_version,valid_only=valid_only,no_update=no_update,create_dir=create_dir,daily_names=daily_names,hourly_names=hourly_names,resolution=res,shiftres=shiftres,no_server=no_server,no_download=no_download,_extra=_extra,public=public,skipoddhours=skipoddhours, datasuf=datasuf tstart = systime(1) if keyword_set(public) and size(daily_names,/type) eq 0 then daily_names = 1 if not keyword_set(shiftres) then shiftres =0 if keyword_set(daily_names) then begin res = round(24*3600L * daily_names) sres= round(24*3600L * shiftres) endif if keyword_set(hourly_names) then begin res = round(3600L * hourly_names) sres= round(3600L * shiftres) endif source = kgy_file_source(src,verbose=verbose,public=public,no_server=no_server,valid_only=valid_only,last_version=last_version,no_update=no_update,_extra=_extra) dprint,dlevel=5,verbose=verbose,phelp=1,source ; display the options if ~keyword_set(files) then begin if keyword_set(res) then begin tr = timerange(trange) str = (tr-sres)/res dtr = (ceil(str[1]) - floor(str[0]) ) > 1 ; must have at least one file times = res * (floor(str[0]) + lindgen(dtr))+sres pathnames = time_string(times,tformat=pathname) pathnames = pathnames[uniq(pathnames)] ; Remove duplicate filenames - assumes they are sorted if keyword_set(skipoddhours) then begin hh = long(time_string(times,tf='hh')) w = where( ~(hh mod 2) , nw ) ;- only even hours if nw gt 0 then pathnames = pathnames[w] endif ;;; obsolete ;;; get public data from the Kaguya data archive ;; if keyword_set(public) then begin ;; if ~keyword_set(datasuf) then datasuf = ['.dat','.gz','','.cdf'] ;; suf = '.dat' ;; dirs = source.local_data_dir+time_string(times,tf='YYYY/MM/') ;; file_mkdir2,dirs,_extra=source ;; for i=0,n_elements(pathnames)-1 do begin ;; f = '' ;; for isuf=0,n_elements(datasuf)-1 do f = [f,file_search(dirs[i]+pathnames[i]+datasuf[isuf])] ;- search existing data files ;; if total(strlen(f)) eq 0 then begin ;- don't check updates, TBD ;; outtar = dirs[i]+'out.tar' ;; if file_test(outtar) then file_delete,outtar ;- delete old temp file, if exists ;; ;;; download files using IDLnetURL (IDL 6.4+) ;; ourl = OBJ_NEW('IDLnetUrl') ;; proxy = getenv('http_proxy') ;; if proxy ne '' then begin ;; pp = parse_url(proxy) ;; ourl->setproperty,proxy_hostname=pp.host,proxy_prot=pp.port ;; endif ;; s = execute('out = ourl->get(filename=outtar,url=source.remote_data_dir+pathnames[i])') ;; if s then dprint,'downloaded '+out+' for '+pathnames[i] ;; obj_destroy,ourl ;; ;;; untar files ;; if file_test(outtar) then begin ;; if tag_exist(source,'file_mode') then file_chmod,outtar,source.file_mode ;; if float(!version.release) ge 8.3 then begin ;; dprint,'file_untar '+outtar ;; s = execute('file_untar,outtar,dirs[i]') ;; endif else begin ;; untarcmd = 'tar xvf '+outtar+' -C '+dirs[i] ;- doesn't work in Windows? ;; dprint,'untar cmd: '+untarcmd ;; s = execute('spawn,untarcmd') ;; endelse ;; outfiles = file_search(dirs[i]+pathnames[i]+'*') ;; if ~s then dprint,'untar failed: '+outtar ;; if s then if tag_exist(source,'file_mode') and total(file_test(outfiles)) then file_chmod,outfiles,source.file_mode ;; file_delete,outtar ;; endif ;; endif ;; for isuf=0,n_elements(datasuf)-1 do begin ;; f = file_search(dirs[i]+pathnames[i]+datasuf[isuf]) ;; if total(strlen(f)) gt 0 then suf = datasuf[isuf] ;; endfor ;; endfor ;; pathnames = time_string(times,tf='YYYY/MM/') + pathnames + suf ;; source.no_server = 1 ;; endif endif else pathnames = pathname if keyword_set(create_dir) then begin files = source.local_data_dir + pathnames file_mkdir2,file_dirname( files ),_extra=source return,files endif ; files = file_retrieve(pathnames,_extra=source) ;- obsolete files = spd_download( remote_file=pathnames,_extra=source ) dprint,dlevel=4,verbose=verbose,systime(1)-tstart,' seconds to retrieve ',n_elements(files),' files' endif return,files end