;+
;TEMPORARY FILE FOR TESTING. NOT FOR COMMITTING
;based on thm_greenmag_l2gen_new, slightly different method for reading ascii file
;NAME:
; thm_greenmag_l2gen_new2
;
;PURPOSE:
;	Creates Level 2 data CDF file of Ground magnetometer data from
;	various greenland stations. This version handles the files in the new format
;	acquired from: http://flux.phys.uit.no/ascii/ in IAGA-2002 format
;	
;INPUT:	
; filename = An ASCII file corresponding to the data for a given site.
;OUTPUT:
; None explicit, cdf files are created for the data in the given file
;KEYWORDS:
; mastercdf_dir = the directory containing the master cdf file for the
;                 given site. The default is
;                 '$IDL_BASE_DIR/thmsoc/greenland_gmag/mastercdfs/'
; out_dir = the output directory for the cdf files
;
;
;-

function read_tgo_iaga2002, filename

openr,lun,filename,/get_lun
; There are 12 mandatory header lines
header = strarr(12)
readf, lun, header
print, header
; There are an indefinite number of comment lines
; followed by the data type header line
numcommentlines = 0
readcomment = 1
while readcomment do begin
  line=''
  readf, lun, line
  print, line
  if strcmp(line, ' #', 2) then begin
    numcommentlines++
  endif else begin
    if strcmp(line, 'DATE', 4) then begin
      readcomment = 0
    endif else begin
      free_lun, lun
      message, 'Error reading comment and header lines.'
      return, -1 
    endelse
  endelse
endwhile
totalheaderlines = 13+numcommentlines
nrows = file_lines(filename) - totalheaderlines

; the explicit format of the strings is just here for your information 
datastr = {DATE:'YYYY-MM-DD', TIME:'HH:MM:SS.SSS', DOY:0L, D:0.0, H:0.0, Z:0.0, F:0.0}
datatmp = replicate(datastr, nrows)
format = '(a10,1x,a12,1x,i3,4x,f9.2,1x,f9.2,1x,f9.2,1x,f9.2)'
readf, lun, datatmp, format=format
data = {DATE:datatmp.(0),TIME:datatmp.(1), DOY:datatmp.(2), D:datatmp.(3), H:datatmp.(4),F:datatmp.(5)}
free_lun, lun

return, data

end

Pro thm_greenmag_l2gen_new2, filename, mastercdf_dir = mastercdf_dir, $
                        out_dir = out_dir, diagnostics = diagnostics, $
                        _extra = _extra


  If(keyword_set(mastercdf_dir)) Then mdir = mastercdf_dir $
  Else mdir = './mastercdf/'

  If(keyword_set(out_dir)) Then odir = out_dir Else odir = './'

  thm_init

  ;catch errors from incorrectly formatted files
  catch,err
  if err eq 0 then begin
    data = read_tgo_iaga2002(filename)
  endif else begin
    dprint,"Error reading: " + filename
    dprint,"Error: " + !ERROR_STATE.MSG
    return 
  endelse
  catch,/cancel

  if ~is_struct(data) || finite(data.(3)[0],/nan) then begin
    dprint,"Error reading: " + filename
    return
  endif
  
  ;number of data points
  ;last line of new format includes garbage html formatting commands - not true now that I am retrieving unix format
  ;n = n_elements(data.(0))-1
  n = n_elements(data.(0)) 
   
  if n lt 1 then begin
    dprint,"Error not enough lines in input file"
    return
  endif

  ;last line of new format includes garbage html formatting commands - no more
  ;tstr = data.(0)[0:n-1] + '/' + data.(1)[0:n-1]
  tstr = data.(0) + '/' + data.(1)

  t = time_double(tstr)
  date = t[0]
  sec_o_day = t-time_double(data.(0)[0]) ; used for ascii output
  
  b = dblarr(n, 3)
  ;b[*, 0] = data.(4)[0:n-1] ; H
  b[*, 0] = data.(4) ; H
  ;b[*, 1] = data.(4)[0:n-1]*tan(!DTOR*data.(3)[0:n-1]/60)
  b[*, 1] = data.(4)*tan(!DTOR*data.(3)/60)  ;D, according to website documentation data is stored in minutes of arc.  This converts to nT 
                                             ;See: http://www.ngdc.noaa.gov/IAGA/vdat/iagaformat.html 
  ;b[*, 2] = data.(5)[0:n-1] ; Z
  b[*, 2] = data.(5) ; Z

;Get the station and the date for the master cdf file
  ;fff = strsplit(file_basename(filename,'.html'), '_', /extract)
  fff = strsplit(file_basename(filename,'.dat'), '_', /extract)
  ;note website formatting commands added to end of site field,  2 characters
  site = strmid(fff[1],0,3) 
  date_nodash = fff[2]
  mastercdf = mdir+'thg_l2_mag_'+site+'_00000000_v01.cdf'

;read in master CDF file
;-----------------------------------------------------------------------------
  l2_structure = cdf_load_vars(mastercdf, /all) ;create l2 structure for eventual output

;insert data into fit_l2_structure
;-----------------------------------------------------------------------------
  mastertags = l2_structure.vars.name
  tindex = where(mastertags Eq 'thg_mag_'+site+'_time')
  bindex = where(mastertags Eq 'thg_mag_'+site)
  If(tindex[0] Eq -1 Or bindex[0] Eq -1) Then $
    message, 'Variables not found in master CDF?'
  l2_structure.vars[tindex].dataptr = ptr_new(t)
  l2_structure.vars[bindex].dataptr = ptr_new(b)

;range_epoch
;-----------
  range_epoch_vals = [time_string(date), $
                      time_string(time_double(date)+86400.0d0)]
  reval_strs = time_string(range_epoch_vals)
  year = strmid(reval_strs, 0, 4)
  month = strmid(reval_strs, 5, 2)
  day = strmid(reval_strs, 8, 2)
  hour = strmid(reval_strs, 11, 2)
  minu = strmid(reval_strs, 14, 2)
  sec = strmid(reval_strs, 17, 2)
  cdf_epoch, range_epoch_min, year[0], month[0], day[0], $
    hour[0], minu[0], sec[0], /compute_epoch
  cdf_epoch, range_epoch_max, year[1], month[1], day[1], $
    hour[1], minu[1], sec[1], /compute_epoch
  range_epoch_values = [range_epoch_min, range_epoch_max]
  index = where(strjoin('range_epoch') eq mastertags) ; find the range epoch index
  l2_structure.vars[index].dataptr = ptr_new(range_epoch_values) 

;add generation date
;-------------------
  l2_structure.g_attributes.generation_date = systime() 

;add software version
;--------------------
  vno = thm_l2gen_version_info(_extra = _extra)
  If(vno[0] Ne -1) Then Begin
    l2_structure.g_attributes.software_version = $
      strcompress(/remove_all, string(vno))
  Endif

; write fit_l2_structure to L2 CDF file
;--------------------------------------
  out_filename = odir+'thg_l2_mag_'+site+'_'+date_nodash+'_v01.cdf'
  dummy = cdf_save_vars(l2_structure, out_filename)
  print, 'Greenland L2 GMAG CDF file written!   Filename: ', out_filename 

; Write data to ascii file
;-------------------------
	ascii_name = odir+'thg_l2_mag_'+site+'_'+date_nodash+'_v01.txt'
	;stime=time_double(string(yyyy(0))+'-'+string(mm(0))+'-'+string(dd(0)))
	;sec_o_day=tt-stim
	
	write_ascii,[[sec_o_day],[b[*,0]],[b[*,1]],[b[*,2]]],ascii_name,'(f10.3,3(f10.3))'


;diagnostics, first the list of variables in the l2_structure that
;were filled--not filled
;-----------------------------------------------------------------
  If(keyword_set(diagnostics)) Then Begin
    For j = 0, n_elements(l2_structure.vars)-1 Do Begin
      If(ptr_valid(l2_structure.vars[j].dataptr)) Then Begin
        print, l2_structure.vars[j].name, '   Filled'
      Endif Else Begin
        print, l2_structure.vars[j].name, '   *Not* Filled'
      Endelse
    Endfor
  Endif

;clean up
;--------
  unused_ptrs = ptr_extract(l2_structure)
  ptr_free, unused_ptrs
End