; (c) brian jackel 2006
; themis_imager_readfile.pro (c) 2006 Brian Jackel
; renamed thm_asi_imager_readfile ; hfrey 02/05/2007
;
; This program may be freely distributed. Use at your own risk.
; Bug reports are welcome, ideally with the name of an example
; data file that causes the problem.
;+
;
; NAME: thm_asi_Imager_Readfile
;
; PURPOSE: This is intended to be a general tool for reading THEMIS-GBO imager data.
; At the moment it only deals with PGM files and has only been lightly
; tested on full (256x256) and row2 (376x1) frames.
;
; CATEGORY: Themis, Imager, File reading
;
; CALLING SEQUENCE: THM_ASI_IMAGER_READFILE,Filename,Images,[Metadata]
;
; INPUTS: Filename a string or array of strings containing valid image filenames
; or wildcard search expressions as used in "findfile".
;
; KEYWORDS: COUNT returns the number of image frames
; ALL_METADATA set to obtain more metadata, much slower (default=0)
; DEBUG set to increase verbosity (default=0)
;
; OUTPUTS: Images a WIDTH x HEIGHT x NFRAMES array of unsigned integers or bytes
; Metadata a NFRAMES element array of structures
;
; RESTRICTIONS: TBD
;
; EXAMPLES:
;
; Get one file, watch frames as movie
;
; filename="20051225_1100_whit_themis07_full.pgm.gz"
; THM_ASI_IMAGER_READFILE,filename,images,metadata,COUNT=nframes
; FOR indx=0,nframes-1 DO TVSCL,images[*,*,indx]
;
;
; Get 1 hour of data, summarize and display as keogram
;
; directory="\\themis-data\data\themis\imager\stream0\2005\12\25\whit_themis07\ut11\"
; THM_ASI_IMAGER_READFILE,directory+"*full.*",images,metadata,COUNT=nframes,/DEBUG
; keogram= TRANSPOSE(TOTAL(images[96:159,*,*],1))
; TVSCL,keogram,ORDER=1
;
;
; NOTES:
;
; PGM format is described on NetPBM home page at http://netpbm.sourceforge.net
;
; gzipped files (*.pgm.gz) can be read directly in IDL, bzip2 cannot
;
; early data (eg. ATHA 2004) were in PNG format with 1 frame per file
;
;
; MODIFICATION HISTORY:
; 2006-03-09 Bjj assembly and preliminary documentation
;
; VERSION:
; $LastChangedBy$
; $LastChangedDate$
; $LastChangedRevision$
; $URL$
;-
PRO THEMIS_IMAGER_METADATA__DEFINE
COMPILE_OPT HIDDEN
dummy = {themis_imager_metadata $
,site_uid:'' $
,imager_uid:'' $
,exposure_time_string:'' $
,exposure_time_cdf:0.0d0 $
,exposure_time_offset:0.0 $
,exposure_duration_request:0 $
,exposure_duration_actual:0.0 $
,ccd_offset:[0,0] $
,ccd_size:[0,0] $
,ccd_binning:[0,0] $
,comments:'' $
}
END
;TO DO -add more I/O error handling
;
FUNCTION THEMIS_IMAGER_PNM_READFILE,lun,image,comments,DEBUG=debug
COMPILE_OPT HIDDEN
magicnumber= BYTARR(2)
READU,lun,magicnumber
CASE STRING(magicnumber) OF
'P1':type=1
'P2':type=2
'P3':type=3
'P4':type=4
'P5':type=5
'P6':type=6
ELSE: BEGIN
dprint,'Error- unrecognized filetype, magic number is: '+STRING(magicnumber)
RETURN,1
END
ENDCASE
line='' & comments='' & width=0 & height=0 & maxval=-1L & nvalues=0 & values=LONARR(3)
REPEAT BEGIN
READF,lun,line
poundpos= STRPOS(line,'#')
IF (poundpos EQ -1) THEN comment='' ELSE IF (poundpos EQ 0) THEN BEGIN
comment=line & line=''
ENDIF ELSE BEGIN
comment= STRMID(line,pos,999)
line= STRMID(line,0,pos)
ENDELSE
IF (STRLEN(comment) GT 0) THEN comments= comments + comment + STRING(13b)
IF (debug GE 2) THEN dprint, ' -',comment
line= STRCOMPRESS(STRTRIM(line,2))
IF (STRLEN(line) GT 0) THEN BEGIN
vals= STRSPLIT(line,' ',/EXTRACT)
nvals= N_ELEMENTS(vals)
values[nvalues]= vals[0:(nvals-1)<2]
nvalues= nvalues + nvals
ENDIF
ENDREP UNTIL (nvalues GE 3) OR EOF(lun)
IF EOF(lun) AND (nvalues NE 3) THEN BEGIN
dprint,'Error- end of file while reading header'
RETURN,1
ENDIF
width= values[0] & height= values[1] & maxvalue= values[2]
CASE type OF
2: BEGIN
IF (maxvalue GT 255) THEN image=UINTARR(width,height,/NOZERO) ELSE image=BYTARR(width,height,/NOZERO)
READF,lun,image
IF (height EQ 1) THEN image= REFORM(image,width,height)
END
3: BEGIN
IF (maxvalue GT 255) THEN image=UINTARR(3,width,height) ELSE image=BYTARR(3,width,height,/NOZERO)
READF,lun,image
IF (height EQ 1) THEN image= REFORM(image,3,width,height)
END
5: BEGIN
IF (maxvalue GT 255) THEN image=UINTARR(width,height,/NOZERO) ELSE image=BYTARR(width,height,/NOZERO)
READU,lun,image
BYTEORDER,image,/HTONS
IF (height EQ 1) THEN image= REFORM(image,width,height)
END
6: BEGIN
IF (maxvalue GT 255) THEN image=UINTARR(3,width,height,/NOZERO) ELSE image=BYTARR(3,width,height,/NOZERO)
READU,lun,image
BYTEORDER,image,/HTONS
IF (height EQ 1) THEN image= REFORM(image,3,width,height)
END
ELSE:MESSAGE,'Sorry, PNM image type '+STRING(type)+' not yet implemented'
ENDCASE
RETURN,0
END
; Digging through header strings with IDL6.1 regular expressions
; takes more than half the total time. Replacing various ranges
; (eg. {4,16} -> + ) helped a bit, but it is still slow. Default
; to returning subset of essential information (eg. time); only
; parse everything if /ALL_METADATA switch is set.
;
FUNCTION THEMIS_IMAGER_PARSE_COMMENTS,comments,metadata,ALL_METADATA=all_metadata
COMPILE_OPT HIDDEN
metadata= {THEMIS_IMAGER_METADATA}
metadata.comments= comments
tmp= STRLOWCASE(comments) ;avoid doing /FOLD_CASE for all STREGEX
timestring= (STREGEX(tmp,'"image request start" *([^#]+) utc',/SUBEXPR,/EXTRACT))[1]
metadata.exposure_time_string= (STRSPLIT(timestring,'.',/EXTRACT))[0]
year=0 & month=0 & day=0 & hour=0 & minute=0 & second=0 & fraction=""
READS,timestring,year,month,day,hour,minute,second,fraction,FORMAT='(I4,X,I2,X,I2,X,I2,X,I2,X,I2,X,A)'
metadata.exposure_time_offset= fraction/10.0^STRLEN(fraction)
CDF_EPOCH,epoch,year,month,day,hour,minute,second,/COMPUTE
metadata.exposure_time_cdf=epoch
IF KEYWORD_SET(ALL_METADATA) THEN BEGIN
metadata.site_uid= (STREGEX(tmp,'"site unique *id" ([a-z0-9-]+)',/SUBEXPR,/EXTRACT))[1]
metadata.imager_uid= (STREGEX(tmp,'"imager unique *id" ([a-z0-9-]+)',/SUBEXPR,/EXTRACT))[1]
exposure= STREGEX(tmp,'"exposure options"[^#]*',/EXTRACT)
metadata.ccd_size= (STREGEX(exposure,'width=([0-9]+).*height=([0-9]+)',/SUBEXPR,/EXTRACT))[1:2]
metadata.ccd_offset= (STREGEX(exposure,'xoffset=([0-9]+).*yoffset=([0-9]+)',/SUBEXPR,/EXTRACT))[1:2]
metadata.ccd_binning= (STREGEX(exposure,'xbin=([0-9]+).*ybin=([0-9]+)',/SUBEXPR,/EXTRACT))[1:2]
metadata.exposure_duration_request= (STREGEX(exposure,'msec=([0-9]+)',/SUBEXPR,/EXTRACT))[1]
metadata.exposure_duration_actual= (STREGEX(tmp,'"exposure.*plus.*readout" *([0-9\.]+) ms',/SUBEXPR,/EXTRACT))[1]
ENDIF
RETURN,0
END
PRO THM_ASI_IMAGER_READFILE,filename,images,metadata,COUNT=n_frames,DEBUG=debug,ALL_METADATA=all_metadata
;PRO THEMIS_IMAGER_READFILE,filename,images,metadata,COUNT=n_frames,DEBUG=debug,ALL_METADATA=all_metadata
IF (N_ELEMENTS(debug) EQ 0) THEN debug=0
time0= SYSTIME(1)
filenames='' & nfiles= 0
FOR indx=0,N_ELEMENTS(filename)-1 DO BEGIN
fname= FILE_SEARCH(filename[indx],COUNT=nf)
IF (nf GT 0) THEN filenames=[filenames,fname]
ENDFOR
nfiles= N_ELEMENTS(filenames)-1
IF (nfiles EQ 0) THEN BEGIN
dprint,'Error- files not found:'+filename[0]
n_frames=0
RETURN
ENDIF
filenames= filenames[1:nfiles]
filenames= filenames[SORT(filenames)]
IF (debug GT 0) THEN dprint, N_ELEMENTS(filenames),FORMAT='("found ",I," files")'
;pre-allocating memory significantly increases speed
nchunk= 20 & nstart= (nchunk*nfiles)<2400
n_frames= 0 & n_bytes= 0
FOR indx=0,nfiles-1 DO BEGIN
IF (debug GT 0) THEN dprint, ' reading file: '+filenames[indx]
OPENR,lun,filenames[indx],/GET_LUN,COMPRESS=STREGEX(STRUPCASE(filenames[indx]),'.*\.GZ$',/BOOLEAN)
WHILE NOT EOF(lun) DO BEGIN
IF (debug GT 1) THEN dprint, ' -reading frame: '+STRING(n_frames)
IF THEMIS_IMAGER_PNM_READFILE(lun,image,comments,DEBUG=debug) THEN BREAK
IF THEMIS_IMAGER_PARSE_COMMENTS(comments,mdata,ALL_METADATA=all_metadata) THEN BREAK
IF (n_frames EQ 0) THEN BEGIN
isize= SIZE(image,/STR) ;& stop
dimensions= isize.dimensions[0:isize.n_dimensions]
dimensions[isize.n_dimensions]= nstart
images= MAKE_ARRAY(dimensions,TYPE=isize.type,/NOZERO)
metadata= REPLICATE({THEMIS_IMAGER_METADATA},nstart)
dimensions[isize.n_dimensions]= nchunk
ENDIF ELSE IF (n_frames GE nstart) THEN BEGIN ;need to expand the arrays
images= [ [[images]], [[MAKE_ARRAY(dimensions,TYPE=isize.type,/NOZERO)]] ]
metadata= [metadata, REPLICATE({THEMIS_IMAGER_METADATA},nchunk)]
nstart= nstart+nchunk
ENDIF
;copy previous metadata that may not be present in every record
IF (n_frames GT 0) THEN BEGIN
mdata.site_uid= metadata[n_frames-1].site_uid
mdata.imager_uid= metadata[n_frames-1].imager_uid
ENDIF
; TO DO: check for size differences
metadata[n_frames]= mdata
images[0,0,n_frames]= image
n_frames= n_frames+1
ENDWHILE
n_bytes= n_bytes + (FSTAT(lun)).cur_ptr
FREE_LUN,lun
ENDFOR
;remove extra unused memory
metadata= metadata[0:n_frames-1]
images= images[*,*,0:n_frames-1]
IF (debug GT 0) THEN BEGIN
dtime= (SYSTIME(1)-time0)>1 & prefix=''
IF (n_bytes GT 1024L*9) THEN BEGIN n_bytes=n_bytes/1024.0 & prefix='kilo' & ENDIF
IF (n_bytes GT 1024L*9) THEN BEGIN n_bytes=n_bytes/1024.0 & prefix='Mega' & ENDIF
IF (n_bytes GT 1024L*9) THEN BEGIN n_bytes=n_bytes/1024.0 & prefix='Giga' & ENDIF
infoline=STRING(n_bytes,prefix,dtime,8*n_bytes/dtime,prefix $
,FORMAT='(" read ",F6.1,X,A,"bytes in ",I," seconds: ",F7.1,X,A,"bits/second")')
DPRINT, STRCOMPRESS(infoline)
ENDIF
END
;Benchmark/optimizing (note ALL_METADATA=0)
;Windows share is bandwidth limited by UCalgary 100 Mbps network:
;
; f=findfile('\data\themis\imager\stream0\2005\12\09\atha_themis02\ut08\*full.*',COUNT=nf)
; THEMIS_IMAGER_READFILE,f[*],images,metadata,COUNT=nframes,DEBUG=1
; reading file: \\themis-data\data\themis\imager\stream0\2005\12\09\atha_themis02\ut08\20051209_0800_atha_themis02_full.pgm.gz
; ...
; reading file: \\themis-data\data\themis\imager\stream0\2005\12\09\atha_themis02\ut08\20051209_0859_atha_themis02_full.pgm.gz
; read 150.5 Megabytes in 13 seconds: 88.4 Megabits/second
;Local disk is significantly faster:
;
; f=findfile('\data\themis\imager\stream0\2005\12\09\atha_themis02\ut08\*full.*',COUNT=nf)
; THEMIS_IMAGER_READFILE,f[*],images,metadata,COUNT=nframes,DEBUG=1
; reading file: \data\themis\imager\stream0\2005\12\09\atha_themis02\ut08\20051209_0800_atha_themis02_full.pgm.gz
; ...
; reading file: \data\themis\imager\stream0\2005\12\09\atha_themis02\ut08\20051209_0859_atha_themis02_full.pgm.gz
; read 150.5 Megabytes in 4 seconds: 269.4 Megabits/second