;+
;NAME:
; spd_ui_loaded_data::dproc
;
;PURPOSE:
; extracts tplot variables from active data, performs data processing
; tasks, creates new variables, optionally sets those variables to
; active variables
;
;CALLING SEQUENCE:
; success = loaded_data_obj -> dproc(dp_task, dp_pars,callSequence, names_out=names_out, no_setactive=no_setactive)
;
;INPUT:
; dp_task = a string variable specifying the task to be carried
;           out. The options are ['subavg', 'submed', 'smooth',
;           'blkavg','clip','deflag','degap','spike','deriv',
;           'pwrspc','wave','hpfilt']
; dp_pars = an anonymous structure containing the input parameters for
; the task, this will be unpacked in this routine and the parameters
; are passed through. Note that, since this is only called from the
; thm_GUI_new routine, there is no error checking for
; content, it is expected that the calling routine passes through the
; proper parameters in each case.
;
; callSequence = Object to store previous dproc operations for replay
;
;OUTPUT:
; success = a byte, 0b if the process was unsuccessful or cancelled,
;           1b if the process was completed
;
;KEYWORDS:
; names_out = the tplot names of the created data variables
; no_setactive = if set, the new variables will no be set to active at
;                the end of the process.
; hwin, sbar = history window and status bar objects for updates
; gui_id = the id of the calling widget - to pass into warning pop-ups
;
;HISTORY:
; 16-oct-2008, jmm, jimm@ssl.berkeley.edu
; switched output from message to byte, 29-oct-2008,jmm
; 12-Dec-2008,prc Fixed bug where dproc was not reading data stored in
; loaded data,but instead was reading non-gui-data.
; Fixed bug where data produced by dproc was not inheriting any meta-data.
; 23-jan-2009, jmm, deletes any tplot variables that are created
;                   during processing, added catch, so that deletion
;                   of tplot variables is done if an error bonks a
;                   process.
; 10-Feb-2009, jmm, Added hwin, sbar keywords
; 24-Apr-2015, af, updating plugins, reformatting code
;
;$LastChangedBy: nikos $
;$LastChangedDate: 2016-03-07 10:51:31 -0800 (Mon, 07 Mar 2016) $
;$LastChangedRevision: 20343 $
;$URL: svn+ssh://thmsvn@ambrosia.ssl.berkeley.edu/repos/spdsoft/tags/spedas_4_0/spedas_gui/objects/spd_ui_loaded_data__dproc.pro $
Function spd_ui_loaded_data::dproc, dp_task, dp_pars,callSequence=callSequence,replay=replay,in_vars=in_vars, names_out = names_out, $
                           no_setactive = no_setactive, hwin = hwin, sbar = sbar, gui_id = gui_id, $
;when replaying data, need to remember user interaction responses if we're gonna get it right
                           overwrite_selections=overwrite_selections,degap_selections=degap_selections,blkavg_selections=blkavg_selections,$
                           smooth_selections=smooth_selections, hpf_selections=hpf_selections,$
                           _extra = _extra

compile_opt idl2, hidden



;Catch errors during processing
;  -allows dproc window to persist in case of error?
;------------------------------------------------
err = 0
catch, err
If(err Ne 0) Then Begin
  catch, /cancel
  Help, /Last_Message, Output=error
  if obj_valid(hwin) then hwin->update, error
  out_msg = 'Warning: An error occured during processing.  Check the history window for details.'
  ok = error_message(traceback = 1, /noname, title = 'Error in Data Processing: ',/center)
  if is_string(tnames_in) then begin
    spd_ui_cleanup_tplot, tnames_in, del_vars=new_names
    store_data, new_names, /delete
  endif
  return, 0b
Endif


;Initialize tracking variables
;------------------------------------------------
overwrite_selection =''
degap_selection=''
blkavg_selection=''
smooth_selection=''
hpf_selection=''

overwrite_count = 0
degap_count = 0
blkavg_count = 0
smooth_count = 0
hpf_count = 0
  
if ~keyword_set(replay) then begin
  overwrite_selections=''
  degap_selections=''
  blkavg_selections=''
  smooth_selections=''
  hpf_selections=''
endif

if ~is_string(in_vars) then begin
  return,0b
endif

task = strtrim(strlowcase(dp_task), 2)

;store all current tplot variable names
;  -new variables created while processing data will be removed later
tnames_in = tnames()

;create a display object to house the status bar and history windows
;  -this object will be passed to the underlying analysis routines
;   to allow them to report messages to the gui via dprint
display_object = obj_new('spd_ui_dprint_display', statusbar=sbar, historywin=hwin)


;Export active data to tplot and verify
;  -all metadata should be included in tplot var's dlimits structure
;------------------------------------------------
for i=0, n_elements(in_vars)-1 do begin

  tvar = self->getTvarData(in_vars[i])
  
  if is_string(tvar) then begin
    exported = array_concat(tvar,exported)
  endif else begin
    spd_ui_message, 'Failed to export '+in_vars[i]+' to tplot', sb=sbar, hw=hwin
  Endelse

endfor

;check that something exported
if undefined(exported) then begin
  spd_ui_message, 'Failed to export all active data to tplot', sb=sbar, hw=hwin
  return, 0b
endif


;Abstract operation so that a single loop can handle all cases
;  -join and plugin operations process all variables simultaneously
;  -all other operations currently take single variable inputs 
;------------------------------------------------
if task eq 'join' or task eq 'plugin' then begin
  operation = {task: task, input: exported}
endif else begin
  operation = replicate( {task:task, input:''}, n_elements(exported) )
  operation.input = exported
endelse


;Loop over operations
;------------------------------------------------
For j = 0, n_elements(operation)-1 Do Begin


  canceled = 0b  ; set/reset canceled flag
  skipped = 0b  ; set/reset skipped flag

  ;copy current task's input
  input = operation[j].input

  ;ensure output from last loop is cleared
  undefine, nn ;new names
  undefine, sn ;support names


  ;Process the data
  ;  -call appropriate routine to operate on exported tplot variable(s)
  ;------------------------------------------------
  Case operation[j].task Of

    ;plugins
    ;------------------
    'plugin': begin
        if ~tag_exist(dp_pars, 'dproc_routine') then begin
          dprint, dlevel = 0, 'Error, the dproc_routine tag must be set in the structure returned by the plugin window.'
          return, -1
        endif
        
        if tag_exist(dp_pars, 'keywords') then begin
          dproc_keywords = dp_pars.keywords
        endif
              
        call_procedure, dp_pars.dproc_routine, input, $
                                               output_names=nn, $
                                               support_names=sn, $
                                               status_bar=sbar, $
                                               history_window=hwin, $
                                               _extra = dproc_keywords
    end
    
    ;split variable
    ;------------------
    'split': Begin
        split_vec, input, names_out = nn, inset='_split', display_object=display_object
    End
    
    ;join variables
    ;------------------
    'join': Begin
        join_vec, input, dp_pars.new_name[0], display_object=display_object, fail=jfail
        if keyword_set(jfail) then begin
          spd_ui_message, 'Could not join selected variables.', sb=sbar, hw=hwin
        endif else begin
          nn = dp_pars.new_name
        endelse
    End
    
    ;subtract average
    ;------------------
    'subavg': Begin
        tsub_average, input, nn, new_name = (input+'-d')[0], display_object=display_object
        nn = tnames(input+'-d')
    End
    
    ;subtract median
    ;------------------
    'submed': Begin
        tsub_average, input, nn, new_name = (input+'-m')[0], /median
        nn = tnames(input+'-m')
    End
    
    ;time derivative
    ;------------------
    'deriv': Begin
        get_data, input, ttt
        If(n_elements(temporary(ttt)) Gt 3) Then Begin
           deriv_data, input, newname = input+dp_pars.suffix[0], display_object=display_object, $
                       _extra={nsmooth:( dp_pars.setswidth ? dp_pars.swidth:0b)}
           nn = tnames(input+dp_pars.suffix)
        Endif Else Begin
           spd_ui_message,'Unable to get time derivative for '+input+'; not enough time elements.',sb=sbar, hw=hwin
           canceled = 1b
        Endelse
    End
    
    ;clean spikes
    ;------------------
    'spike': Begin
        clean_spikes, input, new_name = input+dp_pars.suffix[0], display_object=display_object, $
                      nsmooth = dp_pars.swidth[0], $
                      thresh = dp_pars.thresh[0]
        nn = tnames(input+dp_pars.suffix)
    End
    
    ;smooth data
    ;------------------
    'smooth': Begin
        get_data, input, t
        noktimes =  n_elements(t)
        if noktimes gt 1 then begin ; operation not valid on single time element
            av_dt = median(t[1:*]-t)
            If(av_dt Gt dp_pars.dt) then begin
                if smooth_selection ne 'yestoall' && smooth_selection ne 'notoall' Then Begin
                    smooth_selection=''
                    if ~keyword_set(replay) then begin
                        lbl = ['Note that the median value of the time resolution for '+input+' is:'+strcompress(string(av_dt))+' sec.', $
                               'The value that you have chosen for the averaging time resolution is smaller: '+$
                               strcompress(string(dp_pars.dt))+' sec.', $
                               'This will have non-intuitive and possibly non-plottable results. Do you want to continue?']
                                                    
                        smooth_selection = spd_ui_prompt_widget(gui_id,sbar,hwin,prompt=strjoin(lbl,ssl_newline()),title='SPEDAS GUI SMOOTH TEST',/yes,/no,/allyes,/allno, frame_attr=8)
                        smooth_selections = array_concat_wrapper(smooth_selection,smooth_selections)
                    endif else begin
                        if smooth_count ge n_elements(smooth_selections) then begin
                            spd_ui_message, "ERROR:Discrepancy in spedas document, may have lead to a document load error", sb=sbar, hw=hwin
                            smooth_selection = "yestoall"
                        endif else begin
                            smooth_selection = smooth_selections[smooth_count]
                        endelse
                    endelse
                endif
                smooth_count++
            Endif
            If (av_dt le dp_pars.dt[0] || (smooth_selection ne 'notoall' && smooth_selection ne 'no')) Then Begin
                dt = dp_pars.dt[0]
                _extra = {forward:dp_pars.dttype[1], $
                          backward:dp_pars.dttype[2], $
                          no_time_interp:dp_pars.opts[0], $
                          true_t_integration:dp_pars.opts[1], $
                          smooth_nans:dp_pars.opts[2]}
                if dp_pars.setICad then str_element, _extra, 'interp_resolution', dp_pars.icad, /add_replace
                If(dt Lt 1.0) Then dtchar = strcompress(/remove_all, dt) $
                Else dtchar = strcompress(/remove_all, fix(dt))
                tsmooth_in_time, input, dt, newname = input+dp_pars.suffix[0], display_object=display_object, _extra=_extra, $
                  interactive_varname=input,/interactive_warning,warning_result=warning_result
                if n_elements(warning_result) gt 0 && warning_result eq 1 then begin
                    nn = tnames(input+dp_pars.suffix)
                endif
            Endif Else Begin
                spd_ui_message,'Smooth process for: '+input+' cancelled.', sb=sbar, hw=hwin
                canceled = 1b ;prevent variable from being added later
            Endelse
        endif else begin
            spd_ui_message,'Unable to smooth '+input+' not enough elements in time range.', sb=sbar, hw=hwin
            canceled = 1b
        endelse
    End
    
    ;block average
    ;------------------
    'blkavg': Begin
    ;need another sanity test
      get_data, input, t
      ;test trange here
      if dp_pars.limit Eq 1 then begin
         oktimes =  where(t Gt dp_pars.trange[0] And t Le dp_pars.trange[1],  noktimes)
       endif else noktimes =  n_elements(t)
      
      if noktimes gt 1 then begin ; operation not valid on single time element
        av_dt = median(t[1:*]-t)
        
        If(av_dt Gt dp_pars.dt) then begin
          if blkavg_selection ne 'yestoall' && blkavg_selection ne 'notoall' Then Begin
            blkavg_selection=''
            if ~keyword_set(replay) then begin
              lbl = ['Note that the median value of the time resolution for '+input+' is:'+strcompress(string(av_dt))+' sec.', $
                     'The value that you have chosen for the averaging time resolution is smaller: '+$
                     strcompress(string(dp_pars.dt))+' sec.', $
                     'This will have non-intuitive and possibly non-plottable results. Do you want to continue?']
          
              blkavg_selection = spd_ui_prompt_widget(gui_id,sbar,hwin,prompt=strjoin(lbl,ssl_newline()),title='SPEDAS GUI BLK_AVG TEST',/yes,/no,/allyes,/allno, frame_attr=8)
              blkavg_selections = array_concat_wrapper(blkavg_selection,blkavg_selections)
            endif else begin
              if blkavg_count ge n_elements(blkavg_selections) then begin
                spd_ui_message, "ERROR:Discrepancy in spedas document, may have lead to a document load error", sb=sbar, hw=hwin
                blkavg_selection = "yestoall"
              endif else begin
                blkavg_selection = blkavg_selections[blkavg_count]
              endelse
            endelse
          endif
          
          blkavg_count++
        Endif
        If  (av_dt le dp_pars.dt || (blkavg_selection ne 'notoall' && blkavg_selection ne 'no')) Then Begin
            dt = dp_pars.dt[0]
            avg_data, input, dt, newname = input+dp_pars.suffix[0], display_object=display_object, $
                      _extra = {trange:( dp_pars.limit ? dp_pars.trange:0b)}
            nn = tnames(input+dp_pars.suffix)
        Endif Else Begin
            spd_ui_message,'Block Average process for: '+input+' cancelled.', sb=sbar, hw=hwin
            canceled = 1b ;prevent variable from being added later
        Endelse
      endif else begin
        spd_ui_message,'Unable to block average '+input+' not enough elements in time range.', sb=sbar, hw=hwin
        canceled = 1b
      endelse
    End
    
    ;clip y axis
    ;------------------
    'clip': Begin
        tclip, input, dp_pars.minc[0], dp_pars.maxc[0], newname = input+dp_pars.suffix[0], display_object=display_object, $
               _extra = {clip_adjacent: dp_pars.opts[0], $
                         flag: (dp_pars.opts[1] ? dp_pars.flag:0b)}
        nn = tnames(input+dp_pars.suffix)
    End
    
    ;deflat / remove flags
    ;------------------
    'deflag': Begin
        tdeflag, input, (dp_pars.method[0] ? 'repeat':'linear'), $
                 newname = input+dp_pars.suffix[0], display_object=display_object, $
                 _extra = {flag: (dp_pars.opts[0] ? dp_pars.flag:0b), $
                           maxgap: (dp_pars.opts[1] ? dp_pars.maxgap:0b)}
        nn = tnames(input+dp_pars.suffix)
    End
    
    ;degap / remove time gaps
    ;------------------
    'degap': Begin
    ;need another sanity test
      get_data, input, t
      
      if n_elements(t) gt 1 then begin ; operation not valid on single time element
        dt = t[1:*]-t[0:n_elements(t)-2]
        av_dt = median(dt)
        max_dt = max(dt)
        
        ;filter variables who no dt larger than threshold+margin (seems to be how xdegap works)
        if (dp_pars.dt[0] + dp_pars.margin[0]) gt max_dt then begin
          spd_ui_message,'No gaps below threshold in '+input+'.', sb=sbar, hw=hwin
          canceled = 1b ;prevent variable from being added later
          break
        endif
              
        If(av_dt Gt dp_pars.dt) then begin
          if degap_selection ne 'yestoall' && degap_selection ne 'notoall' Then Begin
            degap_selection=''
            if ~keyword_set(replay) then begin
              lbl = ['Note that the median value of the time resolution for '+input+' is:'+strcompress(string(av_dt))+' sec.', $
                     'The value that you have chosen for the degap time resolution is smaller: '+$
                     strcompress(string(dp_pars.dt))+' sec.', $
                     'This will have non-intuitive and possibly non-plottable results. Do you want to continue?']
                     
              degap_selection = spd_ui_prompt_widget(gui_id,sbar,hwin,prompt=strjoin(lbl,ssl_newline()),title='SPEDAS GUI DEGAP TEST',/yes,/no,/allyes,/allno, frame_attr=8) 
              degap_selections = array_concat_wrapper(degap_selection,degap_selections)
            endif else begin
              if degap_count ge n_elements(degap_selections) then begin
                spd_ui_message, "ERROR:Discrepancy in spedas document, may have lead to a document load error", sb=sbar, hw=hwin
                degap_selection = "yestoall"
              endif else begin
                degap_selection = degap_selections[degap_count]
              endelse
            endelse
          endif
          
          degap_count++
        Endif
              
        If (av_dt le dp_pars.dt || (degap_selection ne 'notoall' && degap_selection ne 'no')) Then Begin
            if dp_pars.opts[0] then str_element,_extra,'flag',dp_pars.flag, /add_replace
            if dp_pars.opts[1] then str_element,_extra,'maxgap',dp_pars.maxgap, /add_replace
            tdegap, input, dt = dp_pars.dt[0], $
                    margin = dp_pars.margin[0], $
                    ;maxgap = dp_pars.maxgap[0], $
                    newname = input+dp_pars.suffix[0], $
                    display_object=display_object, $
                    _extra = _extra
            nn = tnames(input+dp_pars.suffix)

        Endif Else Begin
            spd_ui_message,'Degap process for: '+input+' cancelled.', sb=sbar, hw=hwin
            canceled = 1b ;prevent variable from being added later
        Endelse
      endif else begin
        spd_ui_message,'Unable to process '+input+' not enough elements', sb=sbar, hw=hwin
        canceled = 1b
      endelse
        
    End
    
    ;wavelet transform
    ;------------------
    'wave': Begin
        get_data, input, t
        sstx = where(t Ge dp_pars.trange[0] And $
                     t Lt dp_pars.trange[1], nsstx)
        If(nsstx Gt 0) Then Begin
           spd_ui_message,'Processing Wavelet for: '+input, sb=sbar, hw=hwin
           ;Here just increase maxpoints to be larger than nsstx, the memory
           ;check should do enough so that the user knows when he has memory
           ;issues, jmm 2015-01-20
           spd_ui_wavelet, input, nn, dp_pars.trange, $
                           maxpoints=nsstx+16, $
                           temp_names = temp_names, $
                           display_object=display_object, prange=dp_pars.prange
           if is_string(temp_names) then begin
              store_data, temp_names, /delete
           endif
           options, nn, spec=1, /default
        Endif Else Begin
           spd_ui_message, 'Wavelet process for: '+input+' cancelled.', sb=sbar, hw=hwin
           canceled = 1b ;prevent variable from being added later
        Endelse
    End
    
    ;high pass filter
    ;------------------
    'hpfilt': Begin
        get_data, input, t
        noktimes =  n_elements(t)
        if noktimes gt 1 then begin ; operation not valid on single time element
            av_dt = median(t[1:*]-t)
            If(av_dt Gt dp_pars.dt) then begin
                if hpf_selection ne 'yestoall' && hpf_selection ne 'notoall' Then Begin
                    hpf_selection=''
                    if ~keyword_set(replay) then begin
                        lbl = ['Note that the median value of the time resolution for '+input+' is:'+strcompress(string(av_dt))+' sec.', $
                               'The value that you have chosen for the averaging time resolution is smaller: '+$
                               strcompress(string(dp_pars.dt))+' sec.', $
                               'This will have non-intuitive and possibly non-plottable results. Do you want to continue?']
                        
                        hpf_selection = spd_ui_prompt_widget(gui_id,sbar,hwin,prompt=strjoin(lbl,ssl_newline()),title='SPEDAS GUI HPFILTER TEST',/yes,/no,/allyes,/allno, frame_attr=8)
                        hpf_selections = array_concat_wrapper(hpf_selection,hpf_selections)
                    endif else begin
                        if hpf_count ge n_elements(hpf_selections) then begin
                            spd_ui_message, "ERROR:Discrepancy in spedas document, may have lead to a document load error", sb=sbar, hw=hwin
                            hpf_selection = "yestoall"
                        endif else begin
                            hpf_selection = hpf_selections[hpf_count]
                        endelse
                    endelse
                endif
                hpf_count++
            Endif
            If (av_dt le dp_pars.dt[0] || (hpf_selection ne 'notoall' && hpf_selection ne 'no')) Then Begin
                dt = dp_pars.dt[0]
                thigh_pass_filter, input, dt, newname = input+dp_pars.suffix[0], display_object=display_object, $
                  /interactive_warning,warning_result=warning_result, $
                  _extra = {interp_resolution: (dp_pars.seticad ? dp_pars.icad:0b)}
                if n_elements(warning_result) gt 0 && warning_result eq 0 then break
                nn = tnames(input+dp_pars.suffix)
            Endif Else Begin
                spd_ui_message,'High Pass Filter process for: '+input+' cancelled.', sb=sbar, hw=hwin
                canceled = 1b ;prevent variable from being added later
            Endelse
        endif else begin
            spd_ui_message,'Unable to High Pass filter '+input+' not enough elements in time range.', sb=sbar, hw=hwin
            canceled = 1b
        endelse
    End
    else: 
  Endcase ; ------------- end of requested task --------------

  
    
  ;Add output from processing routine to the GUI
  ;------------------------------------------------
  if is_string(nn) and ~canceled Then Begin

    for k = 0, n_elements(nn)-1 do begin
      
      spd_ui_check_overwrite_data,nn[k],self,gui_id,sbar,hwin,overwrite_selection,overwrite_count,$
                           replay=replay,overwrite_selections=overwrite_selections
      if strmid(overwrite_selection, 0, 2) eq 'no' then continue   
           
      ;add tplot variable to the gui
      ;  -this should properly capture metadata, if not then then it should be fixed elsewhere rather than kludged here
      add_success = self->add(nn[k])
        
      msg = add_success ? 'Added variable: ' : 'Failed to add variable: '
      spd_ui_message, msg+nn[k], sb=sbar, hw=hwin
      
      if ~add_success then continue

      ;add output variables to list of valid outputs
      names_out = array_concat(nn[k],names_out)
      
      ;add support variables to list of valid support vars (excluded from deletion later)
      if is_string(sn) then begin
        support_names = array_concat(sn, support_names)
      endif

    endfor

  endif else begin

    ;notify user if no valid output was found
    spd_ui_message, strjoin(input,', ')+' not processed', sb=sbar, hw=hwin
    skipped = 1b

  endelse

  
  if canceled or skipped then oops = 1b ; set flag to notify user later
  if double(!version.release) lt 8.0d then heap_gc ;clean-up memory


Endfor ; --------- end of loop over active data ----------



;if output names exist then at least one opperation was successful
success = is_string(names_out)


;set any output as the new active data
if success && ~keyword_set(no_setactive) then begin
  self->clearallactive
  for j = 0, n_elements(names_out)-1 do begin
    self->setactive, names_out[j]
  endfor
endif


;add this call to the call sequence 
if success && ~keyword_set(replay) then begin
  callSequence->addDprocOp,dp_task,in_vars,params=dp_pars,overwrite_selections,degap_selections,blkavg_selections
endif


;dump any tplot variables that were not previously present
;  -exclude explicitly designated support dat from plugins
spd_ui_cleanup_tplot, tnames_in, del_vars=new_names
if is_string(support_names) then begin
  new_names = ssl_set_complement(support_names, new_names)
endif
store_data, new_names, /delete


;notify user if any quantities were excluded (this should be the last output message)
if keyword_set(oops) then begin
  spd_ui_message, 'Finished.  Some quantities were not processed.  '+ $
                  '(Scroll back in status bar or check history window for details)', $
                  sb=sbar, hw=hwin
endif


return, success


End