;+
; Procedure: tplot_force_monotonic
;
; Purpose:
;    This routine checks tplot variables for sample time (abscissa: data.x) monotonicity
;     and forces them so through removal or replacement of non-monotonic segments, if requested;
;     the corresponding elements of data.y and data.v are also repaired.
;    Indices of consecutively repeated header times and piece-wise monotonic segment "negative jumps"
;     are identified and used to determine monotonicity of tplot variables. If checking,
;     the routine will report cause of failures (i.e. negative jumps, repeats).  
;    Three repair methods are available: /forward, /reverse and /sort (see Keywords) 
;    
; Inputs:   [optional] tplot variable name(s) string/array or wild-card name string or tplot variable number(s); same as input for tnames()
;                    
; Keywords:
;           forward:      the recommended repair method, which keeps the older time elements of over-lapping time segments
;           reverse:      repair method which keeps the newer time elements over-lapping time segments
;           sort:         repair method which sorts time-lines chronologically, using bsort()
;           
;           keep_repeats: [optional] do not remove consecutively repeated header times
;           replace_NaN:  [optional] instead of removing 'bad' elements, replace them with NaN (only applied to data.y and data.v elements, data.x is not modified)
;           
;              
; Outputs:
;           If checking, PASS/FAIL console message per tplot variable, including cause of failures
;           If repairing, PASS/FAIL console message per tplot variable and repaired tplot variable(s) (via store_data) 
;              
; Examples:
;           tplot_force_monotonic                           ; check all tplot variables
;           tplot_force_monotonic,'*'                       ; check all tplot variables
;           tplot_force_monotonic,'var1'                    ; check tplot variable named var1
;           tplot_force_monotonic,['var1','var2']           ; check tplot variables named var1 and var2
;           tplot_force_monotonic,[6,7,9]                   ; check tplot variables 6, 7 and 9
;           tplot_force_monotonic,/forward                  ; repair all tplot variables using 'forward' method, discarding repeated sample times (recommended repair method)
;           tplot_force_monotonic,/reverse,/keep_repeats    ; repair all tplot variables using 'reverse' method, keeping repeated sample times
;           tplot_force_monotonic,/sort                     ; repair all tplot variables using 'sort' method, discarding consecutively repeated sample times
;           tplot_force_monotonic,'var?',/forward,/replace_nan  ; repair tplot variable(s) in var? using 'forward' method and replace bad elements with NaNs
;        
; Notes:
;     1. tplot variables repaired with the /replace_nan keyword will not pass a monotonicity check.
;     2. tplot variables repaired with the /keep_repeats keyword may not pass a monotonicity check.
;     3. A warning is issued if more than 10% of tplot variable elements are removed or replaced.
;      
; ToDo: nothing yet
;
; $LastChangedBy: egrimes $
; $LastChangedDate: 2016-04-25 12:07:51 -0700 (Mon, 25 Apr 2016) $
; $LastChangedRevision: 20911 $
; $URL: svn+ssh://thmsvn@ambrosia.ssl.berkeley.edu/repos/spdsoft/tags/spedas_4_1/general/tplot/tplot_force_monotonic.pro $
;-

function keep_overlap, i_keep, i_new, time_array, REVERSE=REVERSE, KEEP_REPEATS=KEEP_REPEATS
;;; returns the indices of overlapping time segments to keep
if keyword_set(reverse) then begin
  if keyword_set(KEEP_REPEATS) then begin i_keep_new = where( time_array[i_new] le min(time_array[i_keep]) ,c_new)
  endif else i_keep_new = where( time_array[i_new] lt min(time_array[i_keep]) ,c_new)
endif else begin
  if keyword_set(KEEP_REPEATS) then begin i_keep_new = where( time_array[i_new] ge max(time_array[i_keep]) ,c_new)
  endif else i_keep_new = where( time_array[i_new] gt max(time_array[i_keep]) ,c_new)
endelse

if c_new gt 0 then begin
  if keyword_set(reverse) then begin
    i_keep = [i_new[i_keep_new], i_keep]
  endif else i_keep = [i_keep, i_new[i_keep_new]]
endif

return, i_keep
end

function set_difference, set_a, set_b
;;; returns the elements present in set_a but not in set_b (if their intersection is not empty)
;;; returns -1 if intersection of set_a and set_b is empty
mm = minmax(set_a)
r = where((histogram(set_a, Min=mm[0], Max=mm[1]) ne 0) and (histogram(set_b, Min=mm[0], Max=mm[1]) eq 0))
diff = r + mm[0]
if n_elements(set_a) eq n_elements(diff) then begin
  return, -1
endif else return, diff
end

pro tplot_force_monotonic, tplot_vars, FORWARD=FORWARD, REVERSE=REVERSE, SORT=SORT, KEEP_REPEATS=KEEP_REPEATS, REPLACE_NAN=REPLACE_NAN

;;; Get and check tplot variable names/numbers
tplot_vars = tnames(tplot_vars)
if strlen(strjoin(tplot_vars)) eq 0 then begin
  dprint, 'Invalid tplot variable name(s) or number(s)'
  return
endif
max_var_len = 32

;;; Loop over requested tplot variables
for j=0L,n_elements(tplot_vars)-1L do begin
  tplot_var = tplot_vars[j]
  
  ; the following fixes a crash when the tplot variable name is the same length as max_var_len
  if strlen(tplot_var) ge max_var_len then max_var_len = strlen(tplot_var) + 1
  
  get_data,tplot_var,data=data,dlimits=dlimits,limits=limits
  
  ;;; Check if data is a structure
  if not is_struct(data) then begin
    dprint,' ',tplot_var,':'+strjoin(strarr(max_var_len-strlen(tplot_var))+' ')+'FAIL - not a valid tplot variable'
    continue
  endif
  time_array = data.x
  n_time_array = n_elements(time_array)
  ;;; Check if time array has only one element
  if (n_time_array eq 1) then begin
    dprint,tplot_var,':'+strjoin(strarr(max_var_len-strlen(tplot_var))+' ')+'PASS, variable has one time element'
    continue
  endif
  
  ;;; Identify consecutive repeats and boundaries of piece-wise monotonic segments "negative jumps"
  i_non_monotonic = where((time_array[1:n_time_array-1]-time_array[0:n_time_array-2]) lt 0d, c_non_monotonic) +1L
  i_repeat = where((time_array[1:n_time_array-1]-time_array[0:n_time_array-2]) eq 0d, c_repeat) +1L

  ;;; If time_array is monotonic (i.e. has no negative jumps) and has no repeats, then it PASSES
  if ((c_non_monotonic eq 0) and (c_repeat eq 0)) then begin 
    dprint,tplot_var,':'+strjoin(strarr(max_var_len-strlen(tplot_var))+' ')+'PASS'
    continue
  endif else begin
    
    ;;; case: tplot variable is non-monotonic
    if keyword_set(forward) or keyword_set(reverse) or keyword_set(SORT) then begin
      if c_non_monotonic eq 0 then GOTO, ONLY_REPEATS
      
      ;;; Keep forward portion of overlaps
      if keyword_set(forward) then begin
        if i_non_monotonic[0] eq 0 then begin
          i_keep = 0L
        endif else i_keep = lindgen(i_non_monotonic[0])
        for k = 0L, c_non_monotonic-2L do begin
            i_new = lindgen(i_non_monotonic[k+1] - i_non_monotonic[k]) + i_non_monotonic[k]
            i_keep = keep_overlap(i_keep,i_new,time_array,KEEP_REPEATS=KEEP_REPEATS)
        endfor
        i_new = lindgen(n_time_array - i_non_monotonic[k] ) + i_non_monotonic[k]
        i_keep = keep_overlap(i_keep,i_new,time_array,KEEP_REPEATS=KEEP_REPEATS)
      endif 
    
      ;;; Keep reverse portion of overlaps
      if keyword_set(reverse) then begin
        n = c_non_monotonic-1
        if i_non_monotonic[n] eq 0 then begin
          i_keep = lindgen(n_time_array -1L) +1L
        endif else i_keep = lindgen( n_time_array - i_non_monotonic[n]) +i_non_monotonic[n]
        for k = c_non_monotonic-2L,0,-1 do begin
            i_new = lindgen(i_non_monotonic[k+1]-i_non_monotonic[k]) + i_non_monotonic[k]
            i_keep = keep_overlap(i_keep,i_new,time_array,/reverse,KEEP_REPEATS=KEEP_REPEATS)
        endfor
        if i_non_monotonic[0] eq 0 then begin
          i_new = 0L
        endif else i_new = lindgen(i_non_monotonic[0])
        i_keep = keep_overlap(i_keep,i_new,time_array,/reverse,KEEP_REPEATS=KEEP_REPEATS)
      endif
      
      ;;; Sort time-series using bsort()
      if keyword_set(SORT) then i_keep = bsort(time_array)

      ;;; Keep repeats or not
      ONLY_REPEATS: if c_non_monotonic eq 0 then i_keep = lindgen(n_time_array)
      if not keyword_set(KEEP_REPEATS) then begin
        if (c_repeat gt 0) then begin
          i_non_repeats = set_difference([i_keep],[i_repeat])
          if min(i_non_repeats) ne -1 then i_keep = i_non_repeats
        endif
      endif

      ;;; Repair tplot variable and print summary
        ;;; Remove "bad" elements of data.x, data.y and data.v
      if not keyword_set(replace_nan) then begin
        x = data.x[i_keep]
        y_dim = size(data.y,/n_dimensions)
        if y_dim eq 1 then begin
          y = data.y[i_keep]
          if tag_exist(data,'v') then begin
            v = data.v
            if n_elements(data.y) eq n_elements(data.v) then v = data.v[i_keep]
          endif 
        endif
        if y_dim eq 2 then begin
          y = data.y[i_keep,*]
          if tag_exist(data,'v') then begin
            v_dim = size(data.v,/n_dimensions)
            if v_dim eq 2 then begin
              v = data.v[i_keep,*]
            endif else v = data.v
          endif
        endif
        if y_dim eq 3 then begin
          y = data.y[i_keep,*,*]
          if tag_exist(data,'v1') then begin
            v1 = data.v1
            v2 = data.v2
          endif
        endif
        ;;; Replace "bad" elements of data.y and data.v with NaN
      endif else begin
        x = data.x
        i_all = lindgen(n_elements(x))
        i_replace = set_difference([i_all],[i_keep])
        if min(i_replace) ne -1 then begin
          y_dim = size(data.y,/n_dimensions)
          if y_dim eq 1 then begin
            y = data.y
            y[i_replace] = 'NaN'
            if tag_exist(data,'v') then begin
              v = data.v
              if n_elements(data.y) eq n_elements(data.v) then v[i_replace] = 'NaN'
            endif
          endif
          if y_dim eq 2 then begin
            y = data.y
            y[i_replace,*] = 'NaN'
            if tag_exist(data,'v') then begin
              v = data.v
              v_dim = size(data.v,/n_dimensions)
              if v_dim eq 2 then v[i_replace,*] = 'NaN'
            endif
          endif
          if y_dim eq 3 then begin
            y = data.y
            y[i_replace,*,*] = 'NaN'
            if tag_exist(data,'v1') then begin
              v1 = data.v1
              v2 = data.v2
            endif
          endif
        endif            
       endelse
      
      ;;; Write modified data to tplot variable
      if (tag_exist(data,'v') or tag_exist(data,'v1')) then begin
        if tag_exist(data,'v') then store_data,tplot_var,data={x:x,y:y,v:v},dlimits=dlimits,limits=limits
        if tag_exist(data,'v1') then store_data,tplot_var,data={x:x,y:y,v1:v1,v2:v2},dlimits=dlimits,limits=limits
      endif else store_data,tplot_var,data={x:x,y:y},dlimits=dlimits,limits=limits
      
      ;;; Determine percentage of elements removed or replaced, and report them
      percent_removed = (n_time_array - n_elements(i_keep)) / float(n_time_array) * float(100)
      if not keyword_set(replace_nan) then begin
        dprint,tplot_var,':'+strjoin(strarr(max_var_len-strlen(tplot_var))+' ')+'FAIL - repaired: '+ strtrim(percent_removed,2) + ' % removed'
      endif else dprint,tplot_var,':'+strjoin(strarr(max_var_len-strlen(tplot_var))+' ')+'FAIL - repaired: '+ strtrim(percent_removed,2) + ' % replaced with NaN'   
      if percent_removed gt 10d then dprint,'********** Warning: Over 10% removed or replaced from '+strtrim(tplot_var,2)+' **********'
    endif else begin
      ;;; Checking case: report causes of failures
      if ((c_repeat gt 0) and (c_non_monotonic) gt 0) then dprint,tplot_var,':'+strjoin(strarr(max_var_len-strlen(tplot_var))+' ')+'FAIL - '+strtrim(c_non_monotonic,2)+' negative jump(s), '+strtrim(c_repeat,2)+' repeat(s)'
      if ((c_repeat gt 0) and (c_non_monotonic) eq 0) then dprint,tplot_var,':'+strjoin(strarr(max_var_len-strlen(tplot_var))+' ')+'FAIL - '+strtrim(c_repeat,2)+' repeat(s)'
      if ((c_repeat eq 0) and (c_non_monotonic) gt 0) then dprint,tplot_var,':'+strjoin(strarr(max_var_len-strlen(tplot_var))+' ')+'FAIL - '+strtrim(c_non_monotonic,2)+' negative jump(s)'
      
    endelse
  endelse

endfor  ;;; end of loop over tplot variables
end