;+
;
;Procedure: lex
;
;Purpose:  Turns a string into a list of token structs with some basic
;          processing performed, for the mini language.  The tokens are 
;          generated by get_token.  The processing performed is to remove blanks
;          and check for and illegal token types(token types supported by IDL but
;          not the mini language, like @statements)
;          
; Inputs: s:  the string to be tokenized
; 
; Keywords: token_list:  The token_list is returned through this variable
;           error:  If an error occurs, a struct describing it will be returned through this variable
;
; $LastChangedBy: pcruce $
; $LastChangedDate: 2016-10-25 17:48:17 -0700 (Tue, 25 Oct 2016) $
; $LastChangedRevision: 22195 $
; $URL: svn+ssh://thmsvn@ambrosia.ssl.berkeley.edu/repos/spdsoft/tags/spedas_3_1/general/mini/lex.pro $
;


pro lex,s,token_list=token_list,error=error

  compile_opt idl2

  mini_routines ;compiles procedures associated with library routines for the mini language
  
  mini_predicates ;compiles predicates for mini language

  n = 0
  len = strlen(s)
  
  error = 0
  
  ;unset token_list
  if keyword_set(token_list) then begin
    t = temporary(token_list)
  endif
  
  ;unset error
  if keyword_set(error) then begin
    t = temporary(error)
  endif

  repeat begin
  
    if keyword_set(token) && ~is_blank_type(token) then begin
      previous = token
    endif

    token = get_token(strmid(s,n,len-n))
    
    n += strlen(token.value)
    
    ;this set of ifs transforms the tokens into types for use with the parser
    if is_invalid_type(token) then begin
      error = {type:token.type,name:token.name,value:token.value,position:n}
      return
    endif
       
    if ~is_blank_type(token) then begin
    
      if ~is_struct(token_list) then begin
        token_list = [token]
      endif else begin
        token_list = [token_list,token]
      endelse
    
    endif
  
  endrep until is_endline_type(token)
  

end