definition module SaplTokenizer

import StdString, Unicode

/**
* Possible token types of a SAPL program
*/
:: Token = TIdentifier String 
	     | TStrictIdentifier String
	     | TComment String 
	     | TInlineAnnotation	     
	     | TAssignmentOp
	     | TSelectAssignmentOp
	     | TMacroAssignmentOp
	     | TCAFAssignmentOp	     
     	 | TLambda
     	 | TColon
     	 | TVerticalBar     	 
		 | TOpenParenthesis
 		 | TCloseParenthesis
 		 | TOpenBracket
 		 | TCloseBracket
 		 | TTypeDef
		 | TLit Literal
		 | TSelectKeyword
		 | TIfKeyword
		 | TLetKeyword
		 | TInKeyword
		 | TEndOfLine

instance toString Literal	
instance toString Token

/**
* Token wrapped around position information (line and column numbers)
*/
:: PosToken = PosToken Int Int Token

// String and Char constants may contain Clean escape sequences. If the target
// language uses different escaping technique the code generator must replace the
// escape sequences
:: Literal = LString UString 
		   | LChar UString 
		   | LInt Int 
		   | LReal Real 
		   | LBool Bool

/**
* Low level function to read a token from a given position of the input string.
* 
* @param start position (from zero)
* @param input string
* @return start position of the token (can differ from the argument)
* @return start position of next token
* @return the read token
*/
read_token :: !Int !String -> (!Int, !Int, !Token)

/**
* Convert a given SAPL string to a list of tokens.
*/
tokens              :: !String -> [Token]		// used by linker
tokensWithPositions :: !String -> [PosToken]	// used by parser