type token_type =
  | LeftParen
  | RightParen
  | LeftBrace
  | RightBrace
  | Plus
  | Minus
  | Star
  | Slash
  | Bang
  | Dot
  | Comma
  | Semicolon
  | Equal
  | EqualEqual
  | BangEqual
  | Greater
  | Less
  | GreaterEqual
  | LessEqual
  | Identifier of string
  | String of string
  | Number of float
  | And
  | Break
  | Class
  | Continue
  | Else
  | False
  | Fun
  | For
  | If
  | Nil
  | Or
  | Print
  | Return
  | Super
  | This
  | True
  | Var
  | While
  | Comment of string
  | Eof

val pp_token_type : Format.formatter -> token_type -> unit
val show_token_type : token_type -> string
val keywords : (string, token_type) Hashtbl.t

type token = { token_type : token_type; pos : Error.code_pos }

val show_token : token -> string

type lexer_result = (token list, Error.lexer_error list) result

val tokenize : string -> lexer_result