diff --git a/lib/lexer.ml b/lib/lexer.ml index 507548a..b2a7afc 100644 --- a/lib/lexer.ml +++ b/lib/lexer.ml @@ -128,6 +128,17 @@ module State = struct let tt = lexeme |> Hashtbl.find_opt keywords |> Option.value ~default:(Identifier lexeme) in append_token code_pos tt state + let rec parse_block_commend (state : state) : state = + (* "- 2" since we already consumed the "/*" *) + let pos = { line = state.line; col = state.col - 2 } in + let found, state = advance_until '*' state in + if not found then append_error pos "Unterminated block commend" state + else if peek state = Some '/' then + let state = snd @@ advance state in + let lexeme = get_lexeme state (state.start_pos + 2) (state.cur_pos - 2) in + append_token pos (Comment lexeme) state + else parse_block_commend state + let rec tokenize_rec (state : state) : state = let pos = { line = state.line; col = state.col } in let append_token = append_token pos in @@ -166,15 +177,16 @@ module State = struct fun state -> let b, state = advance_if '=' state in append_token (if b then GreaterEqual else Greater) state - | '/' -> + | '/' -> ( fun state -> - let found, state = advance_if '/' state in - if not found then append_token Slash state - else - let start_pos = state.cur_pos in - let _, state = advance_until '\n' state in - let lexeme = String.trim @@ get_lexeme state start_pos state.cur_pos in - append_token (Comment lexeme) state + match peek state with + | Some '/' -> + let start_pos = state.cur_pos in + let _, state = advance_until '\n' state in + let lexeme = String.trim @@ get_lexeme state start_pos state.cur_pos in + append_token (Comment lexeme) state + | Some '*' -> parse_block_commend (snd @@ advance state) + | _ -> append_token Slash state) | '"' -> fun state -> let found, state = advance_until '"' state in