monoid/small-set-of-ml
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: parse with lexer context

Browse source
This commit is contained in:
monoid 2025-01-29 22:36:57 +09:00
parent b29331c246
commit 5ab22e2f68
3 changed files with 6 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
lib/eval.ml
View file

@ -94,7 +94,6 @@ and eval_bin_op_expr scope op left_expr right_expr =
let eval_str (str: string): value_type = let eval_str (str: string): value_type =
let tokens = Lexer.lex_tokens_seq str in let tokens = Lexer.lex_tokens_seq str in
let tokens = tokens |> Seq.map (fun (x,_) -> x) in
let expr = Parser.get_expr_tree_from_tokens tokens in let expr = Parser.get_expr_tree_from_tokens tokens in
match expr with match expr with
| Some e -> eval_expr { parent = None; bindings = VariableBindingMap.empty } e | Some e -> eval_expr { parent = None; bindings = VariableBindingMap.empty } e

2
lib/lexer.ml
View file

@ -257,7 +257,7 @@ let%test "test lex_token 2" =
let lex_tokens_seq (total: string): (token * lexer_context) Seq.t = let lex_tokens_seq (total: string): (token * lexer_context) Seq.t =
let rec aux ctx = let rec aux ctx =
let token, next_ctx = lex_token ctx in let token, next_ctx = lex_token ctx in
if token.token_type = Eof then if token.token_type = Eof then
Seq.Cons ((token, next_ctx), fun () -> Seq.Nil) Seq.Cons ((token, next_ctx), fun () -> Seq.Nil)
else else
Seq.Cons ((token, next_ctx), fun () -> aux next_ctx) in Seq.Cons ((token, next_ctx), fun () -> aux next_ctx) in

11
lib/parser.ml
View file

@ -1,7 +1,7 @@
open Lexer open Lexer
type parser_context = { type parser_context = {
seq: Lexer.token Seq.t; seq: (Lexer.token * Lexer.lexer_context) Seq.t;
errors: string list; errors: string list;
} }
@ -33,10 +33,10 @@ let or_parser (a: 'a parser) (b: 'a parser): 'a parser = fun (ctx: parser_contex
let (<|>) = or_parser let (<|>) = or_parser
let peek_token: token parser = fun (ctx: parser_context) -> let peek_token: token parser = fun (ctx: parser_context) ->
Seq.uncons ctx.seq |> Option.map (fun (t,_) -> (t,ctx)) Seq.uncons ctx.seq |> Option.map (fun ((t, _),_) -> (t,ctx))
let next_token: token parser = fun (ctx: parser_context) -> let next_token: token parser = fun (ctx: parser_context) ->
Seq.uncons ctx.seq |> Option.map (fun (t, s) -> (t, Seq.uncons ctx.seq |> Option.map (fun ((t,_), s) -> (t,
{ ctx with seq = s} { ctx with seq = s}
)) ))
@ -201,8 +201,8 @@ and expr (): expr_tree parser =
(parse_if_expr() |> fmap (fun x -> IfExpr x)) <|> parse_level3() in (parse_if_expr() |> fmap (fun x -> IfExpr x)) <|> parse_level3() in
return e return e
let get_expr_tree_from_tokens (tokens: Lexer.token Seq.t): expr_tree option = let get_expr_tree_from_tokens (tokens: (Lexer.token * Lexer.lexer_context) Seq.t): expr_tree option =
let ntokens = Seq.filter (fun x -> let ntokens = Seq.filter (fun (x,_) ->
match x.token_type with match x.token_type with
| Lexer.Comment(_) -> false | Lexer.Comment(_) -> false
| _ -> true | _ -> true
@ -214,7 +214,6 @@ let get_expr_tree_from_tokens (tokens: Lexer.token Seq.t): expr_tree option =
let%test "test get_expr_tree_from_tokens 1" = let%test "test get_expr_tree_from_tokens 1" =
let tokens = Lexer.lex_tokens_seq "let x = 1 in\n x" in let tokens = Lexer.lex_tokens_seq "let x = 1 in\n x" in
let tokens = tokens |> Seq.map (fun (x,_) -> x) in
match get_expr_tree_from_tokens tokens with match get_expr_tree_from_tokens tokens with
| Some e -> expr2str e = "let x = 1 in\n x" | Some e -> expr2str e = "let x = 1 in\n x"
| None -> false | None -> false