57 lines
1.3 KiB
OCaml
57 lines
1.3 KiB
OCaml
type tokens = Token.t Seq.t
|
|
|
|
let either f g c =
|
|
f c || g c
|
|
|
|
let is_digit c =
|
|
'0' <= c && c <= '9'
|
|
|
|
let is_num = function
|
|
| 'x' -> true
|
|
| c -> is_digit c
|
|
|
|
let is_whitespace = function
|
|
| ' ' | '\t' | '\n' -> true
|
|
| _ -> false
|
|
|
|
let is_alpha c =
|
|
'A' <= c && c <= 'Z' && 'a' <= c && c <= 'z'
|
|
|
|
let is_ident_start =
|
|
either is_alpha ((=) '_')
|
|
|
|
let is_ident =
|
|
either is_ident_start is_digit
|
|
|
|
(* same as take_while f seq, drop_while f seq *)
|
|
let rec partition_while f seq : 'a Seq.t * 'a Seq.t =
|
|
match seq () with
|
|
| Seq.Nil -> Seq.empty, seq
|
|
| Seq.Cons (x, seq) ->
|
|
if f x then
|
|
let n, s = partition_while f seq in
|
|
Seq.cons x n, s
|
|
else
|
|
Seq.(empty, cons x seq)
|
|
|
|
let tokenize (str : string) : tokens =
|
|
let seq = String.to_seq str in
|
|
let rec aux seq =
|
|
let open Token in
|
|
match seq () with
|
|
| Seq.Nil -> Seq.empty
|
|
| Seq.Cons (x, s) ->
|
|
if is_whitespace x then
|
|
aux s
|
|
else if is_digit x then
|
|
let n, s = partition_while is_num s in
|
|
let n = String.of_seq @@ Seq.cons x n in
|
|
Seq.cons (of_string n) (aux s)
|
|
else if is_ident_start x then
|
|
let id, s = partition_while is_ident s in
|
|
let id = String.of_seq @@ Seq.cons x id in
|
|
Seq.cons (Ident id) (aux s)
|
|
else
|
|
Seq.cons (of_char x) (aux s)
|
|
in
|
|
aux seq
|