diff --git a/dune-project b/dune-project index 5421cec..c5fd611 100644 --- a/dune-project +++ b/dune-project @@ -27,4 +27,5 @@ extension.") (ppxlib (>= 0.26.0)) gen (ppx_expect :with-test) + (qcheck-core :with-test) (menhir :with-test))) diff --git a/sedlex.opam b/sedlex.opam index 7b917f4..9f98931 100644 --- a/sedlex.opam +++ b/sedlex.opam @@ -22,6 +22,7 @@ depends: [ "ppxlib" {>= "0.26.0"} "gen" "ppx_expect" {with-test} + "qcheck-core" {with-test} "menhir" {with-test} "odoc" {with-doc} ] diff --git a/test/oracle/dune b/test/oracle/dune new file mode 100644 index 0000000..b0fe7f4 --- /dev/null +++ b/test/oracle/dune @@ -0,0 +1,8 @@ +(library + (name oracle_test) + (libraries sedlex.compiler qcheck-core) + (inline_tests) + (enabled_if + (>= %{ocaml_version} 4.14)) + (preprocess + (pps ppx_expect))) diff --git a/test/oracle/oracle.ml b/test/oracle/oracle.ml new file mode 100644 index 0000000..2381c67 --- /dev/null +++ b/test/oracle/oracle.ml @@ -0,0 +1,411 @@ +(* The package sedlex is released under the terms of an MIT-like license. *) +(* See the attached LICENSE file. *) +(* Copyright 2026, Hugo Heuzard *) + +(** Test oracle for the tagged-DFA compiler. + + Two independent implementations of sedlex's matching semantics are run on + the same (rules, input) pairs and their results compared: + + - {!ref_match}: a brute-force backtracking matcher over {!Ir.t}. It + enumerates parses in leftmost-greedy priority order (alternation prefers + the left branch, repetition prefers more iterations) and keeps the + highest-priority parse among those of maximal length. This is the + specification of capture semantics. + + - {!dfa_match}: an interpreter for the output of {!Sedlex.compile_ir} that + runs it the way PPX-generated code does: tag operations on transitions and + on accept, mark/backtrack, end of input read once, and binding extraction + via discriminator cells. + + {!oracle} prints one line per input, prefixed with ERROR when the two + disagree. {!qcheck} runs a deterministic random sweep and prints failing + cases through {!oracle}, so an empty expect block means full agreement. *) + +open Sedlex_compiler + +(* ------------------------------------------------------------------ *) +(* Ir combinators *) +(* ------------------------------------------------------------------ *) + +let ok = function Ok x -> x | Error e -> failwith e +let lit c = Ir.chars (Cset.singleton (Char.code c)) +let cls lo hi = Ir.chars (Cset.interval (Char.code lo) (Char.code hi)) +let eof = Ir.chars Cset.eof +let seq r1 r2 = ok (Ir.seq r1 r2) +let ( ^. ) = seq +let alt r1 r2 = ok (Ir.alt r1 r2) +let star r = ok (Ir.star r) +let plus r = ok (Ir.plus r) +let rep r n m = ok (Ir.rep r n m) +let opt r = rep r 0 1 +let capture name r = ok (Ir.capture name r) + +let cset_of = function + | Ir.Chars c -> c + | _ -> failwith "single-character regexp expected" + +let compl r = Ir.chars (Cset.difference Cset.any (cset_of r)) +let sub r1 r2 = Ir.chars (Cset.difference (cset_of r1) (cset_of r2)) +let inter r1 r2 = Ir.chars (Cset.intersection (cset_of r1) (cset_of r2)) + +(* ------------------------------------------------------------------ *) +(* Common result shape *) +(* ------------------------------------------------------------------ *) + +type match_result = { + rule : int; + len : int; + bindings : (string * (int option * int option)) list; + (** Sorted by name. [None] means the position comes from a memory cell + that was never written (only the DFA side can produce this). *) +} + +(* ------------------------------------------------------------------ *) +(* Reference matcher *) +(* ------------------------------------------------------------------ *) + +(* [parses ir input] lazily enumerates the (end position, capture environment) + of every parse of [ir] starting at position 0, in leftmost-greedy priority + order. Captures cannot occur under repetition (enforced by the Ir smart + constructors), so repetition bodies never extend the environment and + iterations that consume nothing can be cut without losing bindings. *) +let parses (ir : Ir.t) (input : int array) : + (int * (string * (int * int)) list) Seq.t = + let len = Array.length input in + let rec go ir pos env = + match ir with + | Ir.Chars cs -> + (* The input is followed by one end-of-input marker, read once: + position [len + 1] means it was consumed. *) + if pos < len && Cset.mem input.(pos) cs then Seq.return (pos + 1, env) + else if pos = len && Cset.mem (-1) cs then Seq.return (len + 1, env) + else Seq.empty + | Ir.Eps -> Seq.return (pos, env) + | Ir.Seq elems -> + List.fold_left + (fun acc elem -> Seq.concat_map (fun (p, e) -> go elem p e) acc) + (Seq.return (pos, env)) + elems + | Ir.Alt branches -> + Seq.concat_map (fun b -> go b pos env) (List.to_seq branches) + | Ir.Star r -> iter r pos env + | Ir.Plus r -> Seq.concat_map (fun (p, e) -> iter r p e) (go r pos env) + | Ir.Rep (r, n, m) -> bounded r n m pos env + | Ir.Capture (name, r) -> + (* End of input has no width: positions are capped at [len]. *) + Seq.map + (fun (p, e) -> (p, (name, (min pos len, min p len)) :: e)) + (go r pos env) + and iter r pos env = + (* Greedy: continuing the loop has priority over exiting. *) + Seq.append + (Seq.concat_map + (fun (p, e) -> if p = pos then Seq.empty else iter r p e) + (go r pos env)) + (Seq.return (pos, env)) + and bounded r n m pos env = + if m = 0 then Seq.return (pos, env) + else if n > 0 then + Seq.concat_map + (fun (p, e) -> bounded r (n - 1) (m - 1) p e) + (go r pos env) + else + Seq.append + (Seq.concat_map (fun (p, e) -> bounded r 0 (m - 1) p e) (go r pos env)) + (Seq.return (pos, env)) + in + go ir 0 [] + +(* Highest-priority parse among those of maximal length: enumeration order is + priority order, so the first parse reaching the maximum wins ties. *) +let best_parse ir input = + (* Longest lexeme first; end of input adds no length, so among parses of + the same length the first in priority order wins, whether or not it + read end of input. *) + let len = Array.length input in + Seq.fold_left + (fun best (p, e) -> + let p = min p len in + match best with Some (bp, _) when bp >= p -> best | _ -> Some (p, e)) + None (parses ir input) + +let ref_match (rules : Ir.t array) (input : int array) : match_result option = + let best = ref None in + Array.iteri + (fun i ir -> + match best_parse ir input with + | None -> () + | Some (p, e) -> ( + (* Longest match wins; the lowest-numbered rule wins ties. *) + match !best with + | Some (_, bp, _) when bp >= p -> () + | _ -> best := Some (i, p, e))) + rules; + Option.map + (fun (i, p, e) -> + let bindings = + List.sort compare + (List.map (fun (n, (s, ep)) -> (n, (Some s, Some ep))) e) + in + { rule = i; len = p; bindings }) + !best + +(* ------------------------------------------------------------------ *) +(* DFA interpreter *) +(* ------------------------------------------------------------------ *) + +(* Replicates the runtime behavior of PPX-generated code (see [gen_state] and + [gen_definition] in ppx_sedlex.ml): + - memory cells start at -1 ([__private__init_mem]); + - on entering an accepting state, its final operations run and the rule and + position are saved ([Sedlexing.mark]); the cells they write are only + written again on the next accept, so no snapshot is taken; + - a transition's tag operations record the position before the code point + that selected it ([__private__set_mem_prev_pos]); + - end of input is read once, as the code point -1, without advancing; + - on a dead end, the last mark wins ([Sedlexing.backtrack]). *) + +let eval_pos mem ~len (pe : Sedlex.pos_expr) = + match pe with + | Sedlex.Tag { tag; offset } -> + let v = mem.(tag) in + if v < 0 then None else Some (v + offset) + | Sedlex.Start_plus n -> Some n + | Sedlex.End_minus n -> Some (len - n) + +(* Mirrors [gen_binding_code]: bindings are grouped by name; for a name with + several (start, end, disc) alternatives the generated code emits an if/else + chain on the discriminator cells, falling through to the last alternative + unconditionally. *) +let extract_bindings (bindings : Sedlex.compiled_binding list) mem ~len = + let names = + List.rev + (List.fold_left + (fun acc (b : Sedlex.compiled_binding) -> + if List.mem b.name acc then acc else b.name :: acc) + [] bindings) + in + List.sort compare + (List.map + (fun name -> + let entries = + List.filter + (fun (b : Sedlex.compiled_binding) -> b.name = name) + bindings + in + let rec select = function + | [] -> assert false + | [(b : Sedlex.compiled_binding)] -> b + | (b : Sedlex.compiled_binding) :: rest -> + if List.for_all (fun (cell, v) -> mem.(cell) = v) b.disc then b + else select rest + in + let b = select entries in + (name, (eval_pos mem ~len b.start_pos, eval_pos mem ~len b.end_pos))) + names) + +let dfa_match (compiled : Sedlex.compiled_ir) (input : int array) : + match_result option = + let len = Array.length input in + let mem = Array.make (max compiled.num_tags 0) (-1) in + let apply pos ops = + (* A parallel move: copies read the cells as they were before the list. *) + let before = Array.copy mem in + List.iter + (fun (op : Sedlex.tag_op) -> + match op with + | Set_position { dst } -> mem.(dst) <- pos + | Set_value { dst; value } -> mem.(dst) <- value + | Copy { dst; src } -> mem.(dst) <- before.(src)) + ops + in + let marked = ref None in + let rec loop ?(eofed = false) state pos = + let st = compiled.dfa.(state) in + (* End of input is read once: what it leads to must be terminal. *) + if eofed && st.trans <> [||] then failwith "eof target with transitions"; + (match st.accept with + | Some { rule; final_ops } -> + apply pos final_ops; + marked := Some (rule, pos, mem) + | None -> ()); + if not eofed then ( + (* At end of input the runtime feeds the EOF code point (-1), without + advancing. *) + let ch = if pos < len then input.(pos) else -1 in + let tr = + Array.fold_left + (fun acc (cs, tgt, ops) -> + match acc with + | Some _ -> acc + | None -> if Cset.mem ch cs then Some (tgt, ops) else None) + None st.trans + in + match tr with + | None -> () + | Some (tgt, ops) -> + (* TDFA(1): the operations of a transition belong before the + character that selected it. *) + apply pos ops; + if pos < len then loop tgt (pos + 1) else loop ~eofed:true tgt pos) + in + loop 0 0; + Option.map + (fun (r, p, m) -> + { + rule = r; + len = p; + bindings = extract_bindings compiled.bindings.(r) m ~len:p; + }) + !marked + +(* ------------------------------------------------------------------ *) +(* Comparison and printing *) +(* ------------------------------------------------------------------ *) + +let show_result input (r : match_result option) = + match r with + | None -> "no match" + | Some { rule; len; bindings } -> + let buf = Buffer.create 32 in + Printf.bprintf buf "rule %d, len %d" rule len; + if bindings <> [] then ( + Buffer.add_string buf ", ["; + List.iteri + (fun i (name, (s, e)) -> + if i > 0 then Buffer.add_string buf ", "; + match (s, e) with + | Some s, Some e + when 0 <= s && s <= e && e <= Array.length input -> + Printf.bprintf buf "%s=%S" name + (String.init (e - s) (fun i -> + let c = input.(s + i) in + if c >= 32 && c < 127 then Char.chr c else '?')) + | s, e -> + let p = function + | None -> "unset" + | Some n -> string_of_int n + in + Printf.bprintf buf "%s=<%s..%s>" name (p s) (p e)) + bindings; + Buffer.add_string buf "]"); + Buffer.contents buf + +let codes s = Array.init (String.length s) (fun i -> Char.code s.[i]) + +let compile rules = + try Ok (Sedlex.compile_ir rules) with exn -> Error (Printexc.to_string exn) + +let oracle rules input_str = + let input = codes input_str in + let ref_s = show_result input (ref_match rules input) in + match compile rules with + | Error e -> Printf.printf "ERROR %S -> compile_ir raised: %s\n" input_str e + | Ok compiled -> + let dfa_s = show_result input (dfa_match compiled input) in + if String.equal ref_s dfa_s then + Printf.printf "%S -> %s\n" input_str ref_s + else Printf.printf "ERROR %S -> ref=%s dfa=%s\n" input_str ref_s dfa_s + +let check rules input_str = + let input = codes input_str in + match compile rules with + | Error _ -> false + | Ok compiled -> + String.equal + (show_result input (ref_match rules input)) + (show_result input (dfa_match compiled input)) + +(* ------------------------------------------------------------------ *) +(* Random pattern generation *) +(* ------------------------------------------------------------------ *) + +module G = QCheck2.Gen + +let gen_char = G.char_range 'a' 'd' + +let gen_cset = + G.oneof + [ + G.map (fun c -> Cset.singleton (Char.code c)) gen_char; + G.return Cset.eof; + G.map + (fun c -> Cset.union Cset.eof (Cset.singleton (Char.code c))) + gen_char; + G.map2 + (fun a b -> + let a = Char.code a and b = Char.code b in + Cset.interval (min a b) (max a b)) + gen_char gen_char; + ] + +(* Capture-free regexp of bounded depth. *) +let rec gen_simple depth = + if depth = 0 then G.map Ir.chars gen_cset + else ( + let sub = gen_simple (depth - 1) in + G.oneof_weighted + [ + (3, G.map Ir.chars gen_cset); + (2, G.map2 seq sub sub); + (2, G.map2 alt sub sub); + (1, G.map star sub); + (1, G.map plus sub); + ( 1, + G.map2 + (fun r (n, m) -> rep r n m) + sub + (G.map2 (fun n k -> (n, n + k)) (G.int_range 0 2) (G.int_range 0 2)) + ); + ]) + +let names = [| "x"; "y"; "z"; "w" |] + +(* A rule: a sequence of up to 4 elements with at least one capture. Captures + sit at the top level of the sequence (the Ir smart constructors reject them + under repetition), each binding a distinct name. *) +let gen_rule = + G.bind (G.int_range 1 4) (fun n -> + G.bind + (G.int_range 0 (n - 1)) + (fun cap_at -> + let gen_elem i = + let simple = G.bind (G.int_range 0 2) gen_simple in + if i = cap_at then G.map (capture names.(i)) simple + else + G.oneof_weighted + [(3, simple); (1, G.map (capture names.(i)) simple)] + in + let rec build i acc = + if i = n then acc else build (i + 1) (G.map2 seq acc (gen_elem i)) + in + build 1 (gen_elem 0))) + +(* An or-pattern rule: both branches bind the same name (discriminators). *) +let gen_or_rule = + let branch = G.bind (G.int_range 0 2) gen_simple in + G.map2 (fun a b -> alt (capture "x" a) (capture "x" b)) branch branch + +let gen_ir = G.oneof_weighted [(4, gen_rule); (1, gen_or_rule)] +let gen_input = G.string_size ~gen:gen_char (G.int_range 0 5) + +(* Deterministic property runner: fixed seed, no shrinking. Failing cases are + printed through [oracle] so expect blocks capture them; an empty expect + block means reference and DFA agree on every generated case. *) +let qcheck ?(count = 2000) ?(max_print = 5) gen = + let rand = Random.State.make [| 0x5ed1ec5 |] in + let failures = ref 0 in + for _ = 1 to count do + let rules, input = G.generate1 ~rand gen in + if not (check rules input) then ( + incr failures; + if !failures <= max_print then ( + Array.iteri + (fun i ir -> Printf.printf " rule%d: %s\n" i (Ir.show ir)) + rules; + oracle rules input)) + done; + if !failures > 0 then + Printf.printf "%d/%d cases failed (printed at most %d)\n" !failures count + max_print diff --git a/test/oracle/test_oracle.ml b/test/oracle/test_oracle.ml new file mode 100644 index 0000000..20b6ece --- /dev/null +++ b/test/oracle/test_oracle.ml @@ -0,0 +1,254 @@ +(* The package sedlex is released under the terms of an MIT-like license. *) +(* See the attached LICENSE file. *) +(* Copyright 2026, Hugo Heuzard *) + +(* Oracle tests: the brute-force reference matcher vs the compiled DFA. + Lines prefixed with ERROR mark disagreements, i.e. compiler bugs. The + random sweeps print nothing when every case agrees. *) + +open Oracle + +(* ================================================================== *) +(* Hand-written: basic capture shapes *) +(* ================================================================== *) + +let%expect_test "simple capture" = + oracle [| lit 'a' ^. capture "x" (plus (lit 'b')) ^. lit 'c' |] "abbc"; + [%expect {| "abbc" -> rule 0, len 4, [x="bb"] |}] + +let%expect_test "whole-match capture" = + oracle [| capture "x" (plus (cls 'a' 'z')) |] "hello"; + [%expect {| "hello" -> rule 0, len 5, [x="hello"] |}] + +let%expect_test "two captures in sequence" = + oracle + [| capture "x" (plus (lit 'a')) ^. capture "y" (plus (lit 'b')) |] + "aaabb"; + [%expect {| "aaabb" -> rule 0, len 5, [x="aaa", y="bb"] |}] + +let%expect_test "variable-length capture" = + oracle [| lit 'd' ^. capture "c" (star (cls 'a' 'c')) ^. lit 'd' |] "dabcd"; + [%expect {| "dabcd" -> rule 0, len 5, [c="abc"] |}] + +let%expect_test "no match" = + oracle [| lit 'a' ^. capture "x" (plus (lit 'b')) ^. lit 'c' |] "xyz"; + [%expect {| "xyz" -> no match |}] + +let%expect_test "multi-rule" = + oracle + [| + lit 'd' ^. capture "x" (plus (lit 'b')) ^. lit 'e'; + capture "x" (plus (cls 'a' 'c')); + |] + "dbbe"; + oracle + [| + lit 'd' ^. capture "x" (plus (lit 'b')) ^. lit 'e'; + capture "x" (plus (cls 'a' 'c')); + |] + "abc"; + [%expect + {| + "dbbe" -> rule 0, len 4, [x="bb"] + "abc" -> rule 1, len 3, [x="abc"] + |}] + +let%expect_test "backtrack restores tags" = + (* Rule 0 needs a trailing 'c'; on "aaabb" the DFA overruns into rule 0 + territory, fails, and must backtrack to rule 1's accepting state with + rule 1's captures. *) + oracle + [| + capture "x" (plus (lit 'a')) ^. plus (lit 'b') ^. lit 'c'; + capture "x" (plus (lit 'a')) ^. plus (lit 'b'); + |] + "aaabb"; + [%expect {| "aaabb" -> rule 1, len 5, [x="aaa"] |}] + +let%expect_test "bounded repetition" = + oracle [| rep (lit 'a') 2 4 ^. capture "x" (plus (lit 'b')) |] "aaabbb"; + oracle [| rep (lit 'a') 2 4 ^. capture "x" (plus (lit 'b')) |] "abbb"; + [%expect + {| + "aaabbb" -> rule 0, len 6, [x="bbb"] + "abbb" -> no match + |}] + +let%expect_test "complement, subtraction, intersection" = + oracle [| lit 'd' ^. capture "x" (compl (cls 'a' 'c')) ^. lit 'd' |] "dxd"; + oracle [| lit 'd' ^. capture "x" (compl (cls 'a' 'c')) ^. lit 'd' |] "dad"; + oracle [| capture "x" (plus (sub (cls 'a' 'e') (cls 'c' 'e'))) |] "abcde"; + oracle [| capture "x" (plus (inter (cls 'a' 'd') (cls 'c' 'f'))) |] "cdabe"; + [%expect + {| + "dxd" -> rule 0, len 3, [x="x"] + "dad" -> no match + "abcde" -> rule 0, len 2, [x="ab"] + "cdabe" -> rule 0, len 2, [x="cd"] + |}] + +let%expect_test "Rep(0,1) capture" = + oracle [| capture "z" (opt (cls 'a' 'c')) ^. star (lit 'b') |] "b"; + oracle [| capture "z" (opt (cls 'a' 'c')) ^. star (lit 'b') |] "ab"; + oracle [| capture "z" (opt (cls 'a' 'c')) ^. star (lit 'b') |] "bb"; + [%expect + {| + "b" -> rule 0, len 1, [z="b"] + "ab" -> rule 0, len 2, [z="a"] + "bb" -> rule 0, len 2, [z="b"] + |}] + +(* ================================================================== *) +(* Or-patterns and discriminators *) +(* ================================================================== *) + +let%expect_test "or-pattern with discriminator" = + oracle [| alt (capture "x" (lit 'a')) (capture "x" (lit 'b')) |] "a"; + oracle [| alt (capture "x" (lit 'a')) (capture "x" (lit 'b')) |] "b"; + oracle [| alt (capture "x" (lit 'a')) (capture "x" (lit 'b')) |] "c"; + [%expect + {| + "a" -> rule 0, len 1, [x="a"] + "b" -> rule 0, len 1, [x="b"] + "c" -> no match + |}] + +let%expect_test "or-pattern with variable-length branches" = + let rules () = + [| + alt + (capture "x" (plus (lit 'a'))) + (capture "x" (seq (lit 'b') (plus (lit 'c')))); + |] + in + oracle (rules ()) "aaa"; + oracle (rules ()) "bccc"; + [%expect + {| + "aaa" -> rule 0, len 3, [x="aaa"] + "bccc" -> rule 0, len 4, [x="bccc"] + |}] + +(* ================================================================== *) +(* Greedy disambiguation *) +(* ================================================================== *) + +let%expect_test "Rep greedy disambiguation" = + (* Bounded repetition should greedily take as many iterations as the + overall longest match allows. *) + oracle [| capture "z" (rep (cls 'a' 'c') 0 2) ^. plus (lit 'a') |] "aaa"; + oracle [| capture "z" (rep (cls 'a' 'c') 0 2) ^. plus (lit 'a') |] "aa"; + oracle [| capture "z" (rep (cls 'a' 'c') 0 2) ^. lit 'a' |] "aa"; + oracle [| capture "z" (rep (cls 'a' 'c') 0 2) ^. lit 'a' |] "aaa"; + [%expect + {| + "aaa" -> rule 0, len 3, [z="aa"] + "aa" -> rule 0, len 2, [z="a"] + "aa" -> rule 0, len 2, [z="a"] + "aaa" -> rule 0, len 3, [z="aa"] + |}] + +let%expect_test "Rep with even-parity tail" = + (* plus(seq('a','a')) needs an even number of trailing chars: for "aaaaa" + z=1 is the only repetition count that lets the tail match. *) + oracle + [| capture "z" (rep (lit 'a') 0 2) ^. plus (lit 'a' ^. lit 'a') |] + "aaaaa"; + [%expect {| "aaaaa" -> rule 0, len 5, [z="a"] |}] + +let%expect_test "star/capture ambiguity (greedy spec)" = + (* Greedy: the star takes as much as it can, the capture gets the rest. *) + oracle [| star (lit 'a') ^. capture "x" (plus (lit 'a')) |] "aaa"; + [%expect {| "aaa" -> rule 0, len 3, [x="a"] |}] + +(* ================================================================== *) +(* Loop preceding a capture (#199) *) +(* ================================================================== *) + +let%expect_test "star before variable-length capture" = + (* Unambiguous: the only complete parse is star="a", x="abb". The path + still in the star and the path inside the capture share DFA states, so + the capture's start must not follow the star's iterations. *) + oracle [| star (lit 'a') ^. capture "x" (lit 'a' ^. plus (lit 'b')) |] "aabb"; + [%expect {| "aabb" -> rule 0, len 4, [x="abb"] |}] + +let%expect_test "capture start after characters inside the capture" = + (* The star path stays alive while characters are consumed inside the + capture; x must still be a value Plus 'a' can match. *) + oracle [| star (lit 'a') ^. capture "x" (plus (lit 'a')) ^. lit 'b' |] "aab"; + [%expect {| "aab" -> rule 0, len 3, [x="a"] |}] + +let%expect_test "star before capture with overlapping alt" = + oracle + [| + star (lit 'a') + ^. capture "y" (alt (lit 'a' ^. cls 'a' 'c') (star (cls 'a' 'c'))); + |] + "aba"; + [%expect {| "aba" -> rule 0, len 3, [y="ba"] |}] + +let%expect_test "stars around a single-char capture" = + oracle [| star (lit 'a') ^. capture "y" (lit 'b') ^. star (lit 'b') |] "bb"; + [%expect {| "bb" -> rule 0, len 2, [y="b"] |}] + +(* ================================================================== *) +(* End of input *) +(* ================================================================== *) + +let%expect_test "capture before eof" = + oracle [| capture "x" (star (lit 'a')) ^. eof |] "aa"; + oracle [| capture "x" (star (lit 'a')) ^. eof |] ""; + oracle [| capture "x" (star (lit 'a')) ^. eof |] "ab"; + [%expect + {| + "aa" -> rule 0, len 2, [x="aa"] + "" -> rule 0, len 0, [x=""] + "ab" -> no match + |}] + +let%expect_test "eof adds no length: rule priority decides" = + oracle + [| capture "x" (plus (lit 'a')); capture "y" (plus (lit 'a')) ^. eof |] + "aa"; + oracle + [| capture "x" (plus (lit 'a')) ^. eof; capture "y" (plus (lit 'a')) |] + "aa"; + [%expect + {| + "aa" -> rule 0, len 2, [x="aa"] + "aa" -> rule 0, len 2, [x="aa"] + |}] + +let%expect_test "eof tie within a rule" = + (* Both branches match "a"; the left one wins. *) + oracle [| alt (capture "x" (lit 'a')) (capture "x" (lit 'a' ^. eof)) |] "a"; + oracle [| alt (capture "x" (lit 'a' ^. eof)) (capture "x" (lit 'a')) |] "a"; + [%expect + {| + "a" -> rule 0, len 1, [x="a"] + "a" -> rule 0, len 1, [x="a"] + |}] + +let%expect_test "eof is read once" = + oracle [| capture "x" (plus eof) |] ""; + oracle [| capture "x" (star (alt (lit 'a') eof)) |] "aa"; + oracle [| capture "x" (eof ^. eof) |] ""; + [%expect + {| + "" -> rule 0, len 0, [x=""] + "aa" -> rule 0, len 2, [x="aa"] + "" -> no match + |}] + +(* ================================================================== *) +(* Random sweeps (deterministic seed) *) +(* ================================================================== *) + +let%expect_test "qcheck: single rule" = + qcheck (G.map2 (fun r s -> ([| r |], s)) gen_ir gen_input); + [%expect {| |}] + +let%expect_test "qcheck: two rules" = + qcheck ~count:1000 + (G.map3 (fun a b s -> ([| a; b |], s)) gen_ir gen_ir gen_input); + [%expect {| |}]