noslop
/
noslop-sedlex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: noslop:main
Branches Tags
noslop:main
noslop:upstream/v3.7
noslop:upstream/v3.6
noslop:upstream/v3.5
noslop:upstream/v3.4
noslop:upstream/v3.3
noslop:upstream/v3.2
noslop:upstream/v3.1
noslop:upstream/v3.0
noslop:upstream/v2.6
noslop:upstream/v2.5
noslop:upstream/v2.4
noslop:upstream/v2.3
noslop:upstream/v2.2
noslop:upstream/v2.1
noslop:upstream/v2.0
noslop:upstream/v1.99.4
noslop:upstream/v1.99.3
noslop:upstream/sedlex-1.99.2
..
compare: noslop:upstream/v2.5
Branches Tags
noslop:main
noslop:upstream/v3.7
noslop:upstream/v3.6
noslop:upstream/v3.5
noslop:upstream/v3.4
noslop:upstream/v3.3
noslop:upstream/v3.2
noslop:upstream/v3.1
noslop:upstream/v3.0
noslop:upstream/v2.6
noslop:upstream/v2.5
noslop:upstream/v2.4
noslop:upstream/v2.3
noslop:upstream/v2.2
noslop:upstream/v2.1
noslop:upstream/v2.0
noslop:upstream/v1.99.4
noslop:upstream/v1.99.3
noslop:upstream/sedlex-1.99.2

No commits in common. "main" and "upstream/v2.5" have entirely different histories.
main ... upstream/v

62 changed files with 5759 additions and 7336 deletions
Show Stats Expand all files Collapse all files

1
.envrc
View File

@ -1 +0,0 @@
use flake

1
.gitattributes vendored
View File

@ -1 +0,0 @@
test/stress_test.ml text diff

3
.github/CODEOWNERS vendored Normal file
View File

@ -0,0 +1,3 @@
# These are the default owners for everything in the repo. They will
# be requested for review when someone opens a pull request.
* @alainfrisch @Drup @pmetzger

20
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,20 @@
name: CI
on: [push]
jobs:
build:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
steps:
- uses: actions/checkout@v1
- name: Setup OCaml
uses: avsm/setup-ocaml@master
- name: Install locally
run: opam install -y .
- name: Build locally
run: opam exec dune build
- name: Run tests locally
run: opam exec dune runtest

1
.gitignore vendored
View File

@ -19,4 +19,3 @@ examples/complement
examples/tokenizer examples/tokenizer
examples/subtraction examples/subtraction
_opam _opam
.direnv

10
.ocamlformat
View File

@ -1,10 +0,0 @@
version=0.29.0
profile = conventional
break-separators = after
space-around-lists = false
doc-comments = before
match-indent = 2
match-indent-nested = always
parens-ite
exp-grouping = preserve
module-item-spacing = compact

19
.travis.yml Normal file
View File

@ -0,0 +1,19 @@
language: c
sudo: required
install: test -e .travis.opam.sh || wget https://raw.githubusercontent.com/ocaml/ocaml-ci-scripts/master/.travis-opam.sh
script: bash -ex .travis-opam.sh
env:
global:
- PACKAGE=sedlex
matrix:
- OCAML_VERSION=4.04
- OCAML_VERSION=4.05
- OCAML_VERSION=4.06
- OCAML_VERSION=4.07
- OCAML_VERSION=4.08
- OCAML_VERSION=4.09
- OCAML_VERSION=4.10
- OCAML_VERSION=4.11
os:
- linux
- osx

84
CHANGES Normal file
View File

@ -0,0 +1,84 @@
2.5:
* Fix exponential compilation time, thanks to @mnxn for reporting in #97
and @fangyi-zhou for fixing in #106
* Update unicode support for `14.0.0`.
2.4
* Update `dune` support to `2.8`, add auto-generated `opam` files.
* Optimize generated code, thanks to @bobzhang
* Update unicode version to 13.0.0
2.3
* Switch to ppxlib
2.2
* Support for OCaml 4.08
2.1
* GPR#78: Auto-generate unicode data
2.0
* GPR#70: Switch to dune, opam v2
* GPR#60: Breaking change: switch from int codepoints to Uchar.t
codepoints
* GPR#59: Track lexing position
-------------------------------------------------------------------------------
1.99.4
* GPR#47: Switch to ocaml-migrate-parsetree (contributed by Adrien Guatto)
* GPR#42: Added 'Rep' (repeat operator) (contributed by jpathy)
1.99.3
* Update to work with 4.03 (4.02 still supported)
1.99.2
* First official release of sedlex
1.99.1
* Support for new Ast_mapper registration API, follow OCaml trunk after
the inclusion of the extension_point branch
1.99
* First version of sedlex. The history below refers to ulex, the ancestor
or sedlex implemented with Camlp4.
-------------------------------------------------------------------------------
1.1
* Generate (more) globally unique identifiers to avoid conflicts when open'ing another module
processed by ulex (issue reported by Gerd Stolpmann)
1.0
* Update to the new Camlp4 and to ocamlbuild (release for OCaml 3.10
only), by Nicolas Pouillard.
0.8
* Really make it work with OCaml 3.09.
* Support for Utf-16.
0.7 released May 24 2005
* Bug fixes
* Update to OCaml 3.09 (currently CVS). Still works with OCaml 3.08.
* MIT-like license (used to LGPL)
0.5 release Jul. 8 2004
* Document how to use a custom implementation for lex buffers
* Update to OCaml 3.08
0.4 released Jan. 10 2004
* Bug fix (accept 1114111 as valid Unicode code point)
* Add the rollback function
0.3 released Oct. 8 2003
* Bug fix
* Add a new predefined class for ISO identifiers
0.2 released Sep. 22 2003
* Changed the names of predefined regexp
* Fix max_code = 0x10ffff
* Lexers that changes encoding on the fly
* Documentation of the interface Ulexing
0.1 released Sep. 20 2003
* Initial release

128
CHANGES.md
View File

@ -1,128 +0,0 @@
# 3.7 (2025-10-06)
- Update to unicode 17.0.0
# 3.6 (2025-01-05)
- Fixed one of the ranges implementing
Implement Corrigendum #1: UTF-8 Shortest Form
for 4-bytes long characters (#171)
# 3.5 (2025-05-29)
- Implement Corrigendum #1: UTF-8 Shortest Form
- Add utf8 support for string literal (#127)
# 3.4 (2025-03-28)
- Make the library compatibility with ppxlib.0.36 (#166)
# 3.3 (2024-10-29)
- Add support for unicode `16.0.0` (#157)
- Add API for retrieving start and stop positions separately (#155)
# 3.2 (2023-06-28):
- Restore compatibility with OCaml 4.08
- Use `Sedlexing.{Utf8,Utf16}.from_gen` to initialize UTF8 (resp. UTF16) lexing buffers from
string.
- Delay raising Malformed until actually reading the malformed part of the imput. (#140)
- Count lines in all cases (#130). Previously, certain functions for initiating the
lexical buffer would disable lines counting.
- Check and fix invariants from Cset. The codebase was not respecting
invariants documented in the Cset module which could break code
relying on it. The code generated by sedlex.ppx could be affected.
- Do not rely on comments from unicode UCD files
- Add API to track position in bytes. Should be opt-in and backward compatible. (#146)
# 3.1:
- Fix directly nested sedlex matches (@smuenzel, PR #117, fixes: #12)
- Use explicit stdlib in generated code (@hhugo, PR #122, fixes: #115)
- Preserve location of lexbuf (@hhugo, PR #118, fixes: #19)
- Don't use gen to consume channels (@hhugo, PR #124, fixes: #45)
- New expect_test testsuite (@hhugo, PR #124)
- Properly recognize malformed truncated input (@hhugo, PR #124)
- Raise `Malformed` instead of `Invalid_arg` (@hhugo, PR #126, fixes: #91)
- Updated unicode support to `15.0.0`
# 3.0:
- Dropped `Stream` api which was removed in `4.14.0` ahead of the `5.0`
release.
2.6:
- Adapted to ppxlib `0.26`, thanks to @pitag-ha
2.5:
- Fix exponential compilation time, thanks to @mnxn for reporting in #97
and @fangyi-zhou for fixing in #106
- Update unicode support for `14.0.0`.
# 2.4
- Update `dune` support to `2.8`, add auto-generated `opam` files.
- Optimize generated code, thanks to @bobzhang
- Update unicode version to 13.0.0
# 2.3
- Switch to ppxlib
# 2.2
- Support for OCaml 4.08
# 2.1
- GPR#78: Auto-generate unicode data
# 2.0
- GPR#70: Switch to dune, opam v2
- GPR#60: Breaking change: switch from int codepoints to Uchar.t
codepoints
- GPR#59: Track lexing position
# 1.99.4
- GPR#47: Switch to ocaml-migrate-parsetree (contributed by Adrien Guatto)
- GPR#42: Added 'Rep' (repeat operator) (contributed by jpathy)
# 1.99.3
- Update to work with 4.03 (4.02 still supported)
# 1.99.2
- First official release of sedlex
# 1.99.1
- Support for new Ast_mapper registration API, follow OCaml trunk after
the inclusion of the extension_point branch
# 1.99
- First version of sedlex. The history below refers to ulex, the ancestor
or sedlex implemented with Camlp4.
# 1.1
- Generate (more) globally unique identifiers to avoid conflicts when open'ing another module
processed by ulex (issue reported by Gerd Stolpmann)
# 1.0
- Update to the new Camlp4 and to ocamlbuild (release for OCaml 3.10
only), by Nicolas Pouillard.
# 0.8
- Really make it work with OCaml 3.09.
- Support for Utf-16.
# 0.7 released May 24 2005
- Bug fixes
- Update to OCaml 3.09 (currently CVS). Still works with OCaml 3.08.
- MIT-like license (used to LGPL)
# 0.5 release Jul. 8 2004
- Document how to use a custom implementation for lex buffers
- Update to OCaml 3.08
# 0.4 released Jan. 10 2004
- Bug fix (accept 1114111 as valid Unicode code point)
- Add the rollback function-
# 0.3 released Oct. 8 2003
- Bug fix
- Add a new predefined class for ISO identifiers
# 0.2 released Sep. 22 2003
- Changed the names of predefined regexp
- Fix max_code = 0x10ffff
- Lexers that changes encoding on the fly
- Documentation of the interface Ulexing
# 0.1 released Sep. 20 2003
- Initial release

26
LICENSE
View File

@ -1,6 +1,22 @@
it's proprietary. all rights reserved. if you use this code in a way i don't like i will personally The MIT License (MIT)
make your car explode into hammers
Portions of this code are based on "sedlex", Copyright 2005, 2014 by Alain Copyright 2005, 2014 by Alain Frisch and LexiFi.
Frisch and LexiFi, released under the terms of the MIT license which can be
found in <licenses/LICENSE.sedlex> Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

27
Makefile Normal file
View File

@ -0,0 +1,27 @@
# The package sedlex is released under the terms of an MIT-like license.
# See the attached LICENSE file.
# Copyright 2005, 2013 by Alain Frisch and LexiFi.
INSTALL_ARGS := $(if $(PREFIX),--prefix $(PREFIX),)
.PHONY: build install uninstall clean doc test all
build:
dune build @install
install:
dune install $(INSTALL_ARGS)
uninstall:
dune uninstall $(INSTALL_ARGS)
clean:
dune clean
doc:
dune build @doc
test:
dune build @runtest
all: build test doc

28
README.md
View File

@ -1,6 +1,6 @@
# sedlex # sedlex
[![build](https://github.com/ocaml-community/sedlex/actions/workflows/build.yml/badge.svg)](https://github.com/ocaml-community/sedlex/actions/workflows/build.yml) [![Build Status](https://travis-ci.com/ocaml-community/sedlex.svg?branch=master)](https://travis-ci.com/ocaml-community/sedlex)
Unicode-friendly lexer generator for OCaml. Unicode-friendly lexer generator for OCaml.
@ -79,25 +79,12 @@ where:
Unlike ocamllex, lexers work on stream of Unicode codepoints, not Unlike ocamllex, lexers work on stream of Unicode codepoints, not
bytes. bytes.
Like ocamllex, sedlex uses **longest match** with **first rule priority**:
- The lexer always tries to match the longest possible prefix of the
input. It does so by continuing to read characters as long as some
rule can still match a longer string, while remembering the last
position at which a rule did match.
- When two or more rules match the same longest prefix (a tie), the
rule that appears first in the `match%sedlex` definition wins. For
example, given the rules `| "if" -> ...` and `| Plus ('a'..'z') -> ...`,
the input `"if"` is matched by the first rule because it is listed
first, even though the second rule also accepts `"if"`.
The actions can call functions from the Sedlexing module to extract The actions can call functions from the Sedlexing module to extract
(parts of) the matched lexeme, in the desired encoding. (parts of) the matched lexeme, in the desired encoding.
Regular expressions are syntactically OCaml patterns: Regular expressions are syntactically OCaml patterns:
- `"...."` (string constant): recognize the specified string. - `"...."` (string constant): recognize the specified string
- `'....'` (character constant) : recognize the specified character - `'....'` (character constant) : recognize the specified character
- `i` (integer constant) : recognize the specified codepoint - `i` (integer constant) : recognize the specified codepoint
- `'...' .. '...'`: character range - `'...' .. '...'`: character range
@ -116,9 +103,6 @@ Regular expressions are syntactically OCaml patterns:
and recognize the set of items in `R1` but not in `R2` ("subtract") and recognize the set of items in `R1` but not in `R2` ("subtract")
- `Intersect (R1,R2)` : assume that `R` is a single-character length regexp (see - `Intersect (R1,R2)` : assume that `R` is a single-character length regexp (see
below) and recognize the set of items which are in both `R1` and `R2` below) and recognize the set of items which are in both `R1` and `R2`
- `Utf8 R` : string literals inside R are assumed to be utf-8 encoded.
- `Latin1 R` : string literals inside R are assumed to be latin1 encoded.
- `Ascii R` : string literals inside R are assumed to be ascii encoded.
- `lid` (lowercase identifier) : reference a named regexp (see below) - `lid` (lowercase identifier) : reference a named regexp (see below)
A single-character length regexp is a regexp which does not contain (after A single-character length regexp is a regexp which does not contain (after
@ -128,9 +112,8 @@ with a length different from one.
Note: Note:
- The OCaml source is assumed to be encoded in UTF-8. - The OCaml source is assumed to be encoded in Latin1 (for string
- Strings and chars litterals will be interpreted in ASCII unless otherwise and character literals).
specified by the `Latin1`,`Ascii` and `Utf8` constructors in patterns.
It is possible to define named regular expressions with the following It is possible to define named regular expressions with the following
@ -258,6 +241,3 @@ The `examples/` subdirectory contains several samples of sedlex in use.
- improvements to the build system - improvements to the build system
- switched parts of ppx_sedlex to using concrete syntax (with ppx_metaquot) - switched parts of ppx_sedlex to using concrete syntax (with ppx_metaquot)
- Steffen Smolka: port to dune - Steffen Smolka: port to dune
- Romain Beauxis:
- Implementation of the unicode table extractors
- General maintenance

30
dune-project
View File

@ -1,29 +1,25 @@
(lang dune 3.20) (lang dune 2.8)
(version 2.5)
(version "3.7") (name sedlex)
(source (github ocaml-community/sedlex))
(name noslop-sedlex) (license MIT)
(source (uri "https://git.lain.faith/noslop/noslop-sedlex.git")) (authors "Alain Frisch <alain.frisch@lexifi.com>"
(license "LicenseRef-Proprietary") "https://github.com/ocaml-community/sedlex/graphs/contributors")
(authors "xenia <xenia@awoo.systems>") (maintainers "Alain Frisch <alain.frisch@lexifi.com>")
(maintainers "xenia <xenia@awoo.systems") (homepage "https://github.com/ocaml-community/sedlex")
(homepage "https://git.lain.faith/noslop/noslop-sedlex.git")
(maintenance_intent "(latest)")
(documentation "https://git.lain.faith/noslop/noslop-sedlex/wiki")
(generate_opam_files true) (generate_opam_files true)
(executables_implicit_empty_intf true)
(package (package
(name noslop-sedlex) (name sedlex)
(synopsis "An OCaml lexer generator for Unicode") (synopsis "An OCaml lexer generator for Unicode")
(description "sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports (description "sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports
Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular
OCaml source files. Lexing specific constructs are provided via a ppx syntax OCaml source files. Lexing specific constructs are provided via a ppx syntax
extension.") extension.")
(depends (depends
(ocaml (>= 4.08)) (ocaml (>= 4.04))
dune dune
(ppxlib (>= 0.26.0)) (ppxlib (>= 0.18.0))
gen gen
(ppx_expect :with-test))) uchar))

16
examples/complement.ml
View File

@ -1,17 +1,11 @@
let ucase = [%sedlex.regexp? 'A' .. 'Z'] let ucase = [%sedlex.regexp? 'A'..'Z']
let lcase = [%sedlex.regexp? 'a' .. 'z'] let lcase = [%sedlex.regexp? 'a'..'z']
let rec token buf = let rec token buf =
match%sedlex buf with match%sedlex buf with
| lcase -> | lcase -> print_char 'L';token buf
print_char 'L'; | Compl (ucase | lcase) -> print_char '?'; token buf
token buf | ucase -> print_char 'U';token buf
| Compl (ucase | lcase) ->
print_char '?';
token buf
| ucase ->
print_char 'U';
token buf
| eof -> print_endline "." | eof -> print_endline "."
| _ -> assert false | _ -> assert false

5
examples/dune
View File

@ -1,8 +1,9 @@
(executables (executables
(names tokenizer regressions complement subtraction repeat performance) (names tokenizer regressions complement subtraction repeat performance)
(libraries noslop-sedlex noslop-sedlex.ppx) (libraries sedlex sedlex_ppx)
(preprocess (preprocess
(pps noslop-sedlex.ppx))) (pps sedlex.ppx))
(flags :standard -w +39))
(rule (rule
(alias runtest) (alias runtest)

17
examples/performance.ml
View File

@ -1,16 +1,16 @@
let rec token buf = let rec token buf =
match%sedlex buf with any -> token buf | eof -> () | _ -> assert false match%sedlex buf with
| any -> token buf
| eof -> ()
| _ -> assert false
let time f x = let time f x =
let rec acc f x = function let rec acc f x = function
| 0 -> f x | 0 -> f x
| n -> | n -> f x|>ignore; acc f x (n-1) in
f x |> ignore; let t = Sys.time() in
acc f x (n - 1)
in
let t = Sys.time () in
let fx = acc f x 10 in let fx = acc f x 10 in
Printf.printf "Execution time: %fs\n" (Sys.time () -. t); Printf.printf "Execution time: %fs\n" (Sys.time() -. t);
fx fx
let () = let () =
@ -19,5 +19,4 @@ let () =
let lexbuf = Sedlexing.Latin1.from_string long_str in let lexbuf = Sedlexing.Latin1.from_string long_str in
(* let () = Sedlexing.set_curr_p lexbuf Lexing.dummy_pos in *) (* let () = Sedlexing.set_curr_p lexbuf Lexing.dummy_pos in *)
token lexbuf token lexbuf
in in time token_from long_str
time token_from long_str

85
examples/regressions.ml
View File

@ -1,58 +1,49 @@
(* This test that unicode_old.ml is a strict sub-set of new unicode.ml. *) (* This test that unicode_old.ml is a strict sub-set of
* new unicode.ml. *)
module CSet = Sedlex_ppx.Sedlex_cset let test_versions = ("13.0.0","14.0.0")
module Unicode = Sedlex_ppx.Unicode
let test_versions = ("16.0.0", "17.0.0") let regressions = []
let regressions = let interval s e =
[ (* Example *) Array.to_list
(* ("lt", CSet.union (CSet.singleton 0x1c5) (CSet.singleton (0x0001))) *) ] (Array.init (e-s) (fun pos -> s + pos))
let compare name (old_ : CSet.t) (new_ : CSet.t) = exception Found
let diff = CSet.difference old_ new_ in
let regressions = let test_exception name x =
match List.assoc name regressions with try
| exception Not_found -> CSet.empty let l = List.assoc name regressions in
| x -> x List.iter (fun (s,e) ->
if s<=x && x<=e then raise Found) l
with Not_found -> ()
let compare name old_l new_l =
let code_points = List.fold_left (fun res (s,e) ->
res@(interval s e)) [] old_l
in in
let regressions_intersect = CSet.intersection regressions old_ in let test x =
let regressions = CSet.difference regressions regressions_intersect in try
let regressions_useless = CSet.difference regressions new_ in test_exception name x;
let diff = CSet.difference diff regressions in List.iter (fun (s,e) ->
Seq.iter if s<=x && x<=e then raise Found) new_l;
(fun x -> false
Printf.printf with Found -> true
"Invalid regression for 0x%x in %s: already present in old set.\n" x in
name) List.iter (fun x ->
(CSet.to_seq regressions_intersect); if not (test x) then
Seq.iter Printf.printf "Code point 0x%x missing in %s!\n" x name)
(fun x -> code_points
Printf.printf "Invalid regression for 0x%x in %s: absent in new set.\n" x
name)
(CSet.to_seq regressions_useless);
Seq.iter
(fun x -> Printf.printf "Code point 0x%x missing in %s!\n" x name)
(CSet.to_seq diff)
let test new_l (name, old_l) = let test new_l (name, old_l) =
(* Cn is for unassigned code points, which are allowed to be (* Cn is for unassigned code points, which are allowed to be
* used in future version. *) * used in future version. *)
let old_l = Sedlex_utils.Cset.to_list old_l in if name <> "cn" then
if name <> "cn" then ( compare name old_l (List.assoc name new_l)
let old_l =
List.fold_left
(fun acc (a, b) -> CSet.union acc (CSet.interval a b))
CSet.empty old_l
in
compare name old_l (List.assoc name new_l))
let () = let () =
if (Unicode_old.version, Unicode.version) <> test_versions then if (Unicode_old.version,Sedlex_ppx.Unicode.version) <> test_versions then
failwith failwith (Printf.sprintf "Test written for versions: %s => %s\n%!" Unicode_old.version Sedlex_ppx.Unicode.version);
(Printf.sprintf "Test written for versions: %s => %s\n%!" Printf.printf "Testing Unicode regression: %s => %s\n%!" Unicode_old.version Sedlex_ppx.Unicode.version;
Unicode_old.version Unicode.version); List.iter (test Sedlex_ppx.Unicode.Categories.list) Unicode_old.Categories.list;
Printf.printf "Testing Unicode regression: %s => %s\n%!" Unicode_old.version List.iter (test Sedlex_ppx.Unicode.Properties.list) Unicode_old.Properties.list
Unicode.version;
List.iter (test Unicode.Categories.list) Unicode_old.Categories.list;
List.iter (test Unicode.Properties.list) Unicode_old.Properties.list

25
examples/repeat.ml
View File

@ -1,26 +1,15 @@
let rec token buf = let rec token buf =
match%sedlex buf with match%sedlex buf with
| white_space -> | white_space -> print_endline "\tWhitespace"; token buf
print_endline "\tWhitespace"; | 'a', Rep(white_space, 1) -> print_endline "a\n\tWhitespace"; token buf
token buf | Rep("bc", 2) -> print_endline "bcbc"; token buf
| 'a', Rep (white_space, 1) -> | Rep("d", 1 .. 1) -> print_endline "d"; token buf
print_endline "a\n\tWhitespace"; | Rep("ef", 1 .. 3) -> Printf.printf "%s\n" (Sedlexing.Utf8.lexeme buf); token buf
token buf
| Rep ("bc", 2) ->
print_endline "bcbc";
token buf
| Rep ("d", 1 .. 1) ->
print_endline "d";
token buf
| Rep ("ef", 1 .. 3) ->
Printf.printf "%s\n" (Sedlexing.Utf8.lexeme buf);
token buf
| eof -> print_endline "\tEnd" | eof -> print_endline "\tEnd"
| any -> | any -> print_endline "Other"; token buf
print_endline "Other";
token buf
| _ -> failwith "Internal failure: Reached impossible place" | _ -> failwith "Internal failure: Reached impossible place"
let () = let () =
let lexbuf = Sedlexing.Utf8.from_string "a bcbc d ef efef efefef" in let lexbuf = Sedlexing.Utf8.from_string "a bcbc d ef efef efefef" in
token lexbuf token lexbuf

21
examples/subtraction.ml
View File

@ -1,23 +1,14 @@
let rec token buf = let rec token buf =
match%sedlex buf with match%sedlex buf with
| white_space -> | white_space -> print_endline "\tWhitespace"; token buf
print_endline "\tWhitespace"; | Sub (Chars "ab","b") -> print_endline "a"; token buf
token buf | (Chars "ab"|"c") -> print_endline "abc"; token buf
| Sub (Chars "ab", "b") -> | Intersect ("d", Chars "abd") -> print_endline "d"; token buf
print_endline "a";
token buf
| Chars "ab" | "c" ->
print_endline "abc";
token buf
| Intersect ("d", Chars "abd") ->
print_endline "d";
token buf
| eof -> print_endline "\tEnd" | eof -> print_endline "\tEnd"
| any -> | any -> print_endline "Other"; token buf
print_endline "Other";
token buf
| _ -> failwith "Internal failure: Reached impossible place" | _ -> failwith "Internal failure: Reached impossible place"
let () = let () =
let lexbuf = Sedlexing.Utf8.from_string "a b c d e" in let lexbuf = Sedlexing.Utf8.from_string "a b c d e" in
token lexbuf token lexbuf

16
examples/tokenizer.ml
View File

@ -1,19 +1,13 @@
let digit = [%sedlex.regexp? '0' .. '9'] let digit = [%sedlex.regexp? '0'..'9']
let number = [%sedlex.regexp? Plus digit] let number = [%sedlex.regexp? Plus digit]
let rec token buf = let rec token buf =
let letter = [%sedlex.regexp? 'a' .. 'z' | 'A' .. 'Z'] in let letter = [%sedlex.regexp? 'a'..'z'|'A'..'Z'] in
match%sedlex buf with match%sedlex buf with
| number -> | number -> Printf.printf "Number %s\n" (Sedlexing.Latin1.lexeme buf); token buf
Printf.printf "Number %s\n" (Sedlexing.Latin1.lexeme buf); | letter, Star ('A'..'Z' | 'a'..'z' | digit) -> Printf.printf "Ident %s\n" (Sedlexing.Latin1.lexeme buf); token buf
token buf
| letter, Star ('A' .. 'Z' | 'a' .. 'z' | digit) ->
Printf.printf "Ident %s\n" (Sedlexing.Latin1.lexeme buf);
token buf
| Plus xml_blank -> token buf | Plus xml_blank -> token buf
| Plus (Chars "+*-/") -> | Plus (Chars "+*-/") -> Printf.printf "Op %s\n" (Sedlexing.Latin1.lexeme buf); token buf
Printf.printf "Op %s\n" (Sedlexing.Latin1.lexeme buf);
token buf
| 128 .. 255 -> print_endline "Non ASCII" | 128 .. 255 -> print_endline "Non ASCII"
| eof -> print_endline "EOF" | eof -> print_endline "EOF"
| _ -> failwith "Unexpected character" | _ -> failwith "Unexpected character"

3650
examples/unicode_old.ml
View File

File diff suppressed because it is too large Load Diff

45
flake.lock
View File

@ -1,45 +0,0 @@
{
"nodes": {
"dragnpkgs": {
"inputs": {
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1783018382,
"narHash": "sha256-Pw7GM1fh2AnQNCitswXTHPfsswiDnEjXPN17Ge8egz4=",
"ref": "nixos-26.05",
"rev": "085892ad486428b8e51b549539a5430f0c2da1a8",
"revCount": 247,
"type": "git",
"url": "https://git.lain.faith/haskal/dragnpkgs.git"
},
"original": {
"id": "dragnpkgs",
"type": "indirect"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1782847225,
"narHash": "sha256-JC9PjqKYG9ve5U8aDOLQipp3+KLANBHUvGdLZlxzdKI=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "95ca1e203c0750115fd4a6f17d5a245dfe6b1edd",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-26.05",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"dragnpkgs": "dragnpkgs"
}
}
},
"root": "root",
"version": 7
}

23
flake.nix
View File

@ -1,23 +0,0 @@
{
description = "Flake for noslop-sedlex";
outputs = { self, dragnpkgs } @ inputs: dragnpkgs.lib.mkFlake {
packages.default = { ocamlPackages }: ocamlPackages.callPackage ./package.nix {};
devShells.default = { ocamlPackages, stdenv, mkShell }: mkShell {
packages = with ocamlPackages; [
ocaml
dune_3
utop
odoc
alcotest
ocamlformat
] ++ (self.packages.${stdenv.hostPlatform.system}.default.propagatedBuildInputs)
++ (self.packages.${stdenv.hostPlatform.system}.default.nativeBuildInputs);
shellHook = ''
export OCAMLRUNPARAM=b
'';
};
};
}

22
licenses/LICENSE.sedlex
View File

@ -1,22 +0,0 @@
The MIT License (MIT)
Copyright 2005, 2014 by Alain Frisch and LexiFi.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

52
package.nix
View File

@ -1,52 +0,0 @@
{
lib,
fetchurl,
buildDunePackage,
gen,
ppxlib,
ppx_expect,
}: let
unicodeVersion = "17.0.0";
baseUrl = "https://www.unicode.org/Public/${unicodeVersion}";
DerivedCoreProperties = fetchurl {
url = "${baseUrl}/ucd/DerivedCoreProperties.txt";
hash = "sha256-JMf+0RlcSC+q79XB5+uCHF7h+23gfs26pktWqZ2iLAg=";
};
DerivedGeneralCategory = fetchurl {
url = "${baseUrl}/ucd/extracted/DerivedGeneralCategory.txt";
hash = "sha256-1i5bq3DKdPCZND9xIk+gUcsf3WGhq0XASIxEz8C2EC4=";
};
PropList = fetchurl {
url = "${baseUrl}/ucd/PropList.txt";
hash = "sha256-Ew3N3Kra8HEAi9/OHndD4E/fvJEIhvAX2fmskx2MZN0=";
};
in
buildDunePackage (finalAttrs: {
pname = "noslop-sedlex";
version = "3.7+DEV";
minimalOCamlVersion = "5.3";
src = ./.;
propagatedBuildInputs = [
gen
ppxlib
];
preBuild = ''
rm src/generator/data/dune
ln -s ${DerivedCoreProperties} src/generator/data/DerivedCoreProperties.txt
ln -s ${DerivedGeneralCategory} src/generator/data/DerivedGeneralCategory.txt
ln -s ${PropList} src/generator/data/PropList.txt
'';
checkInputs = [
ppx_expect
];
doCheck = true;
dontStrip = true;
})

27
noslop-sedlex.opam → sedlex.opam
View File

@ -1,23 +1,26 @@
# This file is generated by dune, edit dune-project instead # This file is generated by dune, edit dune-project instead
opam-version: "2.0" opam-version: "2.0"
version: "3.7" version: "2.5"
synopsis: "An OCaml lexer generator for Unicode" synopsis: "An OCaml lexer generator for Unicode"
description: """ description: """
sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports
Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular
OCaml source files. Lexing specific constructs are provided via a ppx syntax OCaml source files. Lexing specific constructs are provided via a ppx syntax
extension.""" extension."""
maintainer: ["xenia <xenia@awoo.systems"] maintainer: ["Alain Frisch <alain.frisch@lexifi.com>"]
authors: ["xenia <xenia@awoo.systems>"] authors: [
license: "LicenseRef-Proprietary" "Alain Frisch <alain.frisch@lexifi.com>"
homepage: "https://git.lain.faith/noslop/noslop-sedlex.git" "https://github.com/ocaml-community/sedlex/graphs/contributors"
doc: "https://git.lain.faith/noslop/noslop-sedlex/wiki" ]
license: "MIT"
homepage: "https://github.com/ocaml-community/sedlex"
bug-reports: "https://github.com/ocaml-community/sedlex/issues"
depends: [ depends: [
"ocaml" {>= "4.08"} "ocaml" {>= "4.04"}
"dune" {>= "3.20"} "dune" {>= "2.8"}
"ppxlib" {>= "0.26.0"} "ppxlib" {>= "0.18.0"}
"gen" "gen"
"ppx_expect" {with-test} "uchar"
"odoc" {with-doc} "odoc" {with-doc}
] ]
build: [ build: [
@ -34,5 +37,5 @@ build: [
"@doc" {with-doc} "@doc" {with-doc}
] ]
] ]
dev-repo: "https://git.lain.faith/noslop/noslop-sedlex.git" dev-repo: "git+https://github.com/ocaml-community/sedlex.git"
x-maintenance-intent: ["(latest)"] doc: "https://ocaml-community.github.io/sedlex/index.html"

1
sedlex.opam.template Normal file
View File

@ -0,0 +1 @@
doc: "https://ocaml-community.github.io/sedlex/index.html"

78
src/common/cset.ml
View File

@ -1,78 +0,0 @@
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
(* Character sets are represented as lists of intervals. The
intervals must be non-overlapping and not collapsable, and the list
must be ordered in increasing order. *)
type t = (int * int) list
let rec range_to_seq a b next () =
if a = b then Seq.Cons (a, next) else Seq.Cons (a, range_to_seq (a + 1) b next)
let rec to_seq x () =
match x with [] -> Seq.Nil | (a, b) :: xs -> range_to_seq a b (to_seq xs) ()
let check_invariant l =
let rec loop prev = function
| [] -> ()
| (a, b) :: xs ->
if a < prev then
failwith
(Printf.sprintf
"Sedlex_cset.of_list: not in increasing order or overlapping. \
[_-%d]-[%d-%d]"
prev a b);
if a = prev then
failwith
(Printf.sprintf
"Sedlex_cset.of_list: adjacent range. [_-%d]-[%d-%d]" prev a b);
if a > b then
failwith
(Printf.sprintf "Sedlex_cset.of_list: malformed range. [%d-%d]" a b);
loop b xs
in
loop (-1) l
let of_list l =
check_invariant l;
l
let to_list l = l
let max_code = 0x10ffff (* must be < max_int *)
let min_code = -1
let empty = []
let singleton i = [(i, i)]
let is_empty = function [] -> true | _ -> false
let interval i j = if i <= j then [(i, j)] else [(j, i)]
let eof = singleton (-1)
let any = interval 0 max_code
let rec union c1 c2 =
match (c1, c2) with
| [], _ -> c2
| _, [] -> c1
| ((i1, j1) as s1) :: r1, (i2, j2) :: r2 ->
if i1 <= i2 then
if j1 + 1 < i2 then s1 :: union r1 c2
else if j1 < j2 then union r1 ((i1, j2) :: r2)
else union c1 r2
else union c2 c1
let union_list : t list -> t = function
| [] -> empty
| [x] -> x
| l ->
List.concat l
|> List.sort (fun a b -> compare b a)
|> List.fold_left (fun (acc : t) (x : int * int) -> union [x] acc) empty
let complement c =
let rec aux start = function
| [] -> if start <= max_code then [(start, max_code)] else []
| (i, j) :: l -> (start, i - 1) :: aux (succ j) l
in
match c with (-1, j) :: l -> aux (succ j) l | l -> aux (-1) l
let intersection c1 c2 = complement (union (complement c1) (complement c2))
let difference c1 c2 = complement (union (complement c1) c2)

25
src/common/cset.mli
View File

@ -1,25 +0,0 @@
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
(** Representation of sets of unicode code points. *)
(** Character sets are represented as lists of intervals. The intervals must be
non-overlapping and not collapsable, and the list must be ordered in
increasing order. *)
type t = private (int * int) list
val of_list : (int * int) list -> t
val to_list : t -> (int * int) list
val min_code : int
val max_code : int
val empty : t
val any : t
val union : t -> t -> t
val union_list : t list -> t
val difference : t -> t -> t
val intersection : t -> t -> t
val is_empty : t -> bool
val eof : t
val singleton : int -> t
val interval : int -> int -> t
val to_seq : t -> int Seq.t

3
src/common/dune
View File

@ -1,3 +0,0 @@
(library
(name sedlex_utils)
(public_name noslop-sedlex.utils))

2
src/generator/data/base_url
View File

@ -1 +1 @@
https://www.unicode.org/Public/17.0.0 https://www.unicode.org/Public/14.0.0

33
src/generator/data/dune
View File

@ -1,35 +1,14 @@
(rule (rule
(target DerivedCoreProperties.txt) (targets DerivedCoreProperties.txt)
(deps base_url) (deps base_url)
(action (action (run curl -L -s %{read:base_url}/ucd/DerivedCoreProperties.txt -o DerivedCoreProperties.txt)))
(run
curl
-L
-s
%{read:base_url}/ucd/DerivedCoreProperties.txt
-o
%{target})))
(rule (rule
(target DerivedGeneralCategory.txt) (targets DerivedGeneralCategory.txt)
(deps base_url) (deps base_url)
(action (action (run curl -L -s %{read:base_url}/ucd/extracted/DerivedGeneralCategory.txt -o DerivedGeneralCategory.txt)))
(run
curl
-L
-s
%{read:base_url}/ucd/extracted/DerivedGeneralCategory.txt
-o
%{target})))
(rule (rule
(target PropList.txt) (targets PropList.txt)
(deps base_url) (deps base_url)
(action (action (run curl -L -s %{read:base_url}/ucd/PropList.txt -o PropList.txt)))
(run curl -L -s %{read:base_url}/ucd/PropList.txt -o %{target})))
(rule
(target UnicodeData.txt)
(deps base_url)
(action
(run curl -L -s %{read:base_url}/ucd/UnicodeData.txt -o %{target})))

8
src/generator/dune
View File

@ -1,3 +1,9 @@
(executable (executable
(name gen_unicode) (name gen_unicode)
(libraries str noslop-sedlex.utils)) (libraries str))
(rule
(targets gen_unicode.ml)
(deps gen_unicode.ml.inc)
(action (run cp gen_unicode.ml.inc gen_unicode.ml)))

209
src/generator/gen_unicode.ml
View File

@ -1,209 +0,0 @@
(* This file generates unicode data from
* the files exported at https://www.unicode.org/Public/<unicode version>
* and stored at src/generator/data. *)
open Sedlex_utils
module SSet = Set.Make (String)
let target = Sys.argv.(1)
let categories = Hashtbl.create 1024
let labels = Hashtbl.create 1024
(* Drop comments and split semi-column separated fields *)
let parse_line l =
let l =
match String.index_opt l '#' with None -> l | Some i -> String.sub l 0 i
in
String.split_on_char ';' l |> List.map String.trim
let parse_code s =
try int_of_string (Printf.sprintf "0x%s" s)
with _ -> failwith (Printf.sprintf "invalid code %s" s)
let parse_category x = String.lowercase_ascii (String.trim x)
let parse_prop x = String.lowercase_ascii (String.trim x)
let parse_interval s =
match String.split_on_char '.' (String.trim s) with
| [] -> assert false
| [x] ->
let x = parse_code x in
Cset.singleton x
| [x; ""; y] ->
let x = parse_code x and y = parse_code y in
Cset.interval x y
| _ -> failwith (Printf.sprintf "invalid interval %s" s)
let print_elements ch hashtbl cats =
let cats_set = SSet.of_list cats in
let all_keys = SSet.of_seq (Hashtbl.to_seq_keys hashtbl) in
let missing = SSet.diff cats_set all_keys in
let ignoring = SSet.diff all_keys cats_set in
let len = List.length cats in
List.iter
(fun c ->
let entries =
List.map
(fun (b, e) -> Printf.sprintf "0x%x, 0x%x" b e)
(Cset.union_list (Hashtbl.find_all hashtbl c) :> (int * int) list)
in
Printf.fprintf ch " let %s = Sedlex_cset.of_list\n [" c;
List.iteri
(fun i x ->
if i > 0 then
if i mod 5 = 0 then Printf.fprintf ch ";\n "
else Printf.fprintf ch "; ";
Printf.fprintf ch "%s" x)
entries;
Printf.fprintf ch "]\n\n")
cats;
Printf.fprintf ch " let list = [\n";
List.iteri
(fun pos c ->
Printf.fprintf ch " (%S, %s)%s\n" c c
(if pos == len - 1 then "" else ";"))
cats;
Printf.fprintf ch " ]\n\n";
if not (SSet.is_empty ignoring) then (
Printf.fprintf ch "(* ignoring:\n";
SSet.iter (fun s -> Printf.fprintf ch " - %s\n" s) ignoring;
Printf.fprintf ch "*)\n");
if not (SSet.is_empty missing) then (
Printf.fprintf ch "(* missing:\n";
SSet.iter (fun s -> Printf.fprintf ch " - %s\n" s) missing;
Printf.fprintf ch "*)\n")
let files =
[
( "PropList.txt",
fun s ->
match parse_line s with
| [""] -> ()
| [interval; prop] ->
let interval = parse_interval interval in
let prop = parse_prop prop in
Hashtbl.add labels prop interval
| _ -> assert false );
( "DerivedCoreProperties.txt",
fun s ->
match parse_line s with
| [""] -> ()
| [interval; prop] ->
let interval = parse_interval interval in
let prop = parse_prop prop in
Hashtbl.add labels prop interval
| [_interval; "InCB"; ("Extend" | "Consonant" | "Linker")] ->
(* TODO: support non-binary properties? *)
()
| _ -> assert false );
( "DerivedGeneralCategory.txt",
fun s ->
match parse_line s with
| [""] -> ()
| [interval; cat] ->
let interval = parse_interval interval in
let cat = parse_category cat in
Hashtbl.add categories cat interval
| _ -> assert false );
( "UnicodeData.txt",
fun s ->
match parse_line s with
| [""] -> ()
| interval :: _ :: cat :: _ ->
let interval = parse_interval interval in
let cat = parse_category cat in
Hashtbl.add categories cat interval
| _ -> assert false );
]
let read_version fname =
let version_rex =
Str.regexp "^# PropList-\\([0-9]+\\.[0-9]+\\.[0-9]+\\)\\.txt"
in
let ch = open_in_bin fname in
let s = input_line ch in
close_in ch;
ignore (Str.string_match version_rex s 0);
Str.matched_group 1 s
let exported_categories =
[
"cc";
"cf";
"cn";
"co";
"cs";
"ll";
"lm";
"lo";
"lt";
"lu";
"mc";
"me";
"mn";
"nd";
"nl";
"no";
"pc";
"pd";
"pe";
"pf";
"pi";
"po";
"ps";
"sc";
"sk";
"sm";
"so";
"zl";
"zp";
"zs";
]
let exported_properties =
[
"alphabetic";
"ascii_hex_digit";
"hex_digit";
"id_continue";
"id_start";
"lowercase";
"math";
"other_alphabetic";
"other_lowercase";
"other_math";
"other_uppercase";
"uppercase";
"white_space";
"xid_continue";
"xid_start";
]
let () =
let base_dir =
Filename.concat (Filename.dirname Sys.executable_name) "data"
in
let version = read_version (Filename.concat base_dir "PropList.txt") in
List.iter
(fun (fname, fn) ->
let ch = open_in_bin (Filename.concat base_dir fname) in
try
while true do
let ret = input_line ch in
fn ret
done
with End_of_file -> close_in ch)
files;
let ch = open_out_bin target in
Printf.fprintf ch {|[@@@ocamlformat "disable"]|};
Printf.fprintf ch "\n\n";
Printf.fprintf ch
"(* This file was automatically generated, do not edit. *)\n";
Printf.fprintf ch "(* Edit gen_unicode.ml.inc instead. *)\n\n";
Printf.fprintf ch "\n\nlet version = %S\n\n" version;
Printf.fprintf ch "module Categories = struct\n\n";
print_elements ch categories exported_categories;
Printf.fprintf ch "end\n\n";
Printf.fprintf ch "module Properties = struct\n\n";
print_elements ch labels exported_properties;
Printf.fprintf ch "end\n";
close_out ch

145
src/generator/gen_unicode.ml.inc Normal file
View File

@ -0,0 +1,145 @@
(* This file generates unicode data from
* the files exported at https://www.unicode.org/Public/<unicode version>
* and stored at src/generator/data. *)
let target = Sys.argv.(1)
let categories =
Hashtbl.create 1024
let labels =
Hashtbl.create 1024
(* Categories and Properties that we're keeping. *)
let keepers =
["cc"; "cf"; "cn"; "co"; "cs"; "ll"; "lm"; "lo"; "lt"; "lu"; "mc"; "me";
"mn"; "nd"; "nl"; "no"; "pc"; "pd"; "pe"; "pf"; "pi"; "po"; "ps"; "sc";
"sk"; "sm"; "so"; "zl"; "zp"; "zs"; "alphabetic"; "ascii_hex_digit";
"hex_digit"; "id_continue"; "id_start"; "lowercase"; "math";
"other_alphabetic"; "other_lowercase"; "other_math"; "other_uppercase";
"uppercase"; "white_space"; "xid_continue"; "xid_start"]
let prop_interval_rex =
Str.regexp "^\\([0-9a-fA-F]+\\)\\.\\.\\([0-9a-fA-F]+\\)[ ]*;[ ]+\\([a-zA-Z_]+\\)[ ]+#[ ]+\\([a-zA-Z][a-zA-Z&]\\)"
let prop_single_rex =
Str.regexp "^\\([0-9a-fA-F]+\\)[ ]*;[ ]+\\([a-zA-Z_]+\\)[ ]+#[ ]+\\([a-zA-Z][a-zA-Z&]\\)"
let derived_interval_rex =
Str.regexp "^\\([0-9a-fA-F]+\\)\\.\\.\\([0-9a-fA-F]+\\)[ ]*;[ ]+\\([a-zA-Z_]+\\)"
let derived_single_rex =
Str.regexp "^\\([0-9a-fA-F]+\\)[ ]*;[ ]+\\([a-zA-Z_]+\\)"
let add_entry hashtbl (b,e) name =
let mk s =
int_of_string (Printf.sprintf "0x%s" s)
in
let interval = (mk b, mk e) in
let label = String.lowercase_ascii name in
if List.mem label keepers then
Hashtbl.add hashtbl label interval
let match_interval s =
if (Str.string_match prop_interval_rex s 0) then
let interval = Str.matched_group 1 s, Str.matched_group 2 s in
add_entry labels interval (Str.matched_group 3 s);
add_entry categories interval (Str.matched_group 4 s)
let match_single s =
if (Str.string_match prop_single_rex s 0) then
let interval = Str.matched_group 1 s, Str.matched_group 1 s in
add_entry labels interval (Str.matched_group 2 s);
add_entry categories interval (Str.matched_group 3 s)
let match_derived_interval s =
if (Str.string_match derived_interval_rex s 0) then
let interval = Str.matched_group 1 s, Str.matched_group 2 s in
add_entry categories interval (Str.matched_group 3 s)
let match_derived_single s =
if (Str.string_match derived_single_rex s 0) then
let interval = Str.matched_group 1 s, Str.matched_group 1 s in
add_entry categories interval (Str.matched_group 2 s)
let split list n =
let rec aux acc rem =
match acc, rem with
| [], el::rem ->
aux [[el]] rem
| l::acc, el::rem when List.length l = n ->
aux ([el]::(List.rev l)::acc) rem
| l::acc, el::rem ->
aux ((el::l)::acc) rem
| _, [] -> List.rev acc
in
aux [] list
let print_elements ch hashtbl =
let cats =
List.sort_uniq compare
(Hashtbl.fold (fun cat _ l -> cat::l) hashtbl [])
in
let len = List.length cats in
List.iter (fun c ->
let entries =
List.map (fun (b,e) -> Printf.sprintf "0x%x, 0x%x" b e)
(List.sort_uniq compare
(Hashtbl.find_all hashtbl c))
in
let entries =
List.map (String.concat "; ") (split entries 5)
in
let entries =
String.concat ";\n " entries
in
Printf.fprintf ch " let %s =\n [%s]\n\n" c entries) cats;
Printf.fprintf ch " let list = [\n";
List.iteri (fun pos c ->
Printf.fprintf ch " (%S, %s)%s\n" c c (if pos == len - 1 then "" else ";"))
cats;
Printf.fprintf ch " ]\n\n"
let files = [
("PropList.txt", [match_interval; match_single]);
("DerivedCoreProperties.txt", [match_interval; match_single]);
("DerivedGeneralCategory.txt", [match_derived_interval; match_derived_single])
]
let read_version fname =
let version_rex =
Str.regexp "^# PropList-\\([0-9]+\\.[0-9]+\\.[0-9]+\\)\\.txt"
in
let ch = open_in_bin fname in
let s = input_line ch in
close_in ch;
ignore(Str.string_match version_rex s 0);
Str.matched_group 1 s
let () =
let base_dir =
Filename.concat (Filename.dirname Sys.executable_name) "data"
in
let version =
read_version (Filename.concat base_dir "PropList.txt")
in
List.iter (fun (fname, fns) ->
let ch =
open_in_bin
(Filename.concat base_dir fname)
in
try
while true do
let ret = input_line ch in
List.iter (fun fn -> fn ret) fns
done
with End_of_file -> close_in ch) files;
let ch = open_out_bin target in
Printf.fprintf ch "let version = %S\n\n" version;
Printf.fprintf ch "module Categories = struct\n\n";
print_elements ch categories;
Printf.fprintf ch "end\n\n";
Printf.fprintf ch "module Properties = struct\n\n";
print_elements ch labels;
Printf.fprintf ch "end\n";
close_out ch

5
src/lib/dune
View File

@ -1,5 +1,6 @@
(library (library
(name sedlex) (name sedlex)
(public_name noslop-sedlex) (public_name sedlex)
(wrapped false) (wrapped false)
(libraries gen)) (libraries gen uchar)
(flags :standard -w +A-4-9 -safe-string))

813
src/lib/sedlexing.ml
View File

File diff suppressed because it is too large Load Diff

413
src/lib/sedlexing.mli
View File

@ -1,300 +1,269 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
(** Runtime support for lexers generated by [sedlex]. *) (** Runtime support for lexers generated by [sedlex]. *)
(** This module is roughly equivalent to the module Lexing from the OCaml (** This module is roughly equivalent to the module Lexing from the
standard library, except that its lexbuffers handle Unicode code points OCaml standard library, except that its lexbuffers handle Unicode
(OCaml type: {!Uchar.t} in the range [0..0x10ffff]) instead of bytes (OCaml code points (OCaml type: {!Uchar.t} in the range [0..0x10ffff])
type: [char]). instead of bytes (OCaml type: [char]).
It is possible to have sedlex-generated lexers work on a custom It is possible to have sedlex-generated lexers work on a custom
implementation for lex buffers. To do this, define a module [L] which implementation for lex buffers. To do this, define a module [L]
implements the [start], [next], [mark] and [backtrack] functions (See the which implements the [start], [next], [mark] and [backtrack]
Internal Interface section below for a specification). They need not work on functions (See the Internal Interface section below for a
a type named [lexbuf]: you can use the type name you want. Then, just do in specification). They need not work on a type named [lexbuf]: you
your sedlex-processed source, bind this module to the name [Sedlexing] (for can use the type name you want. Then, just do in your
instance, with a local module definition: [let module Sedlexing = L in ...]. sedlex-processed source, bind this module to the name [Sedlexing]
(for instance, with a local module definition: [let module Sedlexing
= L in ...].
Of course, you'll probably want to define functions like [lexeme] to be used Of course, you'll probably want to define functions like [lexeme] to
in the lexers semantic actions. *) be used in the lexers semantic actions. *)
(** The type of lexer buffers. A lexer buffer is the argument passed to the
scanning functions defined by the generated lexers. The lexer buffer holds
the internal information for the scanners, including the code points of the
token currently scanned, its position from the beginning of the input
stream, and the current position of the lexer. *)
type lexbuf type lexbuf
(** The type of lexer buffers. A lexer buffer is the argument passed
to the scanning functions defined by the generated lexers.
The lexer buffer holds the internal information for the
scanners, including the code points of the token currently scanned,
its position from the beginning of the input stream,
and the current position of the lexer. *)
(** Raised by some functions to signal that some code point is not compatible
with a specified encoding. *)
exception InvalidCodepoint of int exception InvalidCodepoint of int
(** Raised by some functions to signal that some code point is not
compatible with a specified encoding. *)
(** Raised by functions in the [Utf8] and [Utf16] modules to report strings
which do not comply to the encoding. *)
exception MalFormed exception MalFormed
(** Raised by functions in the [Utf8] and [Utf16] modules to report
strings which do not comply to the encoding. *)
(** {6 Creating generic lexbufs} *) (** {6 Creating generic lexbufs} *)
(** Create a generic lexer buffer. When the lexer needs more characters, it will val create: (Uchar.t array -> int -> int -> int) -> lexbuf
call the given function, giving it an array of Uchars [a], a position [pos] (** Create a generic lexer buffer. When the lexer needs more
and a code point count [n]. The function should put [n] code points or less characters, it will call the given function, giving it an array of
in [a], starting at position [pos], and return the number of characters Uchars [a], a position [pos] and a code point count [n]. The
provided. A return value of 0 means end of input. [bytes_per_char] argument function should put [n] code points or less in [a], starting at
is optional. If unspecified, byte positions are the same as code point position [pos], and return the number of characters provided. A
position. *) return value of 0 means end of input. *)
val create :
?bytes_per_char:(Uchar.t -> int) ->
(Uchar.t array -> int -> int -> int) ->
lexbuf
(** set the initial tracked input position, in code point, for [lexbuf]. If val set_position: lexbuf -> Lexing.position -> unit
unspecified, byte postion is set to the same value as code point position. (** set the initial tracked input position for [lexbuf].
*) If set to [Lexing.dummy_pos], Sedlexing will not track position
val set_position : information for you. *)
?bytes_position:Lexing.position -> lexbuf -> Lexing.position -> unit
(** [set_filename lexbuf file] sets the filename to [file] in [lexbuf]. It also val set_filename: lexbuf -> string -> unit
sets the {!Lexing.pos_fname} field in returned {!Lexing.position} records. (** [set_filename lexbuf file] sets the filename to [file] in
*) [lexbuf]. It also sets the {!Lexing.pos_fname} field in
val set_filename : lexbuf -> string -> unit returned {!Lexing.position} records. *)
(** Create a lexbuf from a stream of Unicode code points. [bytes_per_char] is val from_gen: Uchar.t Gen.t -> lexbuf
optional. If unspecified, byte positions are the same as code point (** Create a lexbuf from a stream of Unicode code points. *)
positions. *)
val from_gen : ?bytes_per_char:(Uchar.t -> int) -> Uchar.t Gen.t -> lexbuf
(** Create a lexbuf from an array of Unicode code points. [bytes_per_char] is val from_stream: Uchar.t Stream.t -> lexbuf
optional. If unspecified, byte positions are the same as code point [@@ocaml.deprecated "Use [Sedlexing.from_gen] instead."]
positions. *) (** Create a lexbuf from a stream of Unicode code points. *)
val from_int_array : ?bytes_per_char:(Uchar.t -> int) -> int array -> lexbuf
(** Create a lexbuf from an array of Unicode code points. [bytes_per_char] is val from_int_array: int array -> lexbuf
optional. If unspecified, byte positions are the same as code point (** Create a lexbuf from an array of Unicode code points. *)
positions. *)
val from_uchar_array : val from_uchar_array: Uchar.t array -> lexbuf
?bytes_per_char:(Uchar.t -> int) -> Uchar.t array -> lexbuf (** Create a lexbuf from an array of Unicode code points. *)
(** {6 Interface for lexers semantic actions} *) (** {6 Interface for lexers semantic actions} *)
(** The following functions can be called from the semantic actions of lexer (** The following functions can be called from the semantic actions of
definitions. They give access to the character string matched by the regular lexer definitions. They give access to the character string matched
expression associated with the semantic action. *) by the regular expression associated with the semantic action. *)
(** [Sedlexing.lexeme_start lexbuf] returns the offset in the input stream of val lexeme_start: lexbuf -> int
the first code point of the matched string. The first code point of the (** [Sedlexing.lexeme_start lexbuf] returns the offset in the
stream has offset 0. *) input stream of the first code point of the matched string.
val lexeme_start : lexbuf -> int The first code point of the stream has offset 0. *)
(** [Sedlexing.lexeme_start lexbuf] returns the offset in the input stream of val lexeme_end: lexbuf -> int
the first byte of the matched string. The first code point of the stream has (** [Sedlexing.lexeme_end lexbuf] returns the offset in the input
offset 0. *) stream of the character following the last code point of the
val lexeme_bytes_start : lexbuf -> int matched string. The first character of the stream has offset
0. *)
(** [Sedlexing.lexeme_end lexbuf] returns the offset in the input stream of the val loc: lexbuf -> int * int
character following the last code point of the matched string. The first (** [Sedlexing.loc lexbuf] returns the pair
character of the stream has offset 0. *) [(Sedlexing.lexeme_start lexbuf,Sedlexing.lexeme_end
val lexeme_end : lexbuf -> int lexbuf)]. *)
(** [Sedlexing.lexeme_end lexbuf] returns the offset in the input stream of the val lexeme_length: lexbuf -> int
byte following the last code point of the matched string. The first (** [Sedlexing.lexeme_length lexbuf] returns the difference
character of the stream has offset 0. *) [(Sedlexing.lexeme_end lexbuf) - (Sedlexing.lexeme_start
val lexeme_bytes_end : lexbuf -> int lexbuf)], that is, the length (in code points) of the matched
string. *)
(** [Sedlexing.loc lexbuf] returns the pair val lexing_positions : lexbuf -> Lexing.position*Lexing.position
[(Sedlexing.lexeme_start lexbuf,Sedlexing.lexeme_end lexbuf)]. *) (** [Sedlexing.lexing_positions lexbuf] returns the start and end
val loc : lexbuf -> int * int positions of the current token, using a record of type
[Lexing.position]. This is intended for consumption
by parsers like those generated by [Menhir]. *)
(** [Sedlexing.bytes_loc lexbuf] returns the pair val new_line: lexbuf -> unit
[(Sedlexing.lexeme_bytes_start lexbuf,Sedlexing.lexeme_bytes_end lexbuf)]. (** [Sedlexing.new_line lexbuf] increments the line count and
*) sets the beginning of line to the current position, as though
val bytes_loc : lexbuf -> int * int a newline character had been encountered in the input. *)
(** [Sedlexing.lexeme_length lexbuf] returns the difference val lexeme: lexbuf -> Uchar.t array
[(Sedlexing.lexeme_end lexbuf) - (Sedlexing.lexeme_start lexbuf)], that is, (** [Sedlexing.lexeme lexbuf] returns the string matched by the
the length (in code points) of the matched string. *) regular expression as an array of Unicode code point. *)
val lexeme_length : lexbuf -> int
(** [Sedlexing.lexeme_length lexbuf] returns the difference val lexeme_char: lexbuf -> int -> Uchar.t
[(Sedlexing.lexeme_bytes_end lexbuf) - (Sedlexing.lexeme_bytes_start (** [Sedlexing.lexeme_char lexbuf pos] returns code point number [pos] in
lexbuf)], that is, the length (in bytes) of the matched string. *) the matched string. *)
val lexeme_bytes_length : lexbuf -> int
(** [Sedlexing.lexing_positions lexbuf] returns the start and end positions, in
code points, of the current token, using a record of type [Lexing.position].
This is intended for consumption by parsers like those generated by
[Menhir]. *)
val lexing_positions : lexbuf -> Lexing.position * Lexing.position
(** [Sedlexing.lexing_position_start lexbuf] returns the start position, in code
points, of the current token. *)
val lexing_position_start : lexbuf -> Lexing.position
(** [Sedlexing.lexing_position_curr lexbuf] returns the end position, in code
points, of the current token. *)
val lexing_position_curr : lexbuf -> Lexing.position
(** [Sedlexing.lexing_bytes_positions lexbuf] returns the start and end
positions, in bytes, of the current token, using a record of type
[Lexing.position]. This is intended for consumption by parsers like those
generated by [Menhir]. *)
val lexing_bytes_positions : lexbuf -> Lexing.position * Lexing.position
(** [Sedlexing.lexing_bytes_position_start lexbuf] returns the start position,
in bytes, of the current token. *)
val lexing_bytes_position_start : lexbuf -> Lexing.position
(** [Sedlexing.lexing_bytes_position_curr lexbuf] returns the end position, in
bytes, of the current token. *)
val lexing_bytes_position_curr : lexbuf -> Lexing.position
(** [Sedlexing.new_line lexbuf] increments the line count and sets the beginning
of line to the current position, as though a newline character had been
encountered in the input. *)
val new_line : lexbuf -> unit
(** [Sedlexing.lexeme lexbuf] returns the string matched by the regular
expression as an array of Unicode code point. *)
val lexeme : lexbuf -> Uchar.t array
(** [Sedlexing.lexeme_char lexbuf pos] returns code point number [pos] in the
matched string. *)
val lexeme_char : lexbuf -> int -> Uchar.t
val sub_lexeme: lexbuf -> int -> int -> Uchar.t array
(** [Sedlexing.sub_lexeme lexbuf pos len] returns a substring of the string (** [Sedlexing.sub_lexeme lexbuf pos len] returns a substring of the string
matched by the regular expression as an array of Unicode code point. *) matched by the regular expression as an array of Unicode code point. *)
val sub_lexeme : lexbuf -> int -> int -> Uchar.t array
val rollback: lexbuf -> unit
(** [Sedlexing.rollback lexbuf] puts [lexbuf] back in its configuration before (** [Sedlexing.rollback lexbuf] puts [lexbuf] back in its configuration before
the last lexeme was matched. It is then possible to use another lexer to the last lexeme was matched. It is then possible to use another
parse the same characters again. The other functions above in this section lexer to parse the same characters again. The other functions
should not be used in the semantic action after a call to above in this section should not be used in the semantic action
[Sedlexing.rollback]. *) after a call to [Sedlexing.rollback]. *)
val rollback : lexbuf -> unit
(** {6 Internal interface} *) (** {6 Internal interface} *)
(** These functions are used internally by the lexers. They could be used to (** These functions are used internally by the lexers. They could be used
write lexers by hand, or with a lexer generator different from [sedlex]. The to write lexers by hand, or with a lexer generator different from
lexer buffers have a unique internal slot that can store an integer. They [sedlex]. The lexer buffers have a unique internal slot that can store
also store a "backtrack" position. *) an integer. They also store a "backtrack" position.
*)
(** [start t] informs the lexer buffer that any code points until the current val start: lexbuf -> unit
position can be discarded. The current position become the "start" position (** [start t] informs the lexer buffer that any
as returned by [Sedlexing.lexeme_start]. Moreover, the internal slot is set code points until the current position can be discarded.
to [-1] and the backtrack position is set to the current position. *) The current position become the "start" position as returned
val start : lexbuf -> unit by [Sedlexing.lexeme_start]. Moreover, the internal slot is set to
[-1] and the backtrack position is set to the current position.
*)
(** [next lexbuf] extracts the next code point from the lexer buffer and val next: lexbuf -> Uchar.t option
increments to current position. If the input stream is exhausted, the (** [next lexbuf] extracts the next code point from the
function returns [None]. If a ['\n'] is encountered, the tracked line number lexer buffer and increments to current position. If the input stream
is incremented. *) is exhausted, the function returns [None].
val next : lexbuf -> Uchar.t option If a ['\n'] is encountered, the tracked line number is incremented. *)
(** [__private__next_int lexbuf] extracts the next code point from the lexer
buffer and increments to current position. If the input stream is exhausted,
the function returns -1. If a ['\n'] is encountered, the tracked line number
is incremented.
This is a private API, it should not be used by code using this module's API
and can be removed at any time. *)
val __private__next_int : lexbuf -> int val __private__next_int : lexbuf -> int
(** [__private__next_int lexbuf] extracts the next code point from the
lexer buffer and increments to current position. If the input stream
is exhausted, the function returns -1.
If a ['\n'] is encountered, the tracked line number is incremented.
(** [mark lexbuf i] stores the integer [i] in the internal slot. The backtrack This is a private API, it should not be used by code using this module's
position is set to the current position. *) API and can be removed at any time. *)
val mark : lexbuf -> int -> unit
(** [backtrack lexbuf] returns the value stored in the internal slot of the val mark: lexbuf -> int -> unit
buffer, and performs backtracking (the current position is set to the value (** [mark lexbuf i] stores the integer [i] in the internal
of the backtrack position). *) slot. The backtrack position is set to the current position. *)
val backtrack : lexbuf -> int
(** [with_tokenizer tokenizer lexbuf] given a lexer and a lexbuf, returns a val backtrack: lexbuf -> int
generator of tokens annotated with positions. This generator can be used (** [backtrack lexbuf] returns the value stored in the
with the Menir parser generator's incremental API. *) internal slot of the buffer, and performs backtracking
val with_tokenizer : (the current position is set to the value of the backtrack position). *)
(lexbuf -> 'token) ->
lexbuf -> val with_tokenizer: (lexbuf -> 'token) -> lexbuf -> (unit -> 'token * Lexing.position * Lexing.position)
unit -> (** [with_tokenizer tokenizer lexbuf] given a lexer and a lexbuf,
'token * Lexing.position * Lexing.position returns a generator of tokens annotated with positions.
This generator can be used with the Menir parser generator's
incremental API. *)
(** {6 Support for common encodings} *) (** {6 Support for common encodings} *)
module Latin1 : sig module Latin1: sig
(** Create a lexbuf from a Latin1 encoded stream (ie a stream of Unicode code val from_gen: char Gen.t -> lexbuf
points in the range [0..255]) *) (** Create a lexbuf from a Latin1 encoded stream (ie a stream
val from_gen : char Gen.t -> lexbuf of Unicode code points in the range [0..255]) *)
(** Create a lexbuf from a Latin1 encoded input channel. The client is val from_stream: char Stream.t -> lexbuf
responsible for closing the channel. *) [@@ocaml.deprecated "Use [Sedlexing.Latin1.from_gen] instead."]
val from_channel : in_channel -> lexbuf (** Create a lexbuf from a Latin1 encoded stream (ie a stream
of Unicode code points in the range [0..255]) *)
val from_channel: in_channel -> lexbuf
(** Create a lexbuf from a Latin1 encoded input channel.
The client is responsible for closing the channel. *)
val from_string: string -> lexbuf
(** Create a lexbuf from a Latin1 encoded string. *) (** Create a lexbuf from a Latin1 encoded string. *)
val from_string : string -> lexbuf
(** As [Sedlexing.lexeme] with a result encoded in Latin1. This function
throws an exception [InvalidCodepoint] if it is not possible to encode the
result in Latin1. *)
val lexeme : lexbuf -> string
(** As [Sedlexing.sub_lexeme] with a result encoded in Latin1. This function val lexeme: lexbuf -> string
throws an exception [InvalidCodepoint] if it is not possible to encode the (** As [Sedlexing.lexeme] with a result encoded in Latin1. This
result in Latin1. *) function throws an exception [InvalidCodepoint] if it is not
val sub_lexeme : lexbuf -> int -> int -> string possible to encode the result in Latin1. *)
(** As [Sedlexing.lexeme_char] with a result encoded in Latin1. This function val sub_lexeme: lexbuf -> int -> int -> string
throws an exception [InvalidCodepoint] if it is not possible to encode the (** As [Sedlexing.sub_lexeme] with a result encoded in Latin1.
result in Latin1. *) This function throws an exception [InvalidCodepoint] if it
val lexeme_char : lexbuf -> int -> char is not possible to encode the result in Latin1. *)
val lexeme_char: lexbuf -> int -> char
(** As [Sedlexing.lexeme_char] with a result encoded in Latin1.
This function throws an exception [InvalidCodepoint] if it
is not possible to encode the result in Latin1. *)
end end
module Utf8 : sig
module Utf8: sig
val from_gen: char Gen.t -> lexbuf
(** Create a lexbuf from a UTF-8 encoded stream. *) (** Create a lexbuf from a UTF-8 encoded stream. *)
val from_gen : char Gen.t -> lexbuf
val from_stream: char Stream.t -> lexbuf
[@@ocaml.deprecated "Use [Sedlexing.Utf8.from_gen] instead."]
(** Create a lexbuf from a UTF-8 encoded stream. *)
val from_channel: in_channel -> lexbuf
(** Create a lexbuf from a UTF-8 encoded input channel. *) (** Create a lexbuf from a UTF-8 encoded input channel. *)
val from_channel : in_channel -> lexbuf
val from_string: string -> lexbuf
(** Create a lexbuf from a UTF-8 encoded string. *) (** Create a lexbuf from a UTF-8 encoded string. *)
val from_string : string -> lexbuf
val lexeme: lexbuf -> string
(** As [Sedlexing.lexeme] with a result encoded in UTF-8. *) (** As [Sedlexing.lexeme] with a result encoded in UTF-8. *)
val lexeme : lexbuf -> string
val sub_lexeme: lexbuf -> int -> int -> string
(** As [Sedlexing.sub_lexeme] with a result encoded in UTF-8. *) (** As [Sedlexing.sub_lexeme] with a result encoded in UTF-8. *)
val sub_lexeme : lexbuf -> int -> int -> string
module Helper : sig
val width : char -> int
val check_two : int -> int -> int
val check_three : int -> int -> int -> int
val check_four : int -> int -> int -> int -> int
end
end end
module Utf16 : sig
module Utf16: sig
type byte_order = Little_endian | Big_endian type byte_order = Little_endian | Big_endian
(** [Utf16.from_gen s opt_bo] creates a lexbuf from an UTF-16 encoded stream. val from_gen: char Gen.t -> byte_order option -> lexbuf
If [opt_bo] matches with [None] the function expects a BOM (Byte Order (** [Utf16.from_gen s opt_bo] creates a lexbuf from an UTF-16
Mark), and takes the byte order as [Utf16.Big_endian] if it cannot find encoded stream. If [opt_bo] matches with [None] the function
one. When [opt_bo] matches with [Some bo], [bo] is taken as byte order. In expects a BOM (Byte Order Mark), and takes the byte order as
this case a leading BOM is kept in the stream - the lexer has to ignore it [Utf16.Big_endian] if it cannot find one. When [opt_bo]
and a `wrong' BOM ([0xfffe]) will raise Utf16.InvalidCodepoint. *) matches with [Some bo], [bo] is taken as byte order. In this
val from_gen : char Gen.t -> byte_order option -> lexbuf case a leading BOM is kept in the stream - the lexer has to
ignore it and a `wrong' BOM ([0xfffe]) will raise
Utf16.InvalidCodepoint. *)
val from_stream: char Stream.t -> byte_order option -> lexbuf
[@@ocaml.deprecated "Use [Sedlexing.Utf16.from_gen] instead."]
(** Works as [Utf16.from_gen] with a [stream]. *)
val from_channel: in_channel -> byte_order option-> lexbuf
(** Works as [Utf16.from_gen] with an [in_channel]. *) (** Works as [Utf16.from_gen] with an [in_channel]. *)
val from_channel : in_channel -> byte_order option -> lexbuf
val from_string: string -> byte_order option -> lexbuf
(** Works as [Utf16.from_gen] with a [string]. *) (** Works as [Utf16.from_gen] with a [string]. *)
val from_string : string -> byte_order option -> lexbuf
(** [utf16_lexeme lb bo bom] as [Sedlexing.lexeme] with a result encoded in val lexeme: lexbuf -> byte_order -> bool -> string
UTF-16 in byte_order [bo] and starting with a BOM if [bom = true]. *) (** [utf16_lexeme lb bo bom] as [Sedlexing.lexeme] with a result
val lexeme : lexbuf -> byte_order -> bool -> string encoded in UTF-16 in byte_order [bo] and starting with a BOM
if [bom = true]. *)
(** [sub_lexeme lb pos len bo bom] as [Sedlexing.sub_lexeme] with a result val sub_lexeme: lexbuf -> int -> int -> byte_order -> bool -> string
encoded in UTF-16 with byte order [bo] and starting with a BOM if (** [sub_lexeme lb pos len bo bom] as
[bom=true] *) [Sedlexing.sub_lexeme] with a result encoded in UTF-16 with
val sub_lexeme : lexbuf -> int -> int -> byte_order -> bool -> string byte order [bo] and starting with a BOM if [bom=true] *)
end end

16
src/syntax/dune
View File

@ -1,17 +1,21 @@
(library (library
(name sedlex_ppx) (name sedlex_ppx)
(public_name noslop-sedlex.ppx) (public_name sedlex.ppx)
(kind ppx_rewriter) (kind ppx_rewriter)
(libraries ppxlib noslop-sedlex noslop-sedlex.utils) (libraries ppxlib sedlex)
(ppx_runtime_libraries noslop-sedlex) (ppx_runtime_libraries sedlex)
(preprocess (preprocess
(pps ppxlib.metaquot))) (pps ppxlib.metaquot))
(flags
(:standard -w -9)))
(rule (rule
(targets unicode.ml) (targets unicode.ml)
(mode promote) (mode promote-until-clean)
(deps (deps
(:gen ../generator/gen_unicode.exe) (:gen ../generator/gen_unicode.exe)
(glob_files ../generator/data/*.txt)) ../generator/data/DerivedCoreProperties.txt
../generator/data/DerivedGeneralCategory.txt
../generator/data/PropList.txt)
(action (action
(run %{gen} %{targets}))) (run %{gen} %{targets})))

217
src/syntax/iso.ml
View File

@ -1,217 +0,0 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
open Sedlex_cset
let tr8876_ident_char =
let l =
[
(* ASCII *)
(0x0041, 0x005a);
(0x0061, 0x007a);
(* Latin *)
(0x00c0, 0x00d6);
(0x00d8, 0x00f6);
(0x00f8, 0x01f5);
(0x01fa, 0x0217);
(0x0250, 0x02a8);
(* Greek *)
(0x0384, 0x0384);
(0x0388, 0x038a);
(0x038c, 0x038c);
(0x038e, 0x03a1);
(0x03a3, 0x03ce);
(0x03d0, 0x03d6);
(0x03da, 0x03da);
(0x03dc, 0x03dc);
(0x03de, 0x03de);
(0x03e0, 0x03e0);
(0x03e2, 0x03f3);
(* Cyrillic *)
(0x0401, 0x040d);
(0x040f, 0x044f);
(0x0451, 0x045c);
(0x045e, 0x0481);
(0x0490, 0x04c4);
(0x04c7, 0x04c4);
(0x04cb, 0x04cc);
(0x04d0, 0x04eb);
(0x04ee, 0x04f5);
(0x04f8, 0x04f9);
(* Armenian *)
(0x0531, 0x0556);
(0x0561, 0x0587);
(0x04d0, 0x04eb);
(* Hebrew *)
(0x05d0, 0x05ea);
(0x05f0, 0x05f4);
(* Arabic *)
(0x0621, 0x063a);
(0x0640, 0x0652);
(0x0670, 0x06b7);
(0x06ba, 0x06be);
(0x06c0, 0x06ce);
(0x06e5, 0x06e7);
(* Devanagari *)
(0x0905, 0x0939);
(0x0958, 0x0962);
(* Bengali *)
(0x0985, 0x098c);
(0x098f, 0x0990);
(0x0993, 0x09a8);
(0x09aa, 0x09b0);
(0x09b2, 0x09b2);
(0x09b6, 0x09b9);
(0x09dc, 0x09dd);
(0x09df, 0x09e1);
(0x09f0, 0x09f1);
(* Gurmukhi *)
(0x0a05, 0x0a0a);
(0x0a0f, 0x0a10);
(0x0a13, 0x0a28);
(0x0a2a, 0x0a30);
(0x0a32, 0x0a33);
(0x0a35, 0x0a36);
(0x0a38, 0x0a39);
(0x0a59, 0x0a5c);
(0x0a5e, 0x0a5e);
(* Gunjarati *)
(0x0a85, 0x0a8b);
(0x0a8d, 0x0a8d);
(0x0a8f, 0x0a91);
(0x0a93, 0x0aa8);
(0x0aaa, 0x0ab0);
(0x0ab2, 0x0ab3);
(0x0ab5, 0x0ab9);
(0x0ae0, 0x0ae0);
(* Oriya *)
(0x0b05, 0x0b0c);
(0x0b0f, 0x0b10);
(0x0b13, 0x0b28);
(0x0b2a, 0x0b30);
(0x0b32, 0x0b33);
(0x0b36, 0x0b39);
(0x0b5c, 0x0b5d);
(0x0b5f, 0x0b61);
(* Tamil *)
(0x0b85, 0x0b8a);
(0x0b8e, 0x0b90);
(0x0b92, 0x0b95);
(0x0b99, 0x0b9a);
(0x0b9c, 0x0b9c);
(0x0b9e, 0x0b9f);
(0x0ba3, 0x0ba4);
(0x0ba8, 0x0baa);
(0x0bae, 0x0bb5);
(0x0bb7, 0x0bb9);
(* Telugu *)
(0x0c05, 0x0c0c);
(0x0c0e, 0x0c10);
(0x0c12, 0x0c28);
(0x0c2a, 0x0c33);
(0x0c35, 0x0c39);
(0x0c60, 0x0c61);
(* Kannada *)
(0x0c85, 0x0c8c);
(0x0c8e, 0x0c90);
(0x0c92, 0x0ca8);
(0x0caa, 0x0cb3);
(0x0cb5, 0x0cb9);
(0x0ce0, 0x0ce1);
(* Malayam *)
(0x0d05, 0x0d0c);
(0x0d0e, 0x0d10);
(0x0d12, 0x0d28);
(0x0d2a, 0x0d39);
(0x0d60, 0x0d61);
(* Thai *)
(0x0e01, 0x0e30);
(0x0e32, 0x0e33);
(0x0e40, 0x0e46);
(0x0e4f, 0x0e5b);
(* Lao *)
(0x0e81, 0x0e82);
(0x0e84, 0x0e84);
(0x0e87, 0x0e88);
(0x0e8a, 0x0e8a);
(0x0e0d, 0x0e0d);
(0x0e94, 0x0e97);
(0x0e99, 0x0e9f);
(0x0ea1, 0x0ea3);
(0x0ea5, 0x0ea5);
(0x0ea7, 0x0ea7);
(0x0eaa, 0x0eab);
(0x0ead, 0x0eb0);
(0x0eb2, 0x0eb3);
(0x0ebd, 0x0ebd);
(0x0ec0, 0x0ec4);
(0x0ec6, 0x0ec6);
(* Georgian *)
(0x10a0, 0x10c5);
(0x10d0, 0x10f6);
(* Hangul Jamo *)
(0x1100, 0x1159);
(0x1161, 0x11a2);
(0x11a8, 0x11f9);
(0x11d0, 0x11f6);
(* Latin extensions *)
(0x1e00, 0x1e9a);
(0x1ea0, 0x1ef9);
(* Greek extended *)
(0x1f00, 0x1f15);
(0x1f18, 0x1f1d);
(0x1f20, 0x1f45);
(0x1f48, 0x1f4d);
(0x1f50, 0x1f57);
(0x1f59, 0x1f59);
(0x1f5b, 0x1f5b);
(0x1f5d, 0x1f5d);
(0x1f5f, 0x1f7d);
(0x1f80, 0x1fb4);
(0x1fb6, 0x1fbc);
(0x1fc2, 0x1fc4);
(0x1fc6, 0x1fcc);
(0x1fd0, 0x1fd3);
(0x1fd6, 0x1fdb);
(0x1fe0, 0x1fec);
(0x1ff2, 0x1ff4);
(0x1ff6, 0x1ffc);
(* Hiragana *)
(0x3041, 0x3094);
(0x309b, 0x309e);
(* Katakana *)
(0x30a1, 0x30fe);
(* Bopmofo *)
(0x3105, 0x312c);
(* CJK Unified Ideographs *)
(0x4e00, 0x9fa5);
(* CJK Compatibility Ideographs *)
(0xf900, 0xfa2d);
(* Arabic Presentation Forms *)
(0xfb1f, 0xfb36);
(0xfb38, 0xfb3c);
(0xfb3e, 0xfb3e);
(0xfb40, 0xfb41);
(0xfb42, 0xfb44);
(0xfb46, 0xfbb1);
(0xfbd3, 0xfd35);
(* Arabic Presentation Forms-A *)
(0xfd50, 0xfd85);
(0xfd92, 0xfbc7);
(0xfdf0, 0xfdfb);
(* Arabic Presentation Forms-B *)
(0xfe70, 0xfe72);
(0xfe74, 0xfe74);
(0xfe76, 0xfefc);
(* Half width and Fullwidth Forms *)
(0xff21, 0xff3a);
(0xff41, 0xff5a);
(0xff66, 0xffbe);
(0xffc2, 0xffc7);
(0xffca, 0xffcf);
(0xffd2, 0xffd7);
(0xffd2, 0xffd7);
(0xffda, 0xffdc);
]
in
union_list (List.map (fun (a, b) -> interval a b) l)

10
src/syntax/iso.mli
View File

@ -1,10 +0,0 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
open Sedlex_cset
(** Letters to be used in identifiers, as specified by ISO .... *)
(* Data provided by John M. Skaller *)
val tr8876_ident_char : t

521
src/syntax/ppx_sedlex.ml
View File

@ -1,9 +1,9 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
open Ppxlib open Ppxlib
open Ast_builder.Default open Ast_builder.Default
open Ast_helper
(* let ocaml_version = Versions.ocaml_408 *) (* let ocaml_version = Versions.ocaml_408 *)
@ -13,27 +13,35 @@ module Cset = Sedlex_cset
let default_loc = Location.none let default_loc = Location.none
let lident_loc ~loc s = {
loc;
txt= lident s
}
type decision_tree = type decision_tree =
| Lte of int * decision_tree * decision_tree | Lte of int * decision_tree * decision_tree
| Table of int * int array | Table of int * int array
| Return of int | Return of int
let rec simplify_decision_tree (x : decision_tree) = let rec simplify_decision_tree ( x : decision_tree) =
match x with match x with
| Table _ | Return _ -> x | Table _ | Return _ -> x
| Lte (_, (Return a as l), Return b) when a = b -> l | Lte (_, (Return a as l), Return b) when a = b -> l
| Lte (i, l, r) -> ( | Lte (i, l, r) ->
let l = simplify_decision_tree l in let l = simplify_decision_tree l in
let r = simplify_decision_tree r in let r = simplify_decision_tree r in
match (l, r) with match l, r with
| Return a, Return b when a = b -> l | Return a, Return b when a = b -> l
| _ -> Lte (i, l, r)) | _ -> Lte (i, l,r)
let decision l = let decision l =
let l = List.map (fun (a, b, i) -> (a, b, Return i)) l in let l = List.map (fun (a, b, i) -> (a, b, Return i)) l in
let rec merge2 = function let rec merge2 = function
| (a1, b1, d1) :: (a2, b2, d2) :: rest -> | (a1, b1, d1) :: (a2, b2, d2) :: rest ->
let x = if b1 + 1 = a2 then d2 else Lte (a2 - 1, Return (-1), d2) in let x =
if b1 + 1 = a2 then d2
else Lte (a2 - 1, Return (-1), d2)
in
(a1, b2, Lte (b1, d1, x)) :: merge2 rest (a1, b2, Lte (b1, d1, x)) :: merge2 rest
| rest -> rest | rest -> rest
in in
@ -48,44 +56,38 @@ let limit = 8192
let decision_table l = let decision_table l =
let rec aux m accu = function let rec aux m accu = function
| ((a, b, i) as x) :: rem when b < limit && i < 255 -> | ((a, b, i) as x)::rem when b < limit && i < 255->
aux (min a m) (x :: accu) rem aux (min a m) (x :: accu) rem
| rem -> (m, accu, rem) | rem -> m, accu, rem
in in
let min, table, rest = aux max_int [] l in let (min, table, rest) = aux max_int [] l in
match table with match table with
| [] -> decision l | [] -> decision l
| [(min, max, i)] -> | [(min, max, i)] ->
Lte (min - 1, Return (-1), Lte (max, Return i, decision rest)) Lte (min - 1, Return (-1), (Lte (max, Return i, decision rest)))
| (_, max, _) :: _ -> | (_, max, _) :: _ ->
let arr = Array.make (max - min + 1) 0 in let arr = Array.make (max - min + 1) 0 in
let set (a, b, i) = let set (a, b, i) = for j = a to b do arr.(j - min) <- i + 1 done in
for j = a to b do
arr.(j - min) <- i + 1
done
in
List.iter set table; List.iter set table;
Lte (min - 1, Return (-1), Lte (max, Table (min, arr), decision rest)) Lte (min - 1, Return (-1), Lte (max, Table (min, arr), decision rest))
let rec simplify min max = function let rec simplify min max = function
| Lte (i, yes, no) -> | Lte (i,yes,no) ->
if i >= max then simplify min max yes if i >= max then simplify min max yes
else if i < min then simplify min max no else if i < min then simplify min max no
else Lte (i, simplify min i yes, simplify (i + 1) max no) else Lte (i, simplify min i yes, simplify (i+1) max no)
| x -> x | x -> x
let segments_of_partition p = let segments_of_partition p =
let seg = ref [] in let seg = ref [] in
Array.iteri Array.iteri
(fun i c -> (fun i c -> List.iter (fun (a, b) -> seg := (a, b, i) :: !seg) c)
List.iter
(fun (a, b) -> seg := (a, b, i) :: !seg)
(c : Sedlex_cset.t :> (int * int) list))
p; p;
List.sort (fun (a1, _, _) (a2, _, _) -> compare a1 a2) !seg List.sort (fun (a1,_,_) (a2,_,_) -> compare a1 a2) !seg
let decision_table p = let decision_table p =
simplify (-1) Cset.max_code (decision_table (segments_of_partition p)) simplify (-1) (Cset.max_code) (decision_table (segments_of_partition p))
(* Helpers to build AST *) (* Helpers to build AST *)
@ -95,34 +97,31 @@ let appfun s l =
let glb_value name def = let glb_value name def =
let loc = default_loc in let loc = default_loc in
pstr_value ~loc Nonrecursive pstr_value ~loc Nonrecursive [value_binding ~loc ~pat:(pvar ~loc name) ~expr:def]
[value_binding ~loc ~pat:(pvar ~loc name) ~expr:def]
(* Named regexps *) (* Named regexps *)
module StringMap = Map.Make (struct module StringMap = Map.Make(struct
type t = string type t = string
let compare = compare let compare = compare
end) end)
let builtin_regexps = let builtin_regexps =
List.fold_left List.fold_left (fun acc (n, c) -> StringMap.add n (Sedlex.chars c) acc)
(fun acc (n, c) -> StringMap.add n (Sedlex.chars c) acc)
StringMap.empty StringMap.empty
([ ([
("any", Cset.any); "any", Cset.any;
("eof", Cset.eof); "eof", Cset.eof;
("xml_letter", Xml.letter); "xml_letter", Cset.letter;
("xml_digit", Xml.digit); "xml_digit", Cset.digit;
("xml_extender", Xml.extender); "xml_extender", Cset.extender;
("xml_base_char", Xml.base_char); "xml_base_char", Cset.base_char;
("xml_ideographic", Xml.ideographic); "xml_ideographic", Cset.ideographic;
("xml_combining_char", Xml.combining_char); "xml_combining_char", Cset.combining_char;
("xml_blank", Xml.blank); "xml_blank", Cset.blank;
("tr8876_ident_char", Iso.tr8876_ident_char); "tr8876_ident_char", Cset.tr8876_ident_char] @
] Unicode.Categories.list @
@ Unicode.Categories.list @ Unicode.Properties.list) Unicode.Properties.list)
(* Tables (indexed mapping: codepoint -> next state) *) (* Tables (indexed mapping: codepoint -> next state) *)
@ -141,18 +140,14 @@ let table_name x =
let table (name, v) = let table (name, v) =
let n = Array.length v in let n = Array.length v in
let s = Bytes.create n in let s = Bytes.create n in
for i = 0 to n - 1 do for i = 0 to n - 1 do Bytes.set s i (Char.chr v.(i)) done;
Bytes.set s i (Char.chr v.(i))
done;
glb_value name (estring ~loc:default_loc (Bytes.to_string s)) glb_value name (estring ~loc:default_loc (Bytes.to_string s))
(* Partition (function: codepoint -> next state) *) (* Partition (function: codepoint -> next state) *)
let partitions = Hashtbl.create 31 let partitions = Hashtbl.create 31
let partition_counter = ref 0 let partition_counter = ref 0
let get_partitions () = Hashtbl.fold (fun key x accu -> (x, key) :: accu) partitions []
let get_partitions () =
Hashtbl.fold (fun key x accu -> (x, key) :: accu) partitions []
let partition_name x = let partition_name x =
try Hashtbl.find partitions x try Hashtbl.find partitions x
@ -168,22 +163,17 @@ let partition (name, p) =
let loc = default_loc in let loc = default_loc in
let rec gen_tree = function let rec gen_tree = function
| Lte (i, yes, no) -> | Lte (i, yes, no) ->
[%expr [%expr if c <= [%e eint ~loc i] then [%e gen_tree yes] else [%e gen_tree no]]
if c <= [%e eint ~loc i] then [%e gen_tree yes] else [%e gen_tree no]]
| Return i -> eint ~loc:default_loc i | Return i -> eint ~loc:default_loc i
| Table (offset, t) -> | Table (offset, t) ->
let c = let c = if offset = 0 then [%expr c] else [%expr c - [%e eint ~loc offset]] in
if offset = 0 then [%expr c] else [%expr c - [%e eint ~loc offset]] [%expr Char.code (String.unsafe_get [%e evar ~loc (table_name t)] [%e c]) - 1]
in
[%expr
Char.code (String.unsafe_get [%e evar ~loc (table_name t)] [%e c]) - 1]
in in
let body = gen_tree (simplify_decision_tree (decision_table p)) in let body = gen_tree (simplify_decision_tree (decision_table p)) in
glb_value name glb_value name
[%expr [%expr fun c ->
fun c -> [%e body]
let open! Stdlib in ]
[%e body]]
(* Code generation for the automata *) (* Code generation for the automata *)
@ -197,79 +187,59 @@ let best_final final =
let state_fun state = Printf.sprintf "__sedlex_state_%i" state let state_fun state = Printf.sprintf "__sedlex_state_%i" state
let call_state lexbuf auto state = let call_state lexbuf auto state =
let { Sedlex.trans; finals } = auto.(state) in let (trans, final) = auto.(state) in
if Array.length trans = 0 then ( if Array.length trans = 0
match best_final finals with then match best_final final with
| Some i -> eint ~loc:default_loc i | Some i -> eint ~loc:default_loc i
| None -> assert false) | None -> assert false
else appfun (state_fun state) [lexbuf] else appfun (state_fun state) [evar ~loc:default_loc lexbuf]
let gen_state (lexbuf_name, lexbuf) auto i { Sedlex.trans; finals } = let gen_state lexbuf auto i (trans, final) =
let loc = default_loc in let loc = default_loc in
let partition = Array.map fst trans in let partition = Array.map fst trans in
let cases = let cases = Array.mapi (fun i (_, j) -> case ~lhs:(pint ~loc i) ~guard:None ~rhs:(call_state lexbuf auto j)) trans in
Array.mapi
(fun i (_, j) ->
case ~lhs:(pint ~loc i) ~guard:None ~rhs:(call_state lexbuf auto j))
trans
in
let cases = Array.to_list cases in let cases = Array.to_list cases in
let body () = let body () =
pexp_match ~loc pexp_match ~loc
(appfun (partition_name partition) (appfun (partition_name partition) [[%expr Sedlexing.__private__next_int [%e evar ~loc lexbuf]]])
[[%expr Sedlexing.__private__next_int [%e lexbuf]]]) (cases @ [case ~lhs:[%pat? _] ~guard:None ~rhs:[%expr Sedlexing.backtrack [%e evar ~loc lexbuf]]])
(cases
@ [
case
~lhs:[%pat? _]
~guard:None
~rhs:[%expr Sedlexing.backtrack [%e lexbuf]];
])
in in
let ret body = let ret body = [ value_binding ~loc ~pat:(pvar ~loc (state_fun i)) ~expr:(pexp_function ~loc [case ~lhs:(pvar ~loc lexbuf) ~guard:None ~rhs:body]) ] in
let lhs = pvar ~loc:lexbuf.pexp_loc lexbuf_name in match best_final final with
[
value_binding ~loc
~pat:(pvar ~loc (state_fun i))
~expr:(Exp.fun_ ~loc Nolabel None lhs body);
]
in
match best_final finals with
| None -> ret (body ()) | None -> ret (body ())
| Some _ when Array.length trans = 0 -> [] | Some _ when Array.length trans = 0 -> []
| Some i -> | Some i -> ret [%expr Sedlexing.mark [%e evar ~loc lexbuf] [%e eint ~loc i]; [%e body ()]]
ret
[%expr
Sedlexing.mark [%e lexbuf] [%e eint ~loc i];
[%e body ()]]
let gen_recflag auto = let gen_recflag auto =
(* The generated function is not recursive if the transitions end (* The generated function is not recursive if the transitions end
in states with no further transitions. *) in states with no further transitions. *)
try try
Array.iter Array.iter
(fun { Sedlex.trans; _ } -> (fun (trans_i, _) ->
Array.iter Array.iter
(fun (_, j) -> (fun (_, j) ->
if Array.length auto.(j).Sedlex.trans > 0 then raise Exit) let (trans_j, _) = auto.(j) in
trans) if Array.length trans_j > 0 then raise Exit)
trans_i)
auto; auto;
Nonrecursive Nonrecursive
with Exit -> Recursive with
Exit -> Recursive
let gen_definition ((_, lexbuf) as lexbuf_with_name) auto l error = let gen_definition lexbuf l error =
let loc = default_loc in let loc = default_loc in
let cases = let brs = Array.of_list l in
List.mapi (fun i (_, e) -> case ~lhs:(pint ~loc i) ~guard:None ~rhs:e) l let auto = Sedlex.compile (Array.map fst brs) in
in let cases = Array.to_list (Array.mapi (fun i (_, e) -> case ~lhs:(pint ~loc i) ~guard:None ~rhs:e) brs) in
let states = Array.mapi (gen_state lexbuf_with_name auto) auto in let states = Array.mapi (gen_state lexbuf auto) auto in
let states = List.flatten (Array.to_list states) in let states = List.flatten (Array.to_list states) in
pexp_let ~loc (gen_recflag auto) states pexp_let ~loc (gen_recflag auto) states
(pexp_sequence ~loc (pexp_sequence ~loc
[%expr Sedlexing.start [%e lexbuf]] [%expr Sedlexing.start [%e evar ~loc lexbuf]]
(pexp_match ~loc (pexp_match ~loc (appfun (state_fun 0) [evar ~loc lexbuf])
(appfun (state_fun 0) [lexbuf]) (cases @ [case ~lhs:(ppat_any ~loc) ~guard:None ~rhs:error])
(cases @ [case ~lhs:(ppat_any ~loc) ~guard:None ~rhs:error]))) )
)
(* Lexer specification parser *) (* Lexer specification parser *)
@ -278,46 +248,18 @@ let codepoint i =
failwith (Printf.sprintf "Invalid Unicode code point: %i" i); failwith (Printf.sprintf "Invalid Unicode code point: %i" i);
i i
let char c = Cset.singleton (Char.code c) let regexp_for_char c =
let uchar c = Cset.singleton (Uchar.to_int c) Sedlex.chars (Cset.singleton (Char.code c))
let err loc fmt = let regexp_for_string s =
Printf.ksprintf let rec aux n =
(fun s -> if n = String.length s then Sedlex.eps
raise (Location.Error (Location.Error.createf ~loc "Sedlex: %s" s))) else
fmt Sedlex.seq (regexp_for_char s.[n]) (aux (succ n))
in aux 0
type encoding = Utf8 | Latin1 | Ascii let err loc s =
raise (Location.Error (Location.Error.createf ~loc "Sedlex: %s" s))
let string_of_encoding = function
| Utf8 -> "UTF-8"
| Latin1 -> "Latin-1"
| Ascii -> "ASCII"
let rev_csets_of_string ~loc ~encoding s =
match encoding with
| Utf8 ->
Utf8.fold
~f:(fun acc _ x ->
match x with
| `Malformed _ ->
err loc "Malformed %s string" (string_of_encoding encoding)
| `Uchar c -> uchar c :: acc)
[] s
| Latin1 ->
let l = ref [] in
for i = 0 to String.length s - 1 do
l := char s.[i] :: !l
done;
!l
| Ascii ->
let l = ref [] in
for i = 0 to String.length s - 1 do
match s.[i] with
| '\x00' .. '\x7F' as c -> l := char c :: !l
| _ -> err loc "Malformed %s string" (string_of_encoding encoding)
done;
!l
let rec repeat r = function let rec repeat r = function
| 0, 0 -> Sedlex.eps | 0, 0 -> Sedlex.eps
@ -325,264 +267,197 @@ let rec repeat r = function
| n, m -> Sedlex.seq r (repeat r (n - 1, m - 1)) | n, m -> Sedlex.seq r (repeat r (n - 1, m - 1))
let regexp_of_pattern env = let regexp_of_pattern env =
let rec char_pair_op func name ~encoding ~loc tuple = let rec char_pair_op func name p tuple = (* Construct something like Sub(a,b) *)
(* Construct something like Sub(a,b) *)
match tuple with match tuple with
| Some { ppat_desc = Ppat_tuple [p0; p1]; _ } -> | Some {ppat_desc=Ppat_tuple (p0 :: p1 :: [])} ->
begin match func (aux ~encoding p0) (aux ~encoding p1) with begin match func (aux p0) (aux p1) with
| Some r -> r | Some r -> r
| None -> | None ->
err loc err p.ppat_loc @@
"the %s operator can only applied to single-character length \ "the "^name^" operator can only applied to single-character length regexps"
regexps"
name
end end
| _ -> | _ -> err p.ppat_loc @@ "the "^name^" operator requires two arguments, like "^name^"(a,b)"
err loc "the %s operator requires two arguments, like %s(a,b)" name and aux p = (* interpret one pattern node *)
name
and aux ~encoding p =
(* interpret one pattern node *)
match p.ppat_desc with match p.ppat_desc with
| Ppat_or (p1, p2) -> Sedlex.alt (aux ~encoding p1) (aux ~encoding p2) | Ppat_or (p1, p2) -> Sedlex.alt (aux p1) (aux p2)
| Ppat_tuple (p :: pl) -> | Ppat_tuple (p :: pl) ->
List.fold_left List.fold_left (fun r p -> Sedlex.seq r (aux p))
(fun r p -> Sedlex.seq r (aux ~encoding p)) (aux p)
(aux ~encoding p) pl pl
| Ppat_construct ({ txt = Lident "Star"; _ }, Some (_, p)) -> | Ppat_construct ({txt = Lident "Star"}, Some p) ->
Sedlex.rep (aux ~encoding p) Sedlex.rep (aux p)
| Ppat_construct ({ txt = Lident "Plus"; _ }, Some (_, p)) -> | Ppat_construct ({txt = Lident "Plus"}, Some p) ->
Sedlex.plus (aux ~encoding p) Sedlex.plus (aux p)
| Ppat_construct ({ txt = Lident "Utf8"; _ }, Some (_, p)) ->
aux ~encoding:Utf8 p
| Ppat_construct ({ txt = Lident "Latin1"; _ }, Some (_, p)) ->
aux ~encoding:Latin1 p
| Ppat_construct ({ txt = Lident "Ascii"; _ }, Some (_, p)) ->
aux ~encoding:Ascii p
| Ppat_construct | Ppat_construct
( { txt = Lident "Rep"; _ }, ({txt = Lident "Rep"},
Some Some {ppat_desc=Ppat_tuple[p0; {ppat_desc=Ppat_constant (i1 as i2)|Ppat_interval(i1, i2)}]}) ->
( _, begin match i1, i2 with
{ | Pconst_integer(i1,_), Pconst_integer(i2,_) ->
ppat_desc =
Ppat_tuple
[
p0;
{
ppat_desc =
Ppat_constant (i1 as i2) | Ppat_interval (i1, i2);
_;
};
];
_;
} ) ) ->
begin match (i1, i2) with
| Pconst_integer (i1, _), Pconst_integer (i2, _) ->
let i1 = int_of_string i1 in let i1 = int_of_string i1 in
let i2 = int_of_string i2 in let i2 = int_of_string i2 in
if 0 <= i1 && i1 <= i2 then repeat (aux ~encoding p0) (i1, i2) if 0 <= i1 && i1 <= i2 then repeat (aux p0) (i1, i2)
else err p.ppat_loc "Invalid range for Rep operator" else err p.ppat_loc "Invalid range for Rep operator"
| _ -> | _ ->
err p.ppat_loc "Rep must take an integer constant or interval" err p.ppat_loc "Rep must take an integer constant or interval"
end end
| Ppat_construct ({ txt = Lident "Rep"; _ }, _) -> | Ppat_construct ({txt = Lident "Rep"}, _) ->
err p.ppat_loc "the Rep operator takes 2 arguments" err p.ppat_loc "the Rep operator takes 2 arguments"
| Ppat_construct ({ txt = Lident "Opt"; _ }, Some (_, p)) -> | Ppat_construct ({txt = Lident "Opt"}, Some p) ->
Sedlex.alt Sedlex.eps (aux ~encoding p) Sedlex.alt Sedlex.eps (aux p)
| Ppat_construct ({ txt = Lident "Compl"; _ }, arg) -> | Ppat_construct ({txt = Lident "Compl"}, arg) ->
begin match arg with begin match arg with
| Some (_, p0) -> | Some p0 ->
begin match Sedlex.compl (aux ~encoding p0) with begin match Sedlex.compl (aux p0) with
| Some r -> r | Some r -> r
| None -> | None ->
err p.ppat_loc err p.ppat_loc
"the Compl operator can only applied to a \ "the Compl operator can only applied to a single-character length regexp"
single-character length regexp"
end end
| _ -> err p.ppat_loc "the Compl operator requires an argument" | _ -> err p.ppat_loc "the Compl operator requires an argument"
end end
| Ppat_construct ({ txt = Lident "Sub"; _ }, arg) -> | Ppat_construct ({ txt = Lident "Sub" }, arg) ->
char_pair_op ~encoding Sedlex.subtract "Sub" ~loc:p.ppat_loc char_pair_op Sedlex.subtract "Sub" p arg
(Option.map (fun (_, arg) -> arg) arg) | Ppat_construct ({ txt = Lident "Intersect" }, arg) ->
| Ppat_construct ({ txt = Lident "Intersect"; _ }, arg) -> char_pair_op Sedlex.intersection "Intersect" p arg
char_pair_op ~encoding Sedlex.intersection "Intersect" ~loc:p.ppat_loc | Ppat_construct ({txt = Lident "Chars"}, arg) ->
(Option.map (fun (_, arg) -> arg) arg) let const = match arg with
| Ppat_construct ({ txt = Lident "Chars"; _ }, arg) -> ( | Some {ppat_desc=Ppat_constant const} ->
let const = Some (const)
match arg with
| Some (_, { ppat_desc = Ppat_constant const; _ }) -> Some const
| _ -> None | _ -> None
in in
match const with begin match const with
| Some (Pconst_string (s, _, _)) -> | Some (Pconst_string(s,_, _))->
let l = rev_csets_of_string ~loc:p.ppat_loc ~encoding s in let c = ref Cset.empty in
let chars = List.fold_left Cset.union Cset.empty l in for i = 0 to String.length s - 1 do
Sedlex.chars chars c := Cset.union !c (Cset.singleton (Char.code s.[i]))
| _ -> done;
err p.ppat_loc "the Chars operator requires a string argument") Sedlex.chars !c
| _ -> err p.ppat_loc "the Chars operator requires a string argument"
end
| Ppat_interval (i_start, i_end) -> | Ppat_interval (i_start, i_end) ->
begin match (i_start, i_end) with begin match i_start, i_end with
| Pconst_char c1, Pconst_char c2 -> | Pconst_char c1, Pconst_char c2 -> Sedlex.chars (Cset.interval (Char.code c1) (Char.code c2))
let valid = | Pconst_integer(i1,_), Pconst_integer(i2,_) ->
match encoding with Sedlex.chars (Cset.interval (codepoint (int_of_string i1)) (codepoint (int_of_string i2)))
(* utf8 char interval can only match ascii because
of the OCaml lexer. *)
| Ascii | Utf8 -> (
function '\x00' .. '\x7f' -> true | _ -> false)
| Latin1 -> ( function _ -> true)
in
if not (valid c1 && valid c2) then
err p.ppat_loc
"this pattern is not a valid %s interval regexp"
(string_of_encoding encoding);
Sedlex.chars (Cset.interval (Char.code c1) (Char.code c2))
| Pconst_integer (i1, _), Pconst_integer (i2, _) ->
Sedlex.chars
(Cset.interval
(codepoint (int_of_string i1))
(codepoint (int_of_string i2)))
| _ -> err p.ppat_loc "this pattern is not a valid interval regexp" | _ -> err p.ppat_loc "this pattern is not a valid interval regexp"
end end
| Ppat_constant const -> | Ppat_constant (const) ->
begin match const with begin match const with
| Pconst_string (s, _, _) -> | Pconst_string (s, _, _) -> regexp_for_string s
let rev_l = rev_csets_of_string s ~loc:p.ppat_loc ~encoding in | Pconst_char c -> regexp_for_char c
List.fold_left | Pconst_integer(i,_) -> Sedlex.chars (Cset.singleton (codepoint (int_of_string i)))
(fun acc cset -> Sedlex.seq (Sedlex.chars cset) acc)
Sedlex.eps rev_l
| Pconst_char c -> Sedlex.chars (char c)
| Pconst_integer (i, _) ->
Sedlex.chars (Cset.singleton (codepoint (int_of_string i)))
| _ -> err p.ppat_loc "this pattern is not a valid regexp" | _ -> err p.ppat_loc "this pattern is not a valid regexp"
end end
| Ppat_var { txt = x; _ } -> | Ppat_var {txt=x} ->
begin try StringMap.find x env begin try StringMap.find x env
with Not_found -> err p.ppat_loc "unbound regexp %s" x with Not_found ->
err p.ppat_loc (Printf.sprintf "unbound regexp %s" x)
end end
| _ -> err p.ppat_loc "this pattern is not a valid regexp" | _ ->
err p.ppat_loc "this pattern is not a valid regexp"
in in
aux ~encoding:Ascii aux
let handle_sedlex_match ~env ~map_rhs match_expr =
let lexbuf =
match match_expr with
| { pexp_desc = Pexp_match (lexbuf, _); _ } -> (
match lexbuf with
| { pexp_desc = Pexp_ident { txt = Lident txt; _ }; _ } ->
(txt, lexbuf)
| _ ->
err lexbuf.pexp_loc
"the matched expression must be a single identifier")
| _ ->
err match_expr.pexp_loc
"the %%sedlex extension is only recognized on match expressions"
in
let cases =
match match_expr with
| { pexp_desc = Pexp_match (_, cases); _ } -> cases
| _ -> assert false
in
let cases = List.rev cases in
let error =
match List.hd cases with
| { pc_lhs = [%pat? _]; pc_rhs = e; pc_guard = None } -> map_rhs e
| { pc_lhs = p; _ } ->
err p.ppat_loc "the last branch must be a catch-all error case"
in
let cases = List.rev (List.tl cases) in
let cases =
List.map
(function
| { pc_lhs = p; pc_rhs = e; pc_guard = None } ->
(regexp_of_pattern env p, map_rhs e)
| { pc_guard = Some e; _ } ->
err e.pexp_loc "'when' guards are not supported")
cases
in
let brs = Array.of_list cases in
let auto = Sedlex.compile (Array.map fst brs) in
(gen_definition lexbuf auto cases error, auto)
let previous = ref [] let previous = ref []
let regexps = ref [] let regexps = ref []
let should_set_cookies = ref false let should_set_cookies = ref false
let mapper = let mapper =
object (this) object(this)
inherit Ast_traverse.map as super inherit Ast_traverse.map as super
val env = builtin_regexps val env = builtin_regexps
method define_regexp name p = method define_regexp name p =
{<env = StringMap.add name (regexp_of_pattern env p) env>} {< env = StringMap.add name (regexp_of_pattern env p) env >}
method! expression e = method! expression e =
match e with match e with
| [%expr [%sedlex [%e? { pexp_desc = Pexp_match _; _ } as match_expr]]] | [%expr [%sedlex [%e? {pexp_desc=Pexp_match (lexbuf, cases)}]]] ->
-> let lexbuf =
fst (handle_sedlex_match ~env ~map_rhs:this#expression match_expr) match lexbuf with
| [%expr | {pexp_desc=Pexp_ident{txt=Lident lexbuf}} -> lexbuf
let [%p? { ppat_desc = Ppat_var { txt = name; _ }; _ }] = | _ ->
[%sedlex.regexp? [%p? p]] err lexbuf.pexp_loc "the matched expression must be a single identifier"
in in
[%e? body]] -> let cases = List.rev cases in
(this#define_regexp name p)#expression body let error =
match List.hd cases with
| {pc_lhs = [%pat? _]; pc_rhs = e; pc_guard = None} -> super # expression e
| {pc_lhs = p} ->
err p.ppat_loc "the last branch must be a catch-all error case"
in
let cases = List.rev (List.tl cases) in
let cases =
List.map
(function
| {pc_lhs = p; pc_rhs = e; pc_guard = None} -> regexp_of_pattern env p, super # expression e
| {pc_guard = Some e} ->
err e.pexp_loc "'when' guards are not supported"
) cases
in
gen_definition lexbuf cases error
| [%expr let [%p? {ppat_desc=Ppat_var{txt=name}}] = [%sedlex.regexp? [%p? p]] in [%e? body]] ->
(this # define_regexp name p) # expression body
| [%expr [%sedlex [%e? _]]] -> | [%expr [%sedlex [%e? _]]] ->
err e.pexp_loc err e.pexp_loc "the %sedlex extension is only recognized on match expressions"
"the %%sedlex extension is only recognized on match expressions" | _ -> super # expression e
| _ -> super#expression e
val toplevel = true val toplevel = true
method structure_with_regexps l = method structure_with_regexps l =
let mapper = ref this in let mapper = ref this in
let regexps = ref [] in let regexps = ref [] in
let l = let l = List.concat
List.concat
(List.map (List.map
(function (function
| [%stri | [%stri let [%p? {ppat_desc=Ppat_var{txt=name}}] = [%sedlex.regexp? [%p? p]]] as i ->
let [%p? { ppat_desc = Ppat_var { txt = name; _ }; _ }] =
[%sedlex.regexp? [%p? p]]] as i ->
regexps := i :: !regexps; regexps := i :: !regexps;
mapper := !mapper#define_regexp name p; mapper := !mapper # define_regexp name p;
[] []
| i -> [!mapper#structure_item i]) | i ->
l) [ !mapper # structure_item i ]
in ) l) in
(l, List.rev !regexps) (l, List.rev !regexps)
method! structure l = method! structure l =
if toplevel then ( if toplevel then
let sub = {<toplevel = false>} in let sub = {< toplevel = false >} in
let l, regexps' = sub#structure_with_regexps (!previous @ l) in let l, regexps' = sub # structure_with_regexps (!previous @ l) in
let parts = List.map partition (get_partitions ()) in let parts = List.map partition (get_partitions ()) in
let tables = List.map table (get_tables ()) in let tables = List.map table (get_tables ()) in
regexps := regexps'; regexps := regexps';
should_set_cookies := true; should_set_cookies := true;
tables @ parts @ l) tables @ parts @ l
else fst (this#structure_with_regexps l) else
fst (this # structure_with_regexps l)
end end
let pre_handler cookies = let pre_handler cookies =
previous := previous :=
match Driver.Cookies.get cookies "sedlex.regexps" Ast_pattern.__ with match Driver.Cookies.get cookies "sedlex.regexps" Ast_pattern.__ with
| Some { pexp_desc = Pexp_extension (_, PStr l); _ } -> l | Some {pexp_desc = Pexp_extension (_, PStr l)} -> l
| Some _ -> assert false | Some _ -> assert false
| None -> [] | None -> []
let post_handler cookies = let post_handler cookies =
if !should_set_cookies then ( if !should_set_cookies then
let loc = default_loc in let loc = default_loc in
Driver.Cookies.set cookies "sedlex.regexps" Driver.Cookies.set cookies "sedlex.regexps" (pexp_extension ~loc ( {loc; txt="regexps"}, PStr !regexps))
(pexp_extension ~loc ({ loc; txt = "regexps" }, PStr !regexps)))
let extensions = let extensions =
[ [Extension.declare
Extension.declare "sedlex" Extension.Context.expression "sedlex"
Extension.Context.expression
Ast_pattern.(single_expr_payload __) Ast_pattern.(single_expr_payload __)
(fun ~loc:_ ~path:_ expr -> mapper#expression expr); (fun ~loc:_ ~path:_ expr -> mapper # expression expr);
] ]
let () = let () =
Driver.Cookies.add_handler pre_handler; Driver.Cookies.add_handler pre_handler;
Driver.Cookies.add_post_handler post_handler; Driver.Cookies.add_post_handler post_handler;
Driver.register_transformation "sedlex" ~impl:mapper#structure Driver.register_transformation "sedlex" ~impl:(mapper # structure)

117
src/syntax/sedlex.ml
View File

@ -1,5 +1,6 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
module Cset = Sedlex_cset module Cset = Sedlex_cset
@ -16,7 +17,6 @@ type node = {
type regexp = node -> node type regexp = node -> node
let cur_id = ref 0 let cur_id = ref 0
let new_node () = let new_node () =
incr cur_id; incr cur_id;
{ id = !cur_id; eps = []; trans = [] } { id = !cur_id; eps = []; trans = [] }
@ -24,17 +24,17 @@ let new_node () =
let seq r1 r2 succ = r1 (r2 succ) let seq r1 r2 succ = r1 (r2 succ)
let is_chars final = function let is_chars final = function
| { eps = []; trans = [(c, f)]; _ } when f == final -> Some c | {eps = []; trans = [c, f]} when f == final -> Some c
| _ -> None | _ -> None
let chars c succ = let chars c succ =
let n = new_node () in let n = new_node () in
n.trans <- [(c, succ)]; n.trans <- [c,succ];
n n
let alt r1 r2 succ = let alt r1 r2 succ =
let nr1 = r1 succ and nr2 = r2 succ in let nr1 = r1 succ and nr2 = r2 succ in
match (is_chars succ nr1, is_chars succ nr2) with match is_chars succ nr1, is_chars succ nr2 with
| Some c1, Some c2 -> chars (Cset.union c1 c2) succ | Some c1, Some c2 -> chars (Cset.union c1 c2) succ
| _ -> | _ ->
let n = new_node () in let n = new_node () in
@ -57,18 +57,22 @@ let eps succ = succ (* eps for epsilon *)
let compl r = let compl r =
let n = new_node () in let n = new_node () in
match is_chars n (r n) with match is_chars n (r n) with
| Some c -> Some (chars (Cset.difference Cset.any c)) | Some c ->
| _ -> None Some (chars (Cset.difference Cset.any c))
| _ ->
None
let pair_op f r0 r1 = let pair_op f r0 r1 = (* Construct subtract or intersection *)
(* Construct subtract or intersection *)
let n = new_node () in let n = new_node () in
let to_chars r = is_chars n (r n) in let to_chars r = is_chars n (r n) in
match (to_chars r0, to_chars r1) with match to_chars r0, to_chars r1 with
| Some c0, Some c1 -> Some (chars (f c0 c1)) | Some c0, Some c1 ->
| _ -> None Some (chars (f c0 c1))
| _ ->
None
let subtract = pair_op Cset.difference let subtract = pair_op Cset.difference
let intersection = pair_op Cset.intersection let intersection = pair_op Cset.intersection
let compile_re re = let compile_re re =
@ -78,35 +82,36 @@ let compile_re re =
(* Determinization *) (* Determinization *)
type state = node list type state = node list
(* A state of the DFA corresponds to a set of nodes in the NFA. *) (* A state of the DFA corresponds to a set of nodes in the NFA. *)
let rec add_node state node = let rec add_node state node =
if List.memq node state then state else add_nodes (node :: state) node.eps if List.memq node state then state else add_nodes (node::state) node.eps
and add_nodes state nodes =
List.fold_left add_node state nodes
and add_nodes state nodes = List.fold_left add_node state nodes
let transition (state : state) = let transition (state : state) =
(* Merge transition with the same target *) (* Merge transition with the same target *)
let rec norm = function let rec norm = function
| (c1, n1) :: ((c2, n2) :: q as l) -> | (c1, n1)::((c2, n2)::q as l) ->
if n1 == n2 then norm ((Cset.union c1 c2, n1) :: q) if n1 == n2 then norm ((Cset.union c1 c2, n1)::q)
else (c1, n1) :: norm l else (c1, n1)::(norm l)
| l -> l | l -> l in
in
let t = List.concat (List.map (fun n -> n.trans) state) in let t = List.concat (List.map (fun n -> n.trans) state) in
let t = norm (List.sort (fun (_, n1) (_, n2) -> n1.id - n2.id) t) in let t = norm (List.sort (fun (_, n1) (_, n2) -> n1.id - n2.id) t) in
(* Split char sets so as to make them disjoint *) (* Split char sets so as to make them disjoint *)
let split (all, t) (c0, n0) = let split (all, t) (c0, n0) =
let t = let t =
(Cset.difference c0 all, [n0]) (Cset.difference c0 all, [n0]) ::
:: List.map (fun (c, ns) -> (Cset.intersection c c0, n0 :: ns)) t List.map (fun (c, ns) -> (Cset.intersection c c0, n0::ns)) t @
@ List.map (fun (c, ns) -> (Cset.difference c c0, ns)) t List.map (fun (c, ns) -> (Cset.difference c c0, ns)) t
in in
(Cset.union all c0, List.filter (fun (c, _) -> not (Cset.is_empty c)) t) Cset.union all c0,
List.filter (fun (c, _) -> not (Cset.is_empty c)) t
in in
let _, t = List.fold_left split (Cset.empty, []) t in let (_,t) = List.fold_left split (Cset.empty,[]) t in
(* Epsilon closure of targets *) (* Epsilon closure of targets *)
let t = List.map (fun (c, ns) -> (c, add_nodes [] ns)) t in let t = List.map (fun (c, ns) -> (c, add_nodes [] ns)) t in
@ -116,9 +121,6 @@ let transition (state : state) =
Array.sort (fun (c1, _) (c2, _) -> compare c1 c2) t; Array.sort (fun (c1, _) (c2, _) -> compare c1 c2) t;
t t
type dfa_state = { trans : (Cset.t * int) array; finals : bool array }
type dfa = dfa_state array
let compile rs = let compile rs =
let rs = Array.map compile_re rs in let rs = Array.map compile_re rs in
let counter = ref 0 in let counter = ref 0 in
@ -133,64 +135,11 @@ let compile rs =
let trans = transition state in let trans = transition state in
let trans = Array.map (fun (p, t) -> (p, aux t)) trans in let trans = Array.map (fun (p, t) -> (p, aux t)) trans in
let finals = Array.map (fun (_, f) -> List.memq f state) rs in let finals = Array.map (fun (_, f) -> List.memq f state) rs in
Hashtbl.add states_def i { trans; finals }; Hashtbl.add states_def i (trans, finals);
i i
in in
let init = ref [] in let init = ref [] in
Array.iter (fun (i, _) -> init := add_node !init i) rs; Array.iter (fun (i,_) -> init := add_node !init i) rs;
let i = aux !init in let i = aux !init in
assert (i = 0); assert(i = 0);
Array.init !counter (Hashtbl.find states_def) Array.init !counter (Hashtbl.find states_def)
let cset_to_label cset =
let escape_dot c =
match c with
| '"' -> "\\\""
| '\\' -> "\\\\"
| '<' -> "\\<"
| '>' -> "\\>"
| _ -> String.make 1 c
in
let format_interval (lo, hi) =
if lo = -1 && hi = -1 then "EOF"
else if lo = hi then
if lo >= 32 && lo <= 126 then "'" ^ escape_dot (Char.chr lo) ^ "'"
else Printf.sprintf "U+%04X" lo
else if lo >= 32 && lo <= 126 && hi >= 32 && hi <= 126 then
"'" ^ escape_dot (Char.chr lo) ^ "'-'" ^ escape_dot (Char.chr hi) ^ "'"
else Printf.sprintf "U+%04X-U+%04X" lo hi
in
String.concat ", "
(List.map format_interval (cset : Cset.t :> (int * int) list))
let dfa_to_dot dfa =
let buf = Buffer.create 1024 in
let bprintf = Printf.bprintf in
bprintf buf "digraph {\n";
bprintf buf " rankdir=LR;\n";
bprintf buf " node [shape=circle];\n\n";
bprintf buf " _start [shape=point];\n";
bprintf buf " _start -> state0;\n\n";
Array.iteri
(fun i { trans; finals } ->
let accepted =
let acc = ref [] in
for r = Array.length finals - 1 downto 0 do
if finals.(r) then acc := r :: !acc
done;
!acc
in
(match accepted with
| [] -> bprintf buf " state%d [label=\"%d\"];\n" i i
| rules ->
bprintf buf
" state%d [label=\"%d\\n[rule %s]\", shape=doublecircle];\n" i i
(String.concat "," (List.map string_of_int rules)));
Array.iter
(fun (cset, target) ->
let label = cset_to_label cset in
bprintf buf " state%d -> state%d [label=\"%s\"];\n" i target label)
trans)
dfa;
bprintf buf "}\n";
Buffer.contents buf

34
src/syntax/sedlex.mli
View File

@ -1,28 +1,24 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
type regexp type regexp
val chars : Sedlex_cset.t -> regexp val chars: Sedlex_cset.t -> regexp
val seq : regexp -> regexp -> regexp val seq: regexp -> regexp -> regexp
val alt : regexp -> regexp -> regexp val alt: regexp -> regexp -> regexp
val rep : regexp -> regexp val rep: regexp -> regexp
val plus : regexp -> regexp val plus: regexp -> regexp
val eps : regexp val eps: regexp
val compl : regexp -> regexp option
(* If the argument is a single [chars] regexp, returns a regexp val compl: regexp -> regexp option
(* If the argument is a single [chars] regexp, returns a regexp
which matches the complement set. Otherwise returns [None]. *) which matches the complement set. Otherwise returns [None]. *)
val subtract : regexp -> regexp -> regexp option val subtract: regexp -> regexp -> regexp option
(* If each argument is a single [chars] regexp, returns a regexp
(* If each argument is a single [chars] regexp, returns a regexp
which matches the set (arg1 - arg2). Otherwise returns [None]. *) which matches the set (arg1 - arg2). Otherwise returns [None]. *)
val intersection : regexp -> regexp -> regexp option val intersection: regexp -> regexp -> regexp option
(* If each argument is a single [chars] regexp, returns a regexp (* If each argument is a single [chars] regexp, returns a regexp
which matches the intersection set. Otherwise returns [None]. *) which matches the intersection set. Otherwise returns [None]. *)
type dfa_state = { trans : (Sedlex_cset.t * int) array; finals : bool array } val compile: regexp array -> ((Sedlex_cset.t * int) array * bool array) array
type dfa = dfa_state array
val compile : regexp array -> dfa
val dfa_to_dot : dfa -> string

399
src/syntax/sedlex_cset.ml
View File

@ -1,4 +1,399 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
include Sedlex_utils.Cset (* Character sets are represented as lists of intervals. The
intervals must be non-overlapping and not collapsable, and the list
must be ordered in increasing order. *)
type t = (int * int) list
let max_code = 0x10ffff (* must be < max_int *)
let min_code = -1
let empty = []
let singleton i = [i,i]
let is_empty = function [] -> true | _ -> false
let interval i j = if i <= j then [i,j] else [j,i]
let eof = singleton (-1)
let any = interval 0 max_code
let rec union c1 c2 =
match c1,c2 with
| [], _ -> c2
| _, [] -> c1
| ((i1, j1) as s1)::r1, (i2, j2)::r2 ->
if (i1 <= i2) then
if j1 + 1 < i2 then s1::(union r1 c2)
else if (j1 < j2) then union r1 ((i1, j2)::r2)
else union c1 r2
else union c2 c1
let complement c =
let rec aux start = function
| [] -> if start <= max_code then [start,max_code] else []
| (i, j)::l -> (start, i-1)::(aux (succ j) l)
in
match c with
| (-1,j)::l -> aux (succ j) l
| l -> aux (-1) l
let intersection c1 c2 =
complement (union (complement c1) (complement c2))
let difference c1 c2 =
complement (union (complement c1) c2)
(* Unicode classes from XML *)
let base_char =
[ 0x0041,0x005A; 0x0061,0x007A; 0x00C0,0x00D6; 0x00D8,0x00F6;
0x00F8,0x00FF; 0x0100,0x0131; 0x0134,0x013E; 0x0141,0x0148;
0x014A,0x017E; 0x0180,0x01C3; 0x01CD,0x01F0; 0x01F4,0x01F5;
0x01FA,0x0217; 0x0250,0x02A8; 0x02BB,0x02C1; 0x0386,0x0386;
0x0388,0x038A; 0x038C,0x038C; 0x038E,0x03A1; 0x03A3,0x03CE;
0x03D0,0x03D6; 0x03DA,0x03DA; 0x03DC,0x03DC; 0x03DE,0x03DE;
0x03E0,0x03E0; 0x03E2,0x03F3;
0x0401,0x040C; 0x040E,0x044F; 0x0451,0x045C; 0x045E,0x0481;
0x0490,0x04C4; 0x04C7,0x04C8; 0x04CB,0x04CC; 0x04D0,0x04EB;
0x04EE,0x04F5; 0x04F8,0x04F9; 0x0531,0x0556; 0x0559,0x0559;
0x0561,0x0586; 0x05D0,0x05EA; 0x05F0,0x05F2; 0x0621,0x063A;
0x0641,0x064A; 0x0671,0x06B7; 0x06BA,0x06BE; 0x06C0,0x06CE;
0x06D0,0x06D3; 0x06D5,0x06D5; 0x06E5,0x06E6; 0x0905,0x0939;
0x093D,0x093D;
0x0958,0x0961; 0x0985,0x098C; 0x098F,0x0990; 0x0993,0x09A8;
0x09AA,0x09B0; 0x09B2,0x09B2; 0x09B6,0x09B9; 0x09DC,0x09DD;
0x09DF,0x09E1; 0x09F0,0x09F1; 0x0A05,0x0A0A; 0x0A0F,0x0A10;
0x0A13,0x0A28; 0x0A2A,0x0A30; 0x0A32,0x0A33; 0x0A35,0x0A36;
0x0A38,0x0A39; 0x0A59,0x0A5C; 0x0A5E,0x0A5E; 0x0A72,0x0A74;
0x0A85,0x0A8B; 0x0A8D,0x0A8D; 0x0A8F,0x0A91; 0x0A93,0x0AA8;
0x0AAA,0x0AB0; 0x0AB2,0x0AB3; 0x0AB5,0x0AB9; 0x0ABD,0x0ABD;
0x0AE0,0x0AE0;
0x0B05,0x0B0C; 0x0B0F,0x0B10; 0x0B13,0x0B28; 0x0B2A,0x0B30;
0x0B32,0x0B33; 0x0B36,0x0B39; 0x0B3D,0x0B3D; 0x0B5C,0x0B5D;
0x0B5F,0x0B61; 0x0B85,0x0B8A; 0x0B8E,0x0B90; 0x0B92,0x0B95;
0x0B99,0x0B9A; 0x0B9C,0x0B9C; 0x0B9E,0x0B9F; 0x0BA3,0x0BA4;
0x0BA8,0x0BAA; 0x0BAE,0x0BB5; 0x0BB7,0x0BB9; 0x0C05,0x0C0C;
0x0C0E,0x0C10; 0x0C12,0x0C28; 0x0C2A,0x0C33; 0x0C35,0x0C39;
0x0C60,0x0C61; 0x0C85,0x0C8C; 0x0C8E,0x0C90; 0x0C92,0x0CA8;
0x0CAA,0x0CB3; 0x0CB5,0x0CB9; 0x0CDE,0x0CDE; 0x0CE0,0x0CE1;
0x0D05,0x0D0C; 0x0D0E,0x0D10; 0x0D12,0x0D28; 0x0D2A,0x0D39;
0x0D60,0x0D61; 0x0E01,0x0E2E; 0x0E30,0x0E30; 0x0E32,0x0E33;
0x0E40,0x0E45; 0x0E81,0x0E82; 0x0E84,0x0E84; 0x0E87,0x0E88;
0x0E8A,0x0E8A;
0x0E8D,0x0E8D; 0x0E94,0x0E97; 0x0E99,0x0E9F; 0x0EA1,0x0EA3;
0x0EA5,0x0EA5;
0x0EA7,0x0EA7; 0x0EAA,0x0EAB; 0x0EAD,0x0EAE; 0x0EB0,0x0EB0;
0x0EB2,0x0EB3;
0x0EBD,0x0EBD; 0x0EC0,0x0EC4; 0x0F40,0x0F47; 0x0F49,0x0F69;
0x10A0,0x10C5; 0x10D0,0x10F6; 0x1100,0x1100; 0x1102,0x1103;
0x1105,0x1107; 0x1109,0x1109; 0x110B,0x110C; 0x110E,0x1112;
0x113C,0x113C;
0x113E,0x113E; 0x1140,0x1140; 0x114C,0x114C; 0x114E,0x114E;
0x1150,0x1150; 0x1154,0x1155; 0x1159,0x1159;
0x115F,0x1161; 0x1163,0x1163; 0x1165,0x1165; 0x1167,0x1167;
0x1169,0x1169; 0x116D,0x116E;
0x1172,0x1173; 0x1175,0x1175; 0x119E,0x119E; 0x11A8,0x11A8;
0x11AB,0x11AB; 0x11AE,0x11AF;
0x11B7,0x11B8; 0x11BA,0x11BA; 0x11BC,0x11C2; 0x11EB,0x11EB;
0x11F0,0x11F0; 0x11F9,0x11F9;
0x1E00,0x1E9B; 0x1EA0,0x1EF9; 0x1F00,0x1F15; 0x1F18,0x1F1D;
0x1F20,0x1F45; 0x1F48,0x1F4D; 0x1F50,0x1F57; 0x1F59,0x1F59;
0x1F5B,0x1F5B;
0x1F5D,0x1F5D; 0x1F5F,0x1F7D; 0x1F80,0x1FB4; 0x1FB6,0x1FBC;
0x1FBE,0x1FBE;
0x1FC2,0x1FC4; 0x1FC6,0x1FCC; 0x1FD0,0x1FD3; 0x1FD6,0x1FDB;
0x1FE0,0x1FEC; 0x1FF2,0x1FF4; 0x1FF6,0x1FFC; 0x2126,0x2126;
0x212A,0x212B; 0x212E,0x212E; 0x2180,0x2182; 0x3041,0x3094;
0x30A1,0x30FA; 0x3105,0x312C; 0xAC00,0xD7A3 ]
let ideographic =
[ 0x3007,0x3007; 0x3021,0x3029; 0x4E00,0x9FA5 ]
let combining_char =
[ 0x0300,0x0345; 0x0360,0x0361; 0x0483,0x0486; 0x0591,0x05A1;
0x05A3,0x05B9; 0x05BB,0x05BD; 0x05BF,0x05BF; 0x05C1,0x05C2;
0x05C4,0x05C4; 0x064B,0x0652; 0x0670,0x0670; 0x06D6,0x06DC;
0x06DD,0x06DF; 0x06E0,0x06E4; 0x06E7,0x06E8; 0x06EA,0x06ED;
0x0901,0x0903; 0x093C,0x093C; 0x093E,0x094C; 0x094D,0x094D;
0x0951,0x0954; 0x0962,0x0963; 0x0981,0x0983; 0x09BC,0x09BC;
0x09BE,0x09BE; 0x09BF,0x09BF; 0x09C0,0x09C4; 0x09C7,0x09C8;
0x09CB,0x09CD; 0x09D7,0x09D7; 0x09E2,0x09E3; 0x0A02,0x0A02;
0x0A3C,0x0A3C; 0x0A3E,0x0A3E; 0x0A3F,0x0A3F; 0x0A40,0x0A42;
0x0A47,0x0A48; 0x0A4B,0x0A4D; 0x0A70,0x0A71; 0x0A81,0x0A83;
0x0ABC,0x0ABC; 0x0ABE,0x0AC5; 0x0AC7,0x0AC9; 0x0ACB,0x0ACD;
0x0B01,0x0B03; 0x0B3C,0x0B3C; 0x0B3E,0x0B43; 0x0B47,0x0B48;
0x0B4B,0x0B4D; 0x0B56,0x0B57; 0x0B82,0x0B83; 0x0BBE,0x0BC2;
0x0BC6,0x0BC8; 0x0BCA,0x0BCD; 0x0BD7,0x0BD7; 0x0C01,0x0C03;
0x0C3E,0x0C44; 0x0C46,0x0C48; 0x0C4A,0x0C4D; 0x0C55,0x0C56;
0x0C82,0x0C83; 0x0CBE,0x0CC4; 0x0CC6,0x0CC8; 0x0CCA,0x0CCD;
0x0CD5,0x0CD6; 0x0D02,0x0D03; 0x0D3E,0x0D43; 0x0D46,0x0D48;
0x0D4A,0x0D4D; 0x0D57,0x0D57; 0x0E31,0x0E31; 0x0E34,0x0E3A;
0x0E47,0x0E4E; 0x0EB1,0x0EB1; 0x0EB4,0x0EB9; 0x0EBB,0x0EBC;
0x0EC8,0x0ECD; 0x0F18,0x0F19; 0x0F35,0x0F35; 0x0F37,0x0F37;
0x0F39,0x0F39; 0x0F3E,0x0F3E; 0x0F3F,0x0F3F; 0x0F71,0x0F84;
0x0F86,0x0F8B; 0x0F90,0x0F95; 0x0F97,0x0F97; 0x0F99,0x0FAD;
0x0FB1,0x0FB7; 0x0FB9,0x0FB9; 0x20D0,0x20DC; 0x20E1,0x20E1;
0x302A,0x302F; 0x3099,0x3099; 0x309A,0x309A ]
let digit =
[ 0x0030,0x0039;
0x0660,0x0669; 0x06F0,0x06F9; 0x0966,0x096F; 0x09E6,0x09EF;
0x0A66,0x0A6F; 0x0AE6,0x0AEF; 0x0B66,0x0B6F; 0x0BE7,0x0BEF;
0x0C66,0x0C6F; 0x0CE6,0x0CEF; 0x0D66,0x0D6F; 0x0E50,0x0E59;
0x0ED0,0x0ED9; 0x0F20,0x0F29 ]
let extender =
[ 0x00B7,0x00B7; 0x02D0,0x02D1; 0x0387,0x0387; 0x0640,0x0640;
0x0E46,0x0E46; 0x0EC6,0x0EC6; 0x3005,0x3005; 0x3031,0x3035;
0x309D,0x309E; 0x30FC,0x30FE ]
let blank =
[ 0x0009,0x000A; 0x000D,0x000D; 0x0020,0x0020 ]
let letter = union base_char ideographic
(* Letters to be used in identifiers, as specified
by ISO ....
Data provided by John M. Skaller *)
let tr8876_ident_char = [
(* ASCII *)
(0x0041,0x005a);
(0x0061,0x007a);
(* Latin *)
(0x00c0,0x00d6);
(0x00d8,0x00f6);
(0x00f8,0x01f5);
(0x01fa,0x0217);
(0x0250,0x02a8);
(* Greek *)
(0x0384,0x0384);
(0x0388,0x038a);
(0x038c,0x038c);
(0x038e,0x03a1);
(0x03a3,0x03ce);
(0x03d0,0x03d6);
(0x03da,0x03da);
(0x03dc,0x03dc);
(0x03de,0x03de);
(0x03e0,0x03e0);
(0x03e2,0x03f3);
(* Cyrillic *)
(0x0401,0x040d);
(0x040f,0x044f);
(0x0451,0x045c);
(0x045e,0x0481);
(0x0490,0x04c4);
(0x04c7,0x04c4);
(0x04cb,0x04cc);
(0x04d0,0x04eb);
(0x04ee,0x04f5);
(0x04f8,0x04f9);
(* Armenian *)
(0x0531,0x0556);
(0x0561,0x0587);
(0x04d0,0x04eb);
(* Hebrew *)
(0x05d0,0x05ea);
(0x05f0,0x05f4);
(* Arabic *)
(0x0621,0x063a);
(0x0640,0x0652);
(0x0670,0x06b7);
(0x06ba,0x06be);
(0x06c0,0x06ce);
(0x06e5,0x06e7);
(* Devanagari *)
(0x0905,0x0939);
(0x0958,0x0962);
(* Bengali *)
(0x0985,0x098c);
(0x098f,0x0990);
(0x0993,0x09a8);
(0x09aa,0x09b0);
(0x09b2,0x09b2);
(0x09b6,0x09b9);
(0x09dc,0x09dd);
(0x09df,0x09e1);
(0x09f0,0x09f1);
(* Gurmukhi *)
(0x0a05,0x0a0a);
(0x0a0f,0x0a10);
(0x0a13,0x0a28);
(0x0a2a,0x0a30);
(0x0a32,0x0a33);
(0x0a35,0x0a36);
(0x0a38,0x0a39);
(0x0a59,0x0a5c);
(0x0a5e,0x0a5e);
(* Gunjarati *)
(0x0a85,0x0a8b);
(0x0a8d,0x0a8d);
(0x0a8f,0x0a91);
(0x0a93,0x0aa8);
(0x0aaa,0x0ab0);
(0x0ab2,0x0ab3);
(0x0ab5,0x0ab9);
(0x0ae0,0x0ae0);
(* Oriya *)
(0x0b05,0x0b0c);
(0x0b0f,0x0b10);
(0x0b13,0x0b28);
(0x0b2a,0x0b30);
(0x0b32,0x0b33);
(0x0b36,0x0b39);
(0x0b5c,0x0b5d);
(0x0b5f,0x0b61);
(* Tamil *)
(0x0b85,0x0b8a);
(0x0b8e,0x0b90);
(0x0b92,0x0b95);
(0x0b99,0x0b9a);
(0x0b9c,0x0b9c);
(0x0b9e,0x0b9f);
(0x0ba3,0x0ba4);
(0x0ba8,0x0baa);
(0x0bae,0x0bb5);
(0x0bb7,0x0bb9);
(* Telugu *)
(0x0c05,0x0c0c);
(0x0c0e,0x0c10);
(0x0c12,0x0c28);
(0x0c2a,0x0c33);
(0x0c35,0x0c39);
(0x0c60,0x0c61);
(* Kannada *)
(0x0c85,0x0c8c);
(0x0c8e,0x0c90);
(0x0c92,0x0ca8);
(0x0caa,0x0cb3);
(0x0cb5,0x0cb9);
(0x0ce0,0x0ce1);
(* Malayam *)
(0x0d05,0x0d0c);
(0x0d0e,0x0d10);
(0x0d12,0x0d28);
(0x0d2a,0x0d39);
(0x0d60,0x0d61);
(* Thai *)
(0x0e01,0x0e30);
(0x0e32,0x0e33);
(0x0e40,0x0e46);
(0x0e4f,0x0e5b);
(* Lao *)
(0x0e81,0x0e82);
(0x0e84,0x0e84);
(0x0e87,0x0e88);
(0x0e8a,0x0e8a);
(0x0e0d,0x0e0d);
(0x0e94,0x0e97);
(0x0e99,0x0e9f);
(0x0ea1,0x0ea3);
(0x0ea5,0x0ea5);
(0x0ea7,0x0ea7);
(0x0eaa,0x0eab);
(0x0ead,0x0eb0);
(0x0eb2,0x0eb3);
(0x0ebd,0x0ebd);
(0x0ec0,0x0ec4);
(0x0ec6,0x0ec6);
(* Georgian *)
(0x10a0,0x10c5);
(0x10d0,0x10f6);
(* Hangul Jamo *)
(0x1100,0x1159);
(0x1161,0x11a2);
(0x11a8,0x11f9);
(0x11d0,0x11f6);
(* Latin extensions *)
(0x1e00,0x1e9a);
(0x1ea0,0x1ef9);
(* Greek extended *)
(0x1f00,0x1f15);
(0x1f18,0x1f1d);
(0x1f20,0x1f45);
(0x1f48,0x1f4d);
(0x1f50,0x1f57);
(0x1f59,0x1f59);
(0x1f5b,0x1f5b);
(0x1f5d,0x1f5d);
(0x1f5f,0x1f7d);
(0x1f80,0x1fb4);
(0x1fb6,0x1fbc);
(0x1fc2,0x1fc4);
(0x1fc6,0x1fcc);
(0x1fd0,0x1fd3);
(0x1fd6,0x1fdb);
(0x1fe0,0x1fec);
(0x1ff2,0x1ff4);
(0x1ff6,0x1ffc);
(* Hiragana *)
(0x3041,0x3094);
(0x309b,0x309e);
(* Katakana *)
(0x30a1,0x30fe);
(* Bopmofo *)
(0x3105,0x312c);
(* CJK Unified Ideographs *)
(0x4e00,0x9fa5);
(* CJK Compatibility Ideographs *)
(0xf900,0xfa2d);
(* Arabic Presentation Forms *)
(0xfb1f,0xfb36);
(0xfb38,0xfb3c);
(0xfb3e,0xfb3e);
(0xfb40,0xfb41);
(0xfb42,0xfb44);
(0xfb46,0xfbb1);
(0xfbd3,0xfd35);
(* Arabic Presentation Forms-A *)
(0xfd50,0xfd85);
(0xfd92,0xfbc7);
(0xfdf0,0xfdfb);
(* Arabic Presentation Forms-B *)
(0xfe70,0xfe72);
(0xfe74,0xfe74);
(0xfe76,0xfefc);
(* Half width and Fullwidth Forms *)
(0xff21,0xff3a);
(0xff41,0xff5a);
(0xff66,0xffbe);
(0xffc2,0xffc7);
(0xffca,0xffcf);
(0xffd2,0xffd7);
(0xffd2,0xffd7);
(0xffda,0xffdc)
]

30
src/syntax/sedlex_cset.mli Normal file
View File

@ -0,0 +1,30 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(** Representation of sets of unicode code points. *)
type t = (int * int) list
val min_code: int
val max_code: int
val empty: t
val any: t
val union: t -> t -> t
val difference: t -> t -> t
val intersection: t -> t -> t
val is_empty: t -> bool
val eof: t
val singleton: int -> t
val interval: int -> int -> t
val letter: t
val digit: t
val extender: t
val base_char: t
val ideographic: t
val combining_char: t
val blank: t
val tr8876_ident_char: t

3680
src/syntax/unicode.ml
View File

File diff suppressed because it is too large Load Diff

4
src/syntax/unicode.mli
View File

@ -3,6 +3,7 @@
val version : string val version : string
module Categories : sig module Categories : sig
val cc : Sedlex_cset.t val cc : Sedlex_cset.t
val cf : Sedlex_cset.t val cf : Sedlex_cset.t
val cn : Sedlex_cset.t val cn : Sedlex_cset.t
@ -33,10 +34,12 @@ module Categories : sig
val zl : Sedlex_cset.t val zl : Sedlex_cset.t
val zp : Sedlex_cset.t val zp : Sedlex_cset.t
val zs : Sedlex_cset.t val zs : Sedlex_cset.t
val list : (string * Sedlex_cset.t) list val list : (string * Sedlex_cset.t) list
end end
module Properties : sig module Properties : sig
val alphabetic : Sedlex_cset.t val alphabetic : Sedlex_cset.t
val ascii_hex_digit : Sedlex_cset.t val ascii_hex_digit : Sedlex_cset.t
val hex_digit : Sedlex_cset.t val hex_digit : Sedlex_cset.t
@ -52,5 +55,6 @@ module Properties : sig
val white_space : Sedlex_cset.t val white_space : Sedlex_cset.t
val xid_continue : Sedlex_cset.t val xid_continue : Sedlex_cset.t
val xid_start : Sedlex_cset.t val xid_start : Sedlex_cset.t
val list : (string * Sedlex_cset.t) list val list : (string * Sedlex_cset.t) list
end end

49
src/syntax/utf8.ml
View File

@ -1,49 +0,0 @@
open Sedlexing
let unsafe_byte s j = Char.code (String.unsafe_get s j)
let malformed s j l = `Malformed (String.sub s j l)
let r_utf_8 s j l =
(* assert (0 <= j && 0 <= l && j + l <= String.length s); *)
let uchar c = `Uchar (Uchar.unsafe_of_int c) in
match l with
| 1 -> uchar (unsafe_byte s j)
| 2 -> (
let b0 = unsafe_byte s j in
let b1 = unsafe_byte s (j + 1) in
match Utf8.Helper.check_two b0 b1 with
| i -> uchar i
| exception MalFormed -> malformed s j l)
| 3 -> (
let b0 = unsafe_byte s j in
let b1 = unsafe_byte s (j + 1) in
let b2 = unsafe_byte s (j + 2) in
match Utf8.Helper.check_three b0 b1 b2 with
| i -> uchar i
| exception MalFormed -> malformed s j l)
| 4 -> (
let b0 = unsafe_byte s j in
let b1 = unsafe_byte s (j + 1) in
let b2 = unsafe_byte s (j + 2) in
let b3 = unsafe_byte s (j + 3) in
match Utf8.Helper.check_four b0 b1 b2 b3 with
| i -> uchar i
| exception MalFormed -> malformed s j l)
| _ -> assert false
let fold ~f acc s =
let rec loop acc f s i last =
if i > last then acc
else (
match Utf8.Helper.width (String.unsafe_get s i) with
| exception MalFormed ->
loop (f acc i (malformed s i 1)) f s (i + 1) last
| need ->
let rem = last - i + 1 in
if rem < need then f acc i (malformed s i rem)
else loop (f acc i (r_utf_8 s i need)) f s (i + need) last)
in
let pos = 0 in
let len = String.length s in
let last = pos + len - 1 in
loop acc f s pos last

5
src/syntax/utf8.mli
View File

@ -1,5 +0,0 @@
val fold :
f:('a -> int -> [> `Malformed of string | `Uchar of Uchar.t ] -> 'a) ->
'a ->
string ->
'a

365
src/syntax/xml.ml
View File

@ -1,365 +0,0 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
open Sedlex_cset
(* Unicode classes from XML *)
let base_char =
let l =
[
(0x0041, 0x005A);
(0x0061, 0x007A);
(0x00C0, 0x00D6);
(0x00D8, 0x00F6);
(0x00F8, 0x00FF);
(0x0100, 0x0131);
(0x0134, 0x013E);
(0x0141, 0x0148);
(0x014A, 0x017E);
(0x0180, 0x01C3);
(0x01CD, 0x01F0);
(0x01F4, 0x01F5);
(0x01FA, 0x0217);
(0x0250, 0x02A8);
(0x02BB, 0x02C1);
(0x0386, 0x0386);
(0x0388, 0x038A);
(0x038C, 0x038C);
(0x038E, 0x03A1);
(0x03A3, 0x03CE);
(0x03D0, 0x03D6);
(0x03DA, 0x03DA);
(0x03DC, 0x03DC);
(0x03DE, 0x03DE);
(0x03E0, 0x03E0);
(0x03E2, 0x03F3);
(0x0401, 0x040C);
(0x040E, 0x044F);
(0x0451, 0x045C);
(0x045E, 0x0481);
(0x0490, 0x04C4);
(0x04C7, 0x04C8);
(0x04CB, 0x04CC);
(0x04D0, 0x04EB);
(0x04EE, 0x04F5);
(0x04F8, 0x04F9);
(0x0531, 0x0556);
(0x0559, 0x0559);
(0x0561, 0x0586);
(0x05D0, 0x05EA);
(0x05F0, 0x05F2);
(0x0621, 0x063A);
(0x0641, 0x064A);
(0x0671, 0x06B7);
(0x06BA, 0x06BE);
(0x06C0, 0x06CE);
(0x06D0, 0x06D3);
(0x06D5, 0x06D5);
(0x06E5, 0x06E6);
(0x0905, 0x0939);
(0x093D, 0x093D);
(0x0958, 0x0961);
(0x0985, 0x098C);
(0x098F, 0x0990);
(0x0993, 0x09A8);
(0x09AA, 0x09B0);
(0x09B2, 0x09B2);
(0x09B6, 0x09B9);
(0x09DC, 0x09DD);
(0x09DF, 0x09E1);
(0x09F0, 0x09F1);
(0x0A05, 0x0A0A);
(0x0A0F, 0x0A10);
(0x0A13, 0x0A28);
(0x0A2A, 0x0A30);
(0x0A32, 0x0A33);
(0x0A35, 0x0A36);
(0x0A38, 0x0A39);
(0x0A59, 0x0A5C);
(0x0A5E, 0x0A5E);
(0x0A72, 0x0A74);
(0x0A85, 0x0A8B);
(0x0A8D, 0x0A8D);
(0x0A8F, 0x0A91);
(0x0A93, 0x0AA8);
(0x0AAA, 0x0AB0);
(0x0AB2, 0x0AB3);
(0x0AB5, 0x0AB9);
(0x0ABD, 0x0ABD);
(0x0AE0, 0x0AE0);
(0x0B05, 0x0B0C);
(0x0B0F, 0x0B10);
(0x0B13, 0x0B28);
(0x0B2A, 0x0B30);
(0x0B32, 0x0B33);
(0x0B36, 0x0B39);
(0x0B3D, 0x0B3D);
(0x0B5C, 0x0B5D);
(0x0B5F, 0x0B61);
(0x0B85, 0x0B8A);
(0x0B8E, 0x0B90);
(0x0B92, 0x0B95);
(0x0B99, 0x0B9A);
(0x0B9C, 0x0B9C);
(0x0B9E, 0x0B9F);
(0x0BA3, 0x0BA4);
(0x0BA8, 0x0BAA);
(0x0BAE, 0x0BB5);
(0x0BB7, 0x0BB9);
(0x0C05, 0x0C0C);
(0x0C0E, 0x0C10);
(0x0C12, 0x0C28);
(0x0C2A, 0x0C33);
(0x0C35, 0x0C39);
(0x0C60, 0x0C61);
(0x0C85, 0x0C8C);
(0x0C8E, 0x0C90);
(0x0C92, 0x0CA8);
(0x0CAA, 0x0CB3);
(0x0CB5, 0x0CB9);
(0x0CDE, 0x0CDE);
(0x0CE0, 0x0CE1);
(0x0D05, 0x0D0C);
(0x0D0E, 0x0D10);
(0x0D12, 0x0D28);
(0x0D2A, 0x0D39);
(0x0D60, 0x0D61);
(0x0E01, 0x0E2E);
(0x0E30, 0x0E30);
(0x0E32, 0x0E33);
(0x0E40, 0x0E45);
(0x0E81, 0x0E82);
(0x0E84, 0x0E84);
(0x0E87, 0x0E88);
(0x0E8A, 0x0E8A);
(0x0E8D, 0x0E8D);
(0x0E94, 0x0E97);
(0x0E99, 0x0E9F);
(0x0EA1, 0x0EA3);
(0x0EA5, 0x0EA5);
(0x0EA7, 0x0EA7);
(0x0EAA, 0x0EAB);
(0x0EAD, 0x0EAE);
(0x0EB0, 0x0EB0);
(0x0EB2, 0x0EB3);
(0x0EBD, 0x0EBD);
(0x0EC0, 0x0EC4);
(0x0F40, 0x0F47);
(0x0F49, 0x0F69);
(0x10A0, 0x10C5);
(0x10D0, 0x10F6);
(0x1100, 0x1100);
(0x1102, 0x1103);
(0x1105, 0x1107);
(0x1109, 0x1109);
(0x110B, 0x110C);
(0x110E, 0x1112);
(0x113C, 0x113C);
(0x113E, 0x113E);
(0x1140, 0x1140);
(0x114C, 0x114C);
(0x114E, 0x114E);
(0x1150, 0x1150);
(0x1154, 0x1155);
(0x1159, 0x1159);
(0x115F, 0x1161);
(0x1163, 0x1163);
(0x1165, 0x1165);
(0x1167, 0x1167);
(0x1169, 0x1169);
(0x116D, 0x116E);
(0x1172, 0x1173);
(0x1175, 0x1175);
(0x119E, 0x119E);
(0x11A8, 0x11A8);
(0x11AB, 0x11AB);
(0x11AE, 0x11AF);
(0x11B7, 0x11B8);
(0x11BA, 0x11BA);
(0x11BC, 0x11C2);
(0x11EB, 0x11EB);
(0x11F0, 0x11F0);
(0x11F9, 0x11F9);
(0x1E00, 0x1E9B);
(0x1EA0, 0x1EF9);
(0x1F00, 0x1F15);
(0x1F18, 0x1F1D);
(0x1F20, 0x1F45);
(0x1F48, 0x1F4D);
(0x1F50, 0x1F57);
(0x1F59, 0x1F59);
(0x1F5B, 0x1F5B);
(0x1F5D, 0x1F5D);
(0x1F5F, 0x1F7D);
(0x1F80, 0x1FB4);
(0x1FB6, 0x1FBC);
(0x1FBE, 0x1FBE);
(0x1FC2, 0x1FC4);
(0x1FC6, 0x1FCC);
(0x1FD0, 0x1FD3);
(0x1FD6, 0x1FDB);
(0x1FE0, 0x1FEC);
(0x1FF2, 0x1FF4);
(0x1FF6, 0x1FFC);
(0x2126, 0x2126);
(0x212A, 0x212B);
(0x212E, 0x212E);
(0x2180, 0x2182);
(0x3041, 0x3094);
(0x30A1, 0x30FA);
(0x3105, 0x312C);
(0xAC00, 0xD7A3);
]
in
of_list l
let ideographic =
let l = [(0x3007, 0x3007); (0x3021, 0x3029); (0x4E00, 0x9FA5)] in
of_list l
let combining_char =
let l =
[
(0x0300, 0x0345);
(0x0360, 0x0361);
(0x0483, 0x0486);
(0x0591, 0x05A1);
(0x05A3, 0x05B9);
(0x05BB, 0x05BD);
(0x05BF, 0x05BF);
(0x05C1, 0x05C2);
(0x05C4, 0x05C4);
(0x064B, 0x0652);
(0x0670, 0x0670);
(0x06D6, 0x06DC);
(0x06DD, 0x06DF);
(0x06E0, 0x06E4);
(0x06E7, 0x06E8);
(0x06EA, 0x06ED);
(0x0901, 0x0903);
(0x093C, 0x093C);
(0x093E, 0x094C);
(0x094D, 0x094D);
(0x0951, 0x0954);
(0x0962, 0x0963);
(0x0981, 0x0983);
(0x09BC, 0x09BC);
(0x09BE, 0x09BE);
(0x09BF, 0x09BF);
(0x09C0, 0x09C4);
(0x09C7, 0x09C8);
(0x09CB, 0x09CD);
(0x09D7, 0x09D7);
(0x09E2, 0x09E3);
(0x0A02, 0x0A02);
(0x0A3C, 0x0A3C);
(0x0A3E, 0x0A3E);
(0x0A3F, 0x0A3F);
(0x0A40, 0x0A42);
(0x0A47, 0x0A48);
(0x0A4B, 0x0A4D);
(0x0A70, 0x0A71);
(0x0A81, 0x0A83);
(0x0ABC, 0x0ABC);
(0x0ABE, 0x0AC5);
(0x0AC7, 0x0AC9);
(0x0ACB, 0x0ACD);
(0x0B01, 0x0B03);
(0x0B3C, 0x0B3C);
(0x0B3E, 0x0B43);
(0x0B47, 0x0B48);
(0x0B4B, 0x0B4D);
(0x0B56, 0x0B57);
(0x0B82, 0x0B83);
(0x0BBE, 0x0BC2);
(0x0BC6, 0x0BC8);
(0x0BCA, 0x0BCD);
(0x0BD7, 0x0BD7);
(0x0C01, 0x0C03);
(0x0C3E, 0x0C44);
(0x0C46, 0x0C48);
(0x0C4A, 0x0C4D);
(0x0C55, 0x0C56);
(0x0C82, 0x0C83);
(0x0CBE, 0x0CC4);
(0x0CC6, 0x0CC8);
(0x0CCA, 0x0CCD);
(0x0CD5, 0x0CD6);
(0x0D02, 0x0D03);
(0x0D3E, 0x0D43);
(0x0D46, 0x0D48);
(0x0D4A, 0x0D4D);
(0x0D57, 0x0D57);
(0x0E31, 0x0E31);
(0x0E34, 0x0E3A);
(0x0E47, 0x0E4E);
(0x0EB1, 0x0EB1);
(0x0EB4, 0x0EB9);
(0x0EBB, 0x0EBC);
(0x0EC8, 0x0ECD);
(0x0F18, 0x0F19);
(0x0F35, 0x0F35);
(0x0F37, 0x0F37);
(0x0F39, 0x0F39);
(0x0F3E, 0x0F3E);
(0x0F3F, 0x0F3F);
(0x0F71, 0x0F84);
(0x0F86, 0x0F8B);
(0x0F90, 0x0F95);
(0x0F97, 0x0F97);
(0x0F99, 0x0FAD);
(0x0FB1, 0x0FB7);
(0x0FB9, 0x0FB9);
(0x20D0, 0x20DC);
(0x20E1, 0x20E1);
(0x302A, 0x302F);
(0x3099, 0x3099);
(0x309A, 0x309A);
]
in
of_list l
let digit =
let l =
[
(0x0030, 0x0039);
(0x0660, 0x0669);
(0x06F0, 0x06F9);
(0x0966, 0x096F);
(0x09E6, 0x09EF);
(0x0A66, 0x0A6F);
(0x0AE6, 0x0AEF);
(0x0B66, 0x0B6F);
(0x0BE7, 0x0BEF);
(0x0C66, 0x0C6F);
(0x0CE6, 0x0CEF);
(0x0D66, 0x0D6F);
(0x0E50, 0x0E59);
(0x0ED0, 0x0ED9);
(0x0F20, 0x0F29);
]
in
of_list l
let extender =
let l =
[
(0x00B7, 0x00B7);
(0x02D0, 0x02D1);
(0x0387, 0x0387);
(0x0640, 0x0640);
(0x0E46, 0x0E46);
(0x0EC6, 0x0EC6);
(0x3005, 0x3005);
(0x3031, 0x3035);
(0x309D, 0x309E);
(0x30FC, 0x30FE);
]
in
of_list l
let blank =
let l = [(0x0009, 0x000A); (0x000D, 0x000D); (0x0020, 0x0020)] in
of_list l
let letter = union base_char ideographic

14
src/syntax/xml.mli
View File

@ -1,14 +0,0 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(** Unicode classes from XML *)
open Sedlex_cset
val letter : t
val digit : t
val extender : t
val base_char : t
val ideographic : t
val combining_char : t
val blank : t

BIN
test/UTF-8-test.txt
View File

Binary file not shown.

1062
test/basic.ml
View File

File diff suppressed because it is too large Load Diff

8
test/codegen/dune
View File

@ -1,8 +0,0 @@
(library
(name sedlex_gen_test)
(libraries noslop-sedlex)
(inline_tests)
(enabled_if
(>= %{ocaml_version} 4.14))
(preprocess
(pps ppx_sedlex_test ppx_expect)))

120
test/codegen/test_gen.ml
View File

@ -1,120 +0,0 @@
let%expect_test "simple string match" =
(match%sedlex_test buf with "ab" | "de" -> () | _ -> ());
[%expect
{|
DOT:
digraph {
rankdir=LR;
node [shape=circle];
_start [shape=point];
_start -> state0;
state0 [label="0"];
state0 -> state1 [label="'a'"];
state0 -> state3 [label="'d'"];
state1 [label="1"];
state1 -> state2 [label="'b'"];
state2 [label="2\n[rule 0]", shape=doublecircle];
state3 [label="3"];
state3 -> state2 [label="'e'"];
}
CODE:
let rec __sedlex_state_0 buf =
match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
| 0 -> __sedlex_state_1 buf
| 1 -> __sedlex_state_3 buf
| _ -> Sedlexing.backtrack buf
and __sedlex_state_1 buf =
match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with
| 0 -> 0
| _ -> Sedlexing.backtrack buf
and __sedlex_state_3 buf =
match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with
| 0 -> 0
| _ -> Sedlexing.backtrack buf in
Sedlexing.start buf; (match __sedlex_state_0 buf with | 0 -> () | _ -> ())
|}]
let%expect_test "character class" =
(match%sedlex_test buf with Plus 'a' .. 'z' -> () | _ -> ());
[%expect
{|
DOT:
digraph {
rankdir=LR;
node [shape=circle];
_start [shape=point];
_start -> state0;
state0 [label="0"];
state0 -> state1 [label="'a'-'z'"];
state1 [label="1\n[rule 0]", shape=doublecircle];
state1 -> state1 [label="'a'-'z'"];
}
CODE:
let rec __sedlex_state_0 buf =
match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
| 0 -> __sedlex_state_1 buf
| _ -> Sedlexing.backtrack buf
and __sedlex_state_1 buf =
Sedlexing.mark buf 0;
(match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
| 0 -> __sedlex_state_1 buf
| _ -> Sedlexing.backtrack buf) in
Sedlexing.start buf; (match __sedlex_state_0 buf with | 0 -> () | _ -> ())
|}]
let%expect_test "multi-rule" =
(match%sedlex_test buf with
| "ab" -> ()
| "de" -> ()
| Plus '0' .. '9' -> ()
| _ -> ());
[%expect
{|
DOT:
digraph {
rankdir=LR;
node [shape=circle];
_start [shape=point];
_start -> state0;
state0 [label="0"];
state0 -> state1 [label="'0'-'9'"];
state0 -> state2 [label="'a'"];
state0 -> state4 [label="'d'"];
state1 [label="1\n[rule 2]", shape=doublecircle];
state1 -> state1 [label="'0'-'9'"];
state2 [label="2"];
state2 -> state3 [label="'b'"];
state3 [label="3\n[rule 0]", shape=doublecircle];
state4 [label="4"];
state4 -> state5 [label="'e'"];
state5 [label="5\n[rule 1]", shape=doublecircle];
}
CODE:
let rec __sedlex_state_0 buf =
match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
| 0 -> __sedlex_state_1 buf
| 1 -> __sedlex_state_2 buf
| 2 -> __sedlex_state_4 buf
| _ -> Sedlexing.backtrack buf
and __sedlex_state_1 buf =
Sedlexing.mark buf 2;
(match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with
| 0 -> __sedlex_state_1 buf
| _ -> Sedlexing.backtrack buf)
and __sedlex_state_2 buf =
match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with
| 0 -> 0
| _ -> Sedlexing.backtrack buf
and __sedlex_state_4 buf =
match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with
| 0 -> 1
| _ -> Sedlexing.backtrack buf in
Sedlexing.start buf;
(match __sedlex_state_0 buf with | 0 -> () | 1 -> () | 2 -> () | _ -> ())
|}]

9
test/dune
View File

@ -1,9 +0,0 @@
(library
(name sedlex_test)
(libraries noslop-sedlex)
(inline_tests
(deps UTF-8-test.txt))
(enabled_if
(>= %{ocaml_version} 4.14))
(preprocess
(pps noslop-sedlex.ppx ppx_expect)))

9
test/nested.ml
View File

@ -1,9 +0,0 @@
let%expect_test _ =
let lb = Sedlexing.Utf8.from_string "ab" in
let res =
match%sedlex lb with
| 'a' -> ( match%sedlex lb with 'b' -> "ok" | _ -> "error")
| _ -> "error"
in
print_endline res;
[%expect {| ok |}]

6
test/ppx_test/dune
View File

@ -1,6 +0,0 @@
(library
(name ppx_sedlex_test)
(kind ppx_rewriter)
(libraries ppxlib noslop-sedlex.ppx)
(preprocess
(pps ppxlib.metaquot)))

36
test/ppx_test/ppx_sedlex_test.ml
View File

@ -1,36 +0,0 @@
open Ppxlib
module P = Sedlex_ppx.Ppx_sedlex
module S = Sedlex_ppx.Sedlex
let reset_state () =
P.partition_counter := 0;
P.table_counter := 0;
Hashtbl.clear P.partitions;
Hashtbl.clear P.tables
let clear_tables () =
Hashtbl.clear P.partitions;
Hashtbl.clear P.tables
let expand ~ctxt:_ expr =
reset_state ();
let loc = Location.none in
let code_expr, auto =
P.handle_sedlex_match ~env:P.builtin_regexps ~map_rhs:Fun.id expr
in
let code_str = Pprintast.string_of_expression code_expr in
let dot_str = S.dfa_to_dot auto in
clear_tables ();
[%expr
print_string "DOT:\n";
print_string [%e Ast_builder.Default.estring ~loc dot_str];
print_string "CODE:\n";
print_string [%e Ast_builder.Default.estring ~loc code_str];
print_newline ()]
let ext =
Extension.V3.declare "sedlex_test" Extension.Context.expression
Ast_pattern.(single_expr_payload __)
expand
let () = Driver.register_transformation "sedlex_test" ~extensions:[ext]

BIN
test/stress_test.ml
View File

Binary file not shown.

33
test/utf8.ml
View File

@ -1,33 +0,0 @@
open Printf
let next_tok buf =
let open Sedlexing.Utf8 in
match%sedlex buf with
| "a", Utf8 (Chars "+-×÷") -> sprintf "with Chars: %s" (lexeme buf)
| "b", Utf8 ("+" | "-" | "×" | "÷") ->
sprintf "with or_pattern: %s" (lexeme buf)
| Latin1 "\xc0", Utf8 "À", Ascii (Utf8 (Latin1 (Utf8 (Chars "À")))) ->
sprintf "mixed encoding: %s" (lexeme buf)
| Ascii (Star '\x00' .. '\x7f') -> sprintf "only ascii: %s" (lexeme buf)
| Utf8 (Star '\x00' .. '\x7f') ->
assert false
(* utf8 char interval can only match ascii because of the OCaml lexer. The regexp above should match instead *)
| Latin1 (Star '\x00' .. '\xff') -> sprintf "only latin1: %s" (lexeme buf)
| _ -> failwith (sprintf "Unexpected character: %s" (lexeme buf))
let%expect_test _ =
Sedlexing.Utf8.from_string "a+" |> next_tok |> print_string;
[%expect {| with Chars: a+ |}];
Sedlexing.Utf8.from_string "a÷" |> next_tok |> print_string;
[%expect {| with Chars: a÷ |}];
Sedlexing.Utf8.from_string "b+" |> next_tok |> print_string;
[%expect {| with or_pattern: b+ |}];
Sedlexing.Utf8.from_string "b÷" |> next_tok |> print_string;
[%expect {| with or_pattern: b÷ |}];
Sedlexing.Utf8.from_string "ÀÀÀ" |> next_tok |> print_string;
[%expect {| mixed encoding: ÀÀÀ |}];
Sedlexing.Utf8.from_string "az\x7f"
|> next_tok |> String.escaped |> print_string;
[%expect {| only ascii: az\127 |}];
Sedlexing.Utf8.from_string "az\u{c0}" |> next_tok |> print_string;
[%expect {| only latin1: azÀ |}]