noslop
/
noslop-sedlex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: noslop:main
Branches Tags
noslop:main
noslop:upstream/v3.7
noslop:upstream/v3.6
noslop:upstream/v3.5
noslop:upstream/v3.4
noslop:upstream/v3.3
noslop:upstream/v3.2
noslop:upstream/v3.1
noslop:upstream/v3.0
noslop:upstream/v2.6
noslop:upstream/v2.5
noslop:upstream/v2.4
noslop:upstream/v2.3
noslop:upstream/v2.2
noslop:upstream/v2.1
noslop:upstream/v2.0
noslop:upstream/v1.99.4
noslop:upstream/v1.99.3
noslop:upstream/sedlex-1.99.2
..
compare: noslop:upstream/v1.99.4
Branches Tags
noslop:main
noslop:upstream/v3.7
noslop:upstream/v3.6
noslop:upstream/v3.5
noslop:upstream/v3.4
noslop:upstream/v3.3
noslop:upstream/v3.2
noslop:upstream/v3.1
noslop:upstream/v3.0
noslop:upstream/v2.6
noslop:upstream/v2.5
noslop:upstream/v2.4
noslop:upstream/v2.3
noslop:upstream/v2.2
noslop:upstream/v2.1
noslop:upstream/v2.0
noslop:upstream/v1.99.4
noslop:upstream/v1.99.3
noslop:upstream/sedlex-1.99.2

No commits in common. "main" and "upstream/v1.99.4" have entirely different histories.
main ... upstream/v

64 changed files with 3507 additions and 8618 deletions
Show Stats Expand all files Collapse all files

1
.envrc
View File

@ -1 +0,0 @@
use flake

1
.gitattributes vendored
View File

@ -1 +0,0 @@
test/stress_test.ml text diff

7
.gitignore vendored
View File

@ -8,15 +8,8 @@
*.cmxs *.cmxs
*.cmxa *.cmxa
*.exe *.exe
*.install
*.merlin
.*sw*
_build
libdoc libdoc
src/syntax/ppx_sedlex src/syntax/ppx_sedlex
src/generator/data/*.txt
examples/complement examples/complement
examples/tokenizer examples/tokenizer
examples/subtraction examples/subtraction
_opam
.direnv

10
.ocamlformat
View File

@ -1,10 +0,0 @@
version=0.29.0
profile = conventional
break-separators = after
space-around-lists = false
doc-comments = before
match-indent = 2
match-indent-nested = always
parens-ite
exp-grouping = preserve
module-item-spacing = compact

57
CHANGES Normal file
View File

@ -0,0 +1,57 @@
1.99.4
* GPR#47: Switch to ocaml-migrate-parsetree (contributed by Adrien Guatto)
* GPR#42: Added 'Rep' (repeat operator) (contributed by jpathy)
1.99.3
* Update to work with 4.03 (4.02 still supported)
1.99.2
* First official release of sedlex
1.99.1
* Support for new Ast_mapper registration API, follow OCaml trunk after
the inclusion of the extension_point branch
1.99
* First version of sedlex. The history below refers to ulex, the ancestor
or sedlex implemented with Camlp4.
--------------------------------------------------------------------------
1.1
* Generate (more) globally unique identifiers to avoid conflicts when open'ing another module
processed by ulex (issue reported by Gerd Stolpmann)
1.0
* Update to the new Camlp4 and to ocamlbuild (release for OCaml 3.10
only), by Nicolas Pouillard.
0.8
* Really make it work with OCaml 3.09.
* Support for Utf-16.
0.7 released May 24 2005
* Bug fixes
* Update to OCaml 3.09 (currently CVS). Still works with OCaml 3.08.
* MIT-like license (used to LGPL)
0.5 release Jul. 8 2004
* Document how to use a custom implementation for lex buffers
* Update to OCaml 3.08
0.4 released Jan. 10 2004
* Bug fix (accept 1114111 as valid Unicode code point)
* Add the rollback function
0.3 released Oct. 8 2003
* Bug fix
* Add a new predefined class for ISO identifiers
0.2 released Sep. 22 2003
* Changed the names of predefined regexp
* Fix max_code = 0x10ffff
* Lexers that changes encoding on the fly
* Documentation of the interface Ulexing
0.1 released Sep. 20 2003
* Initial release

128
CHANGES.md
View File

@ -1,128 +0,0 @@
# 3.7 (2025-10-06)
- Update to unicode 17.0.0
# 3.6 (2025-01-05)
- Fixed one of the ranges implementing
Implement Corrigendum #1: UTF-8 Shortest Form
for 4-bytes long characters (#171)
# 3.5 (2025-05-29)
- Implement Corrigendum #1: UTF-8 Shortest Form
- Add utf8 support for string literal (#127)
# 3.4 (2025-03-28)
- Make the library compatibility with ppxlib.0.36 (#166)
# 3.3 (2024-10-29)
- Add support for unicode `16.0.0` (#157)
- Add API for retrieving start and stop positions separately (#155)
# 3.2 (2023-06-28):
- Restore compatibility with OCaml 4.08
- Use `Sedlexing.{Utf8,Utf16}.from_gen` to initialize UTF8 (resp. UTF16) lexing buffers from
string.
- Delay raising Malformed until actually reading the malformed part of the imput. (#140)
- Count lines in all cases (#130). Previously, certain functions for initiating the
lexical buffer would disable lines counting.
- Check and fix invariants from Cset. The codebase was not respecting
invariants documented in the Cset module which could break code
relying on it. The code generated by sedlex.ppx could be affected.
- Do not rely on comments from unicode UCD files
- Add API to track position in bytes. Should be opt-in and backward compatible. (#146)
# 3.1:
- Fix directly nested sedlex matches (@smuenzel, PR #117, fixes: #12)
- Use explicit stdlib in generated code (@hhugo, PR #122, fixes: #115)
- Preserve location of lexbuf (@hhugo, PR #118, fixes: #19)
- Don't use gen to consume channels (@hhugo, PR #124, fixes: #45)
- New expect_test testsuite (@hhugo, PR #124)
- Properly recognize malformed truncated input (@hhugo, PR #124)
- Raise `Malformed` instead of `Invalid_arg` (@hhugo, PR #126, fixes: #91)
- Updated unicode support to `15.0.0`
# 3.0:
- Dropped `Stream` api which was removed in `4.14.0` ahead of the `5.0`
release.
2.6:
- Adapted to ppxlib `0.26`, thanks to @pitag-ha
2.5:
- Fix exponential compilation time, thanks to @mnxn for reporting in #97
and @fangyi-zhou for fixing in #106
- Update unicode support for `14.0.0`.
# 2.4
- Update `dune` support to `2.8`, add auto-generated `opam` files.
- Optimize generated code, thanks to @bobzhang
- Update unicode version to 13.0.0
# 2.3
- Switch to ppxlib
# 2.2
- Support for OCaml 4.08
# 2.1
- GPR#78: Auto-generate unicode data
# 2.0
- GPR#70: Switch to dune, opam v2
- GPR#60: Breaking change: switch from int codepoints to Uchar.t
codepoints
- GPR#59: Track lexing position
# 1.99.4
- GPR#47: Switch to ocaml-migrate-parsetree (contributed by Adrien Guatto)
- GPR#42: Added 'Rep' (repeat operator) (contributed by jpathy)
# 1.99.3
- Update to work with 4.03 (4.02 still supported)
# 1.99.2
- First official release of sedlex
# 1.99.1
- Support for new Ast_mapper registration API, follow OCaml trunk after
the inclusion of the extension_point branch
# 1.99
- First version of sedlex. The history below refers to ulex, the ancestor
or sedlex implemented with Camlp4.
# 1.1
- Generate (more) globally unique identifiers to avoid conflicts when open'ing another module
processed by ulex (issue reported by Gerd Stolpmann)
# 1.0
- Update to the new Camlp4 and to ocamlbuild (release for OCaml 3.10
only), by Nicolas Pouillard.
# 0.8
- Really make it work with OCaml 3.09.
- Support for Utf-16.
# 0.7 released May 24 2005
- Bug fixes
- Update to OCaml 3.09 (currently CVS). Still works with OCaml 3.08.
- MIT-like license (used to LGPL)
# 0.5 release Jul. 8 2004
- Document how to use a custom implementation for lex buffers
- Update to OCaml 3.08
# 0.4 released Jan. 10 2004
- Bug fix (accept 1114111 as valid Unicode code point)
- Add the rollback function-
# 0.3 released Oct. 8 2003
- Bug fix
- Add a new predefined class for ISO identifiers
# 0.2 released Sep. 22 2003
- Changed the names of predefined regexp
- Fix max_code = 0x10ffff
- Lexers that changes encoding on the fly
- Documentation of the interface Ulexing
# 0.1 released Sep. 20 2003
- Initial release

26
LICENSE
View File

@ -1,6 +1,22 @@
it's proprietary. all rights reserved. if you use this code in a way i don't like i will personally The MIT License (MIT)
make your car explode into hammers
Portions of this code are based on "sedlex", Copyright 2005, 2014 by Alain Copyright 2005, 2014 by Alain Frisch and LexiFi.
Frisch and LexiFi, released under the terms of the MIT license which can be
found in <licenses/LICENSE.sedlex> Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

14
META Normal file
View File

@ -0,0 +1,14 @@
version = "1.99.4"
description = "Runtime support for sedlex"
archive(byte) = "sedlexing.cma"
archive(native) = "sedlexing.cmxa"
requires = "gen"
ppx = "./ppx_sedlex --as-ppx"
package "ppx" (
version = "1.99.4"
description = "ppx implementation of sedlex"
requires = "ocaml-migrate-parsetree ppx_tools_versioned gen"
archive(byte) = "sedlex.cma"
archive(native) = "sedlex.cmxa"
)

74
Makefile Normal file
View File

@ -0,0 +1,74 @@
# The package sedlex is released under the terms of an MIT-like license.
# See the attached LICENSE file.
# Copyright 2005, 2013 by Alain Frisch and LexiFi.
include $(shell ocamlc -where)/Makefile.config
VERSION=1.99.2
# Don't forget to change META file as well
.PHONY: all opt clean test install package
all:
(cd src/lib && $(MAKE) all doc)
(cd src/syntax && $(MAKE) all)
opt:
(cd src/syntax && $(MAKE) opt)
(cd src/lib && $(MAKE) opt)
clean:
rm -f *~ *.cm* *.a *.lib *.o *.obj
(cd src/lib && $(MAKE) clean)
(cd src/syntax && $(MAKE) clean)
(cd examples && $(MAKE) clean)
rm -rf libdoc
test: clean all opt
cd examples && $(MAKE) test
INSTALL=META src/syntax/sedlex.cma src/syntax/ppx_sedlex$(EXE) src/lib/sedlexing.cma src/lib/sedlexing.cmi
INSTALL_OPT=src/syntax/sedlex.cmxs src/syntax/sedlex$(EXT_LIB) src/syntax/sedlex.cmxa src/syntax/ppx_sedlex.opt$(EXE) src/lib/sedlexing.cmx src/lib/sedlexing$(EXT_LIB) src/lib/sedlexing.cmxa
install:
ocamlfind install sedlex $(INSTALL) $(INSTALL_OPT)
install_byteonly:
ocamlfind install sedlex $(INSTALL)
uninstall:
ocamlfind remove sedlex
PACKAGE = sedlex-$(VERSION)
DISTRIB = \
CHANGES LICENSE META README.md Makefile \
examples/Makefile \
examples/tokenizer.ml \
src/lib/Makefile \
src/lib/sedlexing.ml \
src/lib/sedlexing.mli \
src/syntax/Makefile \
src/syntax/sedlex_cset.ml \
src/syntax/sedlex_cset.mli \
src/syntax/sedlex.ml \
src/syntax/sedlex.mli \
src/syntax/sedlex_ppx.ml \
src/syntax/unicode63.ml \
src/syntax/unicode63.mli
package: clean
rm -rf sedlex.tar.gz
tar czf sedlex.tar.gz $(DISTRIB)
rm -Rf $(PACKAGE)
mkdir $(PACKAGE)
cd $(PACKAGE) && tar xzf ../sedlex.tar.gz
tar czf $(PACKAGE).tar.gz $(PACKAGE)
rm -Rf $(PACKAGE) sedlex.tar.gz
TARGET=foo:bar/sedlex_dara
upload:
scp $(PACKAGE).tar.gz README CHANGES $(TARGET)/
rsync -avz libdoc $(TARGET)

62
README.md
View File

@ -1,17 +1,10 @@
# sedlex # sedlex
[![build](https://github.com/ocaml-community/sedlex/actions/workflows/build.yml/badge.svg)](https://github.com/ocaml-community/sedlex/actions/workflows/build.yml)
Unicode-friendly lexer generator for OCaml. Unicode-friendly lexer generator for OCaml.
This package is licensed by LexiFi under the terms of the MIT license. This package is licensed by LexiFi under the terms of the MIT license.
sedlex was originally written by Alain Frisch Contact: alain.frisch@lexifi.com
<alain.frisch@lexifi.com> and is now maintained as part of the
ocaml-community repositories on github.
## API
The API is documented [here](https://ocaml-community.github.io/sedlex).
## Overview ## Overview
@ -71,33 +64,20 @@ Guard expressions are not allowed.)
where: where:
- lexbuf is an arbitrary lowercase identifier, which must refer to - lexbuf is an arbitrary lowercase identifier, which must refer to
an existing value of type `Sedlexing.lexbuf`. an existing value of type [Sedlexing.lexbuf].
- the Ri are regular expressions (see below); - the Ri are regular expressions (see below);
- the ei and def are OCaml expressions (called actions) of the same type - the ei and def are OCaml expressions (called actions) of the same type
(the type for the whole lexer definition). (the type for the whole lexer definitioon).
Unlike ocamllex, lexers work on stream of Unicode codepoints, not Unlike ocamllex, lexers work on stream of Unicode codepoints, not
bytes. bytes.
Like ocamllex, sedlex uses **longest match** with **first rule priority**:
- The lexer always tries to match the longest possible prefix of the
input. It does so by continuing to read characters as long as some
rule can still match a longer string, while remembering the last
position at which a rule did match.
- When two or more rules match the same longest prefix (a tie), the
rule that appears first in the `match%sedlex` definition wins. For
example, given the rules `| "if" -> ...` and `| Plus ('a'..'z') -> ...`,
the input `"if"` is matched by the first rule because it is listed
first, even though the second rule also accepts `"if"`.
The actions can call functions from the Sedlexing module to extract The actions can call functions from the Sedlexing module to extract
(parts of) the matched lexeme, in the desired encoding. (parts of) the matched lexeme, in the desired encoding.
Regular expressions are syntactically OCaml patterns: Regular expressions are syntactically OCaml patterns:
- `"...."` (string constant): recognize the specified string. - `"...."` (string constant): recognize the specified string
- `'....'` (character constant) : recognize the specified character - `'....'` (character constant) : recognize the specified character
- `i` (integer constant) : recognize the specified codepoint - `i` (integer constant) : recognize the specified codepoint
- `'...' .. '...'`: character range - `'...' .. '...'`: character range
@ -116,9 +96,6 @@ Regular expressions are syntactically OCaml patterns:
and recognize the set of items in `R1` but not in `R2` ("subtract") and recognize the set of items in `R1` but not in `R2` ("subtract")
- `Intersect (R1,R2)` : assume that `R` is a single-character length regexp (see - `Intersect (R1,R2)` : assume that `R` is a single-character length regexp (see
below) and recognize the set of items which are in both `R1` and `R2` below) and recognize the set of items which are in both `R1` and `R2`
- `Utf8 R` : string literals inside R are assumed to be utf-8 encoded.
- `Latin1 R` : string literals inside R are assumed to be latin1 encoded.
- `Ascii R` : string literals inside R are assumed to be ascii encoded.
- `lid` (lowercase identifier) : reference a named regexp (see below) - `lid` (lowercase identifier) : reference a named regexp (see below)
A single-character length regexp is a regexp which does not contain (after A single-character length regexp is a regexp which does not contain (after
@ -128,9 +105,8 @@ with a length different from one.
Note: Note:
- The OCaml source is assumed to be encoded in UTF-8. - The OCaml source is assumed to be encoded in Latin1 (for string
- Strings and chars litterals will be interpreted in ASCII unless otherwise and character literals).
specified by the `Latin1`,`Ascii` and `Utf8` constructors in patterns.
It is possible to define named regular expressions with the following It is possible to define named regular expressions with the following
@ -196,10 +172,11 @@ The quick way:
Otherwise, the first thing to do is to compile and install sedlex. Otherwise, the first thing to do is to compile and install sedlex.
You need a recent version of OCaml and [dune](https://dune.build/). You need a recent version of OCaml.
``` ```
make make all
make opt (* optional *)
``` ```
### With findlib ### With findlib
@ -207,7 +184,7 @@ You need a recent version of OCaml and [dune](https://dune.build/).
If you have findlib, you can use it to install and use sedlex. If you have findlib, you can use it to install and use sedlex.
The name of the findlib package is "sedlex". The name of the findlib package is "sedlex".
Installation (after "make"): Installation (after "make all" and "make opt"):
``` ```
make install make install
@ -216,13 +193,13 @@ Installation (after "make"):
Compilation of OCaml files with lexer specifications: Compilation of OCaml files with lexer specifications:
``` ```
ocamlfind ocamlc -c -package sedlex.ppx my_file.ml ocamlfind ocamlc -c -package sedlex my_file.ml
``` ```
When linking, you must also include the sedlex package: When linking, you must also include the sedlex package:
``` ```
ocamlfind ocamlc -o my_prog -linkpkg -package sedlex.ppx my_file.cmo ocamlfind ocamlc -o my_prog -linkpkg -package sedlex my_file.cmo
``` ```
@ -238,17 +215,6 @@ source file through -ppx rewriter ppx_sedlex. Moreover, you need to
link the application with the runtime support library for sedlex link the application with the runtime support library for sedlex
(sedlexing.cma / sedlexing.cmxa). (sedlexing.cma / sedlexing.cmxa).
### With utop
Once sedlex is installed as per above, simply type
```
#require "sedlex.ppx";;
```
## Examples
The `examples/` subdirectory contains several samples of sedlex in use.
## Contributors ## Contributors
@ -257,7 +223,3 @@ The `examples/` subdirectory contains several samples of sedlex in use.
- Peter Zotov: - Peter Zotov:
- improvements to the build system - improvements to the build system
- switched parts of ppx_sedlex to using concrete syntax (with ppx_metaquot) - switched parts of ppx_sedlex to using concrete syntax (with ppx_metaquot)
- Steffen Smolka: port to dune
- Romain Beauxis:
- Implementation of the unicode table extractors
- General maintenance

29
dune-project
View File

@ -1,29 +0,0 @@
(lang dune 3.20)
(version "3.7")
(name noslop-sedlex)
(source (uri "https://git.lain.faith/noslop/noslop-sedlex.git"))
(license "LicenseRef-Proprietary")
(authors "xenia <xenia@awoo.systems>")
(maintainers "xenia <xenia@awoo.systems")
(homepage "https://git.lain.faith/noslop/noslop-sedlex.git")
(maintenance_intent "(latest)")
(documentation "https://git.lain.faith/noslop/noslop-sedlex/wiki")
(generate_opam_files true)
(executables_implicit_empty_intf true)
(package
(name noslop-sedlex)
(synopsis "An OCaml lexer generator for Unicode")
(description "sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports
Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular
OCaml source files. Lexing specific constructs are provided via a ppx syntax
extension.")
(depends
(ocaml (>= 4.08))
dune
(ppxlib (>= 0.26.0))
gen
(ppx_expect :with-test)))

36
examples/Makefile Normal file
View File

@ -0,0 +1,36 @@
# The package sedlex is released under the terms of an MIT-like license.
# See the attached LICENSE file.
# Copyright 2005, 2013 by Alain Frisch and LexiFi.
include $(shell ocamlc -where)/Makefile.config
.PHONY: test
test: tokenizer$(EXE) complement$(EXE) subtraction$(EXE) repeat$(EXE)
./tokenizer$(EXE)
./complement$(EXE)
./subtraction$(EXE)
./repeat$(EXE)
tokenizer$(EXE): tokenizer.ml
ocamlfind ocamlc -package gen -ppx "../src/syntax/ppx_sedlex$(EXE) --as-ppx" -I ../src/lib -o tokenizer$(EXE) -linkpkg sedlexing.cma tokenizer.ml
complement$(EXE): complement.ml
ocamlfind ocamlc -package gen -ppx "../src/syntax/ppx_sedlex$(EXE) --as-ppx" -I ../src/lib -o complement$(EXE) -linkpkg sedlexing.cma complement.ml
subtraction$(EXE): subtraction.ml
ocamlfind ocamlc -package gen -ppx "../src/syntax/ppx_sedlex$(EXE) --as-ppx" -I ../src/lib -o subtraction$(EXE) -linkpkg sedlexing.cma subtraction.ml
repeat$(EXE): repeat.ml
ocamlfind ocamlc -package gen -ppx "../src/syntax/ppx_sedlex$(EXE) --as-ppx" -I ../src/lib -o repeat$(EXE) -linkpkg sedlexing.cma repeat.ml
.PHONY: with_driver
with_driver: tokenizer.ml
ocamlfind ocamlc -package gen -ppx "ppx_driver ../src/syntax/sedlex.cma" -I ../src/lib -o tokenizer$(EXE) sedlexing.cma tokenizer.ml
.PHONY: with_findlib
with_findlib: tokenizer.ml
ocamlfind ocamlc -c -package sedlex -package gen tokenizer.ml
ocamlfind ocamlc -o tokenizer$(EXE) -linkpkg -package sedlex -package gen tokenizer.cmo
clean:
rm -f *~ *.cm* *.a *.lib *.o *.obj tokenizer$(EXE) complement$(EXE) subtraction$(EXE) repeat$(EXE)

12
examples/complement.ml
View File

@ -3,15 +3,9 @@ let lcase = [%sedlex.regexp? 'a' .. 'z']
let rec token buf = let rec token buf =
match%sedlex buf with match%sedlex buf with
| lcase -> | lcase -> print_char 'L';token buf
print_char 'L'; | Compl (ucase | lcase) -> print_char '?'; token buf
token buf | ucase -> print_char 'U';token buf
| Compl (ucase | lcase) ->
print_char '?';
token buf
| ucase ->
print_char 'U';
token buf
| eof -> print_endline "." | eof -> print_endline "."
| _ -> assert false | _ -> assert false

47
examples/dune
View File

@ -1,47 +0,0 @@
(executables
(names tokenizer regressions complement subtraction repeat performance)
(libraries noslop-sedlex noslop-sedlex.ppx)
(preprocess
(pps noslop-sedlex.ppx)))
(rule
(alias runtest)
(deps
(:< tokenizer.exe))
(action
(run %{<})))
(rule
(alias runtest)
(deps
(:< regressions.exe))
(action
(run %{<})))
(rule
(alias runtest)
(deps
(:< complement.exe))
(action
(run %{<})))
(rule
(alias runtest)
(deps
(:< subtraction.exe))
(action
(run %{<})))
(rule
(alias runtest)
(deps
(:< repeat.exe))
(action
(run %{<})))
(rule
(alias runtest)
(deps
(:< performance.exe))
(action
(run %{<})))

23
examples/performance.ml
View File

@ -1,23 +0,0 @@
let rec token buf =
match%sedlex buf with any -> token buf | eof -> () | _ -> assert false
let time f x =
let rec acc f x = function
| 0 -> f x
| n ->
f x |> ignore;
acc f x (n - 1)
in
let t = Sys.time () in
let fx = acc f x 10 in
Printf.printf "Execution time: %fs\n" (Sys.time () -. t);
fx
let () =
let long_str = String.make 1000000 '\n' in
let token_from _ =
let lexbuf = Sedlexing.Latin1.from_string long_str in
(* let () = Sedlexing.set_curr_p lexbuf Lexing.dummy_pos in *)
token lexbuf
in
time token_from long_str

58
examples/regressions.ml
View File

@ -1,58 +0,0 @@
(* This test that unicode_old.ml is a strict sub-set of new unicode.ml. *)
module CSet = Sedlex_ppx.Sedlex_cset
module Unicode = Sedlex_ppx.Unicode
let test_versions = ("16.0.0", "17.0.0")
let regressions =
[ (* Example *)
(* ("lt", CSet.union (CSet.singleton 0x1c5) (CSet.singleton (0x0001))) *) ]
let compare name (old_ : CSet.t) (new_ : CSet.t) =
let diff = CSet.difference old_ new_ in
let regressions =
match List.assoc name regressions with
| exception Not_found -> CSet.empty
| x -> x
in
let regressions_intersect = CSet.intersection regressions old_ in
let regressions = CSet.difference regressions regressions_intersect in
let regressions_useless = CSet.difference regressions new_ in
let diff = CSet.difference diff regressions in
Seq.iter
(fun x ->
Printf.printf
"Invalid regression for 0x%x in %s: already present in old set.\n" x
name)
(CSet.to_seq regressions_intersect);
Seq.iter
(fun x ->
Printf.printf "Invalid regression for 0x%x in %s: absent in new set.\n" x
name)
(CSet.to_seq regressions_useless);
Seq.iter
(fun x -> Printf.printf "Code point 0x%x missing in %s!\n" x name)
(CSet.to_seq diff)
let test new_l (name, old_l) =
(* Cn is for unassigned code points, which are allowed to be
* used in future version. *)
let old_l = Sedlex_utils.Cset.to_list old_l in
if name <> "cn" then (
let old_l =
List.fold_left
(fun acc (a, b) -> CSet.union acc (CSet.interval a b))
CSet.empty old_l
in
compare name old_l (List.assoc name new_l))
let () =
if (Unicode_old.version, Unicode.version) <> test_versions then
failwith
(Printf.sprintf "Test written for versions: %s => %s\n%!"
Unicode_old.version Unicode.version);
Printf.printf "Testing Unicode regression: %s => %s\n%!" Unicode_old.version
Unicode.version;
List.iter (test Unicode.Categories.list) Unicode_old.Categories.list;
List.iter (test Unicode.Properties.list) Unicode_old.Properties.list

25
examples/repeat.ml
View File

@ -1,26 +1,15 @@
let rec token buf = let rec token buf =
match%sedlex buf with match%sedlex buf with
| white_space -> | white_space -> print_endline "\tWhitespace"; token buf
print_endline "\tWhitespace"; | 'a', Rep(white_space, 1) -> print_endline "a\n\tWhitespace"; token buf
token buf | Rep("bc", 2) -> print_endline "bcbc"; token buf
| 'a', Rep (white_space, 1) -> | Rep("d", 1 .. 1) -> print_endline "d"; token buf
print_endline "a\n\tWhitespace"; | Rep("ef", 1 .. 3) -> Printf.printf "%s\n" (Sedlexing.Utf8.lexeme buf); token buf
token buf
| Rep ("bc", 2) ->
print_endline "bcbc";
token buf
| Rep ("d", 1 .. 1) ->
print_endline "d";
token buf
| Rep ("ef", 1 .. 3) ->
Printf.printf "%s\n" (Sedlexing.Utf8.lexeme buf);
token buf
| eof -> print_endline "\tEnd" | eof -> print_endline "\tEnd"
| any -> | any -> print_endline "Other"; token buf
print_endline "Other";
token buf
| _ -> failwith "Internal failure: Reached impossible place" | _ -> failwith "Internal failure: Reached impossible place"
let () = let () =
let lexbuf = Sedlexing.Utf8.from_string "a bcbc d ef efef efefef" in let lexbuf = Sedlexing.Utf8.from_string "a bcbc d ef efef efefef" in
token lexbuf token lexbuf

21
examples/subtraction.ml
View File

@ -1,23 +1,14 @@
let rec token buf = let rec token buf =
match%sedlex buf with match%sedlex buf with
| white_space -> | white_space -> print_endline "\tWhitespace"; token buf
print_endline "\tWhitespace"; | Sub (Chars "ab","b") -> print_endline "a"; token buf
token buf | (Chars "ab"|"c") -> print_endline "abc"; token buf
| Sub (Chars "ab", "b") -> | Intersect ("d", Chars "abd") -> print_endline "d"; token buf
print_endline "a";
token buf
| Chars "ab" | "c" ->
print_endline "abc";
token buf
| Intersect ("d", Chars "abd") ->
print_endline "d";
token buf
| eof -> print_endline "\tEnd" | eof -> print_endline "\tEnd"
| any -> | any -> print_endline "Other"; token buf
print_endline "Other";
token buf
| _ -> failwith "Internal failure: Reached impossible place" | _ -> failwith "Internal failure: Reached impossible place"
let () = let () =
let lexbuf = Sedlexing.Utf8.from_string "a b c d e" in let lexbuf = Sedlexing.Utf8.from_string "a b c d e" in
token lexbuf token lexbuf

12
examples/tokenizer.ml
View File

@ -4,16 +4,10 @@ let number = [%sedlex.regexp? Plus digit]
let rec token buf = let rec token buf =
let letter = [%sedlex.regexp? 'a'..'z'|'A'..'Z'] in let letter = [%sedlex.regexp? 'a'..'z'|'A'..'Z'] in
match%sedlex buf with match%sedlex buf with
| number -> | number -> Printf.printf "Number %s\n" (Sedlexing.Latin1.lexeme buf); token buf
Printf.printf "Number %s\n" (Sedlexing.Latin1.lexeme buf); | letter, Star ('A'..'Z' | 'a'..'z' | digit) -> Printf.printf "Ident %s\n" (Sedlexing.Latin1.lexeme buf); token buf
token buf
| letter, Star ('A' .. 'Z' | 'a' .. 'z' | digit) ->
Printf.printf "Ident %s\n" (Sedlexing.Latin1.lexeme buf);
token buf
| Plus xml_blank -> token buf | Plus xml_blank -> token buf
| Plus (Chars "+*-/") -> | Plus (Chars "+*-/") -> Printf.printf "Op %s\n" (Sedlexing.Latin1.lexeme buf); token buf
Printf.printf "Op %s\n" (Sedlexing.Latin1.lexeme buf);
token buf
| 128 .. 255 -> print_endline "Non ASCII" | 128 .. 255 -> print_endline "Non ASCII"
| eof -> print_endline "EOF" | eof -> print_endline "EOF"
| _ -> failwith "Unexpected character" | _ -> failwith "Unexpected character"

2164
examples/unicode_old.ml
View File

File diff suppressed because it is too large Load Diff

45
flake.lock
View File

@ -1,45 +0,0 @@
{
"nodes": {
"dragnpkgs": {
"inputs": {
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1783018382,
"narHash": "sha256-Pw7GM1fh2AnQNCitswXTHPfsswiDnEjXPN17Ge8egz4=",
"ref": "nixos-26.05",
"rev": "085892ad486428b8e51b549539a5430f0c2da1a8",
"revCount": 247,
"type": "git",
"url": "https://git.lain.faith/haskal/dragnpkgs.git"
},
"original": {
"id": "dragnpkgs",
"type": "indirect"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1782847225,
"narHash": "sha256-JC9PjqKYG9ve5U8aDOLQipp3+KLANBHUvGdLZlxzdKI=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "95ca1e203c0750115fd4a6f17d5a245dfe6b1edd",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-26.05",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"dragnpkgs": "dragnpkgs"
}
}
},
"root": "root",
"version": 7
}

23
flake.nix
View File

@ -1,23 +0,0 @@
{
description = "Flake for noslop-sedlex";
outputs = { self, dragnpkgs } @ inputs: dragnpkgs.lib.mkFlake {
packages.default = { ocamlPackages }: ocamlPackages.callPackage ./package.nix {};
devShells.default = { ocamlPackages, stdenv, mkShell }: mkShell {
packages = with ocamlPackages; [
ocaml
dune_3
utop
odoc
alcotest
ocamlformat
] ++ (self.packages.${stdenv.hostPlatform.system}.default.propagatedBuildInputs)
++ (self.packages.${stdenv.hostPlatform.system}.default.nativeBuildInputs);
shellHook = ''
export OCAMLRUNPARAM=b
'';
};
};
}

22
licenses/LICENSE.sedlex
View File

@ -1,22 +0,0 @@
The MIT License (MIT)
Copyright 2005, 2014 by Alain Frisch and LexiFi.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

38
noslop-sedlex.opam
View File

@ -1,38 +0,0 @@
# This file is generated by dune, edit dune-project instead
opam-version: "2.0"
version: "3.7"
synopsis: "An OCaml lexer generator for Unicode"
description: """
sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports
Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular
OCaml source files. Lexing specific constructs are provided via a ppx syntax
extension."""
maintainer: ["xenia <xenia@awoo.systems"]
authors: ["xenia <xenia@awoo.systems>"]
license: "LicenseRef-Proprietary"
homepage: "https://git.lain.faith/noslop/noslop-sedlex.git"
doc: "https://git.lain.faith/noslop/noslop-sedlex/wiki"
depends: [
"ocaml" {>= "4.08"}
"dune" {>= "3.20"}
"ppxlib" {>= "0.26.0"}
"gen"
"ppx_expect" {with-test}
"odoc" {with-doc}
]
build: [
["dune" "subst"] {dev}
[
"dune"
"build"
"-p"
name
"-j"
jobs
"@install"
"@runtest" {with-test}
"@doc" {with-doc}
]
]
dev-repo: "https://git.lain.faith/noslop/noslop-sedlex.git"
x-maintenance-intent: ["(latest)"]

20
opam Normal file
View File

@ -0,0 +1,20 @@
opam-version: "1.2"
name: "sedlex"
version: "1.99.4"
maintainer: "Alain Frisch <alain.frisch@lexifi.com>"
authors: [
"Alain Frisch <alain.frisch@lexifi.com>"
]
build: [ [make "all"] [make "opt"] ]
install: [make "install"]
remove: [["ocamlfind" "remove" "sedlex"]]
depends: ["ocamlfind" {build}
"ppx_tools_versioned"
"ocaml-migrate-parsetree"
"gen"
]
available: [ ocaml-version >= "4.02.3" ]
homepage: "https://github.com/alainfrisch/sedlex"
bug-reports: "https://github.com/alainfrisch/sedlex/issues"
license: "MIT"
dev-repo: "https://github.com/alainfrisch/sedlex.git"

52
package.nix
View File

@ -1,52 +0,0 @@
{
lib,
fetchurl,
buildDunePackage,
gen,
ppxlib,
ppx_expect,
}: let
unicodeVersion = "17.0.0";
baseUrl = "https://www.unicode.org/Public/${unicodeVersion}";
DerivedCoreProperties = fetchurl {
url = "${baseUrl}/ucd/DerivedCoreProperties.txt";
hash = "sha256-JMf+0RlcSC+q79XB5+uCHF7h+23gfs26pktWqZ2iLAg=";
};
DerivedGeneralCategory = fetchurl {
url = "${baseUrl}/ucd/extracted/DerivedGeneralCategory.txt";
hash = "sha256-1i5bq3DKdPCZND9xIk+gUcsf3WGhq0XASIxEz8C2EC4=";
};
PropList = fetchurl {
url = "${baseUrl}/ucd/PropList.txt";
hash = "sha256-Ew3N3Kra8HEAi9/OHndD4E/fvJEIhvAX2fmskx2MZN0=";
};
in
buildDunePackage (finalAttrs: {
pname = "noslop-sedlex";
version = "3.7+DEV";
minimalOCamlVersion = "5.3";
src = ./.;
propagatedBuildInputs = [
gen
ppxlib
];
preBuild = ''
rm src/generator/data/dune
ln -s ${DerivedCoreProperties} src/generator/data/DerivedCoreProperties.txt
ln -s ${DerivedGeneralCategory} src/generator/data/DerivedGeneralCategory.txt
ln -s ${PropList} src/generator/data/PropList.txt
'';
checkInputs = [
ppx_expect
];
doCheck = true;
dontStrip = true;
})

78
src/common/cset.ml
View File

@ -1,78 +0,0 @@
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
(* Character sets are represented as lists of intervals. The
intervals must be non-overlapping and not collapsable, and the list
must be ordered in increasing order. *)
type t = (int * int) list
let rec range_to_seq a b next () =
if a = b then Seq.Cons (a, next) else Seq.Cons (a, range_to_seq (a + 1) b next)
let rec to_seq x () =
match x with [] -> Seq.Nil | (a, b) :: xs -> range_to_seq a b (to_seq xs) ()
let check_invariant l =
let rec loop prev = function
| [] -> ()
| (a, b) :: xs ->
if a < prev then
failwith
(Printf.sprintf
"Sedlex_cset.of_list: not in increasing order or overlapping. \
[_-%d]-[%d-%d]"
prev a b);
if a = prev then
failwith
(Printf.sprintf
"Sedlex_cset.of_list: adjacent range. [_-%d]-[%d-%d]" prev a b);
if a > b then
failwith
(Printf.sprintf "Sedlex_cset.of_list: malformed range. [%d-%d]" a b);
loop b xs
in
loop (-1) l
let of_list l =
check_invariant l;
l
let to_list l = l
let max_code = 0x10ffff (* must be < max_int *)
let min_code = -1
let empty = []
let singleton i = [(i, i)]
let is_empty = function [] -> true | _ -> false
let interval i j = if i <= j then [(i, j)] else [(j, i)]
let eof = singleton (-1)
let any = interval 0 max_code
let rec union c1 c2 =
match (c1, c2) with
| [], _ -> c2
| _, [] -> c1
| ((i1, j1) as s1) :: r1, (i2, j2) :: r2 ->
if i1 <= i2 then
if j1 + 1 < i2 then s1 :: union r1 c2
else if j1 < j2 then union r1 ((i1, j2) :: r2)
else union c1 r2
else union c2 c1
let union_list : t list -> t = function
| [] -> empty
| [x] -> x
| l ->
List.concat l
|> List.sort (fun a b -> compare b a)
|> List.fold_left (fun (acc : t) (x : int * int) -> union [x] acc) empty
let complement c =
let rec aux start = function
| [] -> if start <= max_code then [(start, max_code)] else []
| (i, j) :: l -> (start, i - 1) :: aux (succ j) l
in
match c with (-1, j) :: l -> aux (succ j) l | l -> aux (-1) l
let intersection c1 c2 = complement (union (complement c1) (complement c2))
let difference c1 c2 = complement (union (complement c1) c2)

25
src/common/cset.mli
View File

@ -1,25 +0,0 @@
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
(** Representation of sets of unicode code points. *)
(** Character sets are represented as lists of intervals. The intervals must be
non-overlapping and not collapsable, and the list must be ordered in
increasing order. *)
type t = private (int * int) list
val of_list : (int * int) list -> t
val to_list : t -> (int * int) list
val min_code : int
val max_code : int
val empty : t
val any : t
val union : t -> t -> t
val union_list : t list -> t
val difference : t -> t -> t
val intersection : t -> t -> t
val is_empty : t -> bool
val eof : t
val singleton : int -> t
val interval : int -> int -> t
val to_seq : t -> int Seq.t

3
src/common/dune
View File

@ -1,3 +0,0 @@
(library
(name sedlex_utils)
(public_name noslop-sedlex.utils))

28
src/generator/README.md
View File

@ -1,28 +0,0 @@
# Unicode specification extraction
The file `src/syntax/unicode.ml` is generated using the data available at
[unicode.org](https://www.unicode.org/Public/).
The rule with `targets unicode.ml` at `src/syntax/dune` is the main entry point for this process.
It specifies how `unicode.ml` should be generated when running `dune @build` and triggers:
* download of data files at `src/generator/data`
* build of `src/generator/gen_unicode.exe`
* generation `unicode.ml` and places a copy in the source tree and a copy in the build tree
The rule is ignored when using the `--ignore-promoted-rules` option. This option is also implied
when using `-p`/`--for-release-of-packages` which is used for production build so production build
do not download the text data and re-generate `unicode.ml`.
However, each development build re-generates a `unicode.ml` file which is placed into the source
tree and, thus, can be easily commited when it is updated.
See: [dune documentation](https://dune.readthedocs.io/en/latest/dune-files.html#modes) for more
information.
## Update to new Unicode versions
To update the supported version, update the URL at `src/generator/data/base_url`. Make sure to
not include a trailing new line so that it is properly read in `src/generator/data/dune`.
Finally, place a copy of the old `unicode.ml` at `examples/unicode_old.ml` and update
`test_versions` and `regressions` in `examples/regressions.ml`.

1
src/generator/data/base_url
View File

@ -1 +0,0 @@
https://www.unicode.org/Public/17.0.0

35
src/generator/data/dune
View File

@ -1,35 +0,0 @@
(rule
(target DerivedCoreProperties.txt)
(deps base_url)
(action
(run
curl
-L
-s
%{read:base_url}/ucd/DerivedCoreProperties.txt
-o
%{target})))
(rule
(target DerivedGeneralCategory.txt)
(deps base_url)
(action
(run
curl
-L
-s
%{read:base_url}/ucd/extracted/DerivedGeneralCategory.txt
-o
%{target})))
(rule
(target PropList.txt)
(deps base_url)
(action
(run curl -L -s %{read:base_url}/ucd/PropList.txt -o %{target})))
(rule
(target UnicodeData.txt)
(deps base_url)
(action
(run curl -L -s %{read:base_url}/ucd/UnicodeData.txt -o %{target})))

3
src/generator/dune
View File

@ -1,3 +0,0 @@
(executable
(name gen_unicode)
(libraries str noslop-sedlex.utils))

209
src/generator/gen_unicode.ml
View File

@ -1,209 +0,0 @@
(* This file generates unicode data from
* the files exported at https://www.unicode.org/Public/<unicode version>
* and stored at src/generator/data. *)
open Sedlex_utils
module SSet = Set.Make (String)
let target = Sys.argv.(1)
let categories = Hashtbl.create 1024
let labels = Hashtbl.create 1024
(* Drop comments and split semi-column separated fields *)
let parse_line l =
let l =
match String.index_opt l '#' with None -> l | Some i -> String.sub l 0 i
in
String.split_on_char ';' l |> List.map String.trim
let parse_code s =
try int_of_string (Printf.sprintf "0x%s" s)
with _ -> failwith (Printf.sprintf "invalid code %s" s)
let parse_category x = String.lowercase_ascii (String.trim x)
let parse_prop x = String.lowercase_ascii (String.trim x)
let parse_interval s =
match String.split_on_char '.' (String.trim s) with
| [] -> assert false
| [x] ->
let x = parse_code x in
Cset.singleton x
| [x; ""; y] ->
let x = parse_code x and y = parse_code y in
Cset.interval x y
| _ -> failwith (Printf.sprintf "invalid interval %s" s)
let print_elements ch hashtbl cats =
let cats_set = SSet.of_list cats in
let all_keys = SSet.of_seq (Hashtbl.to_seq_keys hashtbl) in
let missing = SSet.diff cats_set all_keys in
let ignoring = SSet.diff all_keys cats_set in
let len = List.length cats in
List.iter
(fun c ->
let entries =
List.map
(fun (b, e) -> Printf.sprintf "0x%x, 0x%x" b e)
(Cset.union_list (Hashtbl.find_all hashtbl c) :> (int * int) list)
in
Printf.fprintf ch " let %s = Sedlex_cset.of_list\n [" c;
List.iteri
(fun i x ->
if i > 0 then
if i mod 5 = 0 then Printf.fprintf ch ";\n "
else Printf.fprintf ch "; ";
Printf.fprintf ch "%s" x)
entries;
Printf.fprintf ch "]\n\n")
cats;
Printf.fprintf ch " let list = [\n";
List.iteri
(fun pos c ->
Printf.fprintf ch " (%S, %s)%s\n" c c
(if pos == len - 1 then "" else ";"))
cats;
Printf.fprintf ch " ]\n\n";
if not (SSet.is_empty ignoring) then (
Printf.fprintf ch "(* ignoring:\n";
SSet.iter (fun s -> Printf.fprintf ch " - %s\n" s) ignoring;
Printf.fprintf ch "*)\n");
if not (SSet.is_empty missing) then (
Printf.fprintf ch "(* missing:\n";
SSet.iter (fun s -> Printf.fprintf ch " - %s\n" s) missing;
Printf.fprintf ch "*)\n")
let files =
[
( "PropList.txt",
fun s ->
match parse_line s with
| [""] -> ()
| [interval; prop] ->
let interval = parse_interval interval in
let prop = parse_prop prop in
Hashtbl.add labels prop interval
| _ -> assert false );
( "DerivedCoreProperties.txt",
fun s ->
match parse_line s with
| [""] -> ()
| [interval; prop] ->
let interval = parse_interval interval in
let prop = parse_prop prop in
Hashtbl.add labels prop interval
| [_interval; "InCB"; ("Extend" | "Consonant" | "Linker")] ->
(* TODO: support non-binary properties? *)
()
| _ -> assert false );
( "DerivedGeneralCategory.txt",
fun s ->
match parse_line s with
| [""] -> ()
| [interval; cat] ->
let interval = parse_interval interval in
let cat = parse_category cat in
Hashtbl.add categories cat interval
| _ -> assert false );
( "UnicodeData.txt",
fun s ->
match parse_line s with
| [""] -> ()
| interval :: _ :: cat :: _ ->
let interval = parse_interval interval in
let cat = parse_category cat in
Hashtbl.add categories cat interval
| _ -> assert false );
]
let read_version fname =
let version_rex =
Str.regexp "^# PropList-\\([0-9]+\\.[0-9]+\\.[0-9]+\\)\\.txt"
in
let ch = open_in_bin fname in
let s = input_line ch in
close_in ch;
ignore (Str.string_match version_rex s 0);
Str.matched_group 1 s
let exported_categories =
[
"cc";
"cf";
"cn";
"co";
"cs";
"ll";
"lm";
"lo";
"lt";
"lu";
"mc";
"me";
"mn";
"nd";
"nl";
"no";
"pc";
"pd";
"pe";
"pf";
"pi";
"po";
"ps";
"sc";
"sk";
"sm";
"so";
"zl";
"zp";
"zs";
]
let exported_properties =
[
"alphabetic";
"ascii_hex_digit";
"hex_digit";
"id_continue";
"id_start";
"lowercase";
"math";
"other_alphabetic";
"other_lowercase";
"other_math";
"other_uppercase";
"uppercase";
"white_space";
"xid_continue";
"xid_start";
]
let () =
let base_dir =
Filename.concat (Filename.dirname Sys.executable_name) "data"
in
let version = read_version (Filename.concat base_dir "PropList.txt") in
List.iter
(fun (fname, fn) ->
let ch = open_in_bin (Filename.concat base_dir fname) in
try
while true do
let ret = input_line ch in
fn ret
done
with End_of_file -> close_in ch)
files;
let ch = open_out_bin target in
Printf.fprintf ch {|[@@@ocamlformat "disable"]|};
Printf.fprintf ch "\n\n";
Printf.fprintf ch
"(* This file was automatically generated, do not edit. *)\n";
Printf.fprintf ch "(* Edit gen_unicode.ml.inc instead. *)\n\n";
Printf.fprintf ch "\n\nlet version = %S\n\n" version;
Printf.fprintf ch "module Categories = struct\n\n";
print_elements ch categories exported_categories;
Printf.fprintf ch "end\n\n";
Printf.fprintf ch "module Properties = struct\n\n";
print_elements ch labels exported_properties;
Printf.fprintf ch "end\n";
close_out ch

26
src/lib/Makefile Normal file
View File

@ -0,0 +1,26 @@
# The package sedlex is released under the terms of an MIT-like license.
# See the attached LICENSE file.
# Copyright 2005, 2013 by Alain Frisch and LexiFi.
MODS=sedlexing.cmo
all: sedlexing.cma
opt: sedlexing.cmxa
sedlexing.cma: $(MODS:.cmo=.mli) $(MODS:.cmo=.ml)
ocamlfind ocamlc -package gen -w +A-4-9 -safe-string -c $(MODS:.cmo=.mli) $(MODS:.cmo=.ml)
ocamlfind ocamlc -package gen -a -o sedlexing.cma $(MODS)
sedlexing.cmxa: $(MODS:.cmo=.mli) $(MODS:.cmo=.ml)
ocamlfind ocamlopt -package gen -safe-string -c $(MODS:.cmo=.mli) $(MODS:.cmo=.ml)
ocamlfind ocamlopt -package gen -a -o sedlexing.cmxa $(MODS:.cmo=.cmx)
doc:
rm -rf ../../libdoc
mkdir ../../libdoc
ocamlfind ocamldoc -package gen -html sedlexing.mli -d ../../libdoc
clean:
rm -f *~ *.cm* *.a *.lib *.exe *.o *.obj

5
src/lib/dune
View File

@ -1,5 +0,0 @@
(library
(name sedlex)
(public_name noslop-sedlex)
(wrapped false)
(libraries gen))

810
src/lib/sedlexing.ml Normal file → Executable file
View File

@ -1,658 +1,464 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
exception InvalidCodepoint of int exception InvalidCodepoint of int
exception MalFormed exception MalFormed
module Uchar = struct
(* This for compatibility with ocaml < 4.14.0 *)
let utf_8_byte_length u =
match Uchar.to_int u with
| u when u < 0 -> assert false
| u when u <= 0x007F -> 1
| u when u <= 0x07FF -> 2
| u when u <= 0xFFFF -> 3
| u when u <= 0x10FFFF -> 4
| _ -> assert false
let utf_16_byte_length u = let gen_of_channel chan =
match Uchar.to_int u with let f () =
| u when u < 0 -> assert false try Some (input_char chan)
| u when u <= 0xFFFF -> 2 with End_of_file -> None
| u when u <= 0x10FFFF -> 4 in
| _ -> assert false f
let () = let (>>=) o f = match o with
ignore utf_8_byte_length; | Some x -> f x
ignore utf_16_byte_length | None -> None
include Uchar
let of_int x = (* For legacy purposes. *)
if Uchar.is_valid x then Uchar.unsafe_of_int x else raise MalFormed let gen_of_stream stream =
end let f () =
try Some (Stream.next stream)
with Stream.Failure -> None
in f
(* shadow polymorphic equal *) let eof = -1
let ( = ) (a : int) b = a = b
let ( >>| ) o f = match o with Some x -> Some (f x) | None -> None
(* Absolute position from the beginning of the stream *) (* Absolute position from the beginning of the stream *)
type apos = int type apos = int
type lexbuf = { type lexbuf = {
refill : Uchar.t array -> int -> int -> int; refill: (int array -> int -> int -> int);
bytes_per_char : Uchar.t -> int; mutable buf: int array;
mutable buf : Uchar.t array; mutable len: int; (* Number of meaningful char in buffer *)
mutable len : int; mutable offset: apos; (* Position of the first char in buffer
(* Number of meaningful uchar in buffer *)
mutable offset : apos;
(* Number of meaningful bytes in buffer *)
mutable bytes_offset : apos;
(* Position of the first uchar in buffer
in the input stream *) in the input stream *)
mutable pos: int; mutable pos: int;
(* Position of the first byte in buffer mutable start: int; (* First char we need to keep visible *)
in the input stream *)
mutable bytes_pos : int;
(* Position of the beginning of the line in the buffer, in uchar *)
mutable curr_bol : int;
(* Position of the beginning of the line in the buffer, in bytes *)
mutable curr_bytes_bol : int;
(* Index of the current line in the input stream. *)
mutable curr_line : int;
(* starting position, in uchar. *)
mutable start_pos : int;
(* starting position, in bytes. *)
mutable start_bytes_pos : int;
(* First uchar we need to keep visible *)
mutable start_bol : int;
(* First byte we need to keep visible *)
mutable start_bytes_bol : int;
(* start from 1 *)
mutable start_line : int;
mutable marked_pos: int; mutable marked_pos: int;
mutable marked_bytes_pos : int;
mutable marked_bol : int;
mutable marked_bytes_bol : int;
mutable marked_line : int;
mutable marked_val: int; mutable marked_val: int;
mutable filename : string;
mutable finished: bool; mutable finished: bool;
} }
let chunk_size = 512 let chunk_size = 512
let empty_lexbuf bytes_per_char = let empty_lexbuf = {
{
refill = (fun _ _ _ -> assert false); refill = (fun _ _ _ -> assert false);
bytes_per_char;
buf = [| |]; buf = [| |];
len = 0; len = 0;
offset = 0; offset = 0;
bytes_offset = 0;
pos = 0; pos = 0;
bytes_pos = 0; start = 0;
curr_bol = 0;
curr_bytes_bol = 0;
curr_line = 1;
start_pos = 0;
start_bytes_pos = 0;
start_bol = 0;
start_bytes_bol = 0;
start_line = 0;
marked_pos = 0; marked_pos = 0;
marked_bytes_pos = 0;
marked_bol = 0;
marked_bytes_bol = 0;
marked_line = 0;
marked_val = 0; marked_val = 0;
filename = "";
finished = false; finished = false;
} }
let dummy_uchar = Uchar.of_int 0 let create f = {
let nl_uchar = Uchar.of_int 10 empty_lexbuf with
refill = f;
let create ?(bytes_per_char = fun _ -> 1) refill = buf = Array.make chunk_size 0;
{
(empty_lexbuf bytes_per_char) with
refill;
buf = Array.make chunk_size dummy_uchar;
} }
let set_position ?bytes_position lexbuf position =
lexbuf.offset <- position.Lexing.pos_cnum - lexbuf.pos;
lexbuf.curr_bol <- position.Lexing.pos_bol;
lexbuf.curr_line <- position.Lexing.pos_lnum;
let bytes_position = Option.value ~default:position bytes_position in
lexbuf.bytes_offset <- bytes_position.Lexing.pos_cnum - lexbuf.bytes_pos;
lexbuf.curr_bytes_bol <- bytes_position.Lexing.pos_bol
let set_filename lexbuf fname = lexbuf.filename <- fname let fill_buf_from_gen f gen buf pos len =
let rec aux i =
let from_gen ?bytes_per_char gen =
let malformed = ref false in
let refill buf pos len =
let rec loop i =
if !malformed then raise MalFormed;
if i >= len then len if i >= len then len
else ( else match gen () with
match gen () with | Some c -> buf.(pos + i) <- f c ; aux (i+1)
| Some c ->
buf.(pos + i) <- c;
loop (i + 1)
| None -> i | None -> i
| exception MalFormed when i <> 0 ->
malformed := true;
i)
in in
loop 0 aux 0
in
create ?bytes_per_char refill
let from_int_array ?bytes_per_char a = let from_gen s =
from_gen ?bytes_per_char create (fill_buf_from_gen (fun id -> id) s)
(Gen.init ~limit:(Array.length a) (fun i -> Uchar.of_int a.(i)))
let from_uchar_array ?(bytes_per_char = fun _ -> 1) a = let from_stream s = from_gen @@ gen_of_stream s
let from_int_array a =
let len = Array.length a in let len = Array.length a in
{ {
(empty_lexbuf bytes_per_char) with empty_lexbuf with
buf = Array.init len (fun i -> a.(i)); buf = Array.init len (fun i -> a.(i));
len; len = len;
finished = true; finished = true;
} }
let refill lexbuf = let refill lexbuf =
if lexbuf.len + chunk_size > Array.length lexbuf.buf then begin if lexbuf.len + chunk_size > Array.length lexbuf.buf
let s = lexbuf.start_pos in then begin
let s_bytes = lexbuf.start_bytes_pos in let s = lexbuf.start in
let ls = lexbuf.len - s in let ls = lexbuf.len - s in
if ls + chunk_size <= Array.length lexbuf.buf then if ls + chunk_size <= Array.length lexbuf.buf then
Array.blit lexbuf.buf s lexbuf.buf 0 ls Array.blit lexbuf.buf s lexbuf.buf 0 ls
else begin else begin
let newlen = (Array.length lexbuf.buf + chunk_size) * 2 in let newlen = (Array.length lexbuf.buf + chunk_size) * 2 in
let newbuf = Array.make newlen dummy_uchar in let newbuf = Array.make newlen 0 in
Array.blit lexbuf.buf s newbuf 0 ls; Array.blit lexbuf.buf s newbuf 0 ls;
lexbuf.buf <- newbuf lexbuf.buf <- newbuf
end; end;
lexbuf.len <- ls; lexbuf.len <- ls;
lexbuf.offset <- lexbuf.offset + s; lexbuf.offset <- lexbuf.offset + s;
lexbuf.bytes_offset <- lexbuf.bytes_offset + s_bytes;
lexbuf.pos <- lexbuf.pos - s; lexbuf.pos <- lexbuf.pos - s;
lexbuf.bytes_pos <- lexbuf.bytes_pos - s_bytes;
lexbuf.marked_pos <- lexbuf.marked_pos - s; lexbuf.marked_pos <- lexbuf.marked_pos - s;
lexbuf.marked_bytes_pos <- lexbuf.marked_bytes_pos - s_bytes; lexbuf.start <- 0
lexbuf.start_pos <- 0;
lexbuf.start_bytes_pos <- 0
end; end;
let n = lexbuf.refill lexbuf.buf lexbuf.pos chunk_size in let n = lexbuf.refill lexbuf.buf lexbuf.pos chunk_size in
if n = 0 then lexbuf.finished <- true else lexbuf.len <- lexbuf.len + n if (n = 0)
then begin
let new_line lexbuf = lexbuf.buf.(lexbuf.len) <- eof;
lexbuf.curr_line <- lexbuf.curr_line + 1; lexbuf.len <- lexbuf.len + 1;
lexbuf.curr_bol <- lexbuf.pos + lexbuf.offset;
lexbuf.curr_bytes_bol <- lexbuf.bytes_pos + lexbuf.bytes_offset
let[@inline always] next_aux some none lexbuf =
if (not lexbuf.finished) && lexbuf.pos = lexbuf.len then refill lexbuf;
if lexbuf.finished && lexbuf.pos = lexbuf.len then none
else begin
let ret = lexbuf.buf.(lexbuf.pos) in
lexbuf.pos <- lexbuf.pos + 1;
lexbuf.bytes_pos <- lexbuf.bytes_pos + lexbuf.bytes_per_char ret;
if Uchar.equal ret nl_uchar then new_line lexbuf;
some ret
end end
else lexbuf.len <- lexbuf.len + n
let next lexbuf = (next_aux [@inlined]) (fun x -> Some x) None lexbuf let next lexbuf =
let __private__next_int lexbuf = (next_aux [@inlined]) Uchar.to_int (-1) lexbuf let i =
if lexbuf.pos = lexbuf.len then
if lexbuf.finished then eof
else (refill lexbuf; lexbuf.buf.(lexbuf.pos))
else lexbuf.buf.(lexbuf.pos)
in
if i = eof then lexbuf.finished <- true else lexbuf.pos <- lexbuf.pos + 1;
i
let start lexbuf =
lexbuf.start <- lexbuf.pos;
lexbuf.marked_pos <- lexbuf.pos;
lexbuf.marked_val <- (-1)
let mark lexbuf i = let mark lexbuf i =
lexbuf.marked_pos <- lexbuf.pos; lexbuf.marked_pos <- lexbuf.pos;
lexbuf.marked_bytes_pos <- lexbuf.bytes_pos;
lexbuf.marked_bol <- lexbuf.curr_bol;
lexbuf.marked_bytes_bol <- lexbuf.curr_bytes_bol;
lexbuf.marked_line <- lexbuf.curr_line;
lexbuf.marked_val <- i lexbuf.marked_val <- i
let start lexbuf =
lexbuf.start_pos <- lexbuf.pos;
lexbuf.start_bytes_pos <- lexbuf.bytes_pos;
lexbuf.start_bol <- lexbuf.curr_bol;
lexbuf.start_bytes_bol <- lexbuf.curr_bytes_bol;
lexbuf.start_line <- lexbuf.curr_line;
mark lexbuf (-1)
let backtrack lexbuf = let backtrack lexbuf =
lexbuf.pos <- lexbuf.marked_pos; lexbuf.pos <- lexbuf.marked_pos;
lexbuf.bytes_pos <- lexbuf.marked_bytes_pos;
lexbuf.curr_bol <- lexbuf.marked_bol;
lexbuf.curr_bytes_bol <- lexbuf.marked_bytes_bol;
lexbuf.curr_line <- lexbuf.marked_line;
lexbuf.marked_val lexbuf.marked_val
let rollback lexbuf = let rollback lexbuf =
lexbuf.pos <- lexbuf.start_pos; lexbuf.pos <- lexbuf.start
lexbuf.bytes_pos <- lexbuf.start_bytes_pos;
lexbuf.curr_bol <- lexbuf.start_bol;
lexbuf.curr_bytes_bol <- lexbuf.start_bytes_bol;
lexbuf.curr_line <- lexbuf.start_line
let lexeme_start lexbuf = lexbuf.start_pos + lexbuf.offset let lexeme_start lexbuf = lexbuf.start + lexbuf.offset
let lexeme_bytes_start lexbuf = lexbuf.start_bytes_pos + lexbuf.bytes_offset
let lexeme_end lexbuf = lexbuf.pos + lexbuf.offset let lexeme_end lexbuf = lexbuf.pos + lexbuf.offset
let lexeme_bytes_end lexbuf = lexbuf.bytes_pos + lexbuf.bytes_offset
let loc lexbuf = (lexbuf.start_pos + lexbuf.offset, lexbuf.pos + lexbuf.offset)
let bytes_loc lexbuf = let loc lexbuf = (lexbuf.start + lexbuf.offset, lexbuf.pos + lexbuf.offset)
( lexbuf.start_bytes_pos + lexbuf.bytes_offset,
lexbuf.bytes_pos + lexbuf.bytes_offset )
let lexeme_length lexbuf = lexbuf.pos - lexbuf.start_pos let lexeme_length lexbuf = lexbuf.pos - lexbuf.start
let lexeme_bytes_length lexbuf = lexbuf.bytes_pos - lexbuf.start_bytes_pos
let sub_lexeme lexbuf pos len = let sub_lexeme lexbuf pos len =
Array.sub lexbuf.buf (lexbuf.start_pos + pos) len Array.sub lexbuf.buf (lexbuf.start + pos) len
let lexeme lexbuf = let lexeme lexbuf =
Array.sub lexbuf.buf lexbuf.start_pos (lexbuf.pos - lexbuf.start_pos) Array.sub lexbuf.buf (lexbuf.start) (lexbuf.pos - lexbuf.start)
let lexeme_char lexbuf pos = lexbuf.buf.(lexbuf.start_pos + pos) let lexeme_char lexbuf pos =
lexbuf.buf.(lexbuf.start + pos)
let lexing_position_start lexbuf =
{
Lexing.pos_fname = lexbuf.filename;
pos_lnum = lexbuf.start_line;
pos_cnum = lexbuf.start_pos + lexbuf.offset;
pos_bol = lexbuf.start_bol;
}
let lexing_position_curr lexbuf =
{
Lexing.pos_fname = lexbuf.filename;
pos_lnum = lexbuf.curr_line;
pos_cnum = lexbuf.pos + lexbuf.offset;
pos_bol = lexbuf.curr_bol;
}
let lexing_positions lexbuf =
let start_p = lexing_position_start lexbuf
and curr_p = lexing_position_curr lexbuf in
(start_p, curr_p)
let lexing_bytes_position_start lexbuf =
{
Lexing.pos_fname = lexbuf.filename;
pos_lnum = lexbuf.start_line;
pos_cnum = lexbuf.start_bytes_pos + lexbuf.bytes_offset;
pos_bol = lexbuf.start_bytes_bol;
}
let lexing_bytes_position_curr lexbuf =
{
Lexing.pos_fname = lexbuf.filename;
pos_lnum = lexbuf.curr_line;
pos_cnum = lexbuf.bytes_pos + lexbuf.bytes_offset;
pos_bol = lexbuf.curr_bytes_bol;
}
let lexing_bytes_positions lexbuf =
let start_p = lexing_bytes_position_start lexbuf
and curr_p = lexing_bytes_position_curr lexbuf in
(start_p, curr_p)
let with_tokenizer lexer' lexbuf =
let lexer () =
let token = lexer' lexbuf in
let start_p, curr_p = lexing_positions lexbuf in
(token, start_p, curr_p)
in
lexer
module Chan = struct
exception Missing_input
type t = {
b : Bytes.t;
ic : in_channel;
mutable len : int;
mutable pos : int;
}
let min_buffer_size = 64
let create ic len : t =
let len = max len min_buffer_size in
{ b = Bytes.create len; ic; len = 0; pos = 0 }
let available (t : t) = t.len - t.pos
let rec ensure_bytes_available (t : t) ~can_refill n =
if available t >= n then ()
else if can_refill then (
let len = t.len - t.pos in
if len > 0 then Bytes.blit t.b t.pos t.b 0 len;
let read = input t.ic t.b len (Bytes.length t.b - len) in
t.len <- len + read;
t.pos <- 0;
if read = 0 then raise Missing_input
else ensure_bytes_available t ~can_refill n)
else raise Missing_input
let ensure_bytes_available t ~can_refill n =
(* [n] should not exceed the size of the buffer. Here we are
conservative and make sure it doesn't exceed the mininum size
for the buffer. *)
if n <= 0 || n > min_buffer_size then invalid_arg "Sedlexing.Chan.ensure";
ensure_bytes_available t ~can_refill n
let get (t : t) i = Bytes.get t.b (t.pos + i)
let advance (t : t) n =
if t.pos + n > t.len then invalid_arg "advance";
t.pos <- t.pos + n
let raw_buf (t : t) = t.b
let raw_pos (t : t) = t.pos
end
let make_from_channel ?bytes_per_char ic ~max_bytes_per_uchar
~min_bytes_per_uchar ~read_uchar =
let t = Chan.create ic (chunk_size * max_bytes_per_uchar) in
let malformed = ref false in
let refill buf pos len =
let rec loop i =
if !malformed then raise MalFormed;
if i = len then i
else (
match
(* we refill our bytes buffer only if we haven't refilled any uchar yet. *)
let can_refill = i = 0 in
Chan.ensure_bytes_available t ~can_refill min_bytes_per_uchar;
read_uchar ~can_refill t
with
| c ->
buf.(pos + i) <- c;
loop (i + 1)
| exception MalFormed when i <> 0 ->
malformed := true;
i
| exception Chan.Missing_input ->
if i = 0 && Chan.available t > 0 then raise MalFormed;
i)
in
loop 0
in
create ?bytes_per_char refill
module Latin1 = struct module Latin1 = struct
let from_gen s = let from_gen s =
from_gen ~bytes_per_char:(fun _ -> 1) (Gen.map Uchar.of_char s) create (fill_buf_from_gen Char.code s)
let from_stream s = from_gen @@ gen_of_stream s
let from_string s = let from_string s =
let len = String.length s in let len = String.length s in
{ {
(empty_lexbuf (fun _ -> 1)) with empty_lexbuf with
buf = Array.init len (fun i -> Uchar.of_char s.[i]); buf = Array.init len (fun i -> Char.code s.[i]);
len; len = len;
finished = true; finished = true;
} }
let from_channel ic = let from_channel ic =
make_from_channel ic from_gen (gen_of_channel ic)
~bytes_per_char:(fun _ -> 1)
~min_bytes_per_uchar:1 ~max_bytes_per_uchar:1
~read_uchar:(fun ~can_refill:_ t ->
let c = Chan.get t 0 in
Chan.advance t 1;
Uchar.of_char c)
let to_latin1 c = let to_latin1 c =
if Uchar.is_char c then Uchar.to_char c if (c >= 0) && (c < 256)
else raise (InvalidCodepoint (Uchar.to_int c)) then Char.chr c
else raise (InvalidCodepoint c)
let lexeme_char lexbuf pos = to_latin1 (lexeme_char lexbuf pos) let lexeme_char lexbuf pos =
to_latin1 (lexeme_char lexbuf pos)
let sub_lexeme lexbuf pos len = let sub_lexeme lexbuf pos len =
let s = Bytes.create len in let s = Bytes.create len in
for i = 0 to len - 1 do for i = 0 to len - 1 do Bytes.set s i (to_latin1 lexbuf.buf.(lexbuf.start + pos + i)) done;
Bytes.set s i (to_latin1 lexbuf.buf.(lexbuf.start_pos + pos + i))
done;
Bytes.to_string s Bytes.to_string s
let lexeme lexbuf = sub_lexeme lexbuf 0 (lexbuf.pos - lexbuf.start_pos) let lexeme lexbuf =
sub_lexeme lexbuf 0 (lexbuf.pos - lexbuf.start)
end end
module Utf8 = struct module Utf8 = struct
module Helper = struct module Helper = struct
(* http://www.faqs.org/rfcs/rfc3629.html *) (* http://www.faqs.org/rfcs/rfc3629.html *)
let width = function let width = Array.make 256 (-1)
| '\000' .. '\127' -> 1 let () =
| '\192' .. '\223' -> 2 for i = 0 to 127 do width.(i) <- 1 done;
| '\224' .. '\239' -> 3 for i = 192 to 223 do width.(i) <- 2 done;
| '\240' .. '\247' -> 4 for i = 224 to 239 do width.(i) <- 3 done;
| _ -> raise MalFormed for i = 240 to 247 do width.(i) <- 4 done
(* https://www.unicode.org/versions/corrigendum1.html *) let next s i =
let check_two n1 n2 = match s.[i] with
if n1 < 0xc2 || 0xdf < n1 then raise MalFormed; | '\000'..'\127' as c ->
if n2 < 0x80 || 0xbf < n2 then raise MalFormed; Char.code c
if n2 lsr 6 != 0b10 then raise MalFormed; | '\192'..'\223' as c ->
let n1 = Char.code c in
let n2 = Char.code s.[i+1] in
if (n2 lsr 6 != 0b10) then raise MalFormed;
((n1 land 0x1f) lsl 6) lor (n2 land 0x3f) ((n1 land 0x1f) lsl 6) lor (n2 land 0x3f)
| '\224'..'\239' as c ->
let check_three n1 n2 n3 = let n1 = Char.code c in
if n1 = 0xe0 then ( let n2 = Char.code s.[i+1] in
if n2 < 0xa0 || 0xbf < n2 then raise MalFormed; let n3 = Char.code s.[i+2] in
if n3 < 0x80 || 0xbf < n3 then raise MalFormed) if (n2 lsr 6 != 0b10) || (n3 lsr 6 != 0b10) then raise MalFormed;
else (
if n1 < 0xe1 || 0xef < n1 then raise MalFormed;
if n2 < 0x80 || 0xbf < n2 then raise MalFormed;
if n3 < 0x80 || 0xbf < n3 then raise MalFormed);
if n2 lsr 6 != 0b10 || n3 lsr 6 != 0b10 then raise MalFormed;
let p = let p =
((n1 land 0x0f) lsl 12) lor ((n2 land 0x3f) lsl 6) lor (n3 land 0x3f) ((n1 land 0x0f) lsl 12) lor ((n2 land 0x3f) lsl 6) lor (n3 land 0x3f)
in in
if p >= 0xd800 && p <= 0xdfff then raise MalFormed; if (p >= 0xd800) && (p <= 0xdf00) then raise MalFormed;
p p
| '\240'..'\247' as c ->
let check_four n1 n2 n3 n4 = let n1 = Char.code c in
if n1 = 0xf0 then (
if n2 < 0x90 || 0xbf < n2 then raise MalFormed;
if n3 < 0x80 || 0xbf < n3 then raise MalFormed;
if n4 < 0x80 || 0xbf < n4 then raise MalFormed)
else if n1 = 0xf4 then (
if n2 < 0x80 || 0x8f < n2 then raise MalFormed;
if n3 < 0x80 || 0xbf < n3 then raise MalFormed;
if n4 < 0x80 || 0xbf < n4 then raise MalFormed)
else (
if n1 < 0xf1 || 0xf3 < n1 then raise MalFormed;
if n2 < 0x80 || 0xbf < n2 then raise MalFormed;
if n3 < 0x80 || 0xbf < n3 then raise MalFormed;
if n4 < 0x80 || 0xbf < n4 then raise MalFormed);
if n2 lsr 6 != 0b10 || n3 lsr 6 != 0b10 || n4 lsr 6 != 0b10 then
raise MalFormed;
((n1 land 0x07) lsl 18)
lor ((n2 land 0x3f) lsl 12)
lor ((n3 land 0x3f) lsl 6)
lor (n4 land 0x3f)
let next s i =
let c1 = s.[i] in
match width c1 with
| 1 -> Char.code c1
| 2 ->
let n1 = Char.code c1 in
let n2 = Char.code s.[i + 1] in
check_two n1 n2
| 3 ->
let n1 = Char.code c1 in
let n2 = Char.code s.[i + 1] in
let n3 = Char.code s.[i + 2] in
check_three n1 n2 n3
| 4 ->
let n1 = Char.code c1 in
let n2 = Char.code s.[i+1] in let n2 = Char.code s.[i+1] in
let n3 = Char.code s.[i+2] in let n3 = Char.code s.[i+2] in
let n4 = Char.code s.[i+3] in let n4 = Char.code s.[i+3] in
check_four n1 n2 n3 n4 if (n2 lsr 6 != 0b10) || (n3 lsr 6 != 0b10) || (n4 lsr 6 != 0b10)
| _ -> assert false then raise MalFormed;
((n1 land 0x07) lsl 18) lor ((n2 land 0x3f) lsl 12) lor
let gen_from_char_gen s = ((n3 land 0x3f) lsl 6) lor (n4 land 0x3f)
let next_or_fail () =
match Gen.next s with None -> raise MalFormed | Some x -> Char.code x
in
fun () ->
Gen.next s >>| fun c1 ->
match width c1 with
| 1 -> Uchar.of_char c1
| 2 ->
let n1 = Char.code c1 in
let n2 = next_or_fail () in
Uchar.of_int (check_two n1 n2)
| 3 ->
let n1 = Char.code c1 in
let n2 = next_or_fail () in
let n3 = next_or_fail () in
Uchar.of_int (check_three n1 n2 n3)
| 4 ->
let n1 = Char.code c1 in
let n2 = next_or_fail () in
let n3 = next_or_fail () in
let n4 = next_or_fail () in
Uchar.of_int (check_four n1 n2 n3 n4)
| _ -> raise MalFormed | _ -> raise MalFormed
let from_gen s =
Gen.next s >>= function
| '\000'..'\127' as c ->
Some (Char.code c)
| '\192'..'\223' as c ->
let n1 = Char.code c in
Gen.next s >>= fun c2 ->
let n2 = Char.code c2 in
if (n2 lsr 6 != 0b10) then raise MalFormed;
Some (((n1 land 0x1f) lsl 6) lor (n2 land 0x3f))
| '\224'..'\239' as c ->
let n1 = Char.code c in
Gen.next s >>= fun c2 ->
let n2 = Char.code c2 in
Gen.next s >>= fun c3 ->
let n3 = Char.code c3 in
if (n2 lsr 6 != 0b10) || (n3 lsr 6 != 0b10) then raise MalFormed;
Some (((n1 land 0x0f) lsl 12) lor ((n2 land 0x3f) lsl 6) lor (n3 land 0x3f))
| '\240'..'\247' as c ->
let n1 = Char.code c in
Gen.next s >>= fun c2 ->
let n2 = Char.code c2 in
Gen.next s >>= fun c3 ->
let n3 = Char.code c3 in
Gen.next s >>= fun c4 ->
let n4 = Char.code c4 in
if (n2 lsr 6 != 0b10) || (n3 lsr 6 != 0b10) || (n4 lsr 6 != 0b10)
then raise MalFormed;
Some (((n1 land 0x07) lsl 18) lor ((n2 land 0x3f) lsl 12) lor
((n3 land 0x3f) lsl 6) lor (n4 land 0x3f))
| _ -> raise MalFormed
let compute_len s pos bytes =
let rec aux n i =
if i >= pos + bytes then if i = pos + bytes then n else raise MalFormed
else
let w = width.(Char.code s.[i]) in
if w > 0 then aux (succ n) (i + w)
else raise MalFormed
in
aux 0 pos
let rec blit_to_int s spos a apos n =
if n > 0 then begin
a.(apos) <- next s spos;
blit_to_int s (spos + width.(Char.code s.[spos])) a (succ apos) (pred n)
end
let to_int_array s pos bytes =
let n = compute_len s pos bytes in
let a = Array.make n 0 in
blit_to_int s pos a 0 n;
a
(**************************) (**************************)
let to_buffer a apos len b = let store b p =
for i = apos to apos + len - 1 do if p <= 0x7f then
Buffer.add_utf_8_uchar b a.(i) Buffer.add_char b (Char.chr p)
done else if p <= 0x7ff then (
Buffer.add_char b (Char.chr (0xc0 lor (p lsr 6)));
Buffer.add_char b (Char.chr (0x80 lor (p land 0x3f)))
)
else if p <= 0xffff then (
if (p >= 0xd800 && p < 0xe000) then raise MalFormed;
Buffer.add_char b (Char.chr (0xe0 lor (p lsr 12)));
Buffer.add_char b (Char.chr (0x80 lor ((p lsr 6) land 0x3f)));
Buffer.add_char b (Char.chr (0x80 lor (p land 0x3f)))
)
else if p <= 0x10ffff then (
Buffer.add_char b (Char.chr (0xf0 lor (p lsr 18)));
Buffer.add_char b (Char.chr (0x80 lor ((p lsr 12) land 0x3f)));
Buffer.add_char b (Char.chr (0x80 lor ((p lsr 6) land 0x3f)));
Buffer.add_char b (Char.chr (0x80 lor (p land 0x3f)))
)
else raise MalFormed
let from_int_array a apos len =
let b = Buffer.create (len * 4) in
let rec aux apos len =
if len > 0 then (store b a.(apos); aux (succ apos) (pred len))
else Buffer.contents b in
aux apos len
let gen_from_char_gen s = (fun () -> from_gen s)
end end
let from_channel ic = let from_channel ic =
make_from_channel ic ~bytes_per_char:Uchar.utf_8_byte_length from_gen (Helper.gen_from_char_gen (gen_of_channel ic))
~min_bytes_per_uchar:1 ~max_bytes_per_uchar:4
~read_uchar:(fun ~can_refill t ->
let w = Helper.width (Chan.get t 0) in
Chan.ensure_bytes_available t ~can_refill w;
let c =
Helper.next (Bytes.unsafe_to_string (Chan.raw_buf t)) (Chan.raw_pos t)
in
Chan.advance t w;
Uchar.of_int c)
let from_gen s = let from_gen s =
from_gen ~bytes_per_char:Uchar.utf_8_byte_length create (fill_buf_from_gen (fun id -> id)
(Helper.gen_from_char_gen s) (Helper.gen_from_char_gen s))
let from_stream s = from_gen @@ gen_of_stream s
let from_string s = let from_string s =
from_gen (Gen.init ~limit:(String.length s) (fun i -> String.get s i)) from_int_array (Helper.to_int_array s 0 (String.length s))
let sub_lexeme lexbuf pos len = let sub_lexeme lexbuf pos len =
let buf = Buffer.create (len * 4) in Helper.from_int_array lexbuf.buf (lexbuf.start + pos) len
Helper.to_buffer lexbuf.buf (lexbuf.start_pos + pos) len buf;
Buffer.contents buf
let lexeme lexbuf = sub_lexeme lexbuf 0 (lexbuf.pos - lexbuf.start_pos) let lexeme lexbuf =
sub_lexeme lexbuf 0 (lexbuf.pos - lexbuf.start)
end end
module Utf16 = struct module Utf16 = struct
type byte_order = Little_endian | Big_endian type byte_order = Little_endian | Big_endian
module Helper = struct module Helper = struct
(* http://www.ietf.org/rfc/rfc2781.txt *) (* http://www.ietf.org/rfc/rfc2781.txt *)
let number_of_pair bo c1 c2 = let number_of_char_pair bo c1 c2 = match bo with
match bo with | Little_endian -> ((Char.code c2) lsl 8) + (Char.code c1)
| Little_endian -> (c2 lsl 8) + c1 | Big_endian -> ((Char.code c1) lsl 8) + (Char.code c2)
| Big_endian -> (c1 lsl 8) + c2
let get_bo bo c1 c2 = let char_pair_of_number bo num = match bo with
match !bo with | Little_endian ->
| Some o -> o (Char.chr (num land 0xFF), Char.chr ((num lsr 8) land 0xFF ))
| None -> | Big_endian ->
let o = (Char.chr ((num lsr 8) land 0xFF), Char.chr (num land 0xFF))
match (c1, c2) with
| 0xff, 0xfe -> Little_endian
| _ -> Big_endian
in
bo := Some o;
o
let gen_from_char_gen opt_bo s = let next_in_gen bo s =
let next_or_fail () = Gen.next s >>= fun c1 ->
match Gen.next s with None -> raise MalFormed | Some x -> Char.code x Gen.next s >>= fun c2 ->
in Some (number_of_char_pair bo c1 c2)
let bo = ref opt_bo in
fun () -> let from_gen bo s w1 =
Gen.next s >>| fun c1 ->
let n1 = Char.code c1 in
let n2 = next_or_fail () in
let o = get_bo bo n1 n2 in
let w1 = number_of_pair o n1 n2 in
if w1 = 0xfffe then raise (InvalidCodepoint w1); if w1 = 0xfffe then raise (InvalidCodepoint w1);
if w1 < 0xd800 || 0xdfff < w1 then Uchar.of_int w1 if w1 < 0xd800 || 0xdfff < w1 then Some w1
else if w1 <= 0xdbff then ( else if w1 <= 0xdbff
let n3 = next_or_fail () in then
let n4 = next_or_fail () in next_in_gen bo s >>= fun w2 ->
let w2 = number_of_pair o n3 n4 in
if w2 < 0xdc00 || w2 > 0xdfff then raise MalFormed; if w2 < 0xdc00 || w2 > 0xdfff then raise MalFormed;
let upper10 = (w1 land 0x3ff) lsl 10 and lower10 = w2 land 0x3ff in let upper10 = (w1 land 0x3ff) lsl 10
Uchar.of_int (0x10000 + upper10 + lower10)) and lower10 = w2 land 0x3ff in
Some (0x10000 + upper10 + lower10)
else raise MalFormed else raise MalFormed
let to_buffer bo a apos len bom b = let gen_from_char_gen opt_bo s =
let store = let bo = ref opt_bo in
match bo with fun () ->
| Big_endian -> Buffer.add_utf_16be_uchar b Gen.next s >>= fun c1 ->
| Little_endian -> Buffer.add_utf_16le_uchar b Gen.next s >>= fun c2 ->
let o = match !bo with
| Some o -> o
| None ->
let o = match (Char.code c1, Char.code c2) with
| (0xff,0xfe) -> Little_endian
| _ -> Big_endian in
bo := Some o;
o in
from_gen o s (number_of_char_pair o c1 c2)
let compute_len opt_bo str pos bytes =
let s = gen_from_char_gen opt_bo
(Gen.init ~limit:(bytes - pos) (fun i -> (str.[i + pos])))
in in
if bom then store (Uchar.of_int 0xfeff); let l = ref 0 in
(* first, store the BOM *) Gen.iter (fun _ -> incr l) s ;
for i = apos to apos + len - 1 do !l
store a.(i)
done let blit_to_int opt_bo s spos a apos bytes =
let s = gen_from_char_gen opt_bo
(Gen.init ~limit:(bytes - spos) (fun i -> (s.[i + spos]))) in
let p = ref apos in
Gen.iter (fun x -> a.(!p) <- x ; incr p) s
let to_int_array opt_bo s pos bytes =
let len = compute_len opt_bo s pos bytes in
let a = Array.make len 0 in
blit_to_int opt_bo s pos a 0 bytes ;
a
let store bo buf code =
if code < 0x10000
then (
let (c1,c2) = char_pair_of_number bo code in
Buffer.add_char buf c1;
Buffer.add_char buf c2
) else (
let u' = code - 0x10000 in
let w1 = 0xd800 + (u' lsr 10)
and w2 = 0xdc00 + (u' land 0x3ff) in
let (c1,c2) = char_pair_of_number bo w1
and (c3,c4) = char_pair_of_number bo w2 in
Buffer.add_char buf c1;
Buffer.add_char buf c2;
Buffer.add_char buf c3;
Buffer.add_char buf c4
)
let from_int_array bo a apos len bom =
let b = Buffer.create (len * 4) in
if bom then store bo b 0xfeff ; (* first, store the BOM *)
let rec aux apos len =
if len > 0
then (store bo b a.(apos); aux (succ apos) (pred len))
else Buffer.contents b in
aux apos len
end end
let from_channel ic opt_bo =
let bo = ref opt_bo in
make_from_channel ic ~bytes_per_char:Uchar.utf_16_byte_length
~min_bytes_per_uchar:2 ~max_bytes_per_uchar:4
~read_uchar:(fun ~can_refill t ->
let n1 = Char.code (Chan.get t 0) in
let n2 = Char.code (Chan.get t 1) in
let o = Helper.get_bo bo n1 n2 in
let w1 = Helper.number_of_pair o n1 n2 in
if w1 = 0xfffe then raise (InvalidCodepoint w1);
if w1 < 0xd800 || 0xdfff < w1 then (
Chan.advance t 2;
Uchar.of_int w1)
else if w1 <= 0xdbff then (
Chan.ensure_bytes_available t ~can_refill 4;
let n3 = Char.code (Chan.get t 2) in
let n4 = Char.code (Chan.get t 3) in
let w2 = Helper.number_of_pair o n3 n4 in
if w2 < 0xdc00 || w2 > 0xdfff then raise MalFormed;
let upper10 = (w1 land 0x3ff) lsl 10 and lower10 = w2 land 0x3ff in
Chan.advance t 4;
Uchar.of_int (0x10000 + upper10 + lower10))
else raise MalFormed)
let from_gen s opt_bo = let from_gen s opt_bo =
from_gen ~bytes_per_char:Uchar.utf_16_byte_length from_gen (Helper.gen_from_char_gen opt_bo s)
(Helper.gen_from_char_gen opt_bo s)
let from_string s = let from_stream s = from_gen @@ gen_of_stream s
from_gen (Gen.init ~limit:(String.length s) (fun i -> String.get s i))
let from_channel ic opt_bo =
from_gen (gen_of_channel ic) opt_bo
let from_string s opt_bo =
let a = Helper.to_int_array opt_bo s 0 (String.length s) in
from_int_array a
let sub_lexeme lb pos len bo bom = let sub_lexeme lb pos len bo bom =
let buf = Buffer.create ((len * 4) + 2) in Helper.from_int_array bo lb.buf (lb.start + pos) len bom
(* +2 for the BOM *)
Helper.to_buffer bo lb.buf (lb.start_pos + pos) len bom buf;
Buffer.contents buf
let lexeme lb bo bom = sub_lexeme lb 0 (lb.pos - lb.start_pos) bo bom let lexeme lb bo bom =
sub_lexeme lb 0 (lb.pos - lb.start) bo bom
end end

353
src/lib/sedlexing.mli
View File

@ -1,300 +1,231 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
(** Runtime support for lexers generated by [sedlex]. *) (** Runtime support for lexers generated by [sedlex]. *)
(** This module is roughly equivalent to the module Lexing from the OCaml (** This module is roughly equivalent to the module Lexing from the
standard library, except that its lexbuffers handle Unicode code points OCaml standard library, except that its lexbuffers handle Unicode
(OCaml type: {!Uchar.t} in the range [0..0x10ffff]) instead of bytes (OCaml code points (OCaml type: [int] in the range [0..0x10ffff]) instead
type: [char]). of bytes (OCaml type: [char]).
It is possible to have sedlex-generated lexers work on a custom It is possible to have sedlex-generated lexers work on a custom
implementation for lex buffers. To do this, define a module [L] which implementation for lex buffers. To do this, define a module [L]
implements the [start], [next], [mark] and [backtrack] functions (See the which implements the [start], [next], [mark] and [backtrack]
Internal Interface section below for a specification). They need not work on functions (See the Internal Interface section below for a
a type named [lexbuf]: you can use the type name you want. Then, just do in specification). They need not work on a type named [lexbuf]: you
your sedlex-processed source, bind this module to the name [Sedlexing] (for can use the type name you want. Then, just do in your
instance, with a local module definition: [let module Sedlexing = L in ...]. sedlex-processed source, bind this module to the name [Sedlexing]
(for instance, with a local module definition: [let module Sedlexing
= L in ...].
Of course, you'll probably want to define functions like [lexeme] to be used Of course, you'll probably want to define functions like [lexeme] to
in the lexers semantic actions. *) be used in the lexers semantic actions. *)
(** The type of lexer buffers. A lexer buffer is the argument passed to the
scanning functions defined by the generated lexers. The lexer buffer holds
the internal information for the scanners, including the code points of the
token currently scanned, its position from the beginning of the input
stream, and the current position of the lexer. *)
type lexbuf type lexbuf
(** The type of lexer buffers. A lexer buffer is the argument passed
to the scanning functions defined by the generated lexers.
The lexer buffer holds the internal information for the
scanners, including the code points of the token currently scanned,
its position from the beginning of the input stream,
and the current position of the lexer. *)
(** Raised by some functions to signal that some code point is not compatible
with a specified encoding. *)
exception InvalidCodepoint of int exception InvalidCodepoint of int
(** Raised by some functions to signal that some code point is not
compatible with a specified encoding. *)
(** Raised by functions in the [Utf8] and [Utf16] modules to report strings
which do not comply to the encoding. *)
exception MalFormed exception MalFormed
(** Raised by functions in the [Utf8] and [Utf16] modules to report
strings which do not comply to the encoding. *)
(** {6 Creating generic lexbufs} *) (** {6 Creating generic lexbufs} *)
(** Create a generic lexer buffer. When the lexer needs more characters, it will val create: (int array -> int -> int -> int) -> lexbuf
call the given function, giving it an array of Uchars [a], a position [pos] (** Create a generic lexer buffer. When the lexer needs more
and a code point count [n]. The function should put [n] code points or less characters, it will call the given function, giving it an array of
in [a], starting at position [pos], and return the number of characters integers [a], a position [pos] and a code point count [n]. The
provided. A return value of 0 means end of input. [bytes_per_char] argument function should put [n] code points or less in [a], starting at
is optional. If unspecified, byte positions are the same as code point position [pos], and return the number of characters provided. A
position. *) return value of 0 means end of input. *)
val create :
?bytes_per_char:(Uchar.t -> int) ->
(Uchar.t array -> int -> int -> int) ->
lexbuf
(** set the initial tracked input position, in code point, for [lexbuf]. If val from_gen: int Gen.t -> lexbuf
unspecified, byte postion is set to the same value as code point position. (** Create a lexbuf from a stream of Unicode code points. *)
*)
val set_position :
?bytes_position:Lexing.position -> lexbuf -> Lexing.position -> unit
(** [set_filename lexbuf file] sets the filename to [file] in [lexbuf]. It also val from_stream: int Stream.t -> lexbuf
sets the {!Lexing.pos_fname} field in returned {!Lexing.position} records. [@@ocaml.deprecated "Use [Sedlexing.from_gen] instead."]
*) (** Create a lexbuf from a stream of Unicode code points. *)
val set_filename : lexbuf -> string -> unit
(** Create a lexbuf from a stream of Unicode code points. [bytes_per_char] is val from_int_array: int array -> lexbuf
optional. If unspecified, byte positions are the same as code point (** Create a lexbuf from an array of Unicode code points. *)
positions. *)
val from_gen : ?bytes_per_char:(Uchar.t -> int) -> Uchar.t Gen.t -> lexbuf
(** Create a lexbuf from an array of Unicode code points. [bytes_per_char] is
optional. If unspecified, byte positions are the same as code point
positions. *)
val from_int_array : ?bytes_per_char:(Uchar.t -> int) -> int array -> lexbuf
(** Create a lexbuf from an array of Unicode code points. [bytes_per_char] is
optional. If unspecified, byte positions are the same as code point
positions. *)
val from_uchar_array :
?bytes_per_char:(Uchar.t -> int) -> Uchar.t array -> lexbuf
(** {6 Interface for lexers semantic actions} *) (** {6 Interface for lexers semantic actions} *)
(** The following functions can be called from the semantic actions of lexer (** The following functions can be called from the semantic actions of
definitions. They give access to the character string matched by the regular lexer definitions. They give access to the character string matched
expression associated with the semantic action. *) by the regular expression associated with the semantic action. *)
(** [Sedlexing.lexeme_start lexbuf] returns the offset in the input stream of
the first code point of the matched string. The first code point of the
stream has offset 0. *)
val lexeme_start: lexbuf -> int val lexeme_start: lexbuf -> int
(** [Sedlexing.lexeme_start lexbuf] returns the offset in the
input stream of the first code point of the matched string.
The first code point of the stream has offset 0. *)
(** [Sedlexing.lexeme_start lexbuf] returns the offset in the input stream of
the first byte of the matched string. The first code point of the stream has
offset 0. *)
val lexeme_bytes_start : lexbuf -> int
(** [Sedlexing.lexeme_end lexbuf] returns the offset in the input stream of the
character following the last code point of the matched string. The first
character of the stream has offset 0. *)
val lexeme_end: lexbuf -> int val lexeme_end: lexbuf -> int
(** [Sedlexing.lexeme_end lexbuf] returns the offset in the input
stream of the character following the last code point of the
matched string. The first character of the stream has offset
0. *)
(** [Sedlexing.lexeme_end lexbuf] returns the offset in the input stream of the
byte following the last code point of the matched string. The first
character of the stream has offset 0. *)
val lexeme_bytes_end : lexbuf -> int
(** [Sedlexing.loc lexbuf] returns the pair
[(Sedlexing.lexeme_start lexbuf,Sedlexing.lexeme_end lexbuf)]. *)
val loc: lexbuf -> int * int val loc: lexbuf -> int * int
(** [Sedlexing.loc lexbuf] returns the pair
[(Sedlexing.lexeme_start lexbuf,Sedlexing.lexeme_end
lexbuf)]. *)
(** [Sedlexing.bytes_loc lexbuf] returns the pair
[(Sedlexing.lexeme_bytes_start lexbuf,Sedlexing.lexeme_bytes_end lexbuf)].
*)
val bytes_loc : lexbuf -> int * int
(** [Sedlexing.lexeme_length lexbuf] returns the difference
[(Sedlexing.lexeme_end lexbuf) - (Sedlexing.lexeme_start lexbuf)], that is,
the length (in code points) of the matched string. *)
val lexeme_length: lexbuf -> int val lexeme_length: lexbuf -> int
(** [Sedlexing.loc lexbuf] returns the difference
[(Sedlexing.lexeme_end lexbuf) - (Sedlexing.lexeme_start
lexbuf)], that is, the length (in code points) of the matched
string. *)
(** [Sedlexing.lexeme_length lexbuf] returns the difference val lexeme: lexbuf -> int array
[(Sedlexing.lexeme_bytes_end lexbuf) - (Sedlexing.lexeme_bytes_start (** [Sedlexing.lexeme lexbuf] returns the string matched by the
lexbuf)], that is, the length (in bytes) of the matched string. *) regular expression as an array of Unicode code point. *)
val lexeme_bytes_length : lexbuf -> int
(** [Sedlexing.lexing_positions lexbuf] returns the start and end positions, in val lexeme_char: lexbuf -> int -> int
code points, of the current token, using a record of type [Lexing.position]. (** [Sedlexing.lexeme_char lexbuf pos] returns code point number [pos] in
This is intended for consumption by parsers like those generated by the matched string. *)
[Menhir]. *)
val lexing_positions : lexbuf -> Lexing.position * Lexing.position
(** [Sedlexing.lexing_position_start lexbuf] returns the start position, in code val sub_lexeme: lexbuf -> int -> int -> int array
points, of the current token. *) (** [Sedlexing.lexeme lexbuf pos len] returns a substring of the string
val lexing_position_start : lexbuf -> Lexing.position
(** [Sedlexing.lexing_position_curr lexbuf] returns the end position, in code
points, of the current token. *)
val lexing_position_curr : lexbuf -> Lexing.position
(** [Sedlexing.lexing_bytes_positions lexbuf] returns the start and end
positions, in bytes, of the current token, using a record of type
[Lexing.position]. This is intended for consumption by parsers like those
generated by [Menhir]. *)
val lexing_bytes_positions : lexbuf -> Lexing.position * Lexing.position
(** [Sedlexing.lexing_bytes_position_start lexbuf] returns the start position,
in bytes, of the current token. *)
val lexing_bytes_position_start : lexbuf -> Lexing.position
(** [Sedlexing.lexing_bytes_position_curr lexbuf] returns the end position, in
bytes, of the current token. *)
val lexing_bytes_position_curr : lexbuf -> Lexing.position
(** [Sedlexing.new_line lexbuf] increments the line count and sets the beginning
of line to the current position, as though a newline character had been
encountered in the input. *)
val new_line : lexbuf -> unit
(** [Sedlexing.lexeme lexbuf] returns the string matched by the regular
expression as an array of Unicode code point. *)
val lexeme : lexbuf -> Uchar.t array
(** [Sedlexing.lexeme_char lexbuf pos] returns code point number [pos] in the
matched string. *)
val lexeme_char : lexbuf -> int -> Uchar.t
(** [Sedlexing.sub_lexeme lexbuf pos len] returns a substring of the string
matched by the regular expression as an array of Unicode code point. *) matched by the regular expression as an array of Unicode code point. *)
val sub_lexeme : lexbuf -> int -> int -> Uchar.t array
(** [Sedlexing.rollback lexbuf] puts [lexbuf] back in its configuration before
the last lexeme was matched. It is then possible to use another lexer to
parse the same characters again. The other functions above in this section
should not be used in the semantic action after a call to
[Sedlexing.rollback]. *)
val rollback: lexbuf -> unit val rollback: lexbuf -> unit
(** [Sedlexing.rollback lexbuf] puts [lexbuf] back in its configuration before
the last lexeme was matched. It is then possible to use another
lexer to parse the same characters again. The other functions
above in this section should not be used in the semantic action
after a call to [Sedlexing.rollback]. *)
(** {6 Internal interface} *) (** {6 Internal interface} *)
(** These functions are used internally by the lexers. They could be used to (** These functions are used internally by the lexers. They could be used
write lexers by hand, or with a lexer generator different from [sedlex]. The to write lexers by hand, or with a lexer generator different from
lexer buffers have a unique internal slot that can store an integer. They [sedlex]. The lexer buffers have a unique internal slot that can store
also store a "backtrack" position. *) an integer. They also store a "backtrack" position.
*)
(** [start t] informs the lexer buffer that any code points until the current
position can be discarded. The current position become the "start" position
as returned by [Sedlexing.lexeme_start]. Moreover, the internal slot is set
to [-1] and the backtrack position is set to the current position. *)
val start: lexbuf -> unit val start: lexbuf -> unit
(** [start t] informs the lexer buffer that any
code points until the current position can be discarded.
The current position become the "start" position as returned
by [Sedlexing.lexeme_start]. Moreover, the internal slot is set to
[-1] and the backtrack position is set to the current position.
*)
(** [next lexbuf] extracts the next code point from the lexer buffer and val next: lexbuf -> int
increments to current position. If the input stream is exhausted, the (** [next lexbuf] extracts the next code point from the
function returns [None]. If a ['\n'] is encountered, the tracked line number lexer buffer and increments to current position. If the input stream
is incremented. *) is exhausted, the function returns [-1]. *)
val next : lexbuf -> Uchar.t option
(** [__private__next_int lexbuf] extracts the next code point from the lexer
buffer and increments to current position. If the input stream is exhausted,
the function returns -1. If a ['\n'] is encountered, the tracked line number
is incremented.
This is a private API, it should not be used by code using this module's API
and can be removed at any time. *)
val __private__next_int : lexbuf -> int
(** [mark lexbuf i] stores the integer [i] in the internal slot. The backtrack
position is set to the current position. *)
val mark: lexbuf -> int -> unit val mark: lexbuf -> int -> unit
(** [mark lexbuf i] stores the integer [i] in the internal
slot. The backtrack position is set to the current position. *)
(** [backtrack lexbuf] returns the value stored in the internal slot of the
buffer, and performs backtracking (the current position is set to the value
of the backtrack position). *)
val backtrack: lexbuf -> int val backtrack: lexbuf -> int
(** [backtrack lexbuf] returns the value stored in the
internal slot of the buffer, and performs backtracking
(the current position is set to the value of the backtrack position). *)
(** [with_tokenizer tokenizer lexbuf] given a lexer and a lexbuf, returns a
generator of tokens annotated with positions. This generator can be used
with the Menir parser generator's incremental API. *)
val with_tokenizer :
(lexbuf -> 'token) ->
lexbuf ->
unit ->
'token * Lexing.position * Lexing.position
(** {6 Support for common encodings} *) (** {6 Support for common encodings} *)
module Latin1: sig module Latin1: sig
(** Create a lexbuf from a Latin1 encoded stream (ie a stream of Unicode code
points in the range [0..255]) *)
val from_gen: char Gen.t -> lexbuf val from_gen: char Gen.t -> lexbuf
(** Create a lexbuf from a Latin1 encoded stream (ie a stream
of Unicode code points in the range [0..255]) *)
val from_stream: char Stream.t -> lexbuf
[@@ocaml.deprecated "Use [Sedlexing.Latin1.from_gen] instead."]
(** Create a lexbuf from a Latin1 encoded stream (ie a stream
of Unicode code points in the range [0..255]) *)
(** Create a lexbuf from a Latin1 encoded input channel. The client is
responsible for closing the channel. *)
val from_channel: in_channel -> lexbuf val from_channel: in_channel -> lexbuf
(** Create a lexbuf from a Latin1 encoded input channel.
The client is responsible for closing the channel. *)
(** Create a lexbuf from a Latin1 encoded string. *)
val from_string: string -> lexbuf val from_string: string -> lexbuf
(** Create a lexbuf from a Latin1 encoded string. *)
(** As [Sedlexing.lexeme] with a result encoded in Latin1. This function
throws an exception [InvalidCodepoint] if it is not possible to encode the
result in Latin1. *)
val lexeme: lexbuf -> string val lexeme: lexbuf -> string
(** As [Sedlexing.lexeme] with a result encoded in Latin1. This
function throws an exception [InvalidCodepoint] if it is not
possible to encode the result in Latin1. *)
(** As [Sedlexing.sub_lexeme] with a result encoded in Latin1. This function
throws an exception [InvalidCodepoint] if it is not possible to encode the
result in Latin1. *)
val sub_lexeme: lexbuf -> int -> int -> string val sub_lexeme: lexbuf -> int -> int -> string
(** As [Sedlexing.sub_lexeme] with a result encoded in Latin1.
This function throws an exception [InvalidCodepoint] if it
is not possible to encode the result in Latin1. *)
(** As [Sedlexing.lexeme_char] with a result encoded in Latin1. This function
throws an exception [InvalidCodepoint] if it is not possible to encode the
result in Latin1. *)
val lexeme_char: lexbuf -> int -> char val lexeme_char: lexbuf -> int -> char
(** As [Sedlexing.lexeme_char] with a result encoded in Latin1.
This function throws an exception [InvalidCodepoint] if it
is not possible to encode the result in Latin1. *)
end end
module Utf8: sig module Utf8: sig
(** Create a lexbuf from a UTF-8 encoded stream. *)
val from_gen: char Gen.t -> lexbuf val from_gen: char Gen.t -> lexbuf
(** Create a lexbuf from a UTF-8 encoded stream. *)
val from_stream: char Stream.t -> lexbuf
[@@ocaml.deprecated "Use [Sedlexing.Utf8.from_gen] instead."]
(** Create a lexbuf from a UTF-8 encoded stream. *)
(** Create a lexbuf from a UTF-8 encoded input channel. *)
val from_channel: in_channel -> lexbuf val from_channel: in_channel -> lexbuf
(** Create a lexbuf from a UTF-8 encoded input channel. *)
(** Create a lexbuf from a UTF-8 encoded string. *)
val from_string: string -> lexbuf val from_string: string -> lexbuf
(** Create a lexbuf from a UTF-8 encoded string. *)
(** As [Sedlexing.lexeme] with a result encoded in UTF-8. *)
val lexeme: lexbuf -> string val lexeme: lexbuf -> string
(** As [Sedlexing.lexeme] with a result encoded in UTF-8. *)
(** As [Sedlexing.sub_lexeme] with a result encoded in UTF-8. *)
val sub_lexeme: lexbuf -> int -> int -> string val sub_lexeme: lexbuf -> int -> int -> string
(** As [Sedlexing.sub_lexeme] with a result encoded in UTF-8. *)
end
module Helper : sig
val width : char -> int
val check_two : int -> int -> int
val check_three : int -> int -> int -> int
val check_four : int -> int -> int -> int -> int
end
end
module Utf16: sig module Utf16: sig
type byte_order = Little_endian | Big_endian type byte_order = Little_endian | Big_endian
(** [Utf16.from_gen s opt_bo] creates a lexbuf from an UTF-16 encoded stream.
If [opt_bo] matches with [None] the function expects a BOM (Byte Order
Mark), and takes the byte order as [Utf16.Big_endian] if it cannot find
one. When [opt_bo] matches with [Some bo], [bo] is taken as byte order. In
this case a leading BOM is kept in the stream - the lexer has to ignore it
and a `wrong' BOM ([0xfffe]) will raise Utf16.InvalidCodepoint. *)
val from_gen: char Gen.t -> byte_order option -> lexbuf val from_gen: char Gen.t -> byte_order option -> lexbuf
(** [Utf16.from_gen s opt_bo] creates a lexbuf from an UTF-16
encoded stream. If [opt_bo] matches with [None] the function
expects a BOM (Byte Order Mark), and takes the byte order as
[Utf16.Big_endian] if it cannot find one. When [opt_bo]
matches with [Some bo], [bo] is taken as byte order. In this
case a leading BOM is kept in the stream - the lexer has to
ignore it and a `wrong' BOM ([0xfffe]) will raise
Utf16.InvalidCodepoint. *)
val from_stream: char Stream.t -> byte_order option -> lexbuf
[@@ocaml.deprecated "Use [Sedlexing.Utf16.from_gen] instead."]
(** Works as [Utf16.from_gen] with a [stream]. *)
(** Works as [Utf16.from_gen] with an [in_channel]. *)
val from_channel: in_channel -> byte_order option-> lexbuf val from_channel: in_channel -> byte_order option-> lexbuf
(** Works as [Utf16.from_gen] with an [in_channel]. *)
(** Works as [Utf16.from_gen] with a [string]. *)
val from_string: string -> byte_order option -> lexbuf val from_string: string -> byte_order option -> lexbuf
(** Works as [Utf16.from_gen] with a [string]. *)
(** [utf16_lexeme lb bo bom] as [Sedlexing.lexeme] with a result encoded in
UTF-16 in byte_order [bo] and starting with a BOM if [bom = true]. *)
val lexeme: lexbuf -> byte_order -> bool -> string val lexeme: lexbuf -> byte_order -> bool -> string
(** [utf16_lexeme lb bo bom] as [Sedlexing.lexeme] with a result
encoded in UTF-16 in byte_order [bo] and starting with a BOM
if [bom = true]. *)
(** [sub_lexeme lb pos len bo bom] as [Sedlexing.sub_lexeme] with a result
encoded in UTF-16 with byte order [bo] and starting with a BOM if
[bom=true] *)
val sub_lexeme: lexbuf -> int -> int -> byte_order -> bool -> string val sub_lexeme: lexbuf -> int -> int -> byte_order -> bool -> string
(** [sub_lexeme lb pos len bo bom] as
[Sedlexing.sub_lexeme] with a result encoded in UTF-16 with
byte order [bo] and starting with a BOM if [bom=true] *)
end end

54
src/syntax/Makefile Normal file
View File

@ -0,0 +1,54 @@
# The package sedlex is released under the terms of an MIT-like license.
# See the attached LICENSE file.
# Copyright 2005, 2013 by Alain Frisch and LexiFi.
include $(shell ocamlc -where)/Makefile.config
SRCS=sedlex_cset.mli sedlex_cset.ml unicode63.mli unicode63.ml sedlex.mli sedlex.ml ppx_sedlex.ml
OCAMLC=ocamlfind ocamlc
OCAMLOPT=ocamlfind ocamlopt
PACKAGES=-package ppx_tools_versioned.metaquot_405 -package ocaml-migrate-parsetree
OCAMLFLAGS=$(PACKAGES) -linkall -w +A-4-9-42 -annot
all: ppx_sedlex$(EXE) sedlex.cma
opt: ppx_sedlex.opt$(EXE) sedlex.cmxa sedlex.cmxs
sedlex.cma: $(SRCS)
$(OCAMLC) $(OCAMLFLAGS) -a -o sedlex.cma $(SRCS)
sedlex.cmxa: $(SRCS)
$(OCAMLOPT) $(OCAMLFLAGS) -a -o sedlex.cmxa $(SRCS)
sedlex.cmxs: $(SRCS)
$(OCAMLOPT) $(OCAMLFLAGS) -shared -o sedlex.cmxs $(SRCS)
# We need the ocaml-migrate-parsetree.driver-main.cm{,x}a library to come last
# on the command-line when linking, since its initialization code must run
# *after* the one of sedlex.cm{,x}a. To ensure that this is the case, we
# manually pass its file to ocaml{c,opt}, as normally ocamlfind links in all the
# -package libraries *before* user code.
MIGRATE_PARSETREE_MAIN_BYTE= \
$(shell ocamlfind query -predicates byte -a-format ocaml-migrate-parsetree.driver-main)
MIGRATE_PARSETREE_MAIN_NATIVE= \
$(shell ocamlfind query -predicates native -a-format ocaml-migrate-parsetree.driver-main)
ppx_sedlex$(EXE): sedlex.cma
$(OCAMLC) \
-predicates ppx_driver \
-o $@ -linkpkg -linkall \
-package ppx_tools_versioned \
$< \
$(MIGRATE_PARSETREE_MAIN_BYTE)
ppx_sedlex.opt$(EXE): sedlex.cmxa
$(OCAMLOPT) \
-predicates ppx_driver \
-o $@ -linkpkg -linkall \
-package ppx_tools_versioned \
$< \
$(MIGRATE_PARSETREE_MAIN_NATIVE)
clean:
rm -f *~ *.cm* *.a *.lib *.o *.obj *.annot ppx_sedlex$(EXE) ppx_sedlex.opt$(EXE)

17
src/syntax/dune
View File

@ -1,17 +0,0 @@
(library
(name sedlex_ppx)
(public_name noslop-sedlex.ppx)
(kind ppx_rewriter)
(libraries ppxlib noslop-sedlex noslop-sedlex.utils)
(ppx_runtime_libraries noslop-sedlex)
(preprocess
(pps ppxlib.metaquot)))
(rule
(targets unicode.ml)
(mode promote)
(deps
(:gen ../generator/gen_unicode.exe)
(glob_files ../generator/data/*.txt))
(action
(run %{gen} %{targets})))

217
src/syntax/iso.ml
View File

@ -1,217 +0,0 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
open Sedlex_cset
let tr8876_ident_char =
let l =
[
(* ASCII *)
(0x0041, 0x005a);
(0x0061, 0x007a);
(* Latin *)
(0x00c0, 0x00d6);
(0x00d8, 0x00f6);
(0x00f8, 0x01f5);
(0x01fa, 0x0217);
(0x0250, 0x02a8);
(* Greek *)
(0x0384, 0x0384);
(0x0388, 0x038a);
(0x038c, 0x038c);
(0x038e, 0x03a1);
(0x03a3, 0x03ce);
(0x03d0, 0x03d6);
(0x03da, 0x03da);
(0x03dc, 0x03dc);
(0x03de, 0x03de);
(0x03e0, 0x03e0);
(0x03e2, 0x03f3);
(* Cyrillic *)
(0x0401, 0x040d);
(0x040f, 0x044f);
(0x0451, 0x045c);
(0x045e, 0x0481);
(0x0490, 0x04c4);
(0x04c7, 0x04c4);
(0x04cb, 0x04cc);
(0x04d0, 0x04eb);
(0x04ee, 0x04f5);
(0x04f8, 0x04f9);
(* Armenian *)
(0x0531, 0x0556);
(0x0561, 0x0587);
(0x04d0, 0x04eb);
(* Hebrew *)
(0x05d0, 0x05ea);
(0x05f0, 0x05f4);
(* Arabic *)
(0x0621, 0x063a);
(0x0640, 0x0652);
(0x0670, 0x06b7);
(0x06ba, 0x06be);
(0x06c0, 0x06ce);
(0x06e5, 0x06e7);
(* Devanagari *)
(0x0905, 0x0939);
(0x0958, 0x0962);
(* Bengali *)
(0x0985, 0x098c);
(0x098f, 0x0990);
(0x0993, 0x09a8);
(0x09aa, 0x09b0);
(0x09b2, 0x09b2);
(0x09b6, 0x09b9);
(0x09dc, 0x09dd);
(0x09df, 0x09e1);
(0x09f0, 0x09f1);
(* Gurmukhi *)
(0x0a05, 0x0a0a);
(0x0a0f, 0x0a10);
(0x0a13, 0x0a28);
(0x0a2a, 0x0a30);
(0x0a32, 0x0a33);
(0x0a35, 0x0a36);
(0x0a38, 0x0a39);
(0x0a59, 0x0a5c);
(0x0a5e, 0x0a5e);
(* Gunjarati *)
(0x0a85, 0x0a8b);
(0x0a8d, 0x0a8d);
(0x0a8f, 0x0a91);
(0x0a93, 0x0aa8);
(0x0aaa, 0x0ab0);
(0x0ab2, 0x0ab3);
(0x0ab5, 0x0ab9);
(0x0ae0, 0x0ae0);
(* Oriya *)
(0x0b05, 0x0b0c);
(0x0b0f, 0x0b10);
(0x0b13, 0x0b28);
(0x0b2a, 0x0b30);
(0x0b32, 0x0b33);
(0x0b36, 0x0b39);
(0x0b5c, 0x0b5d);
(0x0b5f, 0x0b61);
(* Tamil *)
(0x0b85, 0x0b8a);
(0x0b8e, 0x0b90);
(0x0b92, 0x0b95);
(0x0b99, 0x0b9a);
(0x0b9c, 0x0b9c);
(0x0b9e, 0x0b9f);
(0x0ba3, 0x0ba4);
(0x0ba8, 0x0baa);
(0x0bae, 0x0bb5);
(0x0bb7, 0x0bb9);
(* Telugu *)
(0x0c05, 0x0c0c);
(0x0c0e, 0x0c10);
(0x0c12, 0x0c28);
(0x0c2a, 0x0c33);
(0x0c35, 0x0c39);
(0x0c60, 0x0c61);
(* Kannada *)
(0x0c85, 0x0c8c);
(0x0c8e, 0x0c90);
(0x0c92, 0x0ca8);
(0x0caa, 0x0cb3);
(0x0cb5, 0x0cb9);
(0x0ce0, 0x0ce1);
(* Malayam *)
(0x0d05, 0x0d0c);
(0x0d0e, 0x0d10);
(0x0d12, 0x0d28);
(0x0d2a, 0x0d39);
(0x0d60, 0x0d61);
(* Thai *)
(0x0e01, 0x0e30);
(0x0e32, 0x0e33);
(0x0e40, 0x0e46);
(0x0e4f, 0x0e5b);
(* Lao *)
(0x0e81, 0x0e82);
(0x0e84, 0x0e84);
(0x0e87, 0x0e88);
(0x0e8a, 0x0e8a);
(0x0e0d, 0x0e0d);
(0x0e94, 0x0e97);
(0x0e99, 0x0e9f);
(0x0ea1, 0x0ea3);
(0x0ea5, 0x0ea5);
(0x0ea7, 0x0ea7);
(0x0eaa, 0x0eab);
(0x0ead, 0x0eb0);
(0x0eb2, 0x0eb3);
(0x0ebd, 0x0ebd);
(0x0ec0, 0x0ec4);
(0x0ec6, 0x0ec6);
(* Georgian *)
(0x10a0, 0x10c5);
(0x10d0, 0x10f6);
(* Hangul Jamo *)
(0x1100, 0x1159);
(0x1161, 0x11a2);
(0x11a8, 0x11f9);
(0x11d0, 0x11f6);
(* Latin extensions *)
(0x1e00, 0x1e9a);
(0x1ea0, 0x1ef9);
(* Greek extended *)
(0x1f00, 0x1f15);
(0x1f18, 0x1f1d);
(0x1f20, 0x1f45);
(0x1f48, 0x1f4d);
(0x1f50, 0x1f57);
(0x1f59, 0x1f59);
(0x1f5b, 0x1f5b);
(0x1f5d, 0x1f5d);
(0x1f5f, 0x1f7d);
(0x1f80, 0x1fb4);
(0x1fb6, 0x1fbc);
(0x1fc2, 0x1fc4);
(0x1fc6, 0x1fcc);
(0x1fd0, 0x1fd3);
(0x1fd6, 0x1fdb);
(0x1fe0, 0x1fec);
(0x1ff2, 0x1ff4);
(0x1ff6, 0x1ffc);
(* Hiragana *)
(0x3041, 0x3094);
(0x309b, 0x309e);
(* Katakana *)
(0x30a1, 0x30fe);
(* Bopmofo *)
(0x3105, 0x312c);
(* CJK Unified Ideographs *)
(0x4e00, 0x9fa5);
(* CJK Compatibility Ideographs *)
(0xf900, 0xfa2d);
(* Arabic Presentation Forms *)
(0xfb1f, 0xfb36);
(0xfb38, 0xfb3c);
(0xfb3e, 0xfb3e);
(0xfb40, 0xfb41);
(0xfb42, 0xfb44);
(0xfb46, 0xfbb1);
(0xfbd3, 0xfd35);
(* Arabic Presentation Forms-A *)
(0xfd50, 0xfd85);
(0xfd92, 0xfbc7);
(0xfdf0, 0xfdfb);
(* Arabic Presentation Forms-B *)
(0xfe70, 0xfe72);
(0xfe74, 0xfe74);
(0xfe76, 0xfefc);
(* Half width and Fullwidth Forms *)
(0xff21, 0xff3a);
(0xff41, 0xff5a);
(0xff66, 0xffbe);
(0xffc2, 0xffc7);
(0xffca, 0xffcf);
(0xffd2, 0xffd7);
(0xffd2, 0xffd7);
(0xffda, 0xffdc);
]
in
union_list (List.map (fun (a, b) -> interval a b) l)

10
src/syntax/iso.mli
View File

@ -1,10 +0,0 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
open Sedlex_cset
(** Letters to be used in identifiers, as specified by ISO .... *)
(* Data provided by John M. Skaller *)
val tr8876_ident_char : t

608
src/syntax/ppx_sedlex.ml
View File

@ -1,39 +1,36 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
open Ppxlib open Longident
open Ast_builder.Default open Migrate_parsetree
open Ast_405
open Parsetree
open Asttypes
open Ast_helper open Ast_helper
open Ast_convenience_405
(* let ocaml_version = Versions.ocaml_408 *) module Ast_mapper_class = Ast_mapper_class_405
let ocaml_version = Versions.ocaml_405
module Cset = Sedlex_cset module Cset = Sedlex_cset
(* Decision tree for partitions *) (* Decision tree for partitions *)
let default_loc = Location.none
type decision_tree = type decision_tree =
| Lte of int * decision_tree * decision_tree | Lte of int * decision_tree * decision_tree
| Table of int * int array | Table of int * int array
| Return of int | Return of int
let rec simplify_decision_tree (x : decision_tree) =
match x with
| Table _ | Return _ -> x
| Lte (_, (Return a as l), Return b) when a = b -> l
| Lte (i, l, r) -> (
let l = simplify_decision_tree l in
let r = simplify_decision_tree r in
match (l, r) with
| Return a, Return b when a = b -> l
| _ -> Lte (i, l, r))
let decision l = let decision l =
let l = List.map (fun (a, b, i) -> (a, b, Return i)) l in let l = List.map (fun (a, b, i) -> (a, b, Return i)) l in
let rec merge2 = function let rec merge2 = function
| (a1, b1, d1) :: (a2, b2, d2) :: rest -> | (a1, b1, d1) :: (a2, b2, d2) :: rest ->
let x = if b1 + 1 = a2 then d2 else Lte (a2 - 1, Return (-1), d2) in let x =
if b1 + 1 = a2 then d2
else Lte (a2 - 1, Return (-1), d2)
in
(a1, b2, Lte (b1, d1, x)) :: merge2 rest (a1, b2, Lte (b1, d1, x)) :: merge2 rest
| rest -> rest | rest -> rest
in in
@ -50,20 +47,16 @@ let decision_table l =
let rec aux m accu = function let rec aux m accu = function
| ((a, b, i) as x)::rem when b < limit && i < 255-> | ((a, b, i) as x)::rem when b < limit && i < 255->
aux (min a m) (x :: accu) rem aux (min a m) (x :: accu) rem
| rem -> (m, accu, rem) | rem -> m, accu, rem
in in
let min, table, rest = aux max_int [] l in let (min, table, rest) = aux max_int [] l in
match table with match table with
| [] -> decision l | [] -> decision l
| [(min, max, i)] -> | [(min, max, i)] ->
Lte (min - 1, Return (-1), Lte (max, Return i, decision rest)) Lte (min - 1, Return (-1), (Lte (max, Return i, decision rest)))
| (_, max, _) :: _ -> | (_, max, _) :: _ ->
let arr = Array.make (max - min + 1) 0 in let arr = Array.make (max - min + 1) 0 in
let set (a, b, i) = let set (a, b, i) = for j = a to b do arr.(j - min) <- i + 1 done in
for j = a to b do
arr.(j - min) <- i + 1
done
in
List.iter set table; List.iter set table;
Lte (min - 1, Return (-1), Lte (max, Table (min, arr), decision rest)) Lte (min - 1, Return (-1), Lte (max, Table (min, arr), decision rest))
@ -77,52 +70,90 @@ let rec simplify min max = function
let segments_of_partition p = let segments_of_partition p =
let seg = ref [] in let seg = ref [] in
Array.iteri Array.iteri
(fun i c -> (fun i c -> List.iter (fun (a, b) -> seg := (a, b, i) :: !seg) c)
List.iter
(fun (a, b) -> seg := (a, b, i) :: !seg)
(c : Sedlex_cset.t :> (int * int) list))
p; p;
List.sort (fun (a1,_,_) (a2,_,_) -> compare a1 a2) !seg List.sort (fun (a1,_,_) (a2,_,_) -> compare a1 a2) !seg
let decision_table p = let decision_table p =
simplify (-1) Cset.max_code (decision_table (segments_of_partition p)) simplify (-1) (Cset.max_code) (decision_table (segments_of_partition p))
(* Helpers to build AST *) (* Helpers to build AST *)
let appfun s l = let appfun s l = app (evar s) l
let loc = default_loc in let glb_value name def = Str.value Nonrecursive [Vb.mk (pvar name) def]
eapply ~loc (evar ~loc s) l
let glb_value name def =
let loc = default_loc in
pstr_value ~loc Nonrecursive
[value_binding ~loc ~pat:(pvar ~loc name) ~expr:def]
(* Named regexps *) (* Named regexps *)
module StringMap = Map.Make(struct module StringMap = Map.Make(struct
type t = string type t = string
let compare = compare let compare = compare
end) end)
let builtin_regexps = let builtin_regexps =
List.fold_left List.fold_left (fun acc (n, c) -> StringMap.add n (Sedlex.chars c) acc)
(fun acc (n, c) -> StringMap.add n (Sedlex.chars c) acc)
StringMap.empty StringMap.empty
([ [
("any", Cset.any); "any", Cset.any;
("eof", Cset.eof); "eof", Cset.eof;
("xml_letter", Xml.letter); "xml_letter", Cset.letter;
("xml_digit", Xml.digit); "xml_digit", Cset.digit;
("xml_extender", Xml.extender); "xml_extender", Cset.extender;
("xml_base_char", Xml.base_char); "xml_base_char", Cset.base_char;
("xml_ideographic", Xml.ideographic); "xml_ideographic", Cset.ideographic;
("xml_combining_char", Xml.combining_char); "xml_combining_char", Cset.combining_char;
("xml_blank", Xml.blank); "xml_blank", Cset.blank;
("tr8876_ident_char", Iso.tr8876_ident_char); "tr8876_ident_char", Cset.tr8876_ident_char;
(* Unicode 6.3 categories *)
"cc", Unicode63.Categories.cc;
"cf", Unicode63.Categories.cf;
"cn", Unicode63.Categories.cn;
"co", Unicode63.Categories.co;
"cs", Unicode63.Categories.cs;
"ll", Unicode63.Categories.ll;
"lm", Unicode63.Categories.lm;
"lo", Unicode63.Categories.lo;
"lt", Unicode63.Categories.lt;
"lu", Unicode63.Categories.lu;
"mc", Unicode63.Categories.mc;
"me", Unicode63.Categories.me;
"mn", Unicode63.Categories.mn;
"nd", Unicode63.Categories.nd;
"nl", Unicode63.Categories.nl;
"no", Unicode63.Categories.no;
"pc", Unicode63.Categories.pc;
"pd", Unicode63.Categories.pd;
"pe", Unicode63.Categories.pe;
"pf", Unicode63.Categories.pf;
"pi", Unicode63.Categories.pi;
"po", Unicode63.Categories.po;
"ps", Unicode63.Categories.ps;
"sc", Unicode63.Categories.sc;
"sk", Unicode63.Categories.sk;
"sm", Unicode63.Categories.sm;
"so", Unicode63.Categories.so;
"zl", Unicode63.Categories.zl;
"zp", Unicode63.Categories.zp;
"zs", Unicode63.Categories.zs;
(* Unicode 6.3 properties *)
"alphabetic", Unicode63.Properties.alphabetic;
"ascii_hex_digit", Unicode63.Properties.ascii_hex_digit;
"hex_digit", Unicode63.Properties.hex_digit;
"id_continue", Unicode63.Properties.id_continue;
"id_start", Unicode63.Properties.id_start;
"lowercase", Unicode63.Properties.lowercase;
"math", Unicode63.Properties.math;
"other_alphabetic", Unicode63.Properties.other_alphabetic;
"other_lowercase", Unicode63.Properties.other_lowercase;
"other_math", Unicode63.Properties.other_math;
"other_uppercase", Unicode63.Properties.other_uppercase;
"uppercase", Unicode63.Properties.uppercase;
"white_space", Unicode63.Properties.white_space;
"xid_continue", Unicode63.Properties.xid_continue;
"xid_start", Unicode63.Properties.xid_start;
] ]
@ Unicode.Categories.list @ Unicode.Properties.list)
(* Tables (indexed mapping: codepoint -> next state) *) (* Tables (indexed mapping: codepoint -> next state) *)
@ -141,18 +172,14 @@ let table_name x =
let table (name, v) = let table (name, v) =
let n = Array.length v in let n = Array.length v in
let s = Bytes.create n in let s = Bytes.create n in
for i = 0 to n - 1 do for i = 0 to n - 1 do Bytes.set s i (Char.chr v.(i)) done;
Bytes.set s i (Char.chr v.(i)) glb_value name (str (Bytes.to_string s))
done;
glb_value name (estring ~loc:default_loc (Bytes.to_string s))
(* Partition (function: codepoint -> next state) *) (* Partition (function: codepoint -> next state) *)
let partitions = Hashtbl.create 31 let partitions = Hashtbl.create 31
let partition_counter = ref 0 let partition_counter = ref 0
let get_partitions () = Hashtbl.fold (fun key x accu -> (x, key) :: accu) partitions []
let get_partitions () =
Hashtbl.fold (fun key x accu -> (x, key) :: accu) partitions []
let partition_name x = let partition_name x =
try Hashtbl.find partitions x try Hashtbl.find partitions x
@ -162,28 +189,17 @@ let partition_name x =
Hashtbl.add partitions x s; Hashtbl.add partitions x s;
s s
(* We duplicate the body for the EOF (-1) case rather than creating
an interior utility function. *)
let partition (name, p) = let partition (name, p) =
let loc = default_loc in
let rec gen_tree = function let rec gen_tree = function
| Lte (i, yes, no) -> | Lte (i, yes, no) ->
[%expr [%expr if c <= [%e int i] then [%e gen_tree yes] else [%e gen_tree no]]
if c <= [%e eint ~loc i] then [%e gen_tree yes] else [%e gen_tree no]] | Return i -> int i
| Return i -> eint ~loc:default_loc i
| Table (offset, t) -> | Table (offset, t) ->
let c = let c = if offset = 0 then [%expr c] else [%expr c - [%e int offset]] in
if offset = 0 then [%expr c] else [%expr c - [%e eint ~loc offset]] [%expr Char.code (String.get [%e evar (table_name t)] [%e c]) - 1]
in in
[%expr let body = gen_tree (decision_table p) in
Char.code (String.unsafe_get [%e evar ~loc (table_name t)] [%e c]) - 1] glb_value name (func [pvar "c", body])
in
let body = gen_tree (simplify_decision_tree (decision_table p)) in
glb_value name
[%expr
fun c ->
let open! Stdlib in
[%e body]]
(* Code generation for the automata *) (* Code generation for the automata *)
@ -197,79 +213,41 @@ let best_final final =
let state_fun state = Printf.sprintf "__sedlex_state_%i" state let state_fun state = Printf.sprintf "__sedlex_state_%i" state
let call_state lexbuf auto state = let call_state lexbuf auto state =
let { Sedlex.trans; finals } = auto.(state) in let (trans, final) = auto.(state) in
if Array.length trans = 0 then ( if Array.length trans = 0
match best_final finals with then match best_final final with
| Some i -> eint ~loc:default_loc i | Some i -> int i
| None -> assert false) | None -> assert false
else appfun (state_fun state) [lexbuf] else appfun (state_fun state) [evar lexbuf]
let gen_state (lexbuf_name, lexbuf) auto i { Sedlex.trans; finals } = let gen_state lexbuf auto i (trans, final) =
let loc = default_loc in
let partition = Array.map fst trans in let partition = Array.map fst trans in
let cases = let cases = Array.mapi (fun i (_, j) -> Exp.case(pint i) (call_state lexbuf auto j)) trans in
Array.mapi
(fun i (_, j) ->
case ~lhs:(pint ~loc i) ~guard:None ~rhs:(call_state lexbuf auto j))
trans
in
let cases = Array.to_list cases in let cases = Array.to_list cases in
let body () = let body () =
pexp_match ~loc Exp.match_
(appfun (partition_name partition) (appfun (partition_name partition) [[%expr Sedlexing.next [%e evar lexbuf]]])
[[%expr Sedlexing.__private__next_int [%e lexbuf]]]) (cases @ [Exp.case [%pat? _] [%expr Sedlexing.backtrack [%e evar lexbuf]]])
(cases
@ [
case
~lhs:[%pat? _]
~guard:None
~rhs:[%expr Sedlexing.backtrack [%e lexbuf]];
])
in in
let ret body = let ret body = [ Vb.mk (pvar (state_fun i)) (func [pvar lexbuf, body]) ] in
let lhs = pvar ~loc:lexbuf.pexp_loc lexbuf_name in match best_final final with
[
value_binding ~loc
~pat:(pvar ~loc (state_fun i))
~expr:(Exp.fun_ ~loc Nolabel None lhs body);
]
in
match best_final finals with
| None -> ret (body ()) | None -> ret (body ())
| Some _ when Array.length trans = 0 -> [] | Some _ when Array.length trans = 0 -> []
| Some i -> | Some i -> ret [%expr Sedlexing.mark [%e evar lexbuf] [%e int i]; [%e body ()]]
ret
[%expr
Sedlexing.mark [%e lexbuf] [%e eint ~loc i];
[%e body ()]]
let gen_recflag auto = let gen_definition lexbuf l error =
(* The generated function is not recursive if the transitions end let brs = Array.of_list l in
in states with no further transitions. *) let auto = Sedlex.compile (Array.map fst brs) in
try let cases = Array.to_list (Array.mapi (fun i (_, e) -> Exp.case (pint i) e) brs) in
Array.iter let states = Array.mapi (gen_state lexbuf auto) auto in
(fun { Sedlex.trans; _ } ->
Array.iter
(fun (_, j) ->
if Array.length auto.(j).Sedlex.trans > 0 then raise Exit)
trans)
auto;
Nonrecursive
with Exit -> Recursive
let gen_definition ((_, lexbuf) as lexbuf_with_name) auto l error =
let loc = default_loc in
let cases =
List.mapi (fun i (_, e) -> case ~lhs:(pint ~loc i) ~guard:None ~rhs:e) l
in
let states = Array.mapi (gen_state lexbuf_with_name auto) auto in
let states = List.flatten (Array.to_list states) in let states = List.flatten (Array.to_list states) in
pexp_let ~loc (gen_recflag auto) states Exp.let_ Recursive states
(pexp_sequence ~loc (Exp.sequence
[%expr Sedlexing.start [%e lexbuf]] [%expr Sedlexing.start [%e evar lexbuf]]
(pexp_match ~loc (Exp.match_ (appfun (state_fun 0) [evar lexbuf])
(appfun (state_fun 0) [lexbuf]) (cases @ [Exp.case (Pat.any ()) error])
(cases @ [case ~lhs:(ppat_any ~loc) ~guard:None ~rhs:error]))) )
)
(* Lexer specification parser *) (* Lexer specification parser *)
@ -278,46 +256,18 @@ let codepoint i =
failwith (Printf.sprintf "Invalid Unicode code point: %i" i); failwith (Printf.sprintf "Invalid Unicode code point: %i" i);
i i
let char c = Cset.singleton (Char.code c) let regexp_for_char c =
let uchar c = Cset.singleton (Uchar.to_int c) Sedlex.chars (Cset.singleton (Char.code c))
let err loc fmt = let regexp_for_string s =
Printf.ksprintf let rec aux n =
(fun s -> if n = String.length s then Sedlex.eps
raise (Location.Error (Location.Error.createf ~loc "Sedlex: %s" s))) else
fmt Sedlex.seq (regexp_for_char s.[n]) (aux (succ n))
in aux 0
type encoding = Utf8 | Latin1 | Ascii let err loc s =
raise (Location.Error (Location.error ~loc ("Sedlex: " ^ s)))
let string_of_encoding = function
| Utf8 -> "UTF-8"
| Latin1 -> "Latin-1"
| Ascii -> "ASCII"
let rev_csets_of_string ~loc ~encoding s =
match encoding with
| Utf8 ->
Utf8.fold
~f:(fun acc _ x ->
match x with
| `Malformed _ ->
err loc "Malformed %s string" (string_of_encoding encoding)
| `Uchar c -> uchar c :: acc)
[] s
| Latin1 ->
let l = ref [] in
for i = 0 to String.length s - 1 do
l := char s.[i] :: !l
done;
!l
| Ascii ->
let l = ref [] in
for i = 0 to String.length s - 1 do
match s.[i] with
| '\x00' .. '\x7F' as c -> l := char c :: !l
| _ -> err loc "Malformed %s string" (string_of_encoding encoding)
done;
!l
let rec repeat r = function let rec repeat r = function
| 0, 0 -> Sedlex.eps | 0, 0 -> Sedlex.eps
@ -325,264 +275,178 @@ let rec repeat r = function
| n, m -> Sedlex.seq r (repeat r (n - 1, m - 1)) | n, m -> Sedlex.seq r (repeat r (n - 1, m - 1))
let regexp_of_pattern env = let regexp_of_pattern env =
let rec char_pair_op func name ~encoding ~loc tuple = let rec char_pair_op func name p tuple = (* Construct something like Sub(a,b) *)
(* Construct something like Sub(a,b) *)
match tuple with match tuple with
| Some { ppat_desc = Ppat_tuple [p0; p1]; _ } -> | Some {ppat_desc=Ppat_tuple (p0 :: p1 :: [])} ->
begin match func (aux ~encoding p0) (aux ~encoding p1) with begin match func (aux p0) (aux p1) with
| Some r -> r | Some r -> r
| None -> | None ->
err loc err p.ppat_loc @@
"the %s operator can only applied to single-character length \ "the "^name^" operator can only applied to single-character length regexps"
regexps"
name
end end
| _ -> | _ -> err p.ppat_loc @@ "the "^name^" operator requires two arguments, like "^name^"(a,b)"
err loc "the %s operator requires two arguments, like %s(a,b)" name and aux p = (* interpret one pattern node *)
name
and aux ~encoding p =
(* interpret one pattern node *)
match p.ppat_desc with match p.ppat_desc with
| Ppat_or (p1, p2) -> Sedlex.alt (aux ~encoding p1) (aux ~encoding p2) | Ppat_or (p1, p2) -> Sedlex.alt (aux p1) (aux p2)
| Ppat_tuple (p :: pl) -> | Ppat_tuple (p :: pl) ->
List.fold_left List.fold_left (fun r p -> Sedlex.seq r (aux p))
(fun r p -> Sedlex.seq r (aux ~encoding p)) (aux p)
(aux ~encoding p) pl pl
| Ppat_construct ({ txt = Lident "Star"; _ }, Some (_, p)) -> | Ppat_construct ({txt = Lident "Star"}, Some p) ->
Sedlex.rep (aux ~encoding p) Sedlex.rep (aux p)
| Ppat_construct ({ txt = Lident "Plus"; _ }, Some (_, p)) -> | Ppat_construct ({txt = Lident "Plus"}, Some p) ->
Sedlex.plus (aux ~encoding p) Sedlex.plus (aux p)
| Ppat_construct ({ txt = Lident "Utf8"; _ }, Some (_, p)) ->
aux ~encoding:Utf8 p
| Ppat_construct ({ txt = Lident "Latin1"; _ }, Some (_, p)) ->
aux ~encoding:Latin1 p
| Ppat_construct ({ txt = Lident "Ascii"; _ }, Some (_, p)) ->
aux ~encoding:Ascii p
| Ppat_construct | Ppat_construct
( { txt = Lident "Rep"; _ }, ({txt = Lident "Rep"},
Some Some {ppat_desc=Ppat_tuple[p0; {ppat_desc=Ppat_constant (i1 as i2)|Ppat_interval(i1, i2)}]}) ->
( _, begin match Constant.of_constant i1, Constant.of_constant i2 with
{
ppat_desc =
Ppat_tuple
[
p0;
{
ppat_desc =
Ppat_constant (i1 as i2) | Ppat_interval (i1, i2);
_;
};
];
_;
} ) ) ->
begin match (i1, i2) with
| Pconst_integer(i1,_), Pconst_integer(i2,_) -> | Pconst_integer(i1,_), Pconst_integer(i2,_) ->
let i1 = int_of_string i1 in let i1 = int_of_string i1 in
let i2 = int_of_string i2 in let i2 = int_of_string i2 in
if 0 <= i1 && i1 <= i2 then repeat (aux ~encoding p0) (i1, i2) if 0 <= i1 && i1 <= i2 then repeat (aux p0) (i1, i2)
else err p.ppat_loc "Invalid range for Rep operator" else err p.ppat_loc "Invalid range for Rep operator"
| _ -> | _ ->
err p.ppat_loc "Rep must take an integer constant or interval" err p.ppat_loc "Rep must take an integer constant or interval"
end end
| Ppat_construct ({ txt = Lident "Rep"; _ }, _) -> | Ppat_construct ({txt = Lident "Rep"}, _) ->
err p.ppat_loc "the Rep operator takes 2 arguments" err p.ppat_loc "the Rep operator takes 2 arguments"
| Ppat_construct ({ txt = Lident "Opt"; _ }, Some (_, p)) -> | Ppat_construct ({txt = Lident "Opt"}, Some p) ->
Sedlex.alt Sedlex.eps (aux ~encoding p) Sedlex.alt Sedlex.eps (aux p)
| Ppat_construct ({ txt = Lident "Compl"; _ }, arg) -> | Ppat_construct ({txt = Lident "Compl"}, arg) ->
begin match arg with begin match arg with
| Some (_, p0) -> | Some p0 ->
begin match Sedlex.compl (aux ~encoding p0) with begin match Sedlex.compl (aux p0) with
| Some r -> r | Some r -> r
| None -> | None ->
err p.ppat_loc err p.ppat_loc
"the Compl operator can only applied to a \ "the Compl operator can only applied to a single-character length regexp"
single-character length regexp"
end end
| _ -> err p.ppat_loc "the Compl operator requires an argument" | _ -> err p.ppat_loc "the Compl operator requires an argument"
end end
| Ppat_construct ({ txt = Lident "Sub"; _ }, arg) -> | Ppat_construct ({ txt = Lident "Sub" }, arg) ->
char_pair_op ~encoding Sedlex.subtract "Sub" ~loc:p.ppat_loc char_pair_op Sedlex.subtract "Sub" p arg
(Option.map (fun (_, arg) -> arg) arg) | Ppat_construct ({ txt = Lident "Intersect" }, arg) ->
| Ppat_construct ({ txt = Lident "Intersect"; _ }, arg) -> char_pair_op Sedlex.intersection "Intersect" p arg
char_pair_op ~encoding Sedlex.intersection "Intersect" ~loc:p.ppat_loc | Ppat_construct ({txt = Lident "Chars"}, arg) ->
(Option.map (fun (_, arg) -> arg) arg) let const = match arg with
| Ppat_construct ({ txt = Lident "Chars"; _ }, arg) -> ( | Some {ppat_desc=Ppat_constant const} ->
let const = Some (Constant.of_constant const)
match arg with
| Some (_, { ppat_desc = Ppat_constant const; _ }) -> Some const
| _ -> None | _ -> None
in in
match const with begin match const with
| Some (Pconst_string (s, _, _)) -> | Some (Pconst_string(s,_))->
let l = rev_csets_of_string ~loc:p.ppat_loc ~encoding s in let c = ref Cset.empty in
let chars = List.fold_left Cset.union Cset.empty l in for i = 0 to String.length s - 1 do
Sedlex.chars chars c := Cset.union !c (Cset.singleton (Char.code s.[i]))
| _ -> done;
err p.ppat_loc "the Chars operator requires a string argument") Sedlex.chars !c
| _ -> err p.ppat_loc "the Chars operator requires a string argument"
end
| Ppat_interval (i_start, i_end) -> | Ppat_interval (i_start, i_end) ->
begin match (i_start, i_end) with begin match Constant.of_constant i_start, Constant.of_constant i_end with
| Pconst_char c1, Pconst_char c2 -> | Pconst_char c1, Pconst_char c2 -> Sedlex.chars (Cset.interval (Char.code c1) (Char.code c2))
let valid =
match encoding with
(* utf8 char interval can only match ascii because
of the OCaml lexer. *)
| Ascii | Utf8 -> (
function '\x00' .. '\x7f' -> true | _ -> false)
| Latin1 -> ( function _ -> true)
in
if not (valid c1 && valid c2) then
err p.ppat_loc
"this pattern is not a valid %s interval regexp"
(string_of_encoding encoding);
Sedlex.chars (Cset.interval (Char.code c1) (Char.code c2))
| Pconst_integer(i1,_), Pconst_integer(i2,_) -> | Pconst_integer(i1,_), Pconst_integer(i2,_) ->
Sedlex.chars Sedlex.chars (Cset.interval (codepoint (int_of_string i1)) (codepoint (int_of_string i2)))
(Cset.interval
(codepoint (int_of_string i1))
(codepoint (int_of_string i2)))
| _ -> err p.ppat_loc "this pattern is not a valid interval regexp" | _ -> err p.ppat_loc "this pattern is not a valid interval regexp"
end end
| Ppat_constant const -> | Ppat_constant (const) ->
begin match const with begin match Constant.of_constant const with
| Pconst_string (s, _, _) -> | Pconst_string (s, _) -> regexp_for_string s
let rev_l = rev_csets_of_string s ~loc:p.ppat_loc ~encoding in | Pconst_char c -> regexp_for_char c
List.fold_left | Pconst_integer(i,_) -> Sedlex.chars (Cset.singleton (codepoint (int_of_string i)))
(fun acc cset -> Sedlex.seq (Sedlex.chars cset) acc)
Sedlex.eps rev_l
| Pconst_char c -> Sedlex.chars (char c)
| Pconst_integer (i, _) ->
Sedlex.chars (Cset.singleton (codepoint (int_of_string i)))
| _ -> err p.ppat_loc "this pattern is not a valid regexp" | _ -> err p.ppat_loc "this pattern is not a valid regexp"
end end
| Ppat_var { txt = x; _ } -> | Ppat_var {txt=x} ->
begin try StringMap.find x env begin try StringMap.find x env
with Not_found -> err p.ppat_loc "unbound regexp %s" x with Not_found ->
err p.ppat_loc (Printf.sprintf "unbound regexp %s" x)
end end
| _ -> err p.ppat_loc "this pattern is not a valid regexp" | _ ->
err p.ppat_loc "this pattern is not a valid regexp"
in in
aux ~encoding:Ascii aux
let handle_sedlex_match ~env ~map_rhs match_expr =
let mapper cookies =
object(this)
inherit Ast_mapper_class.mapper as super
val env = builtin_regexps
method define_regexp name p =
{< env = StringMap.add name (regexp_of_pattern env p) env >}
method! expr e =
match e with
| [%expr [%sedlex [%e? {pexp_desc=Pexp_match (lexbuf, cases)}]]] ->
let lexbuf = let lexbuf =
match match_expr with
| { pexp_desc = Pexp_match (lexbuf, _); _ } -> (
match lexbuf with match lexbuf with
| { pexp_desc = Pexp_ident { txt = Lident txt; _ }; _ } -> | {pexp_desc=Pexp_ident{txt=Lident lexbuf}} -> lexbuf
(txt, lexbuf)
| _ -> | _ ->
err lexbuf.pexp_loc err lexbuf.pexp_loc "the matched expression must be a single identifier"
"the matched expression must be a single identifier")
| _ ->
err match_expr.pexp_loc
"the %%sedlex extension is only recognized on match expressions"
in
let cases =
match match_expr with
| { pexp_desc = Pexp_match (_, cases); _ } -> cases
| _ -> assert false
in in
let cases = List.rev cases in let cases = List.rev cases in
let error = let error =
match List.hd cases with match List.hd cases with
| { pc_lhs = [%pat? _]; pc_rhs = e; pc_guard = None } -> map_rhs e | {pc_lhs = [%pat? _]; pc_rhs = e; pc_guard = None} -> super # expr e
| { pc_lhs = p; _ } -> | {pc_lhs = p} ->
err p.ppat_loc "the last branch must be a catch-all error case" err p.ppat_loc "the last branch must be a catch-all error case"
in in
let cases = List.rev (List.tl cases) in let cases = List.rev (List.tl cases) in
let cases = let cases =
List.map List.map
(function (function
| { pc_lhs = p; pc_rhs = e; pc_guard = None } -> | {pc_lhs = p; pc_rhs = e; pc_guard = None} -> regexp_of_pattern env p, super # expr e
(regexp_of_pattern env p, map_rhs e) | {pc_guard = Some e} ->
| { pc_guard = Some e; _ } -> err e.pexp_loc "'when' guards are not supported"
err e.pexp_loc "'when' guards are not supported") ) cases
cases
in in
let brs = Array.of_list cases in gen_definition lexbuf cases error
let auto = Sedlex.compile (Array.map fst brs) in | [%expr let [%p? {ppat_desc=Ppat_var{txt=name}}] = [%sedlex.regexp? [%p? p]] in [%e? body]] ->
(gen_definition lexbuf auto cases error, auto) (this # define_regexp name p) # expr body
let previous = ref []
let regexps = ref []
let should_set_cookies = ref false
let mapper =
object (this)
inherit Ast_traverse.map as super
val env = builtin_regexps
method define_regexp name p =
{<env = StringMap.add name (regexp_of_pattern env p) env>}
method! expression e =
match e with
| [%expr [%sedlex [%e? { pexp_desc = Pexp_match _; _ } as match_expr]]]
->
fst (handle_sedlex_match ~env ~map_rhs:this#expression match_expr)
| [%expr
let [%p? { ppat_desc = Ppat_var { txt = name; _ }; _ }] =
[%sedlex.regexp? [%p? p]]
in
[%e? body]] ->
(this#define_regexp name p)#expression body
| [%expr [%sedlex [%e? _]]] -> | [%expr [%sedlex [%e? _]]] ->
err e.pexp_loc err e.pexp_loc "the %sedlex extension is only recognized on match expressions"
"the %%sedlex extension is only recognized on match expressions" | _ -> super # expr e
| _ -> super#expression e
val toplevel = true val toplevel = true
method structure_with_regexps l = method structure_with_regexps l =
let mapper = ref this in let mapper = ref this in
let regexps = ref [] in let regexps = ref [] in
let l = let l = List.concat
List.concat
(List.map (List.map
(function (function
| [%stri | [%stri let [%p? {ppat_desc=Ppat_var{txt=name}}] = [%sedlex.regexp? [%p? p]]] as i ->
let [%p? { ppat_desc = Ppat_var { txt = name; _ }; _ }] =
[%sedlex.regexp? [%p? p]]] as i ->
regexps := i :: !regexps; regexps := i :: !regexps;
mapper := !mapper # define_regexp name p; mapper := !mapper # define_regexp name p;
[] []
| i -> [!mapper#structure_item i]) | i ->
l) [ !mapper # structure_item i ]
in ) l) in
(l, List.rev !regexps) (l, List.rev !regexps)
method! structure l = method! structure l =
if toplevel then ( if toplevel then
let sub = {< toplevel = false >} in let sub = {< toplevel = false >} in
let l, regexps' = sub#structure_with_regexps (!previous @ l) in let previous =
let parts = List.map partition (get_partitions ()) in match Driver.get_cookie cookies "sedlex.regexps" ocaml_version with
let tables = List.map table (get_tables ()) in | Some {pexp_desc = Pexp_extension (_, PStr l)} -> l
regexps := regexps';
should_set_cookies := true;
tables @ parts @ l)
else fst (this#structure_with_regexps l)
end
let pre_handler cookies =
previous :=
match Driver.Cookies.get cookies "sedlex.regexps" Ast_pattern.__ with
| Some { pexp_desc = Pexp_extension (_, PStr l); _ } -> l
| Some _ -> assert false | Some _ -> assert false
| None -> [] | None -> []
in
let l, regexps = sub # structure_with_regexps (previous @ l) in
let parts = List.map partition (get_partitions ()) in
let tables = List.map table (get_tables ()) in
Driver.set_cookie cookies "sedlex.regexps" ocaml_version (Exp.extension (Location.mknoloc "regexps", PStr regexps));
tables @ parts @ l
else
fst (this # structure_with_regexps l)
let post_handler cookies = end
if !should_set_cookies then (
let loc = default_loc in
Driver.Cookies.set cookies "sedlex.regexps"
(pexp_extension ~loc ({ loc; txt = "regexps" }, PStr !regexps)))
let extensions =
[
Extension.declare "sedlex" Extension.Context.expression
Ast_pattern.(single_expr_payload __)
(fun ~loc:_ ~path:_ expr -> mapper#expression expr);
]
let () = let () =
Driver.Cookies.add_handler pre_handler; Driver.register
Driver.Cookies.add_post_handler post_handler; ~name:"sedlex"
Driver.register_transformation "sedlex" ~impl:mapper#structure ocaml_version
(fun _ cookies -> Ast_mapper_class.to_mapper (mapper cookies))

107
src/syntax/sedlex.ml
View File

@ -1,5 +1,6 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
module Cset = Sedlex_cset module Cset = Sedlex_cset
@ -16,7 +17,6 @@ type node = {
type regexp = node -> node type regexp = node -> node
let cur_id = ref 0 let cur_id = ref 0
let new_node () = let new_node () =
incr cur_id; incr cur_id;
{ id = !cur_id; eps = []; trans = [] } { id = !cur_id; eps = []; trans = [] }
@ -24,21 +24,21 @@ let new_node () =
let seq r1 r2 succ = r1 (r2 succ) let seq r1 r2 succ = r1 (r2 succ)
let is_chars final = function let is_chars final = function
| { eps = []; trans = [(c, f)]; _ } when f == final -> Some c | {eps = []; trans = [c, f]} when f == final -> Some c
| _ -> None | _ -> None
let chars c succ = let chars c succ =
let n = new_node () in let n = new_node () in
n.trans <- [(c, succ)]; n.trans <- [c,succ];
n n
let alt r1 r2 succ = let alt r1 r2 succ =
let nr1 = r1 succ and nr2 = r2 succ in let nr1 = r1 succ and nr2 = r2 succ in
match (is_chars succ nr1, is_chars succ nr2) with match is_chars succ nr1, is_chars succ nr2 with
| Some c1, Some c2 -> chars (Cset.union c1 c2) succ | Some c1, Some c2 -> chars (Cset.union c1 c2) succ
| _ -> | _ ->
let n = new_node () in let n = new_node () in
n.eps <- [nr1; nr2]; n.eps <- [r1 succ; r2 succ];
n n
let rep r succ = let rep r succ =
@ -57,18 +57,22 @@ let eps succ = succ (* eps for epsilon *)
let compl r = let compl r =
let n = new_node () in let n = new_node () in
match is_chars n (r n) with match is_chars n (r n) with
| Some c -> Some (chars (Cset.difference Cset.any c)) | Some c ->
| _ -> None Some (chars (Cset.difference Cset.any c))
| _ ->
None
let pair_op f r0 r1 = let pair_op f r0 r1 = (* Construct subtract or intersection *)
(* Construct subtract or intersection *)
let n = new_node () in let n = new_node () in
let to_chars r = is_chars n (r n) in let to_chars r = is_chars n (r n) in
match (to_chars r0, to_chars r1) with match to_chars r0, to_chars r1 with
| Some c0, Some c1 -> Some (chars (f c0 c1)) | Some c0, Some c1 ->
| _ -> None Some (chars (f c0 c1))
| _ ->
None
let subtract = pair_op Cset.difference let subtract = pair_op Cset.difference
let intersection = pair_op Cset.intersection let intersection = pair_op Cset.intersection
let compile_re re = let compile_re re =
@ -82,31 +86,32 @@ type state = node list
let rec add_node state node = let rec add_node state node =
if List.memq node state then state else add_nodes (node::state) node.eps if List.memq node state then state else add_nodes (node::state) node.eps
and add_nodes state nodes =
List.fold_left add_node state nodes
and add_nodes state nodes = List.fold_left add_node state nodes
let transition (state : state) = let transition (state : state) =
(* Merge transition with the same target *) (* Merge transition with the same target *)
let rec norm = function let rec norm = function
| (c1, n1)::((c2, n2)::q as l) -> | (c1, n1)::((c2, n2)::q as l) ->
if n1 == n2 then norm ((Cset.union c1 c2, n1)::q) if n1 == n2 then norm ((Cset.union c1 c2, n1)::q)
else (c1, n1) :: norm l else (c1, n1)::(norm l)
| l -> l | l -> l in
in
let t = List.concat (List.map (fun n -> n.trans) state) in let t = List.concat (List.map (fun n -> n.trans) state) in
let t = norm (List.sort (fun (_, n1) (_, n2) -> n1.id - n2.id) t) in let t = norm (List.sort (fun (_, n1) (_, n2) -> n1.id - n2.id) t) in
(* Split char sets so as to make them disjoint *) (* Split char sets so as to make them disjoint *)
let split (all, t) (c0, n0) = let split (all, t) (c0, n0) =
let t = let t =
(Cset.difference c0 all, [n0]) (Cset.difference c0 all, [n0]) ::
:: List.map (fun (c, ns) -> (Cset.intersection c c0, n0 :: ns)) t List.map (fun (c, ns) -> (Cset.intersection c c0, n0::ns)) t @
@ List.map (fun (c, ns) -> (Cset.difference c c0, ns)) t List.map (fun (c, ns) -> (Cset.difference c c0, ns)) t
in in
(Cset.union all c0, List.filter (fun (c, _) -> not (Cset.is_empty c)) t) Cset.union all c0,
List.filter (fun (c, _) -> not (Cset.is_empty c)) t
in in
let _, t = List.fold_left split (Cset.empty, []) t in let (_,t) = List.fold_left split (Cset.empty,[]) t in
(* Epsilon closure of targets *) (* Epsilon closure of targets *)
let t = List.map (fun (c, ns) -> (c, add_nodes [] ns)) t in let t = List.map (fun (c, ns) -> (c, add_nodes [] ns)) t in
@ -116,9 +121,6 @@ let transition (state : state) =
Array.sort (fun (c1, _) (c2, _) -> compare c1 c2) t; Array.sort (fun (c1, _) (c2, _) -> compare c1 c2) t;
t t
type dfa_state = { trans : (Cset.t * int) array; finals : bool array }
type dfa = dfa_state array
let compile rs = let compile rs =
let rs = Array.map compile_re rs in let rs = Array.map compile_re rs in
let counter = ref 0 in let counter = ref 0 in
@ -133,7 +135,7 @@ let compile rs =
let trans = transition state in let trans = transition state in
let trans = Array.map (fun (p, t) -> (p, aux t)) trans in let trans = Array.map (fun (p, t) -> (p, aux t)) trans in
let finals = Array.map (fun (_, f) -> List.memq f state) rs in let finals = Array.map (fun (_, f) -> List.memq f state) rs in
Hashtbl.add states_def i { trans; finals }; Hashtbl.add states_def i (trans, finals);
i i
in in
let init = ref [] in let init = ref [] in
@ -141,56 +143,3 @@ let compile rs =
let i = aux !init in let i = aux !init in
assert(i = 0); assert(i = 0);
Array.init !counter (Hashtbl.find states_def) Array.init !counter (Hashtbl.find states_def)
let cset_to_label cset =
let escape_dot c =
match c with
| '"' -> "\\\""
| '\\' -> "\\\\"
| '<' -> "\\<"
| '>' -> "\\>"
| _ -> String.make 1 c
in
let format_interval (lo, hi) =
if lo = -1 && hi = -1 then "EOF"
else if lo = hi then
if lo >= 32 && lo <= 126 then "'" ^ escape_dot (Char.chr lo) ^ "'"
else Printf.sprintf "U+%04X" lo
else if lo >= 32 && lo <= 126 && hi >= 32 && hi <= 126 then
"'" ^ escape_dot (Char.chr lo) ^ "'-'" ^ escape_dot (Char.chr hi) ^ "'"
else Printf.sprintf "U+%04X-U+%04X" lo hi
in
String.concat ", "
(List.map format_interval (cset : Cset.t :> (int * int) list))
let dfa_to_dot dfa =
let buf = Buffer.create 1024 in
let bprintf = Printf.bprintf in
bprintf buf "digraph {\n";
bprintf buf " rankdir=LR;\n";
bprintf buf " node [shape=circle];\n\n";
bprintf buf " _start [shape=point];\n";
bprintf buf " _start -> state0;\n\n";
Array.iteri
(fun i { trans; finals } ->
let accepted =
let acc = ref [] in
for r = Array.length finals - 1 downto 0 do
if finals.(r) then acc := r :: !acc
done;
!acc
in
(match accepted with
| [] -> bprintf buf " state%d [label=\"%d\"];\n" i i
| rules ->
bprintf buf
" state%d [label=\"%d\\n[rule %s]\", shape=doublecircle];\n" i i
(String.concat "," (List.map string_of_int rules)));
Array.iter
(fun (cset, target) ->
let label = cset_to_label cset in
bprintf buf " state%d -> state%d [label=\"%s\"];\n" i target label)
trans)
dfa;
bprintf buf "}\n";
Buffer.contents buf

12
src/syntax/sedlex.mli
View File

@ -1,5 +1,6 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
type regexp type regexp
@ -9,20 +10,15 @@ val alt : regexp -> regexp -> regexp
val rep: regexp -> regexp val rep: regexp -> regexp
val plus: regexp -> regexp val plus: regexp -> regexp
val eps: regexp val eps: regexp
val compl : regexp -> regexp option
val compl: regexp -> regexp option
(* If the argument is a single [chars] regexp, returns a regexp (* If the argument is a single [chars] regexp, returns a regexp
which matches the complement set. Otherwise returns [None]. *) which matches the complement set. Otherwise returns [None]. *)
val subtract: regexp -> regexp -> regexp option val subtract: regexp -> regexp -> regexp option
(* If each argument is a single [chars] regexp, returns a regexp (* If each argument is a single [chars] regexp, returns a regexp
which matches the set (arg1 - arg2). Otherwise returns [None]. *) which matches the set (arg1 - arg2). Otherwise returns [None]. *)
val intersection: regexp -> regexp -> regexp option val intersection: regexp -> regexp -> regexp option
(* If each argument is a single [chars] regexp, returns a regexp (* If each argument is a single [chars] regexp, returns a regexp
which matches the intersection set. Otherwise returns [None]. *) which matches the intersection set. Otherwise returns [None]. *)
type dfa_state = { trans : (Sedlex_cset.t * int) array; finals : bool array } val compile: regexp array -> ((Sedlex_cset.t * int) array * bool array) array
type dfa = dfa_state array
val compile : regexp array -> dfa
val dfa_to_dot : dfa -> string

399
src/syntax/sedlex_cset.ml Normal file → Executable file
View File

@ -1,4 +1,399 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
include Sedlex_utils.Cset (* Character sets are represented as lists of intervals. The
intervals must be non-overlapping and not collapsable, and the list
must be ordered in increasing order. *)
type t = (int * int) list
let max_code = 0x10ffff (* must be < max_int *)
let min_code = -1
let empty = []
let singleton i = [i,i]
let is_empty = function [] -> true | _ -> false
let interval i j = if i <= j then [i,j] else [j,i]
let eof = singleton (-1)
let any = interval 0 max_code
let rec union c1 c2 =
match c1,c2 with
| [], _ -> c2
| _, [] -> c1
| ((i1, j1) as s1)::r1, (i2, j2)::r2 ->
if (i1 <= i2) then
if j1 + 1 < i2 then s1::(union r1 c2)
else if (j1 < j2) then union r1 ((i1, j2)::r2)
else union c1 r2
else union c2 c1
let complement c =
let rec aux start = function
| [] -> if start <= max_code then [start,max_code] else []
| (i, j)::l -> (start, i-1)::(aux (succ j) l)
in
match c with
| (-1,j)::l -> aux (succ j) l
| l -> aux (-1) l
let intersection c1 c2 =
complement (union (complement c1) (complement c2))
let difference c1 c2 =
complement (union (complement c1) c2)
(* Unicode classes from XML *)
let base_char =
[ 0x0041,0x005A; 0x0061,0x007A; 0x00C0,0x00D6; 0x00D8,0x00F6;
0x00F8,0x00FF; 0x0100,0x0131; 0x0134,0x013E; 0x0141,0x0148;
0x014A,0x017E; 0x0180,0x01C3; 0x01CD,0x01F0; 0x01F4,0x01F5;
0x01FA,0x0217; 0x0250,0x02A8; 0x02BB,0x02C1; 0x0386,0x0386;
0x0388,0x038A; 0x038C,0x038C; 0x038E,0x03A1; 0x03A3,0x03CE;
0x03D0,0x03D6; 0x03DA,0x03DA; 0x03DC,0x03DC; 0x03DE,0x03DE;
0x03E0,0x03E0; 0x03E2,0x03F3;
0x0401,0x040C; 0x040E,0x044F; 0x0451,0x045C; 0x045E,0x0481;
0x0490,0x04C4; 0x04C7,0x04C8; 0x04CB,0x04CC; 0x04D0,0x04EB;
0x04EE,0x04F5; 0x04F8,0x04F9; 0x0531,0x0556; 0x0559,0x0559;
0x0561,0x0586; 0x05D0,0x05EA; 0x05F0,0x05F2; 0x0621,0x063A;
0x0641,0x064A; 0x0671,0x06B7; 0x06BA,0x06BE; 0x06C0,0x06CE;
0x06D0,0x06D3; 0x06D5,0x06D5; 0x06E5,0x06E6; 0x0905,0x0939;
0x093D,0x093D;
0x0958,0x0961; 0x0985,0x098C; 0x098F,0x0990; 0x0993,0x09A8;
0x09AA,0x09B0; 0x09B2,0x09B2; 0x09B6,0x09B9; 0x09DC,0x09DD;
0x09DF,0x09E1; 0x09F0,0x09F1; 0x0A05,0x0A0A; 0x0A0F,0x0A10;
0x0A13,0x0A28; 0x0A2A,0x0A30; 0x0A32,0x0A33; 0x0A35,0x0A36;
0x0A38,0x0A39; 0x0A59,0x0A5C; 0x0A5E,0x0A5E; 0x0A72,0x0A74;
0x0A85,0x0A8B; 0x0A8D,0x0A8D; 0x0A8F,0x0A91; 0x0A93,0x0AA8;
0x0AAA,0x0AB0; 0x0AB2,0x0AB3; 0x0AB5,0x0AB9; 0x0ABD,0x0ABD;
0x0AE0,0x0AE0;
0x0B05,0x0B0C; 0x0B0F,0x0B10; 0x0B13,0x0B28; 0x0B2A,0x0B30;
0x0B32,0x0B33; 0x0B36,0x0B39; 0x0B3D,0x0B3D; 0x0B5C,0x0B5D;
0x0B5F,0x0B61; 0x0B85,0x0B8A; 0x0B8E,0x0B90; 0x0B92,0x0B95;
0x0B99,0x0B9A; 0x0B9C,0x0B9C; 0x0B9E,0x0B9F; 0x0BA3,0x0BA4;
0x0BA8,0x0BAA; 0x0BAE,0x0BB5; 0x0BB7,0x0BB9; 0x0C05,0x0C0C;
0x0C0E,0x0C10; 0x0C12,0x0C28; 0x0C2A,0x0C33; 0x0C35,0x0C39;
0x0C60,0x0C61; 0x0C85,0x0C8C; 0x0C8E,0x0C90; 0x0C92,0x0CA8;
0x0CAA,0x0CB3; 0x0CB5,0x0CB9; 0x0CDE,0x0CDE; 0x0CE0,0x0CE1;
0x0D05,0x0D0C; 0x0D0E,0x0D10; 0x0D12,0x0D28; 0x0D2A,0x0D39;
0x0D60,0x0D61; 0x0E01,0x0E2E; 0x0E30,0x0E30; 0x0E32,0x0E33;
0x0E40,0x0E45; 0x0E81,0x0E82; 0x0E84,0x0E84; 0x0E87,0x0E88;
0x0E8A,0x0E8A;
0x0E8D,0x0E8D; 0x0E94,0x0E97; 0x0E99,0x0E9F; 0x0EA1,0x0EA3;
0x0EA5,0x0EA5;
0x0EA7,0x0EA7; 0x0EAA,0x0EAB; 0x0EAD,0x0EAE; 0x0EB0,0x0EB0;
0x0EB2,0x0EB3;
0x0EBD,0x0EBD; 0x0EC0,0x0EC4; 0x0F40,0x0F47; 0x0F49,0x0F69;
0x10A0,0x10C5; 0x10D0,0x10F6; 0x1100,0x1100; 0x1102,0x1103;
0x1105,0x1107; 0x1109,0x1109; 0x110B,0x110C; 0x110E,0x1112;
0x113C,0x113C;
0x113E,0x113E; 0x1140,0x1140; 0x114C,0x114C; 0x114E,0x114E;
0x1150,0x1150; 0x1154,0x1155; 0x1159,0x1159;
0x115F,0x1161; 0x1163,0x1163; 0x1165,0x1165; 0x1167,0x1167;
0x1169,0x1169; 0x116D,0x116E;
0x1172,0x1173; 0x1175,0x1175; 0x119E,0x119E; 0x11A8,0x11A8;
0x11AB,0x11AB; 0x11AE,0x11AF;
0x11B7,0x11B8; 0x11BA,0x11BA; 0x11BC,0x11C2; 0x11EB,0x11EB;
0x11F0,0x11F0; 0x11F9,0x11F9;
0x1E00,0x1E9B; 0x1EA0,0x1EF9; 0x1F00,0x1F15; 0x1F18,0x1F1D;
0x1F20,0x1F45; 0x1F48,0x1F4D; 0x1F50,0x1F57; 0x1F59,0x1F59;
0x1F5B,0x1F5B;
0x1F5D,0x1F5D; 0x1F5F,0x1F7D; 0x1F80,0x1FB4; 0x1FB6,0x1FBC;
0x1FBE,0x1FBE;
0x1FC2,0x1FC4; 0x1FC6,0x1FCC; 0x1FD0,0x1FD3; 0x1FD6,0x1FDB;
0x1FE0,0x1FEC; 0x1FF2,0x1FF4; 0x1FF6,0x1FFC; 0x2126,0x2126;
0x212A,0x212B; 0x212E,0x212E; 0x2180,0x2182; 0x3041,0x3094;
0x30A1,0x30FA; 0x3105,0x312C; 0xAC00,0xD7A3 ]
let ideographic =
[ 0x3007,0x3007; 0x3021,0x3029; 0x4E00,0x9FA5 ]
let combining_char =
[ 0x0300,0x0345; 0x0360,0x0361; 0x0483,0x0486; 0x0591,0x05A1;
0x05A3,0x05B9; 0x05BB,0x05BD; 0x05BF,0x05BF; 0x05C1,0x05C2;
0x05C4,0x05C4; 0x064B,0x0652; 0x0670,0x0670; 0x06D6,0x06DC;
0x06DD,0x06DF; 0x06E0,0x06E4; 0x06E7,0x06E8; 0x06EA,0x06ED;
0x0901,0x0903; 0x093C,0x093C; 0x093E,0x094C; 0x094D,0x094D;
0x0951,0x0954; 0x0962,0x0963; 0x0981,0x0983; 0x09BC,0x09BC;
0x09BE,0x09BE; 0x09BF,0x09BF; 0x09C0,0x09C4; 0x09C7,0x09C8;
0x09CB,0x09CD; 0x09D7,0x09D7; 0x09E2,0x09E3; 0x0A02,0x0A02;
0x0A3C,0x0A3C; 0x0A3E,0x0A3E; 0x0A3F,0x0A3F; 0x0A40,0x0A42;
0x0A47,0x0A48; 0x0A4B,0x0A4D; 0x0A70,0x0A71; 0x0A81,0x0A83;
0x0ABC,0x0ABC; 0x0ABE,0x0AC5; 0x0AC7,0x0AC9; 0x0ACB,0x0ACD;
0x0B01,0x0B03; 0x0B3C,0x0B3C; 0x0B3E,0x0B43; 0x0B47,0x0B48;
0x0B4B,0x0B4D; 0x0B56,0x0B57; 0x0B82,0x0B83; 0x0BBE,0x0BC2;
0x0BC6,0x0BC8; 0x0BCA,0x0BCD; 0x0BD7,0x0BD7; 0x0C01,0x0C03;
0x0C3E,0x0C44; 0x0C46,0x0C48; 0x0C4A,0x0C4D; 0x0C55,0x0C56;
0x0C82,0x0C83; 0x0CBE,0x0CC4; 0x0CC6,0x0CC8; 0x0CCA,0x0CCD;
0x0CD5,0x0CD6; 0x0D02,0x0D03; 0x0D3E,0x0D43; 0x0D46,0x0D48;
0x0D4A,0x0D4D; 0x0D57,0x0D57; 0x0E31,0x0E31; 0x0E34,0x0E3A;
0x0E47,0x0E4E; 0x0EB1,0x0EB1; 0x0EB4,0x0EB9; 0x0EBB,0x0EBC;
0x0EC8,0x0ECD; 0x0F18,0x0F19; 0x0F35,0x0F35; 0x0F37,0x0F37;
0x0F39,0x0F39; 0x0F3E,0x0F3E; 0x0F3F,0x0F3F; 0x0F71,0x0F84;
0x0F86,0x0F8B; 0x0F90,0x0F95; 0x0F97,0x0F97; 0x0F99,0x0FAD;
0x0FB1,0x0FB7; 0x0FB9,0x0FB9; 0x20D0,0x20DC; 0x20E1,0x20E1;
0x302A,0x302F; 0x3099,0x3099; 0x309A,0x309A ]
let digit =
[ 0x0030,0x0039;
0x0660,0x0669; 0x06F0,0x06F9; 0x0966,0x096F; 0x09E6,0x09EF;
0x0A66,0x0A6F; 0x0AE6,0x0AEF; 0x0B66,0x0B6F; 0x0BE7,0x0BEF;
0x0C66,0x0C6F; 0x0CE6,0x0CEF; 0x0D66,0x0D6F; 0x0E50,0x0E59;
0x0ED0,0x0ED9; 0x0F20,0x0F29 ]
let extender =
[ 0x00B7,0x00B7; 0x02D0,0x02D1; 0x0387,0x0387; 0x0640,0x0640;
0x0E46,0x0E46; 0x0EC6,0x0EC6; 0x3005,0x3005; 0x3031,0x3035;
0x309D,0x309E; 0x30FC,0x30FE ]
let blank =
[ 0x0009,0x000A; 0x000D,0x000D; 0x0020,0x0020 ]
let letter = union base_char ideographic
(* Letters to be used in identifiers, as specified
by ISO ....
Data provided by John M. Skaller *)
let tr8876_ident_char = [
(* ASCII *)
(0x0041,0x005a);
(0x0061,0x007a);
(* Latin *)
(0x00c0,0x00d6);
(0x00d8,0x00f6);
(0x00f8,0x01f5);
(0x01fa,0x0217);
(0x0250,0x02a8);
(* Greek *)
(0x0384,0x0384);
(0x0388,0x038a);
(0x038c,0x038c);
(0x038e,0x03a1);
(0x03a3,0x03ce);
(0x03d0,0x03d6);
(0x03da,0x03da);
(0x03dc,0x03dc);
(0x03de,0x03de);
(0x03e0,0x03e0);
(0x03e2,0x03f3);
(* Cyrillic *)
(0x0401,0x040d);
(0x040f,0x044f);
(0x0451,0x045c);
(0x045e,0x0481);
(0x0490,0x04c4);
(0x04c7,0x04c4);
(0x04cb,0x04cc);
(0x04d0,0x04eb);
(0x04ee,0x04f5);
(0x04f8,0x04f9);
(* Armenian *)
(0x0531,0x0556);
(0x0561,0x0587);
(0x04d0,0x04eb);
(* Hebrew *)
(0x05d0,0x05ea);
(0x05f0,0x05f4);
(* Arabic *)
(0x0621,0x063a);
(0x0640,0x0652);
(0x0670,0x06b7);
(0x06ba,0x06be);
(0x06c0,0x06ce);
(0x06e5,0x06e7);
(* Devanagari *)
(0x0905,0x0939);
(0x0958,0x0962);
(* Bengali *)
(0x0985,0x098c);
(0x098f,0x0990);
(0x0993,0x09a8);
(0x09aa,0x09b0);
(0x09b2,0x09b2);
(0x09b6,0x09b9);
(0x09dc,0x09dd);
(0x09df,0x09e1);
(0x09f0,0x09f1);
(* Gurmukhi *)
(0x0a05,0x0a0a);
(0x0a0f,0x0a10);
(0x0a13,0x0a28);
(0x0a2a,0x0a30);
(0x0a32,0x0a33);
(0x0a35,0x0a36);
(0x0a38,0x0a39);
(0x0a59,0x0a5c);
(0x0a5e,0x0a5e);
(* Gunjarati *)
(0x0a85,0x0a8b);
(0x0a8d,0x0a8d);
(0x0a8f,0x0a91);
(0x0a93,0x0aa8);
(0x0aaa,0x0ab0);
(0x0ab2,0x0ab3);
(0x0ab5,0x0ab9);
(0x0ae0,0x0ae0);
(* Oriya *)
(0x0b05,0x0b0c);
(0x0b0f,0x0b10);
(0x0b13,0x0b28);
(0x0b2a,0x0b30);
(0x0b32,0x0b33);
(0x0b36,0x0b39);
(0x0b5c,0x0b5d);
(0x0b5f,0x0b61);
(* Tamil *)
(0x0b85,0x0b8a);
(0x0b8e,0x0b90);
(0x0b92,0x0b95);
(0x0b99,0x0b9a);
(0x0b9c,0x0b9c);
(0x0b9e,0x0b9f);
(0x0ba3,0x0ba4);
(0x0ba8,0x0baa);
(0x0bae,0x0bb5);
(0x0bb7,0x0bb9);
(* Telugu *)
(0x0c05,0x0c0c);
(0x0c0e,0x0c10);
(0x0c12,0x0c28);
(0x0c2a,0x0c33);
(0x0c35,0x0c39);
(0x0c60,0x0c61);
(* Kannada *)
(0x0c85,0x0c8c);
(0x0c8e,0x0c90);
(0x0c92,0x0ca8);
(0x0caa,0x0cb3);
(0x0cb5,0x0cb9);
(0x0ce0,0x0ce1);
(* Malayam *)
(0x0d05,0x0d0c);
(0x0d0e,0x0d10);
(0x0d12,0x0d28);
(0x0d2a,0x0d39);
(0x0d60,0x0d61);
(* Thai *)
(0x0e01,0x0e30);
(0x0e32,0x0e33);
(0x0e40,0x0e46);
(0x0e4f,0x0e5b);
(* Lao *)
(0x0e81,0x0e82);
(0x0e84,0x0e84);
(0x0e87,0x0e88);
(0x0e8a,0x0e8a);
(0x0e0d,0x0e0d);
(0x0e94,0x0e97);
(0x0e99,0x0e9f);
(0x0ea1,0x0ea3);
(0x0ea5,0x0ea5);
(0x0ea7,0x0ea7);
(0x0eaa,0x0eab);
(0x0ead,0x0eb0);
(0x0eb2,0x0eb3);
(0x0ebd,0x0ebd);
(0x0ec0,0x0ec4);
(0x0ec6,0x0ec6);
(* Georgian *)
(0x10a0,0x10c5);
(0x10d0,0x10f6);
(* Hangul Jamo *)
(0x1100,0x1159);
(0x1161,0x11a2);
(0x11a8,0x11f9);
(0x11d0,0x11f6);
(* Latin extensions *)
(0x1e00,0x1e9a);
(0x1ea0,0x1ef9);
(* Greek extended *)
(0x1f00,0x1f15);
(0x1f18,0x1f1d);
(0x1f20,0x1f45);
(0x1f48,0x1f4d);
(0x1f50,0x1f57);
(0x1f59,0x1f59);
(0x1f5b,0x1f5b);
(0x1f5d,0x1f5d);
(0x1f5f,0x1f7d);
(0x1f80,0x1fb4);
(0x1fb6,0x1fbc);
(0x1fc2,0x1fc4);
(0x1fc6,0x1fcc);
(0x1fd0,0x1fd3);
(0x1fd6,0x1fdb);
(0x1fe0,0x1fec);
(0x1ff2,0x1ff4);
(0x1ff6,0x1ffc);
(* Hiragana *)
(0x3041,0x3094);
(0x309b,0x309e);
(* Katakana *)
(0x30a1,0x30fe);
(* Bopmofo *)
(0x3105,0x312c);
(* CJK Unified Ideographs *)
(0x4e00,0x9fa5);
(* CJK Compatibility Ideographs *)
(0xf900,0xfa2d);
(* Arabic Presentation Forms *)
(0xfb1f,0xfb36);
(0xfb38,0xfb3c);
(0xfb3e,0xfb3e);
(0xfb40,0xfb41);
(0xfb42,0xfb44);
(0xfb46,0xfbb1);
(0xfbd3,0xfd35);
(* Arabic Presentation Forms-A *)
(0xfd50,0xfd85);
(0xfd92,0xfbc7);
(0xfdf0,0xfdfb);
(* Arabic Presentation Forms-B *)
(0xfe70,0xfe72);
(0xfe74,0xfe74);
(0xfe76,0xfefc);
(* Half width and Fullwidth Forms *)
(0xff21,0xff3a);
(0xff41,0xff5a);
(0xff66,0xffbe);
(0xffc2,0xffc7);
(0xffca,0xffcf);
(0xffd2,0xffd7);
(0xffd2,0xffd7);
(0xffda,0xffdc)
]

30
src/syntax/sedlex_cset.mli Normal file
View File

@ -0,0 +1,30 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
(** Representation of sets of unicode code points. *)
type t = (int * int) list
val min_code: int
val max_code: int
val empty: t
val any: t
val union: t -> t -> t
val difference: t -> t -> t
val intersection: t -> t -> t
val is_empty: t -> bool
val eof: t
val singleton: int -> t
val interval: int -> int -> t
val letter: t
val digit: t
val extender: t
val base_char: t
val ideographic: t
val combining_char: t
val blank: t
val tr8876_ident_char: t

2181
src/syntax/unicode.ml
View File

File diff suppressed because it is too large Load Diff

1869
src/syntax/unicode63.ml Normal file
View File

File diff suppressed because it is too large Load Diff

10
src/syntax/unicode.mli → src/syntax/unicode63.mli
View File

@ -1,8 +1,5 @@
(* Version is automatically generated from unicode data at
* src/generator/data *)
val version : string
module Categories : sig module Categories : sig
val cc : Sedlex_cset.t val cc : Sedlex_cset.t
val cf : Sedlex_cset.t val cf : Sedlex_cset.t
val cn : Sedlex_cset.t val cn : Sedlex_cset.t
@ -33,10 +30,11 @@ module Categories : sig
val zl : Sedlex_cset.t val zl : Sedlex_cset.t
val zp : Sedlex_cset.t val zp : Sedlex_cset.t
val zs : Sedlex_cset.t val zs : Sedlex_cset.t
val list : (string * Sedlex_cset.t) list
end end
module Properties : sig module Properties : sig
val alphabetic : Sedlex_cset.t val alphabetic : Sedlex_cset.t
val ascii_hex_digit : Sedlex_cset.t val ascii_hex_digit : Sedlex_cset.t
val hex_digit : Sedlex_cset.t val hex_digit : Sedlex_cset.t
@ -52,5 +50,5 @@ module Properties : sig
val white_space : Sedlex_cset.t val white_space : Sedlex_cset.t
val xid_continue : Sedlex_cset.t val xid_continue : Sedlex_cset.t
val xid_start : Sedlex_cset.t val xid_start : Sedlex_cset.t
val list : (string * Sedlex_cset.t) list
end end

49
src/syntax/utf8.ml
View File

@ -1,49 +0,0 @@
open Sedlexing
let unsafe_byte s j = Char.code (String.unsafe_get s j)
let malformed s j l = `Malformed (String.sub s j l)
let r_utf_8 s j l =
(* assert (0 <= j && 0 <= l && j + l <= String.length s); *)
let uchar c = `Uchar (Uchar.unsafe_of_int c) in
match l with
| 1 -> uchar (unsafe_byte s j)
| 2 -> (
let b0 = unsafe_byte s j in
let b1 = unsafe_byte s (j + 1) in
match Utf8.Helper.check_two b0 b1 with
| i -> uchar i
| exception MalFormed -> malformed s j l)
| 3 -> (
let b0 = unsafe_byte s j in
let b1 = unsafe_byte s (j + 1) in
let b2 = unsafe_byte s (j + 2) in
match Utf8.Helper.check_three b0 b1 b2 with
| i -> uchar i
| exception MalFormed -> malformed s j l)
| 4 -> (
let b0 = unsafe_byte s j in
let b1 = unsafe_byte s (j + 1) in
let b2 = unsafe_byte s (j + 2) in
let b3 = unsafe_byte s (j + 3) in
match Utf8.Helper.check_four b0 b1 b2 b3 with
| i -> uchar i
| exception MalFormed -> malformed s j l)
| _ -> assert false
let fold ~f acc s =
let rec loop acc f s i last =
if i > last then acc
else (
match Utf8.Helper.width (String.unsafe_get s i) with
| exception MalFormed ->
loop (f acc i (malformed s i 1)) f s (i + 1) last
| need ->
let rem = last - i + 1 in
if rem < need then f acc i (malformed s i rem)
else loop (f acc i (r_utf_8 s i need)) f s (i + need) last)
in
let pos = 0 in
let len = String.length s in
let last = pos + len - 1 in
loop acc f s pos last

5
src/syntax/utf8.mli
View File

@ -1,5 +0,0 @@
val fold :
f:('a -> int -> [> `Malformed of string | `Uchar of Uchar.t ] -> 'a) ->
'a ->
string ->
'a

365
src/syntax/xml.ml
View File

@ -1,365 +0,0 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
open Sedlex_cset
(* Unicode classes from XML *)
let base_char =
let l =
[
(0x0041, 0x005A);
(0x0061, 0x007A);
(0x00C0, 0x00D6);
(0x00D8, 0x00F6);
(0x00F8, 0x00FF);
(0x0100, 0x0131);
(0x0134, 0x013E);
(0x0141, 0x0148);
(0x014A, 0x017E);
(0x0180, 0x01C3);
(0x01CD, 0x01F0);
(0x01F4, 0x01F5);
(0x01FA, 0x0217);
(0x0250, 0x02A8);
(0x02BB, 0x02C1);
(0x0386, 0x0386);
(0x0388, 0x038A);
(0x038C, 0x038C);
(0x038E, 0x03A1);
(0x03A3, 0x03CE);
(0x03D0, 0x03D6);
(0x03DA, 0x03DA);
(0x03DC, 0x03DC);
(0x03DE, 0x03DE);
(0x03E0, 0x03E0);
(0x03E2, 0x03F3);
(0x0401, 0x040C);
(0x040E, 0x044F);
(0x0451, 0x045C);
(0x045E, 0x0481);
(0x0490, 0x04C4);
(0x04C7, 0x04C8);
(0x04CB, 0x04CC);
(0x04D0, 0x04EB);
(0x04EE, 0x04F5);
(0x04F8, 0x04F9);
(0x0531, 0x0556);
(0x0559, 0x0559);
(0x0561, 0x0586);
(0x05D0, 0x05EA);
(0x05F0, 0x05F2);
(0x0621, 0x063A);
(0x0641, 0x064A);
(0x0671, 0x06B7);
(0x06BA, 0x06BE);
(0x06C0, 0x06CE);
(0x06D0, 0x06D3);
(0x06D5, 0x06D5);
(0x06E5, 0x06E6);
(0x0905, 0x0939);
(0x093D, 0x093D);
(0x0958, 0x0961);
(0x0985, 0x098C);
(0x098F, 0x0990);
(0x0993, 0x09A8);
(0x09AA, 0x09B0);
(0x09B2, 0x09B2);
(0x09B6, 0x09B9);
(0x09DC, 0x09DD);
(0x09DF, 0x09E1);
(0x09F0, 0x09F1);
(0x0A05, 0x0A0A);
(0x0A0F, 0x0A10);
(0x0A13, 0x0A28);
(0x0A2A, 0x0A30);
(0x0A32, 0x0A33);
(0x0A35, 0x0A36);
(0x0A38, 0x0A39);
(0x0A59, 0x0A5C);
(0x0A5E, 0x0A5E);
(0x0A72, 0x0A74);
(0x0A85, 0x0A8B);
(0x0A8D, 0x0A8D);
(0x0A8F, 0x0A91);
(0x0A93, 0x0AA8);
(0x0AAA, 0x0AB0);
(0x0AB2, 0x0AB3);
(0x0AB5, 0x0AB9);
(0x0ABD, 0x0ABD);
(0x0AE0, 0x0AE0);
(0x0B05, 0x0B0C);
(0x0B0F, 0x0B10);
(0x0B13, 0x0B28);
(0x0B2A, 0x0B30);
(0x0B32, 0x0B33);
(0x0B36, 0x0B39);
(0x0B3D, 0x0B3D);
(0x0B5C, 0x0B5D);
(0x0B5F, 0x0B61);
(0x0B85, 0x0B8A);
(0x0B8E, 0x0B90);
(0x0B92, 0x0B95);
(0x0B99, 0x0B9A);
(0x0B9C, 0x0B9C);
(0x0B9E, 0x0B9F);
(0x0BA3, 0x0BA4);
(0x0BA8, 0x0BAA);
(0x0BAE, 0x0BB5);
(0x0BB7, 0x0BB9);
(0x0C05, 0x0C0C);
(0x0C0E, 0x0C10);
(0x0C12, 0x0C28);
(0x0C2A, 0x0C33);
(0x0C35, 0x0C39);
(0x0C60, 0x0C61);
(0x0C85, 0x0C8C);
(0x0C8E, 0x0C90);
(0x0C92, 0x0CA8);
(0x0CAA, 0x0CB3);
(0x0CB5, 0x0CB9);
(0x0CDE, 0x0CDE);
(0x0CE0, 0x0CE1);
(0x0D05, 0x0D0C);
(0x0D0E, 0x0D10);
(0x0D12, 0x0D28);
(0x0D2A, 0x0D39);
(0x0D60, 0x0D61);
(0x0E01, 0x0E2E);
(0x0E30, 0x0E30);
(0x0E32, 0x0E33);
(0x0E40, 0x0E45);
(0x0E81, 0x0E82);
(0x0E84, 0x0E84);
(0x0E87, 0x0E88);
(0x0E8A, 0x0E8A);
(0x0E8D, 0x0E8D);
(0x0E94, 0x0E97);
(0x0E99, 0x0E9F);
(0x0EA1, 0x0EA3);
(0x0EA5, 0x0EA5);
(0x0EA7, 0x0EA7);
(0x0EAA, 0x0EAB);
(0x0EAD, 0x0EAE);
(0x0EB0, 0x0EB0);
(0x0EB2, 0x0EB3);
(0x0EBD, 0x0EBD);
(0x0EC0, 0x0EC4);
(0x0F40, 0x0F47);
(0x0F49, 0x0F69);
(0x10A0, 0x10C5);
(0x10D0, 0x10F6);
(0x1100, 0x1100);
(0x1102, 0x1103);
(0x1105, 0x1107);
(0x1109, 0x1109);
(0x110B, 0x110C);
(0x110E, 0x1112);
(0x113C, 0x113C);
(0x113E, 0x113E);
(0x1140, 0x1140);
(0x114C, 0x114C);
(0x114E, 0x114E);
(0x1150, 0x1150);
(0x1154, 0x1155);
(0x1159, 0x1159);
(0x115F, 0x1161);
(0x1163, 0x1163);
(0x1165, 0x1165);
(0x1167, 0x1167);
(0x1169, 0x1169);
(0x116D, 0x116E);
(0x1172, 0x1173);
(0x1175, 0x1175);
(0x119E, 0x119E);
(0x11A8, 0x11A8);
(0x11AB, 0x11AB);
(0x11AE, 0x11AF);
(0x11B7, 0x11B8);
(0x11BA, 0x11BA);
(0x11BC, 0x11C2);
(0x11EB, 0x11EB);
(0x11F0, 0x11F0);
(0x11F9, 0x11F9);
(0x1E00, 0x1E9B);
(0x1EA0, 0x1EF9);
(0x1F00, 0x1F15);
(0x1F18, 0x1F1D);
(0x1F20, 0x1F45);
(0x1F48, 0x1F4D);
(0x1F50, 0x1F57);
(0x1F59, 0x1F59);
(0x1F5B, 0x1F5B);
(0x1F5D, 0x1F5D);
(0x1F5F, 0x1F7D);
(0x1F80, 0x1FB4);
(0x1FB6, 0x1FBC);
(0x1FBE, 0x1FBE);
(0x1FC2, 0x1FC4);
(0x1FC6, 0x1FCC);
(0x1FD0, 0x1FD3);
(0x1FD6, 0x1FDB);
(0x1FE0, 0x1FEC);
(0x1FF2, 0x1FF4);
(0x1FF6, 0x1FFC);
(0x2126, 0x2126);
(0x212A, 0x212B);
(0x212E, 0x212E);
(0x2180, 0x2182);
(0x3041, 0x3094);
(0x30A1, 0x30FA);
(0x3105, 0x312C);
(0xAC00, 0xD7A3);
]
in
of_list l
let ideographic =
let l = [(0x3007, 0x3007); (0x3021, 0x3029); (0x4E00, 0x9FA5)] in
of_list l
let combining_char =
let l =
[
(0x0300, 0x0345);
(0x0360, 0x0361);
(0x0483, 0x0486);
(0x0591, 0x05A1);
(0x05A3, 0x05B9);
(0x05BB, 0x05BD);
(0x05BF, 0x05BF);
(0x05C1, 0x05C2);
(0x05C4, 0x05C4);
(0x064B, 0x0652);
(0x0670, 0x0670);
(0x06D6, 0x06DC);
(0x06DD, 0x06DF);
(0x06E0, 0x06E4);
(0x06E7, 0x06E8);
(0x06EA, 0x06ED);
(0x0901, 0x0903);
(0x093C, 0x093C);
(0x093E, 0x094C);
(0x094D, 0x094D);
(0x0951, 0x0954);
(0x0962, 0x0963);
(0x0981, 0x0983);
(0x09BC, 0x09BC);
(0x09BE, 0x09BE);
(0x09BF, 0x09BF);
(0x09C0, 0x09C4);
(0x09C7, 0x09C8);
(0x09CB, 0x09CD);
(0x09D7, 0x09D7);
(0x09E2, 0x09E3);
(0x0A02, 0x0A02);
(0x0A3C, 0x0A3C);
(0x0A3E, 0x0A3E);
(0x0A3F, 0x0A3F);
(0x0A40, 0x0A42);
(0x0A47, 0x0A48);
(0x0A4B, 0x0A4D);
(0x0A70, 0x0A71);
(0x0A81, 0x0A83);
(0x0ABC, 0x0ABC);
(0x0ABE, 0x0AC5);
(0x0AC7, 0x0AC9);
(0x0ACB, 0x0ACD);
(0x0B01, 0x0B03);
(0x0B3C, 0x0B3C);
(0x0B3E, 0x0B43);
(0x0B47, 0x0B48);
(0x0B4B, 0x0B4D);
(0x0B56, 0x0B57);
(0x0B82, 0x0B83);
(0x0BBE, 0x0BC2);
(0x0BC6, 0x0BC8);
(0x0BCA, 0x0BCD);
(0x0BD7, 0x0BD7);
(0x0C01, 0x0C03);
(0x0C3E, 0x0C44);
(0x0C46, 0x0C48);
(0x0C4A, 0x0C4D);
(0x0C55, 0x0C56);
(0x0C82, 0x0C83);
(0x0CBE, 0x0CC4);
(0x0CC6, 0x0CC8);
(0x0CCA, 0x0CCD);
(0x0CD5, 0x0CD6);
(0x0D02, 0x0D03);
(0x0D3E, 0x0D43);
(0x0D46, 0x0D48);
(0x0D4A, 0x0D4D);
(0x0D57, 0x0D57);
(0x0E31, 0x0E31);
(0x0E34, 0x0E3A);
(0x0E47, 0x0E4E);
(0x0EB1, 0x0EB1);
(0x0EB4, 0x0EB9);
(0x0EBB, 0x0EBC);
(0x0EC8, 0x0ECD);
(0x0F18, 0x0F19);
(0x0F35, 0x0F35);
(0x0F37, 0x0F37);
(0x0F39, 0x0F39);
(0x0F3E, 0x0F3E);
(0x0F3F, 0x0F3F);
(0x0F71, 0x0F84);
(0x0F86, 0x0F8B);
(0x0F90, 0x0F95);
(0x0F97, 0x0F97);
(0x0F99, 0x0FAD);
(0x0FB1, 0x0FB7);
(0x0FB9, 0x0FB9);
(0x20D0, 0x20DC);
(0x20E1, 0x20E1);
(0x302A, 0x302F);
(0x3099, 0x3099);
(0x309A, 0x309A);
]
in
of_list l
let digit =
let l =
[
(0x0030, 0x0039);
(0x0660, 0x0669);
(0x06F0, 0x06F9);
(0x0966, 0x096F);
(0x09E6, 0x09EF);
(0x0A66, 0x0A6F);
(0x0AE6, 0x0AEF);
(0x0B66, 0x0B6F);
(0x0BE7, 0x0BEF);
(0x0C66, 0x0C6F);
(0x0CE6, 0x0CEF);
(0x0D66, 0x0D6F);
(0x0E50, 0x0E59);
(0x0ED0, 0x0ED9);
(0x0F20, 0x0F29);
]
in
of_list l
let extender =
let l =
[
(0x00B7, 0x00B7);
(0x02D0, 0x02D1);
(0x0387, 0x0387);
(0x0640, 0x0640);
(0x0E46, 0x0E46);
(0x0EC6, 0x0EC6);
(0x3005, 0x3005);
(0x3031, 0x3035);
(0x309D, 0x309E);
(0x30FC, 0x30FE);
]
in
of_list l
let blank =
let l = [(0x0009, 0x000A); (0x000D, 0x000D); (0x0020, 0x0020)] in
of_list l
let letter = union base_char ideographic

14
src/syntax/xml.mli
View File

@ -1,14 +0,0 @@
(* The package sedlex is released under the terms of an MIT-like license. *)
(* See the attached LICENSE file. *)
(** Unicode classes from XML *)
open Sedlex_cset
val letter : t
val digit : t
val extender : t
val base_char : t
val ideographic : t
val combining_char : t
val blank : t

BIN
test/UTF-8-test.txt
View File

Binary file not shown.

1062
test/basic.ml
View File

File diff suppressed because it is too large Load Diff

8
test/codegen/dune
View File

@ -1,8 +0,0 @@
(library
(name sedlex_gen_test)
(libraries noslop-sedlex)
(inline_tests)
(enabled_if
(>= %{ocaml_version} 4.14))
(preprocess
(pps ppx_sedlex_test ppx_expect)))

120
test/codegen/test_gen.ml
View File

@ -1,120 +0,0 @@
let%expect_test "simple string match" =
(match%sedlex_test buf with "ab" | "de" -> () | _ -> ());
[%expect
{|
DOT:
digraph {
rankdir=LR;
node [shape=circle];
_start [shape=point];
_start -> state0;
state0 [label="0"];
state0 -> state1 [label="'a'"];
state0 -> state3 [label="'d'"];
state1 [label="1"];
state1 -> state2 [label="'b'"];
state2 [label="2\n[rule 0]", shape=doublecircle];
state3 [label="3"];
state3 -> state2 [label="'e'"];
}
CODE:
let rec __sedlex_state_0 buf =
match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
| 0 -> __sedlex_state_1 buf
| 1 -> __sedlex_state_3 buf
| _ -> Sedlexing.backtrack buf
and __sedlex_state_1 buf =
match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with
| 0 -> 0
| _ -> Sedlexing.backtrack buf
and __sedlex_state_3 buf =
match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with
| 0 -> 0
| _ -> Sedlexing.backtrack buf in
Sedlexing.start buf; (match __sedlex_state_0 buf with | 0 -> () | _ -> ())
|}]
let%expect_test "character class" =
(match%sedlex_test buf with Plus 'a' .. 'z' -> () | _ -> ());
[%expect
{|
DOT:
digraph {
rankdir=LR;
node [shape=circle];
_start [shape=point];
_start -> state0;
state0 [label="0"];
state0 -> state1 [label="'a'-'z'"];
state1 [label="1\n[rule 0]", shape=doublecircle];
state1 -> state1 [label="'a'-'z'"];
}
CODE:
let rec __sedlex_state_0 buf =
match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
| 0 -> __sedlex_state_1 buf
| _ -> Sedlexing.backtrack buf
and __sedlex_state_1 buf =
Sedlexing.mark buf 0;
(match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
| 0 -> __sedlex_state_1 buf
| _ -> Sedlexing.backtrack buf) in
Sedlexing.start buf; (match __sedlex_state_0 buf with | 0 -> () | _ -> ())
|}]
let%expect_test "multi-rule" =
(match%sedlex_test buf with
| "ab" -> ()
| "de" -> ()
| Plus '0' .. '9' -> ()
| _ -> ());
[%expect
{|
DOT:
digraph {
rankdir=LR;
node [shape=circle];
_start [shape=point];
_start -> state0;
state0 [label="0"];
state0 -> state1 [label="'0'-'9'"];
state0 -> state2 [label="'a'"];
state0 -> state4 [label="'d'"];
state1 [label="1\n[rule 2]", shape=doublecircle];
state1 -> state1 [label="'0'-'9'"];
state2 [label="2"];
state2 -> state3 [label="'b'"];
state3 [label="3\n[rule 0]", shape=doublecircle];
state4 [label="4"];
state4 -> state5 [label="'e'"];
state5 [label="5\n[rule 1]", shape=doublecircle];
}
CODE:
let rec __sedlex_state_0 buf =
match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
| 0 -> __sedlex_state_1 buf
| 1 -> __sedlex_state_2 buf
| 2 -> __sedlex_state_4 buf
| _ -> Sedlexing.backtrack buf
and __sedlex_state_1 buf =
Sedlexing.mark buf 2;
(match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with
| 0 -> __sedlex_state_1 buf
| _ -> Sedlexing.backtrack buf)
and __sedlex_state_2 buf =
match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with
| 0 -> 0
| _ -> Sedlexing.backtrack buf
and __sedlex_state_4 buf =
match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with
| 0 -> 1
| _ -> Sedlexing.backtrack buf in
Sedlexing.start buf;
(match __sedlex_state_0 buf with | 0 -> () | 1 -> () | 2 -> () | _ -> ())
|}]

9
test/dune
View File

@ -1,9 +0,0 @@
(library
(name sedlex_test)
(libraries noslop-sedlex)
(inline_tests
(deps UTF-8-test.txt))
(enabled_if
(>= %{ocaml_version} 4.14))
(preprocess
(pps noslop-sedlex.ppx ppx_expect)))

9
test/nested.ml
View File

@ -1,9 +0,0 @@
let%expect_test _ =
let lb = Sedlexing.Utf8.from_string "ab" in
let res =
match%sedlex lb with
| 'a' -> ( match%sedlex lb with 'b' -> "ok" | _ -> "error")
| _ -> "error"
in
print_endline res;
[%expect {| ok |}]

6
test/ppx_test/dune
View File

@ -1,6 +0,0 @@
(library
(name ppx_sedlex_test)
(kind ppx_rewriter)
(libraries ppxlib noslop-sedlex.ppx)
(preprocess
(pps ppxlib.metaquot)))

36
test/ppx_test/ppx_sedlex_test.ml
View File

@ -1,36 +0,0 @@
open Ppxlib
module P = Sedlex_ppx.Ppx_sedlex
module S = Sedlex_ppx.Sedlex
let reset_state () =
P.partition_counter := 0;
P.table_counter := 0;
Hashtbl.clear P.partitions;
Hashtbl.clear P.tables
let clear_tables () =
Hashtbl.clear P.partitions;
Hashtbl.clear P.tables
let expand ~ctxt:_ expr =
reset_state ();
let loc = Location.none in
let code_expr, auto =
P.handle_sedlex_match ~env:P.builtin_regexps ~map_rhs:Fun.id expr
in
let code_str = Pprintast.string_of_expression code_expr in
let dot_str = S.dfa_to_dot auto in
clear_tables ();
[%expr
print_string "DOT:\n";
print_string [%e Ast_builder.Default.estring ~loc dot_str];
print_string "CODE:\n";
print_string [%e Ast_builder.Default.estring ~loc code_str];
print_newline ()]
let ext =
Extension.V3.declare "sedlex_test" Extension.Context.expression
Ast_pattern.(single_expr_payload __)
expand
let () = Driver.register_transformation "sedlex_test" ~extensions:[ext]

BIN
test/stress_test.ml
View File

Binary file not shown.

33
test/utf8.ml
View File

@ -1,33 +0,0 @@
open Printf
let next_tok buf =
let open Sedlexing.Utf8 in
match%sedlex buf with
| "a", Utf8 (Chars "+-×÷") -> sprintf "with Chars: %s" (lexeme buf)
| "b", Utf8 ("+" | "-" | "×" | "÷") ->
sprintf "with or_pattern: %s" (lexeme buf)
| Latin1 "\xc0", Utf8 "À", Ascii (Utf8 (Latin1 (Utf8 (Chars "À")))) ->
sprintf "mixed encoding: %s" (lexeme buf)
| Ascii (Star '\x00' .. '\x7f') -> sprintf "only ascii: %s" (lexeme buf)
| Utf8 (Star '\x00' .. '\x7f') ->
assert false
(* utf8 char interval can only match ascii because of the OCaml lexer. The regexp above should match instead *)
| Latin1 (Star '\x00' .. '\xff') -> sprintf "only latin1: %s" (lexeme buf)
| _ -> failwith (sprintf "Unexpected character: %s" (lexeme buf))
let%expect_test _ =
Sedlexing.Utf8.from_string "a+" |> next_tok |> print_string;
[%expect {| with Chars: a+ |}];
Sedlexing.Utf8.from_string "a÷" |> next_tok |> print_string;
[%expect {| with Chars: a÷ |}];
Sedlexing.Utf8.from_string "b+" |> next_tok |> print_string;
[%expect {| with or_pattern: b+ |}];
Sedlexing.Utf8.from_string "b÷" |> next_tok |> print_string;
[%expect {| with or_pattern: b÷ |}];
Sedlexing.Utf8.from_string "ÀÀÀ" |> next_tok |> print_string;
[%expect {| mixed encoding: ÀÀÀ |}];
Sedlexing.Utf8.from_string "az\x7f"
|> next_tok |> String.escaped |> print_string;
[%expect {| only ascii: az\127 |}];
Sedlexing.Utf8.from_string "az\u{c0}" |> next_tok |> print_string;
[%expect {| only latin1: azÀ |}]