Compare commits
No commits in common. "main" and "upstream/v3.2" have entirely different histories.
main
...
upstream/v
|
|
@ -1 +0,0 @@
|
||||||
test/stress_test.ml text diff
|
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
# These are the default owners for everything in the repo. They will
|
||||||
|
# be requested for review when someone opens a pull request.
|
||||||
|
* @alainfrisch @Drup @pmetzger
|
||||||
|
|
@ -0,0 +1,97 @@
|
||||||
|
name: build
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
schedule:
|
||||||
|
# Prime the caches every Monday
|
||||||
|
- cron: 0 1 * * MON
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
os:
|
||||||
|
- ubuntu-latest
|
||||||
|
- macos-latest
|
||||||
|
ocaml-compiler:
|
||||||
|
- 4.14.x
|
||||||
|
- 5.0.x
|
||||||
|
skip-test:
|
||||||
|
- false
|
||||||
|
include:
|
||||||
|
- os: ubuntu-latest
|
||||||
|
ocaml-compiler: 4.8.x
|
||||||
|
skip-test: true
|
||||||
|
- os: windows-latest
|
||||||
|
ocaml-compiler: 4.14.x
|
||||||
|
skip-test: false
|
||||||
|
- os: windows-latest
|
||||||
|
ocaml-compiler: ocaml.5.0.0,ocaml-option-mingw
|
||||||
|
skip-test: false
|
||||||
|
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Set git to use LF
|
||||||
|
run: |
|
||||||
|
git config --global core.autocrlf false
|
||||||
|
git config --global core.eol lf
|
||||||
|
git config --global core.ignorecase false
|
||||||
|
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Use OCaml ${{ matrix.ocaml-compiler }}
|
||||||
|
if: runner.os == 'Windows'
|
||||||
|
uses: ocaml/setup-ocaml@v2
|
||||||
|
with:
|
||||||
|
ocaml-compiler: ${{ matrix.ocaml-compiler }}
|
||||||
|
opam-repositories: |
|
||||||
|
dra27: https://github.com/dra27/opam-repository.git#windows-5.0
|
||||||
|
default: https://github.com/ocaml-opam/opam-repository-mingw.git#sunset
|
||||||
|
opam: https://github.com/ocaml/opam-repository.git
|
||||||
|
dune-cache: ${{ matrix.os == 'ubuntu-latest' }}
|
||||||
|
opam-depext: ${{ !matrix.skip-test }}
|
||||||
|
opam-depext-flags: --with-test
|
||||||
|
|
||||||
|
- name: Use OCaml ${{ matrix.ocaml-compiler }}
|
||||||
|
if: runner.os != 'Windows'
|
||||||
|
uses: ocaml/setup-ocaml@v2
|
||||||
|
with:
|
||||||
|
ocaml-compiler: ${{ matrix.ocaml-compiler }}
|
||||||
|
dune-cache: ${{ matrix.os == 'ubuntu-latest' }}
|
||||||
|
opam-depext: ${{ !matrix.skip-test }}
|
||||||
|
opam-depext-flags: --with-test
|
||||||
|
|
||||||
|
|
||||||
|
- run: opam install . --with-test
|
||||||
|
if: ${{ !matrix.skip-test }}
|
||||||
|
|
||||||
|
- run: opam install .
|
||||||
|
if: ${{ matrix.skip-test }}
|
||||||
|
|
||||||
|
- run: opam exec -- make build
|
||||||
|
|
||||||
|
- run: opam exec -- make test
|
||||||
|
if: ${{ !matrix.skip-test }}
|
||||||
|
|
||||||
|
- run: opam exec -- git diff --exit-code
|
||||||
|
|
||||||
|
lint-fmt:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Use OCaml 4.14.x
|
||||||
|
uses: ocaml/setup-ocaml@v2
|
||||||
|
with:
|
||||||
|
ocaml-compiler: 4.14.x
|
||||||
|
dune-cache: true
|
||||||
|
|
||||||
|
- name: Lint fmt
|
||||||
|
uses: ocaml/setup-ocaml/lint-fmt@v2
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
name: Check changelog
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
types:
|
||||||
|
- labeled
|
||||||
|
- opened
|
||||||
|
- reopened
|
||||||
|
- synchronize
|
||||||
|
- unlabeled
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
check-changelog:
|
||||||
|
name: Check changelog
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Check changelog
|
||||||
|
uses: tarides/changelog-check-action@v1
|
||||||
|
|
@ -0,0 +1,28 @@
|
||||||
|
name: Doc build
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build_doc:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v1
|
||||||
|
- name: Setup OCaml
|
||||||
|
uses: ocaml/setup-ocaml@v2
|
||||||
|
with:
|
||||||
|
ocaml-compiler: 4.14.x
|
||||||
|
- name: Pin locally
|
||||||
|
run: opam pin -y add --no-action .
|
||||||
|
- name: Install locally
|
||||||
|
run: opam install -y --with-doc sedlex
|
||||||
|
- name: Build doc
|
||||||
|
run: opam exec dune build @doc
|
||||||
|
- name: Deploy doc
|
||||||
|
uses: JamesIves/github-pages-deploy-action@4.1.4
|
||||||
|
with:
|
||||||
|
branch: gh-pages
|
||||||
|
folder: _build/default/_doc/_html
|
||||||
|
|
@ -19,4 +19,3 @@ examples/complement
|
||||||
examples/tokenizer
|
examples/tokenizer
|
||||||
examples/subtraction
|
examples/subtraction
|
||||||
_opam
|
_opam
|
||||||
.direnv
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
version=0.29.0
|
version=0.25.1
|
||||||
profile = conventional
|
profile = conventional
|
||||||
break-separators = after
|
break-separators = after
|
||||||
space-around-lists = false
|
space-around-lists = false
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
language: c
|
||||||
|
sudo: required
|
||||||
|
install: test -e .travis.opam.sh || wget https://raw.githubusercontent.com/ocaml/ocaml-ci-scripts/master/.travis-opam.sh
|
||||||
|
script: bash -ex .travis-opam.sh
|
||||||
|
env:
|
||||||
|
global:
|
||||||
|
- PACKAGE=sedlex
|
||||||
|
matrix:
|
||||||
|
- OCAML_VERSION=4.04
|
||||||
|
- OCAML_VERSION=4.05
|
||||||
|
- OCAML_VERSION=4.06
|
||||||
|
- OCAML_VERSION=4.07
|
||||||
|
- OCAML_VERSION=4.08
|
||||||
|
- OCAML_VERSION=4.09
|
||||||
|
- OCAML_VERSION=4.10
|
||||||
|
- OCAML_VERSION=4.11
|
||||||
|
os:
|
||||||
|
- linux
|
||||||
|
- osx
|
||||||
19
CHANGES.md
19
CHANGES.md
|
|
@ -1,22 +1,3 @@
|
||||||
# 3.7 (2025-10-06)
|
|
||||||
- Update to unicode 17.0.0
|
|
||||||
|
|
||||||
# 3.6 (2025-01-05)
|
|
||||||
- Fixed one of the ranges implementing
|
|
||||||
Implement Corrigendum #1: UTF-8 Shortest Form
|
|
||||||
for 4-bytes long characters (#171)
|
|
||||||
|
|
||||||
# 3.5 (2025-05-29)
|
|
||||||
- Implement Corrigendum #1: UTF-8 Shortest Form
|
|
||||||
- Add utf8 support for string literal (#127)
|
|
||||||
|
|
||||||
# 3.4 (2025-03-28)
|
|
||||||
- Make the library compatibility with ppxlib.0.36 (#166)
|
|
||||||
|
|
||||||
# 3.3 (2024-10-29)
|
|
||||||
- Add support for unicode `16.0.0` (#157)
|
|
||||||
- Add API for retrieving start and stop positions separately (#155)
|
|
||||||
|
|
||||||
# 3.2 (2023-06-28):
|
# 3.2 (2023-06-28):
|
||||||
- Restore compatibility with OCaml 4.08
|
- Restore compatibility with OCaml 4.08
|
||||||
- Use `Sedlexing.{Utf8,Utf16}.from_gen` to initialize UTF8 (resp. UTF16) lexing buffers from
|
- Use `Sedlexing.{Utf8,Utf16}.from_gen` to initialize UTF8 (resp. UTF16) lexing buffers from
|
||||||
|
|
|
||||||
26
LICENSE
26
LICENSE
|
|
@ -1,6 +1,22 @@
|
||||||
it's proprietary. all rights reserved. if you use this code in a way i don't like i will personally
|
The MIT License (MIT)
|
||||||
make your car explode into hammers
|
|
||||||
|
|
||||||
Portions of this code are based on "sedlex", Copyright 2005, 2014 by Alain
|
Copyright 2005, 2014 by Alain Frisch and LexiFi.
|
||||||
Frisch and LexiFi, released under the terms of the MIT license which can be
|
|
||||||
found in <licenses/LICENSE.sedlex>
|
Permission is hereby granted, free of charge, to any person obtaining
|
||||||
|
a copy of this software and associated documentation files (the
|
||||||
|
"Software"), to deal in the Software without restriction, including
|
||||||
|
without limitation the rights to use, copy, modify, merge, publish,
|
||||||
|
distribute, sublicense, and/or sell copies of the Software, and to
|
||||||
|
permit persons to whom the Software is furnished to do so, subject to
|
||||||
|
the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be
|
||||||
|
included in all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||||
|
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||||
|
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||||
|
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
# The package sedlex is released under the terms of an MIT-like license.
|
||||||
|
# See the attached LICENSE file.
|
||||||
|
# Copyright 2005, 2013 by Alain Frisch and LexiFi.
|
||||||
|
|
||||||
|
INSTALL_ARGS := $(if $(PREFIX),--prefix $(PREFIX),)
|
||||||
|
|
||||||
|
.PHONY: build install uninstall clean doc test all
|
||||||
|
|
||||||
|
build:
|
||||||
|
dune build @install
|
||||||
|
|
||||||
|
install:
|
||||||
|
dune install $(INSTALL_ARGS)
|
||||||
|
|
||||||
|
uninstall:
|
||||||
|
dune uninstall $(INSTALL_ARGS)
|
||||||
|
|
||||||
|
clean:
|
||||||
|
dune clean
|
||||||
|
|
||||||
|
doc:
|
||||||
|
dune build @doc
|
||||||
|
|
||||||
|
test:
|
||||||
|
dune build @runtest
|
||||||
|
|
||||||
|
all: build test doc
|
||||||
26
README.md
26
README.md
|
|
@ -79,25 +79,12 @@ where:
|
||||||
Unlike ocamllex, lexers work on stream of Unicode codepoints, not
|
Unlike ocamllex, lexers work on stream of Unicode codepoints, not
|
||||||
bytes.
|
bytes.
|
||||||
|
|
||||||
Like ocamllex, sedlex uses **longest match** with **first rule priority**:
|
|
||||||
|
|
||||||
- The lexer always tries to match the longest possible prefix of the
|
|
||||||
input. It does so by continuing to read characters as long as some
|
|
||||||
rule can still match a longer string, while remembering the last
|
|
||||||
position at which a rule did match.
|
|
||||||
|
|
||||||
- When two or more rules match the same longest prefix (a tie), the
|
|
||||||
rule that appears first in the `match%sedlex` definition wins. For
|
|
||||||
example, given the rules `| "if" -> ...` and `| Plus ('a'..'z') -> ...`,
|
|
||||||
the input `"if"` is matched by the first rule because it is listed
|
|
||||||
first, even though the second rule also accepts `"if"`.
|
|
||||||
|
|
||||||
The actions can call functions from the Sedlexing module to extract
|
The actions can call functions from the Sedlexing module to extract
|
||||||
(parts of) the matched lexeme, in the desired encoding.
|
(parts of) the matched lexeme, in the desired encoding.
|
||||||
|
|
||||||
Regular expressions are syntactically OCaml patterns:
|
Regular expressions are syntactically OCaml patterns:
|
||||||
|
|
||||||
- `"...."` (string constant): recognize the specified string.
|
- `"...."` (string constant): recognize the specified string
|
||||||
- `'....'` (character constant) : recognize the specified character
|
- `'....'` (character constant) : recognize the specified character
|
||||||
- `i` (integer constant) : recognize the specified codepoint
|
- `i` (integer constant) : recognize the specified codepoint
|
||||||
- `'...' .. '...'`: character range
|
- `'...' .. '...'`: character range
|
||||||
|
|
@ -116,9 +103,6 @@ Regular expressions are syntactically OCaml patterns:
|
||||||
and recognize the set of items in `R1` but not in `R2` ("subtract")
|
and recognize the set of items in `R1` but not in `R2` ("subtract")
|
||||||
- `Intersect (R1,R2)` : assume that `R` is a single-character length regexp (see
|
- `Intersect (R1,R2)` : assume that `R` is a single-character length regexp (see
|
||||||
below) and recognize the set of items which are in both `R1` and `R2`
|
below) and recognize the set of items which are in both `R1` and `R2`
|
||||||
- `Utf8 R` : string literals inside R are assumed to be utf-8 encoded.
|
|
||||||
- `Latin1 R` : string literals inside R are assumed to be latin1 encoded.
|
|
||||||
- `Ascii R` : string literals inside R are assumed to be ascii encoded.
|
|
||||||
- `lid` (lowercase identifier) : reference a named regexp (see below)
|
- `lid` (lowercase identifier) : reference a named regexp (see below)
|
||||||
|
|
||||||
A single-character length regexp is a regexp which does not contain (after
|
A single-character length regexp is a regexp which does not contain (after
|
||||||
|
|
@ -128,9 +112,8 @@ with a length different from one.
|
||||||
|
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
- The OCaml source is assumed to be encoded in UTF-8.
|
- The OCaml source is assumed to be encoded in Latin1 (for string
|
||||||
- Strings and chars litterals will be interpreted in ASCII unless otherwise
|
and character literals).
|
||||||
specified by the `Latin1`,`Ascii` and `Utf8` constructors in patterns.
|
|
||||||
|
|
||||||
|
|
||||||
It is possible to define named regular expressions with the following
|
It is possible to define named regular expressions with the following
|
||||||
|
|
@ -258,6 +241,3 @@ The `examples/` subdirectory contains several samples of sedlex in use.
|
||||||
- improvements to the build system
|
- improvements to the build system
|
||||||
- switched parts of ppx_sedlex to using concrete syntax (with ppx_metaquot)
|
- switched parts of ppx_sedlex to using concrete syntax (with ppx_metaquot)
|
||||||
- Steffen Smolka: port to dune
|
- Steffen Smolka: port to dune
|
||||||
- Romain Beauxis:
|
|
||||||
- Implementation of the unicode table extractors
|
|
||||||
- General maintenance
|
|
||||||
|
|
|
||||||
23
dune-project
23
dune-project
|
|
@ -1,21 +1,18 @@
|
||||||
(lang dune 3.20)
|
(lang dune 3.0)
|
||||||
|
(version 3.2)
|
||||||
(version "3.7")
|
(name sedlex)
|
||||||
|
(source (github ocaml-community/sedlex))
|
||||||
(name noslop-sedlex)
|
(license MIT)
|
||||||
(source (uri "https://git.lain.faith/noslop/noslop-sedlex.git"))
|
(authors "Alain Frisch <alain.frisch@lexifi.com>"
|
||||||
(license "LicenseRef-Proprietary")
|
"https://github.com/ocaml-community/sedlex/graphs/contributors")
|
||||||
(authors "xenia <xenia@awoo.systems>")
|
(maintainers "Alain Frisch <alain.frisch@lexifi.com>")
|
||||||
(maintainers "xenia <xenia@awoo.systems")
|
(homepage "https://github.com/ocaml-community/sedlex")
|
||||||
(homepage "https://git.lain.faith/noslop/noslop-sedlex.git")
|
|
||||||
(maintenance_intent "(latest)")
|
|
||||||
(documentation "https://git.lain.faith/noslop/noslop-sedlex/wiki")
|
|
||||||
|
|
||||||
(generate_opam_files true)
|
(generate_opam_files true)
|
||||||
(executables_implicit_empty_intf true)
|
(executables_implicit_empty_intf true)
|
||||||
|
|
||||||
(package
|
(package
|
||||||
(name noslop-sedlex)
|
(name sedlex)
|
||||||
(synopsis "An OCaml lexer generator for Unicode")
|
(synopsis "An OCaml lexer generator for Unicode")
|
||||||
(description "sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports
|
(description "sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports
|
||||||
Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular
|
Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,9 @@
|
||||||
(executables
|
(executables
|
||||||
(names tokenizer regressions complement subtraction repeat performance)
|
(names tokenizer regressions complement subtraction repeat performance)
|
||||||
(libraries noslop-sedlex noslop-sedlex.ppx)
|
(libraries sedlex sedlex_ppx)
|
||||||
(preprocess
|
(preprocess
|
||||||
(pps noslop-sedlex.ppx)))
|
(pps sedlex.ppx))
|
||||||
|
(flags :standard -w +39))
|
||||||
|
|
||||||
(rule
|
(rule
|
||||||
(alias runtest)
|
(alias runtest)
|
||||||
|
|
|
||||||
|
|
@ -1,58 +1,51 @@
|
||||||
(* This test that unicode_old.ml is a strict sub-set of new unicode.ml. *)
|
(* This test that unicode_old.ml is a strict sub-set of
|
||||||
|
* new unicode.ml. *)
|
||||||
|
|
||||||
module CSet = Sedlex_ppx.Sedlex_cset
|
let test_versions = ("14.0.0", "15.0.0")
|
||||||
module Unicode = Sedlex_ppx.Unicode
|
let regressions = []
|
||||||
|
let interval s e = Array.to_list (Array.init (e - s) (fun pos -> s + pos))
|
||||||
|
|
||||||
let test_versions = ("16.0.0", "17.0.0")
|
exception Found
|
||||||
|
|
||||||
let regressions =
|
let test_exception name x =
|
||||||
[ (* Example *)
|
try
|
||||||
(* ("lt", CSet.union (CSet.singleton 0x1c5) (CSet.singleton (0x0001))) *) ]
|
let l = List.assoc name regressions in
|
||||||
|
List.iter (fun (s, e) -> if s <= x && x <= e then raise Found) l
|
||||||
|
with Not_found -> ()
|
||||||
|
|
||||||
let compare name (old_ : CSet.t) (new_ : CSet.t) =
|
let compare name (old_l : (int * int) list) (new_l : Sedlex_ppx.Sedlex_cset.t) =
|
||||||
let diff = CSet.difference old_ new_ in
|
let new_l = (new_l :> (int * int) list) in
|
||||||
let regressions =
|
let code_points =
|
||||||
match List.assoc name regressions with
|
List.fold_left (fun res (s, e) -> res @ interval s e) [] old_l
|
||||||
| exception Not_found -> CSet.empty
|
|
||||||
| x -> x
|
|
||||||
in
|
in
|
||||||
let regressions_intersect = CSet.intersection regressions old_ in
|
let test x =
|
||||||
let regressions = CSet.difference regressions regressions_intersect in
|
try
|
||||||
let regressions_useless = CSet.difference regressions new_ in
|
test_exception name x;
|
||||||
let diff = CSet.difference diff regressions in
|
List.iter (fun (s, e) -> if s <= x && x <= e then raise Found) new_l;
|
||||||
Seq.iter
|
false
|
||||||
|
with Found -> true
|
||||||
|
in
|
||||||
|
List.iter
|
||||||
(fun x ->
|
(fun x ->
|
||||||
Printf.printf
|
if not (test x) then
|
||||||
"Invalid regression for 0x%x in %s: already present in old set.\n" x
|
Printf.printf "Code point 0x%x missing in %s!\n" x name)
|
||||||
name)
|
code_points
|
||||||
(CSet.to_seq regressions_intersect);
|
|
||||||
Seq.iter
|
|
||||||
(fun x ->
|
|
||||||
Printf.printf "Invalid regression for 0x%x in %s: absent in new set.\n" x
|
|
||||||
name)
|
|
||||||
(CSet.to_seq regressions_useless);
|
|
||||||
Seq.iter
|
|
||||||
(fun x -> Printf.printf "Code point 0x%x missing in %s!\n" x name)
|
|
||||||
(CSet.to_seq diff)
|
|
||||||
|
|
||||||
let test new_l (name, old_l) =
|
let test new_l (name, old_l) =
|
||||||
(* Cn is for unassigned code points, which are allowed to be
|
(* Cn is for unassigned code points, which are allowed to be
|
||||||
* used in future version. *)
|
* used in future version. *)
|
||||||
let old_l = Sedlex_utils.Cset.to_list old_l in
|
if name <> "cn" then compare name old_l (List.assoc name new_l)
|
||||||
if name <> "cn" then (
|
|
||||||
let old_l =
|
|
||||||
List.fold_left
|
|
||||||
(fun acc (a, b) -> CSet.union acc (CSet.interval a b))
|
|
||||||
CSet.empty old_l
|
|
||||||
in
|
|
||||||
compare name old_l (List.assoc name new_l))
|
|
||||||
|
|
||||||
let () =
|
let () =
|
||||||
if (Unicode_old.version, Unicode.version) <> test_versions then
|
if (Unicode_old.version, Sedlex_ppx.Unicode.version) <> test_versions then
|
||||||
failwith
|
failwith
|
||||||
(Printf.sprintf "Test written for versions: %s => %s\n%!"
|
(Printf.sprintf "Test written for versions: %s => %s\n%!"
|
||||||
Unicode_old.version Unicode.version);
|
Unicode_old.version Sedlex_ppx.Unicode.version);
|
||||||
Printf.printf "Testing Unicode regression: %s => %s\n%!" Unicode_old.version
|
Printf.printf "Testing Unicode regression: %s => %s\n%!" Unicode_old.version
|
||||||
Unicode.version;
|
Sedlex_ppx.Unicode.version;
|
||||||
List.iter (test Unicode.Categories.list) Unicode_old.Categories.list;
|
List.iter
|
||||||
List.iter (test Unicode.Properties.list) Unicode_old.Properties.list
|
(test Sedlex_ppx.Unicode.Categories.list)
|
||||||
|
Unicode_old.Categories.list;
|
||||||
|
List.iter
|
||||||
|
(test Sedlex_ppx.Unicode.Properties.list)
|
||||||
|
Unicode_old.Properties.list
|
||||||
|
|
|
||||||
13920
examples/unicode_old.ml
13920
examples/unicode_old.ml
File diff suppressed because it is too large
Load Diff
45
flake.lock
45
flake.lock
|
|
@ -1,45 +0,0 @@
|
||||||
{
|
|
||||||
"nodes": {
|
|
||||||
"dragnpkgs": {
|
|
||||||
"inputs": {
|
|
||||||
"nixpkgs": "nixpkgs"
|
|
||||||
},
|
|
||||||
"locked": {
|
|
||||||
"lastModified": 1783018382,
|
|
||||||
"narHash": "sha256-Pw7GM1fh2AnQNCitswXTHPfsswiDnEjXPN17Ge8egz4=",
|
|
||||||
"ref": "nixos-26.05",
|
|
||||||
"rev": "085892ad486428b8e51b549539a5430f0c2da1a8",
|
|
||||||
"revCount": 247,
|
|
||||||
"type": "git",
|
|
||||||
"url": "https://git.lain.faith/haskal/dragnpkgs.git"
|
|
||||||
},
|
|
||||||
"original": {
|
|
||||||
"id": "dragnpkgs",
|
|
||||||
"type": "indirect"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nixpkgs": {
|
|
||||||
"locked": {
|
|
||||||
"lastModified": 1782847225,
|
|
||||||
"narHash": "sha256-JC9PjqKYG9ve5U8aDOLQipp3+KLANBHUvGdLZlxzdKI=",
|
|
||||||
"owner": "NixOS",
|
|
||||||
"repo": "nixpkgs",
|
|
||||||
"rev": "95ca1e203c0750115fd4a6f17d5a245dfe6b1edd",
|
|
||||||
"type": "github"
|
|
||||||
},
|
|
||||||
"original": {
|
|
||||||
"owner": "NixOS",
|
|
||||||
"ref": "nixos-26.05",
|
|
||||||
"repo": "nixpkgs",
|
|
||||||
"type": "github"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"root": {
|
|
||||||
"inputs": {
|
|
||||||
"dragnpkgs": "dragnpkgs"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"root": "root",
|
|
||||||
"version": 7
|
|
||||||
}
|
|
||||||
23
flake.nix
23
flake.nix
|
|
@ -1,23 +0,0 @@
|
||||||
{
|
|
||||||
description = "Flake for noslop-sedlex";
|
|
||||||
|
|
||||||
outputs = { self, dragnpkgs } @ inputs: dragnpkgs.lib.mkFlake {
|
|
||||||
packages.default = { ocamlPackages }: ocamlPackages.callPackage ./package.nix {};
|
|
||||||
|
|
||||||
devShells.default = { ocamlPackages, stdenv, mkShell }: mkShell {
|
|
||||||
packages = with ocamlPackages; [
|
|
||||||
ocaml
|
|
||||||
dune_3
|
|
||||||
utop
|
|
||||||
odoc
|
|
||||||
alcotest
|
|
||||||
ocamlformat
|
|
||||||
] ++ (self.packages.${stdenv.hostPlatform.system}.default.propagatedBuildInputs)
|
|
||||||
++ (self.packages.${stdenv.hostPlatform.system}.default.nativeBuildInputs);
|
|
||||||
|
|
||||||
shellHook = ''
|
|
||||||
export OCAMLRUNPARAM=b
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
The MIT License (MIT)
|
|
||||||
|
|
||||||
Copyright 2005, 2014 by Alain Frisch and LexiFi.
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining
|
|
||||||
a copy of this software and associated documentation files (the
|
|
||||||
"Software"), to deal in the Software without restriction, including
|
|
||||||
without limitation the rights to use, copy, modify, merge, publish,
|
|
||||||
distribute, sublicense, and/or sell copies of the Software, and to
|
|
||||||
permit persons to whom the Software is furnished to do so, subject to
|
|
||||||
the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be
|
|
||||||
included in all copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
||||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
||||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
||||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
||||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
||||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
||||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
||||||
52
package.nix
52
package.nix
|
|
@ -1,52 +0,0 @@
|
||||||
{
|
|
||||||
lib,
|
|
||||||
fetchurl,
|
|
||||||
buildDunePackage,
|
|
||||||
gen,
|
|
||||||
ppxlib,
|
|
||||||
ppx_expect,
|
|
||||||
}: let
|
|
||||||
unicodeVersion = "17.0.0";
|
|
||||||
baseUrl = "https://www.unicode.org/Public/${unicodeVersion}";
|
|
||||||
|
|
||||||
DerivedCoreProperties = fetchurl {
|
|
||||||
url = "${baseUrl}/ucd/DerivedCoreProperties.txt";
|
|
||||||
hash = "sha256-JMf+0RlcSC+q79XB5+uCHF7h+23gfs26pktWqZ2iLAg=";
|
|
||||||
};
|
|
||||||
DerivedGeneralCategory = fetchurl {
|
|
||||||
url = "${baseUrl}/ucd/extracted/DerivedGeneralCategory.txt";
|
|
||||||
hash = "sha256-1i5bq3DKdPCZND9xIk+gUcsf3WGhq0XASIxEz8C2EC4=";
|
|
||||||
};
|
|
||||||
PropList = fetchurl {
|
|
||||||
url = "${baseUrl}/ucd/PropList.txt";
|
|
||||||
hash = "sha256-Ew3N3Kra8HEAi9/OHndD4E/fvJEIhvAX2fmskx2MZN0=";
|
|
||||||
};
|
|
||||||
in
|
|
||||||
buildDunePackage (finalAttrs: {
|
|
||||||
pname = "noslop-sedlex";
|
|
||||||
version = "3.7+DEV";
|
|
||||||
|
|
||||||
minimalOCamlVersion = "5.3";
|
|
||||||
|
|
||||||
src = ./.;
|
|
||||||
|
|
||||||
propagatedBuildInputs = [
|
|
||||||
gen
|
|
||||||
ppxlib
|
|
||||||
];
|
|
||||||
|
|
||||||
preBuild = ''
|
|
||||||
rm src/generator/data/dune
|
|
||||||
ln -s ${DerivedCoreProperties} src/generator/data/DerivedCoreProperties.txt
|
|
||||||
ln -s ${DerivedGeneralCategory} src/generator/data/DerivedGeneralCategory.txt
|
|
||||||
ln -s ${PropList} src/generator/data/PropList.txt
|
|
||||||
'';
|
|
||||||
|
|
||||||
checkInputs = [
|
|
||||||
ppx_expect
|
|
||||||
];
|
|
||||||
|
|
||||||
doCheck = true;
|
|
||||||
|
|
||||||
dontStrip = true;
|
|
||||||
})
|
|
||||||
|
|
@ -1,20 +1,23 @@
|
||||||
# This file is generated by dune, edit dune-project instead
|
# This file is generated by dune, edit dune-project instead
|
||||||
opam-version: "2.0"
|
opam-version: "2.0"
|
||||||
version: "3.7"
|
version: "3.2"
|
||||||
synopsis: "An OCaml lexer generator for Unicode"
|
synopsis: "An OCaml lexer generator for Unicode"
|
||||||
description: """
|
description: """
|
||||||
sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports
|
sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports
|
||||||
Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular
|
Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular
|
||||||
OCaml source files. Lexing specific constructs are provided via a ppx syntax
|
OCaml source files. Lexing specific constructs are provided via a ppx syntax
|
||||||
extension."""
|
extension."""
|
||||||
maintainer: ["xenia <xenia@awoo.systems"]
|
maintainer: ["Alain Frisch <alain.frisch@lexifi.com>"]
|
||||||
authors: ["xenia <xenia@awoo.systems>"]
|
authors: [
|
||||||
license: "LicenseRef-Proprietary"
|
"Alain Frisch <alain.frisch@lexifi.com>"
|
||||||
homepage: "https://git.lain.faith/noslop/noslop-sedlex.git"
|
"https://github.com/ocaml-community/sedlex/graphs/contributors"
|
||||||
doc: "https://git.lain.faith/noslop/noslop-sedlex/wiki"
|
]
|
||||||
|
license: "MIT"
|
||||||
|
homepage: "https://github.com/ocaml-community/sedlex"
|
||||||
|
bug-reports: "https://github.com/ocaml-community/sedlex/issues"
|
||||||
depends: [
|
depends: [
|
||||||
"ocaml" {>= "4.08"}
|
"ocaml" {>= "4.08"}
|
||||||
"dune" {>= "3.20"}
|
"dune" {>= "3.0"}
|
||||||
"ppxlib" {>= "0.26.0"}
|
"ppxlib" {>= "0.26.0"}
|
||||||
"gen"
|
"gen"
|
||||||
"ppx_expect" {with-test}
|
"ppx_expect" {with-test}
|
||||||
|
|
@ -34,5 +37,5 @@ build: [
|
||||||
"@doc" {with-doc}
|
"@doc" {with-doc}
|
||||||
]
|
]
|
||||||
]
|
]
|
||||||
dev-repo: "https://git.lain.faith/noslop/noslop-sedlex.git"
|
dev-repo: "git+https://github.com/ocaml-community/sedlex.git"
|
||||||
x-maintenance-intent: ["(latest)"]
|
doc: "https://ocaml-community.github.io/sedlex/index.html"
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
doc: "https://ocaml-community.github.io/sedlex/index.html"
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
|
(* The package sedlex is released under the terms of an MIT-like license. *)
|
||||||
|
(* See the attached LICENSE file. *)
|
||||||
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
||||||
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
|
|
||||||
|
|
||||||
(* Character sets are represented as lists of intervals. The
|
(* Character sets are represented as lists of intervals. The
|
||||||
intervals must be non-overlapping and not collapsable, and the list
|
intervals must be non-overlapping and not collapsable, and the list
|
||||||
|
|
@ -7,12 +8,6 @@
|
||||||
|
|
||||||
type t = (int * int) list
|
type t = (int * int) list
|
||||||
|
|
||||||
let rec range_to_seq a b next () =
|
|
||||||
if a = b then Seq.Cons (a, next) else Seq.Cons (a, range_to_seq (a + 1) b next)
|
|
||||||
|
|
||||||
let rec to_seq x () =
|
|
||||||
match x with [] -> Seq.Nil | (a, b) :: xs -> range_to_seq a b (to_seq xs) ()
|
|
||||||
|
|
||||||
let check_invariant l =
|
let check_invariant l =
|
||||||
let rec loop prev = function
|
let rec loop prev = function
|
||||||
| [] -> ()
|
| [] -> ()
|
||||||
|
|
@ -38,7 +33,6 @@ let of_list l =
|
||||||
check_invariant l;
|
check_invariant l;
|
||||||
l
|
l
|
||||||
|
|
||||||
let to_list l = l
|
|
||||||
let max_code = 0x10ffff (* must be < max_int *)
|
let max_code = 0x10ffff (* must be < max_int *)
|
||||||
let min_code = -1
|
let min_code = -1
|
||||||
let empty = []
|
let empty = []
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,15 @@
|
||||||
|
(* The package sedlex is released under the terms of an MIT-like license. *)
|
||||||
|
(* See the attached LICENSE file. *)
|
||||||
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
||||||
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
|
|
||||||
|
|
||||||
(** Representation of sets of unicode code points. *)
|
(** Representation of sets of unicode code points. *)
|
||||||
|
|
||||||
(** Character sets are represented as lists of intervals. The intervals must be
|
(** Character sets are represented as lists of intervals. The
|
||||||
non-overlapping and not collapsable, and the list must be ordered in
|
intervals must be non-overlapping and not collapsable, and the list
|
||||||
increasing order. *)
|
must be ordered in increasing order. *)
|
||||||
type t = private (int * int) list
|
type t = private (int * int) list
|
||||||
|
|
||||||
val of_list : (int * int) list -> t
|
val of_list : (int * int) list -> t
|
||||||
val to_list : t -> (int * int) list
|
|
||||||
val min_code : int
|
val min_code : int
|
||||||
val max_code : int
|
val max_code : int
|
||||||
val empty : t
|
val empty : t
|
||||||
|
|
@ -22,4 +22,3 @@ val is_empty : t -> bool
|
||||||
val eof : t
|
val eof : t
|
||||||
val singleton : int -> t
|
val singleton : int -> t
|
||||||
val interval : int -> int -> t
|
val interval : int -> int -> t
|
||||||
val to_seq : t -> int Seq.t
|
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,3 @@
|
||||||
(library
|
(library
|
||||||
(name sedlex_utils)
|
(name sedlex_utils)
|
||||||
(public_name noslop-sedlex.utils))
|
(public_name sedlex.utils))
|
||||||
|
|
|
||||||
|
|
@ -1 +1 @@
|
||||||
https://www.unicode.org/Public/17.0.0
|
https://www.unicode.org/Public/15.0.0
|
||||||
|
|
@ -1,3 +1,3 @@
|
||||||
(executable
|
(executable
|
||||||
(name gen_unicode)
|
(name gen_unicode)
|
||||||
(libraries str noslop-sedlex.utils))
|
(libraries str sedlex.utils))
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ let parse_line l =
|
||||||
let l =
|
let l =
|
||||||
match String.index_opt l '#' with None -> l | Some i -> String.sub l 0 i
|
match String.index_opt l '#' with None -> l | Some i -> String.sub l 0 i
|
||||||
in
|
in
|
||||||
String.split_on_char ';' l |> List.map String.trim
|
String.split_on_char ';' l
|
||||||
|
|
||||||
let parse_code s =
|
let parse_code s =
|
||||||
try int_of_string (Printf.sprintf "0x%s" s)
|
try int_of_string (Printf.sprintf "0x%s" s)
|
||||||
|
|
@ -91,9 +91,6 @@ let files =
|
||||||
let interval = parse_interval interval in
|
let interval = parse_interval interval in
|
||||||
let prop = parse_prop prop in
|
let prop = parse_prop prop in
|
||||||
Hashtbl.add labels prop interval
|
Hashtbl.add labels prop interval
|
||||||
| [_interval; "InCB"; ("Extend" | "Consonant" | "Linker")] ->
|
|
||||||
(* TODO: support non-binary properties? *)
|
|
||||||
()
|
|
||||||
| _ -> assert false );
|
| _ -> assert false );
|
||||||
( "DerivedGeneralCategory.txt",
|
( "DerivedGeneralCategory.txt",
|
||||||
fun s ->
|
fun s ->
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
(library
|
(library
|
||||||
(name sedlex)
|
(name sedlex)
|
||||||
(public_name noslop-sedlex)
|
(public_name sedlex)
|
||||||
(wrapped false)
|
(wrapped false)
|
||||||
(libraries gen))
|
(libraries gen)
|
||||||
|
(flags :standard -w +A-4-9 -safe-string))
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
|
(* The package sedlex is released under the terms of an MIT-like license. *)
|
||||||
|
(* See the attached LICENSE file. *)
|
||||||
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
||||||
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
|
|
||||||
|
|
||||||
exception InvalidCodepoint of int
|
exception InvalidCodepoint of int
|
||||||
exception MalFormed
|
exception MalFormed
|
||||||
|
|
@ -260,46 +261,40 @@ let lexeme lexbuf =
|
||||||
|
|
||||||
let lexeme_char lexbuf pos = lexbuf.buf.(lexbuf.start_pos + pos)
|
let lexeme_char lexbuf pos = lexbuf.buf.(lexbuf.start_pos + pos)
|
||||||
|
|
||||||
let lexing_position_start lexbuf =
|
let lexing_positions lexbuf =
|
||||||
|
let start_p =
|
||||||
{
|
{
|
||||||
Lexing.pos_fname = lexbuf.filename;
|
Lexing.pos_fname = lexbuf.filename;
|
||||||
pos_lnum = lexbuf.start_line;
|
pos_lnum = lexbuf.start_line;
|
||||||
pos_cnum = lexbuf.start_pos + lexbuf.offset;
|
pos_cnum = lexbuf.start_pos + lexbuf.offset;
|
||||||
pos_bol = lexbuf.start_bol;
|
pos_bol = lexbuf.start_bol;
|
||||||
}
|
}
|
||||||
|
and curr_p =
|
||||||
let lexing_position_curr lexbuf =
|
|
||||||
{
|
{
|
||||||
Lexing.pos_fname = lexbuf.filename;
|
Lexing.pos_fname = lexbuf.filename;
|
||||||
pos_lnum = lexbuf.curr_line;
|
pos_lnum = lexbuf.curr_line;
|
||||||
pos_cnum = lexbuf.pos + lexbuf.offset;
|
pos_cnum = lexbuf.pos + lexbuf.offset;
|
||||||
pos_bol = lexbuf.curr_bol;
|
pos_bol = lexbuf.curr_bol;
|
||||||
}
|
}
|
||||||
|
in
|
||||||
let lexing_positions lexbuf =
|
|
||||||
let start_p = lexing_position_start lexbuf
|
|
||||||
and curr_p = lexing_position_curr lexbuf in
|
|
||||||
(start_p, curr_p)
|
(start_p, curr_p)
|
||||||
|
|
||||||
let lexing_bytes_position_start lexbuf =
|
let lexing_bytes_positions lexbuf =
|
||||||
|
let start_p =
|
||||||
{
|
{
|
||||||
Lexing.pos_fname = lexbuf.filename;
|
Lexing.pos_fname = lexbuf.filename;
|
||||||
pos_lnum = lexbuf.start_line;
|
pos_lnum = lexbuf.start_line;
|
||||||
pos_cnum = lexbuf.start_bytes_pos + lexbuf.bytes_offset;
|
pos_cnum = lexbuf.start_bytes_pos + lexbuf.bytes_offset;
|
||||||
pos_bol = lexbuf.start_bytes_bol;
|
pos_bol = lexbuf.start_bytes_bol;
|
||||||
}
|
}
|
||||||
|
and curr_p =
|
||||||
let lexing_bytes_position_curr lexbuf =
|
|
||||||
{
|
{
|
||||||
Lexing.pos_fname = lexbuf.filename;
|
Lexing.pos_fname = lexbuf.filename;
|
||||||
pos_lnum = lexbuf.curr_line;
|
pos_lnum = lexbuf.curr_line;
|
||||||
pos_cnum = lexbuf.bytes_pos + lexbuf.bytes_offset;
|
pos_cnum = lexbuf.bytes_pos + lexbuf.bytes_offset;
|
||||||
pos_bol = lexbuf.curr_bytes_bol;
|
pos_bol = lexbuf.curr_bytes_bol;
|
||||||
}
|
}
|
||||||
|
in
|
||||||
let lexing_bytes_positions lexbuf =
|
|
||||||
let start_p = lexing_bytes_position_start lexbuf
|
|
||||||
and curr_p = lexing_bytes_position_curr lexbuf in
|
|
||||||
(start_p, curr_p)
|
(start_p, curr_p)
|
||||||
|
|
||||||
let with_tokenizer lexer' lexbuf =
|
let with_tokenizer lexer' lexbuf =
|
||||||
|
|
@ -435,49 +430,6 @@ module Utf8 = struct
|
||||||
| '\240' .. '\247' -> 4
|
| '\240' .. '\247' -> 4
|
||||||
| _ -> raise MalFormed
|
| _ -> raise MalFormed
|
||||||
|
|
||||||
(* https://www.unicode.org/versions/corrigendum1.html *)
|
|
||||||
let check_two n1 n2 =
|
|
||||||
if n1 < 0xc2 || 0xdf < n1 then raise MalFormed;
|
|
||||||
if n2 < 0x80 || 0xbf < n2 then raise MalFormed;
|
|
||||||
if n2 lsr 6 != 0b10 then raise MalFormed;
|
|
||||||
((n1 land 0x1f) lsl 6) lor (n2 land 0x3f)
|
|
||||||
|
|
||||||
let check_three n1 n2 n3 =
|
|
||||||
if n1 = 0xe0 then (
|
|
||||||
if n2 < 0xa0 || 0xbf < n2 then raise MalFormed;
|
|
||||||
if n3 < 0x80 || 0xbf < n3 then raise MalFormed)
|
|
||||||
else (
|
|
||||||
if n1 < 0xe1 || 0xef < n1 then raise MalFormed;
|
|
||||||
if n2 < 0x80 || 0xbf < n2 then raise MalFormed;
|
|
||||||
if n3 < 0x80 || 0xbf < n3 then raise MalFormed);
|
|
||||||
if n2 lsr 6 != 0b10 || n3 lsr 6 != 0b10 then raise MalFormed;
|
|
||||||
let p =
|
|
||||||
((n1 land 0x0f) lsl 12) lor ((n2 land 0x3f) lsl 6) lor (n3 land 0x3f)
|
|
||||||
in
|
|
||||||
if p >= 0xd800 && p <= 0xdfff then raise MalFormed;
|
|
||||||
p
|
|
||||||
|
|
||||||
let check_four n1 n2 n3 n4 =
|
|
||||||
if n1 = 0xf0 then (
|
|
||||||
if n2 < 0x90 || 0xbf < n2 then raise MalFormed;
|
|
||||||
if n3 < 0x80 || 0xbf < n3 then raise MalFormed;
|
|
||||||
if n4 < 0x80 || 0xbf < n4 then raise MalFormed)
|
|
||||||
else if n1 = 0xf4 then (
|
|
||||||
if n2 < 0x80 || 0x8f < n2 then raise MalFormed;
|
|
||||||
if n3 < 0x80 || 0xbf < n3 then raise MalFormed;
|
|
||||||
if n4 < 0x80 || 0xbf < n4 then raise MalFormed)
|
|
||||||
else (
|
|
||||||
if n1 < 0xf1 || 0xf3 < n1 then raise MalFormed;
|
|
||||||
if n2 < 0x80 || 0xbf < n2 then raise MalFormed;
|
|
||||||
if n3 < 0x80 || 0xbf < n3 then raise MalFormed;
|
|
||||||
if n4 < 0x80 || 0xbf < n4 then raise MalFormed);
|
|
||||||
if n2 lsr 6 != 0b10 || n3 lsr 6 != 0b10 || n4 lsr 6 != 0b10 then
|
|
||||||
raise MalFormed;
|
|
||||||
((n1 land 0x07) lsl 18)
|
|
||||||
lor ((n2 land 0x3f) lsl 12)
|
|
||||||
lor ((n3 land 0x3f) lsl 6)
|
|
||||||
lor (n4 land 0x3f)
|
|
||||||
|
|
||||||
let next s i =
|
let next s i =
|
||||||
let c1 = s.[i] in
|
let c1 = s.[i] in
|
||||||
match width c1 with
|
match width c1 with
|
||||||
|
|
@ -485,18 +437,31 @@ module Utf8 = struct
|
||||||
| 2 ->
|
| 2 ->
|
||||||
let n1 = Char.code c1 in
|
let n1 = Char.code c1 in
|
||||||
let n2 = Char.code s.[i + 1] in
|
let n2 = Char.code s.[i + 1] in
|
||||||
check_two n1 n2
|
if n2 lsr 6 != 0b10 then raise MalFormed;
|
||||||
|
((n1 land 0x1f) lsl 6) lor (n2 land 0x3f)
|
||||||
| 3 ->
|
| 3 ->
|
||||||
let n1 = Char.code c1 in
|
let n1 = Char.code c1 in
|
||||||
let n2 = Char.code s.[i + 1] in
|
let n2 = Char.code s.[i + 1] in
|
||||||
let n3 = Char.code s.[i + 2] in
|
let n3 = Char.code s.[i + 2] in
|
||||||
check_three n1 n2 n3
|
if n2 lsr 6 != 0b10 || n3 lsr 6 != 0b10 then raise MalFormed;
|
||||||
|
let p =
|
||||||
|
((n1 land 0x0f) lsl 12)
|
||||||
|
lor ((n2 land 0x3f) lsl 6)
|
||||||
|
lor (n3 land 0x3f)
|
||||||
|
in
|
||||||
|
if p >= 0xd800 && p <= 0xdf00 then raise MalFormed;
|
||||||
|
p
|
||||||
| 4 ->
|
| 4 ->
|
||||||
let n1 = Char.code c1 in
|
let n1 = Char.code c1 in
|
||||||
let n2 = Char.code s.[i + 1] in
|
let n2 = Char.code s.[i + 1] in
|
||||||
let n3 = Char.code s.[i + 2] in
|
let n3 = Char.code s.[i + 2] in
|
||||||
let n4 = Char.code s.[i + 3] in
|
let n4 = Char.code s.[i + 3] in
|
||||||
check_four n1 n2 n3 n4
|
if n2 lsr 6 != 0b10 || n3 lsr 6 != 0b10 || n4 lsr 6 != 0b10 then
|
||||||
|
raise MalFormed;
|
||||||
|
((n1 land 0x07) lsl 18)
|
||||||
|
lor ((n2 land 0x3f) lsl 12)
|
||||||
|
lor ((n3 land 0x3f) lsl 6)
|
||||||
|
lor (n4 land 0x3f)
|
||||||
| _ -> assert false
|
| _ -> assert false
|
||||||
|
|
||||||
let gen_from_char_gen s =
|
let gen_from_char_gen s =
|
||||||
|
|
@ -510,18 +475,29 @@ module Utf8 = struct
|
||||||
| 2 ->
|
| 2 ->
|
||||||
let n1 = Char.code c1 in
|
let n1 = Char.code c1 in
|
||||||
let n2 = next_or_fail () in
|
let n2 = next_or_fail () in
|
||||||
Uchar.of_int (check_two n1 n2)
|
if n2 lsr 6 != 0b10 then raise MalFormed;
|
||||||
|
Uchar.of_int (((n1 land 0x1f) lsl 6) lor (n2 land 0x3f))
|
||||||
| 3 ->
|
| 3 ->
|
||||||
let n1 = Char.code c1 in
|
let n1 = Char.code c1 in
|
||||||
let n2 = next_or_fail () in
|
let n2 = next_or_fail () in
|
||||||
let n3 = next_or_fail () in
|
let n3 = next_or_fail () in
|
||||||
Uchar.of_int (check_three n1 n2 n3)
|
if n2 lsr 6 != 0b10 || n3 lsr 6 != 0b10 then raise MalFormed;
|
||||||
|
Uchar.of_int
|
||||||
|
(((n1 land 0x0f) lsl 12)
|
||||||
|
lor ((n2 land 0x3f) lsl 6)
|
||||||
|
lor (n3 land 0x3f))
|
||||||
| 4 ->
|
| 4 ->
|
||||||
let n1 = Char.code c1 in
|
let n1 = Char.code c1 in
|
||||||
let n2 = next_or_fail () in
|
let n2 = next_or_fail () in
|
||||||
let n3 = next_or_fail () in
|
let n3 = next_or_fail () in
|
||||||
let n4 = next_or_fail () in
|
let n4 = next_or_fail () in
|
||||||
Uchar.of_int (check_four n1 n2 n3 n4)
|
if n2 lsr 6 != 0b10 || n3 lsr 6 != 0b10 || n4 lsr 6 != 0b10 then
|
||||||
|
raise MalFormed;
|
||||||
|
Uchar.of_int
|
||||||
|
(((n1 land 0x07) lsl 18)
|
||||||
|
lor ((n2 land 0x3f) lsl 12)
|
||||||
|
lor ((n3 land 0x3f) lsl 6)
|
||||||
|
lor (n4 land 0x3f))
|
||||||
| _ -> raise MalFormed
|
| _ -> raise MalFormed
|
||||||
|
|
||||||
(**************************)
|
(**************************)
|
||||||
|
|
|
||||||
|
|
@ -1,164 +1,155 @@
|
||||||
|
(* The package sedlex is released under the terms of an MIT-like license. *)
|
||||||
|
(* See the attached LICENSE file. *)
|
||||||
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
||||||
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
|
|
||||||
|
|
||||||
(** Runtime support for lexers generated by [sedlex]. *)
|
(** Runtime support for lexers generated by [sedlex]. *)
|
||||||
|
|
||||||
(** This module is roughly equivalent to the module Lexing from the OCaml
|
(** This module is roughly equivalent to the module Lexing from the
|
||||||
standard library, except that its lexbuffers handle Unicode code points
|
OCaml standard library, except that its lexbuffers handle Unicode
|
||||||
(OCaml type: {!Uchar.t} in the range [0..0x10ffff]) instead of bytes (OCaml
|
code points (OCaml type: {!Uchar.t} in the range [0..0x10ffff])
|
||||||
type: [char]).
|
instead of bytes (OCaml type: [char]).
|
||||||
|
|
||||||
It is possible to have sedlex-generated lexers work on a custom
|
It is possible to have sedlex-generated lexers work on a custom
|
||||||
implementation for lex buffers. To do this, define a module [L] which
|
implementation for lex buffers. To do this, define a module [L]
|
||||||
implements the [start], [next], [mark] and [backtrack] functions (See the
|
which implements the [start], [next], [mark] and [backtrack]
|
||||||
Internal Interface section below for a specification). They need not work on
|
functions (See the Internal Interface section below for a
|
||||||
a type named [lexbuf]: you can use the type name you want. Then, just do in
|
specification). They need not work on a type named [lexbuf]: you
|
||||||
your sedlex-processed source, bind this module to the name [Sedlexing] (for
|
can use the type name you want. Then, just do in your
|
||||||
instance, with a local module definition: [let module Sedlexing = L in ...].
|
sedlex-processed source, bind this module to the name [Sedlexing]
|
||||||
|
(for instance, with a local module definition: [let module Sedlexing
|
||||||
|
= L in ...].
|
||||||
|
|
||||||
Of course, you'll probably want to define functions like [lexeme] to be used
|
Of course, you'll probably want to define functions like [lexeme] to
|
||||||
in the lexers semantic actions. *)
|
be used in the lexers semantic actions. *)
|
||||||
|
|
||||||
(** The type of lexer buffers. A lexer buffer is the argument passed to the
|
(** The type of lexer buffers. A lexer buffer is the argument passed
|
||||||
scanning functions defined by the generated lexers. The lexer buffer holds
|
to the scanning functions defined by the generated lexers.
|
||||||
the internal information for the scanners, including the code points of the
|
The lexer buffer holds the internal information for the
|
||||||
token currently scanned, its position from the beginning of the input
|
scanners, including the code points of the token currently scanned,
|
||||||
stream, and the current position of the lexer. *)
|
its position from the beginning of the input stream,
|
||||||
|
and the current position of the lexer. *)
|
||||||
type lexbuf
|
type lexbuf
|
||||||
|
|
||||||
(** Raised by some functions to signal that some code point is not compatible
|
(** Raised by some functions to signal that some code point is not
|
||||||
with a specified encoding. *)
|
compatible with a specified encoding. *)
|
||||||
exception InvalidCodepoint of int
|
exception InvalidCodepoint of int
|
||||||
|
|
||||||
(** Raised by functions in the [Utf8] and [Utf16] modules to report strings
|
(** Raised by functions in the [Utf8] and [Utf16] modules to report
|
||||||
which do not comply to the encoding. *)
|
strings which do not comply to the encoding. *)
|
||||||
exception MalFormed
|
exception MalFormed
|
||||||
|
|
||||||
(** {6 Creating generic lexbufs} *)
|
(** {6 Creating generic lexbufs} *)
|
||||||
|
|
||||||
(** Create a generic lexer buffer. When the lexer needs more characters, it will
|
(** Create a generic lexer buffer. When the lexer needs more
|
||||||
call the given function, giving it an array of Uchars [a], a position [pos]
|
characters, it will call the given function, giving it an array of
|
||||||
and a code point count [n]. The function should put [n] code points or less
|
Uchars [a], a position [pos] and a code point count [n]. The
|
||||||
in [a], starting at position [pos], and return the number of characters
|
function should put [n] code points or less in [a], starting at
|
||||||
provided. A return value of 0 means end of input. [bytes_per_char] argument
|
position [pos], and return the number of characters provided. A
|
||||||
is optional. If unspecified, byte positions are the same as code point
|
return value of 0 means end of input. [bytes_per_char] argument is
|
||||||
|
optional. If unspecified, byte positions are the same as code point
|
||||||
position. *)
|
position. *)
|
||||||
val create :
|
val create :
|
||||||
?bytes_per_char:(Uchar.t -> int) ->
|
?bytes_per_char:(Uchar.t -> int) ->
|
||||||
(Uchar.t array -> int -> int -> int) ->
|
(Uchar.t array -> int -> int -> int) ->
|
||||||
lexbuf
|
lexbuf
|
||||||
|
|
||||||
(** set the initial tracked input position, in code point, for [lexbuf]. If
|
(** set the initial tracked input position, in code point, for [lexbuf].
|
||||||
unspecified, byte postion is set to the same value as code point position.
|
If unspecified, byte postion is set to the same value as code
|
||||||
*)
|
point position. *)
|
||||||
val set_position :
|
val set_position :
|
||||||
?bytes_position:Lexing.position -> lexbuf -> Lexing.position -> unit
|
?bytes_position:Lexing.position -> lexbuf -> Lexing.position -> unit
|
||||||
|
|
||||||
(** [set_filename lexbuf file] sets the filename to [file] in [lexbuf]. It also
|
(** [set_filename lexbuf file] sets the filename to [file] in
|
||||||
sets the {!Lexing.pos_fname} field in returned {!Lexing.position} records.
|
[lexbuf]. It also sets the {!Lexing.pos_fname} field in
|
||||||
*)
|
returned {!Lexing.position} records. *)
|
||||||
val set_filename : lexbuf -> string -> unit
|
val set_filename : lexbuf -> string -> unit
|
||||||
|
|
||||||
(** Create a lexbuf from a stream of Unicode code points. [bytes_per_char] is
|
(** Create a lexbuf from a stream of Unicode code points. [bytes_per_char] is
|
||||||
optional. If unspecified, byte positions are the same as code point
|
optional. If unspecified, byte positions are the same as code point positions. *)
|
||||||
positions. *)
|
|
||||||
val from_gen : ?bytes_per_char:(Uchar.t -> int) -> Uchar.t Gen.t -> lexbuf
|
val from_gen : ?bytes_per_char:(Uchar.t -> int) -> Uchar.t Gen.t -> lexbuf
|
||||||
|
|
||||||
(** Create a lexbuf from an array of Unicode code points. [bytes_per_char] is
|
(** Create a lexbuf from an array of Unicode code points. [bytes_per_char] is
|
||||||
optional. If unspecified, byte positions are the same as code point
|
optional. If unspecified, byte positions are the same as code point positions. *)
|
||||||
positions. *)
|
|
||||||
val from_int_array : ?bytes_per_char:(Uchar.t -> int) -> int array -> lexbuf
|
val from_int_array : ?bytes_per_char:(Uchar.t -> int) -> int array -> lexbuf
|
||||||
|
|
||||||
(** Create a lexbuf from an array of Unicode code points. [bytes_per_char] is
|
(** Create a lexbuf from an array of Unicode code points. [bytes_per_char] is
|
||||||
optional. If unspecified, byte positions are the same as code point
|
optional. If unspecified, byte positions are the same as code point positions. *)
|
||||||
positions. *)
|
|
||||||
val from_uchar_array :
|
val from_uchar_array :
|
||||||
?bytes_per_char:(Uchar.t -> int) -> Uchar.t array -> lexbuf
|
?bytes_per_char:(Uchar.t -> int) -> Uchar.t array -> lexbuf
|
||||||
|
|
||||||
(** {6 Interface for lexers semantic actions} *)
|
(** {6 Interface for lexers semantic actions} *)
|
||||||
|
|
||||||
(** The following functions can be called from the semantic actions of lexer
|
(** The following functions can be called from the semantic actions of
|
||||||
definitions. They give access to the character string matched by the regular
|
lexer definitions. They give access to the character string matched
|
||||||
expression associated with the semantic action. *)
|
by the regular expression associated with the semantic action. *)
|
||||||
|
|
||||||
(** [Sedlexing.lexeme_start lexbuf] returns the offset in the input stream of
|
(** [Sedlexing.lexeme_start lexbuf] returns the offset in the
|
||||||
the first code point of the matched string. The first code point of the
|
input stream of the first code point of the matched string.
|
||||||
stream has offset 0. *)
|
The first code point of the stream has offset 0. *)
|
||||||
val lexeme_start : lexbuf -> int
|
val lexeme_start : lexbuf -> int
|
||||||
|
|
||||||
(** [Sedlexing.lexeme_start lexbuf] returns the offset in the input stream of
|
(** [Sedlexing.lexeme_start lexbuf] returns the offset in the
|
||||||
the first byte of the matched string. The first code point of the stream has
|
input stream of the first byte of the matched string.
|
||||||
offset 0. *)
|
The first code point of the stream has offset 0. *)
|
||||||
val lexeme_bytes_start : lexbuf -> int
|
val lexeme_bytes_start : lexbuf -> int
|
||||||
|
|
||||||
(** [Sedlexing.lexeme_end lexbuf] returns the offset in the input stream of the
|
(** [Sedlexing.lexeme_end lexbuf] returns the offset in the input
|
||||||
character following the last code point of the matched string. The first
|
stream of the character following the last code point of the
|
||||||
character of the stream has offset 0. *)
|
matched string. The first character of the stream has offset
|
||||||
|
0. *)
|
||||||
val lexeme_end : lexbuf -> int
|
val lexeme_end : lexbuf -> int
|
||||||
|
|
||||||
(** [Sedlexing.lexeme_end lexbuf] returns the offset in the input stream of the
|
(** [Sedlexing.lexeme_end lexbuf] returns the offset in the input
|
||||||
byte following the last code point of the matched string. The first
|
stream of the byte following the last code point of the
|
||||||
character of the stream has offset 0. *)
|
matched string. The first character of the stream has offset
|
||||||
|
0. *)
|
||||||
val lexeme_bytes_end : lexbuf -> int
|
val lexeme_bytes_end : lexbuf -> int
|
||||||
|
|
||||||
(** [Sedlexing.loc lexbuf] returns the pair
|
(** [Sedlexing.loc lexbuf] returns the pair
|
||||||
[(Sedlexing.lexeme_start lexbuf,Sedlexing.lexeme_end lexbuf)]. *)
|
[(Sedlexing.lexeme_start lexbuf,Sedlexing.lexeme_end
|
||||||
|
lexbuf)]. *)
|
||||||
val loc : lexbuf -> int * int
|
val loc : lexbuf -> int * int
|
||||||
|
|
||||||
(** [Sedlexing.bytes_loc lexbuf] returns the pair
|
(** [Sedlexing.bytes_loc lexbuf] returns the pair
|
||||||
[(Sedlexing.lexeme_bytes_start lexbuf,Sedlexing.lexeme_bytes_end lexbuf)].
|
[(Sedlexing.lexeme_bytes_start lexbuf,Sedlexing.lexeme_bytes_end
|
||||||
*)
|
lexbuf)]. *)
|
||||||
val bytes_loc : lexbuf -> int * int
|
val bytes_loc : lexbuf -> int * int
|
||||||
|
|
||||||
(** [Sedlexing.lexeme_length lexbuf] returns the difference
|
(** [Sedlexing.lexeme_length lexbuf] returns the difference
|
||||||
[(Sedlexing.lexeme_end lexbuf) - (Sedlexing.lexeme_start lexbuf)], that is,
|
[(Sedlexing.lexeme_end lexbuf) - (Sedlexing.lexeme_start
|
||||||
the length (in code points) of the matched string. *)
|
lexbuf)], that is, the length (in code points) of the matched
|
||||||
|
string. *)
|
||||||
val lexeme_length : lexbuf -> int
|
val lexeme_length : lexbuf -> int
|
||||||
|
|
||||||
(** [Sedlexing.lexeme_length lexbuf] returns the difference
|
(** [Sedlexing.lexeme_length lexbuf] returns the difference
|
||||||
[(Sedlexing.lexeme_bytes_end lexbuf) - (Sedlexing.lexeme_bytes_start
|
[(Sedlexing.lexeme_bytes_end lexbuf) - (Sedlexing.lexeme_bytes_start
|
||||||
lexbuf)], that is, the length (in bytes) of the matched string. *)
|
lexbuf)], that is, the length (in bytes) of the matched
|
||||||
|
string. *)
|
||||||
val lexeme_bytes_length : lexbuf -> int
|
val lexeme_bytes_length : lexbuf -> int
|
||||||
|
|
||||||
(** [Sedlexing.lexing_positions lexbuf] returns the start and end positions, in
|
(** [Sedlexing.lexing_positions lexbuf] returns the start and end
|
||||||
code points, of the current token, using a record of type [Lexing.position].
|
positions, in code points, of the current token, using a record of type
|
||||||
This is intended for consumption by parsers like those generated by
|
[Lexing.position]. This is intended for consumption
|
||||||
[Menhir]. *)
|
by parsers like those generated by [Menhir]. *)
|
||||||
val lexing_positions : lexbuf -> Lexing.position * Lexing.position
|
val lexing_positions : lexbuf -> Lexing.position * Lexing.position
|
||||||
|
|
||||||
(** [Sedlexing.lexing_position_start lexbuf] returns the start position, in code
|
|
||||||
points, of the current token. *)
|
|
||||||
val lexing_position_start : lexbuf -> Lexing.position
|
|
||||||
|
|
||||||
(** [Sedlexing.lexing_position_curr lexbuf] returns the end position, in code
|
|
||||||
points, of the current token. *)
|
|
||||||
val lexing_position_curr : lexbuf -> Lexing.position
|
|
||||||
|
|
||||||
(** [Sedlexing.lexing_bytes_positions lexbuf] returns the start and end
|
(** [Sedlexing.lexing_bytes_positions lexbuf] returns the start and end
|
||||||
positions, in bytes, of the current token, using a record of type
|
positions, in bytes, of the current token, using a record of type
|
||||||
[Lexing.position]. This is intended for consumption by parsers like those
|
[Lexing.position]. This is intended for consumption
|
||||||
generated by [Menhir]. *)
|
by parsers like those generated by [Menhir]. *)
|
||||||
val lexing_bytes_positions : lexbuf -> Lexing.position * Lexing.position
|
val lexing_bytes_positions : lexbuf -> Lexing.position * Lexing.position
|
||||||
|
|
||||||
(** [Sedlexing.lexing_bytes_position_start lexbuf] returns the start position,
|
(** [Sedlexing.new_line lexbuf] increments the line count and
|
||||||
in bytes, of the current token. *)
|
sets the beginning of line to the current position, as though
|
||||||
val lexing_bytes_position_start : lexbuf -> Lexing.position
|
a newline character had been encountered in the input. *)
|
||||||
|
|
||||||
(** [Sedlexing.lexing_bytes_position_curr lexbuf] returns the end position, in
|
|
||||||
bytes, of the current token. *)
|
|
||||||
val lexing_bytes_position_curr : lexbuf -> Lexing.position
|
|
||||||
|
|
||||||
(** [Sedlexing.new_line lexbuf] increments the line count and sets the beginning
|
|
||||||
of line to the current position, as though a newline character had been
|
|
||||||
encountered in the input. *)
|
|
||||||
val new_line : lexbuf -> unit
|
val new_line : lexbuf -> unit
|
||||||
|
|
||||||
(** [Sedlexing.lexeme lexbuf] returns the string matched by the regular
|
(** [Sedlexing.lexeme lexbuf] returns the string matched by the
|
||||||
expression as an array of Unicode code point. *)
|
regular expression as an array of Unicode code point. *)
|
||||||
val lexeme : lexbuf -> Uchar.t array
|
val lexeme : lexbuf -> Uchar.t array
|
||||||
|
|
||||||
(** [Sedlexing.lexeme_char lexbuf pos] returns code point number [pos] in the
|
(** [Sedlexing.lexeme_char lexbuf pos] returns code point number [pos] in
|
||||||
matched string. *)
|
the matched string. *)
|
||||||
val lexeme_char : lexbuf -> int -> Uchar.t
|
val lexeme_char : lexbuf -> int -> Uchar.t
|
||||||
|
|
||||||
(** [Sedlexing.sub_lexeme lexbuf pos len] returns a substring of the string
|
(** [Sedlexing.sub_lexeme lexbuf pos len] returns a substring of the string
|
||||||
|
|
@ -166,52 +157,56 @@ val lexeme_char : lexbuf -> int -> Uchar.t
|
||||||
val sub_lexeme : lexbuf -> int -> int -> Uchar.t array
|
val sub_lexeme : lexbuf -> int -> int -> Uchar.t array
|
||||||
|
|
||||||
(** [Sedlexing.rollback lexbuf] puts [lexbuf] back in its configuration before
|
(** [Sedlexing.rollback lexbuf] puts [lexbuf] back in its configuration before
|
||||||
the last lexeme was matched. It is then possible to use another lexer to
|
the last lexeme was matched. It is then possible to use another
|
||||||
parse the same characters again. The other functions above in this section
|
lexer to parse the same characters again. The other functions
|
||||||
should not be used in the semantic action after a call to
|
above in this section should not be used in the semantic action
|
||||||
[Sedlexing.rollback]. *)
|
after a call to [Sedlexing.rollback]. *)
|
||||||
val rollback : lexbuf -> unit
|
val rollback : lexbuf -> unit
|
||||||
|
|
||||||
(** {6 Internal interface} *)
|
(** {6 Internal interface} *)
|
||||||
|
|
||||||
(** These functions are used internally by the lexers. They could be used to
|
(** These functions are used internally by the lexers. They could be used
|
||||||
write lexers by hand, or with a lexer generator different from [sedlex]. The
|
to write lexers by hand, or with a lexer generator different from
|
||||||
lexer buffers have a unique internal slot that can store an integer. They
|
[sedlex]. The lexer buffers have a unique internal slot that can store
|
||||||
also store a "backtrack" position. *)
|
an integer. They also store a "backtrack" position.
|
||||||
|
*)
|
||||||
|
|
||||||
(** [start t] informs the lexer buffer that any code points until the current
|
(** [start t] informs the lexer buffer that any
|
||||||
position can be discarded. The current position become the "start" position
|
code points until the current position can be discarded.
|
||||||
as returned by [Sedlexing.lexeme_start]. Moreover, the internal slot is set
|
The current position become the "start" position as returned
|
||||||
to [-1] and the backtrack position is set to the current position. *)
|
by [Sedlexing.lexeme_start]. Moreover, the internal slot is set to
|
||||||
|
[-1] and the backtrack position is set to the current position.
|
||||||
|
*)
|
||||||
val start : lexbuf -> unit
|
val start : lexbuf -> unit
|
||||||
|
|
||||||
(** [next lexbuf] extracts the next code point from the lexer buffer and
|
(** [next lexbuf] extracts the next code point from the
|
||||||
increments to current position. If the input stream is exhausted, the
|
lexer buffer and increments to current position. If the input stream
|
||||||
function returns [None]. If a ['\n'] is encountered, the tracked line number
|
is exhausted, the function returns [None].
|
||||||
is incremented. *)
|
If a ['\n'] is encountered, the tracked line number is incremented. *)
|
||||||
val next : lexbuf -> Uchar.t option
|
val next : lexbuf -> Uchar.t option
|
||||||
|
|
||||||
(** [__private__next_int lexbuf] extracts the next code point from the lexer
|
(** [__private__next_int lexbuf] extracts the next code point from the
|
||||||
buffer and increments to current position. If the input stream is exhausted,
|
lexer buffer and increments to current position. If the input stream
|
||||||
the function returns -1. If a ['\n'] is encountered, the tracked line number
|
is exhausted, the function returns -1.
|
||||||
is incremented.
|
If a ['\n'] is encountered, the tracked line number is incremented.
|
||||||
|
|
||||||
This is a private API, it should not be used by code using this module's API
|
This is a private API, it should not be used by code using this module's
|
||||||
and can be removed at any time. *)
|
API and can be removed at any time. *)
|
||||||
val __private__next_int : lexbuf -> int
|
val __private__next_int : lexbuf -> int
|
||||||
|
|
||||||
(** [mark lexbuf i] stores the integer [i] in the internal slot. The backtrack
|
(** [mark lexbuf i] stores the integer [i] in the internal
|
||||||
position is set to the current position. *)
|
slot. The backtrack position is set to the current position. *)
|
||||||
val mark : lexbuf -> int -> unit
|
val mark : lexbuf -> int -> unit
|
||||||
|
|
||||||
(** [backtrack lexbuf] returns the value stored in the internal slot of the
|
(** [backtrack lexbuf] returns the value stored in the
|
||||||
buffer, and performs backtracking (the current position is set to the value
|
internal slot of the buffer, and performs backtracking
|
||||||
of the backtrack position). *)
|
(the current position is set to the value of the backtrack position). *)
|
||||||
val backtrack : lexbuf -> int
|
val backtrack : lexbuf -> int
|
||||||
|
|
||||||
(** [with_tokenizer tokenizer lexbuf] given a lexer and a lexbuf, returns a
|
(** [with_tokenizer tokenizer lexbuf] given a lexer and a lexbuf,
|
||||||
generator of tokens annotated with positions. This generator can be used
|
returns a generator of tokens annotated with positions.
|
||||||
with the Menir parser generator's incremental API. *)
|
This generator can be used with the Menir parser generator's
|
||||||
|
incremental API. *)
|
||||||
val with_tokenizer :
|
val with_tokenizer :
|
||||||
(lexbuf -> 'token) ->
|
(lexbuf -> 'token) ->
|
||||||
lexbuf ->
|
lexbuf ->
|
||||||
|
|
@ -221,30 +216,30 @@ val with_tokenizer :
|
||||||
(** {6 Support for common encodings} *)
|
(** {6 Support for common encodings} *)
|
||||||
|
|
||||||
module Latin1 : sig
|
module Latin1 : sig
|
||||||
(** Create a lexbuf from a Latin1 encoded stream (ie a stream of Unicode code
|
(** Create a lexbuf from a Latin1 encoded stream (ie a stream
|
||||||
points in the range [0..255]) *)
|
of Unicode code points in the range [0..255]) *)
|
||||||
val from_gen : char Gen.t -> lexbuf
|
val from_gen : char Gen.t -> lexbuf
|
||||||
|
|
||||||
(** Create a lexbuf from a Latin1 encoded input channel. The client is
|
(** Create a lexbuf from a Latin1 encoded input channel.
|
||||||
responsible for closing the channel. *)
|
The client is responsible for closing the channel. *)
|
||||||
val from_channel : in_channel -> lexbuf
|
val from_channel : in_channel -> lexbuf
|
||||||
|
|
||||||
(** Create a lexbuf from a Latin1 encoded string. *)
|
(** Create a lexbuf from a Latin1 encoded string. *)
|
||||||
val from_string : string -> lexbuf
|
val from_string : string -> lexbuf
|
||||||
|
|
||||||
(** As [Sedlexing.lexeme] with a result encoded in Latin1. This function
|
(** As [Sedlexing.lexeme] with a result encoded in Latin1. This
|
||||||
throws an exception [InvalidCodepoint] if it is not possible to encode the
|
function throws an exception [InvalidCodepoint] if it is not
|
||||||
result in Latin1. *)
|
possible to encode the result in Latin1. *)
|
||||||
val lexeme : lexbuf -> string
|
val lexeme : lexbuf -> string
|
||||||
|
|
||||||
(** As [Sedlexing.sub_lexeme] with a result encoded in Latin1. This function
|
(** As [Sedlexing.sub_lexeme] with a result encoded in Latin1.
|
||||||
throws an exception [InvalidCodepoint] if it is not possible to encode the
|
This function throws an exception [InvalidCodepoint] if it
|
||||||
result in Latin1. *)
|
is not possible to encode the result in Latin1. *)
|
||||||
val sub_lexeme : lexbuf -> int -> int -> string
|
val sub_lexeme : lexbuf -> int -> int -> string
|
||||||
|
|
||||||
(** As [Sedlexing.lexeme_char] with a result encoded in Latin1. This function
|
(** As [Sedlexing.lexeme_char] with a result encoded in Latin1.
|
||||||
throws an exception [InvalidCodepoint] if it is not possible to encode the
|
This function throws an exception [InvalidCodepoint] if it
|
||||||
result in Latin1. *)
|
is not possible to encode the result in Latin1. *)
|
||||||
val lexeme_char : lexbuf -> int -> char
|
val lexeme_char : lexbuf -> int -> char
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
@ -263,24 +258,19 @@ module Utf8 : sig
|
||||||
|
|
||||||
(** As [Sedlexing.sub_lexeme] with a result encoded in UTF-8. *)
|
(** As [Sedlexing.sub_lexeme] with a result encoded in UTF-8. *)
|
||||||
val sub_lexeme : lexbuf -> int -> int -> string
|
val sub_lexeme : lexbuf -> int -> int -> string
|
||||||
|
|
||||||
module Helper : sig
|
|
||||||
val width : char -> int
|
|
||||||
val check_two : int -> int -> int
|
|
||||||
val check_three : int -> int -> int -> int
|
|
||||||
val check_four : int -> int -> int -> int -> int
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
||||||
module Utf16 : sig
|
module Utf16 : sig
|
||||||
type byte_order = Little_endian | Big_endian
|
type byte_order = Little_endian | Big_endian
|
||||||
|
|
||||||
(** [Utf16.from_gen s opt_bo] creates a lexbuf from an UTF-16 encoded stream.
|
(** [Utf16.from_gen s opt_bo] creates a lexbuf from an UTF-16
|
||||||
If [opt_bo] matches with [None] the function expects a BOM (Byte Order
|
encoded stream. If [opt_bo] matches with [None] the function
|
||||||
Mark), and takes the byte order as [Utf16.Big_endian] if it cannot find
|
expects a BOM (Byte Order Mark), and takes the byte order as
|
||||||
one. When [opt_bo] matches with [Some bo], [bo] is taken as byte order. In
|
[Utf16.Big_endian] if it cannot find one. When [opt_bo]
|
||||||
this case a leading BOM is kept in the stream - the lexer has to ignore it
|
matches with [Some bo], [bo] is taken as byte order. In this
|
||||||
and a `wrong' BOM ([0xfffe]) will raise Utf16.InvalidCodepoint. *)
|
case a leading BOM is kept in the stream - the lexer has to
|
||||||
|
ignore it and a `wrong' BOM ([0xfffe]) will raise
|
||||||
|
Utf16.InvalidCodepoint. *)
|
||||||
val from_gen : char Gen.t -> byte_order option -> lexbuf
|
val from_gen : char Gen.t -> byte_order option -> lexbuf
|
||||||
|
|
||||||
(** Works as [Utf16.from_gen] with an [in_channel]. *)
|
(** Works as [Utf16.from_gen] with an [in_channel]. *)
|
||||||
|
|
@ -289,12 +279,13 @@ module Utf16 : sig
|
||||||
(** Works as [Utf16.from_gen] with a [string]. *)
|
(** Works as [Utf16.from_gen] with a [string]. *)
|
||||||
val from_string : string -> byte_order option -> lexbuf
|
val from_string : string -> byte_order option -> lexbuf
|
||||||
|
|
||||||
(** [utf16_lexeme lb bo bom] as [Sedlexing.lexeme] with a result encoded in
|
(** [utf16_lexeme lb bo bom] as [Sedlexing.lexeme] with a result
|
||||||
UTF-16 in byte_order [bo] and starting with a BOM if [bom = true]. *)
|
encoded in UTF-16 in byte_order [bo] and starting with a BOM
|
||||||
|
if [bom = true]. *)
|
||||||
val lexeme : lexbuf -> byte_order -> bool -> string
|
val lexeme : lexbuf -> byte_order -> bool -> string
|
||||||
|
|
||||||
(** [sub_lexeme lb pos len bo bom] as [Sedlexing.sub_lexeme] with a result
|
(** [sub_lexeme lb pos len bo bom] as
|
||||||
encoded in UTF-16 with byte order [bo] and starting with a BOM if
|
[Sedlexing.sub_lexeme] with a result encoded in UTF-16 with
|
||||||
[bom=true] *)
|
byte order [bo] and starting with a BOM if [bom=true] *)
|
||||||
val sub_lexeme : lexbuf -> int -> int -> byte_order -> bool -> string
|
val sub_lexeme : lexbuf -> int -> int -> byte_order -> bool -> string
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,18 @@
|
||||||
(library
|
(library
|
||||||
(name sedlex_ppx)
|
(name sedlex_ppx)
|
||||||
(public_name noslop-sedlex.ppx)
|
(public_name sedlex.ppx)
|
||||||
(kind ppx_rewriter)
|
(kind ppx_rewriter)
|
||||||
(libraries ppxlib noslop-sedlex noslop-sedlex.utils)
|
(libraries ppxlib sedlex sedlex.utils)
|
||||||
(ppx_runtime_libraries noslop-sedlex)
|
(ppx_runtime_libraries sedlex)
|
||||||
(preprocess
|
(preprocess
|
||||||
(pps ppxlib.metaquot)))
|
(pps ppxlib.metaquot))
|
||||||
|
(flags
|
||||||
|
(:standard -w -9)))
|
||||||
|
|
||||||
(rule
|
(rule
|
||||||
(targets unicode.ml)
|
(targets unicode.ml)
|
||||||
(mode promote)
|
(mode
|
||||||
|
(promote (until-clean)))
|
||||||
(deps
|
(deps
|
||||||
(:gen ../generator/gen_unicode.exe)
|
(:gen ../generator/gen_unicode.exe)
|
||||||
(glob_files ../generator/data/*.txt))
|
(glob_files ../generator/data/*.txt))
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,8 @@
|
||||||
|
|
||||||
open Sedlex_cset
|
open Sedlex_cset
|
||||||
|
|
||||||
(** Letters to be used in identifiers, as specified by ISO .... *)
|
(** Letters to be used in identifiers, as specified
|
||||||
|
by ISO .... *)
|
||||||
|
|
||||||
(* Data provided by John M. Skaller *)
|
(* Data provided by John M. Skaller *)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,9 @@
|
||||||
|
(* The package sedlex is released under the terms of an MIT-like license. *)
|
||||||
|
(* See the attached LICENSE file. *)
|
||||||
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
||||||
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
|
|
||||||
|
|
||||||
open Ppxlib
|
open Ppxlib
|
||||||
open Ast_builder.Default
|
open Ast_builder.Default
|
||||||
open Ast_helper
|
|
||||||
|
|
||||||
(* let ocaml_version = Versions.ocaml_408 *)
|
(* let ocaml_version = Versions.ocaml_408 *)
|
||||||
|
|
||||||
|
|
@ -12,6 +12,7 @@ module Cset = Sedlex_cset
|
||||||
(* Decision tree for partitions *)
|
(* Decision tree for partitions *)
|
||||||
|
|
||||||
let default_loc = Location.none
|
let default_loc = Location.none
|
||||||
|
let lident_loc ~loc s = { loc; txt = lident s }
|
||||||
|
|
||||||
type decision_tree =
|
type decision_tree =
|
||||||
| Lte of int * decision_tree * decision_tree
|
| Lte of int * decision_tree * decision_tree
|
||||||
|
|
@ -197,14 +198,14 @@ let best_final final =
|
||||||
let state_fun state = Printf.sprintf "__sedlex_state_%i" state
|
let state_fun state = Printf.sprintf "__sedlex_state_%i" state
|
||||||
|
|
||||||
let call_state lexbuf auto state =
|
let call_state lexbuf auto state =
|
||||||
let { Sedlex.trans; finals } = auto.(state) in
|
let trans, final = auto.(state) in
|
||||||
if Array.length trans = 0 then (
|
if Array.length trans = 0 then (
|
||||||
match best_final finals with
|
match best_final final with
|
||||||
| Some i -> eint ~loc:default_loc i
|
| Some i -> eint ~loc:default_loc i
|
||||||
| None -> assert false)
|
| None -> assert false)
|
||||||
else appfun (state_fun state) [lexbuf]
|
else appfun (state_fun state) [lexbuf]
|
||||||
|
|
||||||
let gen_state (lexbuf_name, lexbuf) auto i { Sedlex.trans; finals } =
|
let gen_state (lexbuf_name, lexbuf) auto i (trans, final) =
|
||||||
let loc = default_loc in
|
let loc = default_loc in
|
||||||
let partition = Array.map fst trans in
|
let partition = Array.map fst trans in
|
||||||
let cases =
|
let cases =
|
||||||
|
|
@ -231,10 +232,10 @@ let gen_state (lexbuf_name, lexbuf) auto i { Sedlex.trans; finals } =
|
||||||
[
|
[
|
||||||
value_binding ~loc
|
value_binding ~loc
|
||||||
~pat:(pvar ~loc (state_fun i))
|
~pat:(pvar ~loc (state_fun i))
|
||||||
~expr:(Exp.fun_ ~loc Nolabel None lhs body);
|
~expr:(pexp_function ~loc [case ~lhs ~guard:None ~rhs:body]);
|
||||||
]
|
]
|
||||||
in
|
in
|
||||||
match best_final finals with
|
match best_final final with
|
||||||
| None -> ret (body ())
|
| None -> ret (body ())
|
||||||
| Some _ when Array.length trans = 0 -> []
|
| Some _ when Array.length trans = 0 -> []
|
||||||
| Some i ->
|
| Some i ->
|
||||||
|
|
@ -248,19 +249,25 @@ let gen_recflag auto =
|
||||||
in states with no further transitions. *)
|
in states with no further transitions. *)
|
||||||
try
|
try
|
||||||
Array.iter
|
Array.iter
|
||||||
(fun { Sedlex.trans; _ } ->
|
(fun (trans_i, _) ->
|
||||||
Array.iter
|
Array.iter
|
||||||
(fun (_, j) ->
|
(fun (_, j) ->
|
||||||
if Array.length auto.(j).Sedlex.trans > 0 then raise Exit)
|
let trans_j, _ = auto.(j) in
|
||||||
trans)
|
if Array.length trans_j > 0 then raise Exit)
|
||||||
|
trans_i)
|
||||||
auto;
|
auto;
|
||||||
Nonrecursive
|
Nonrecursive
|
||||||
with Exit -> Recursive
|
with Exit -> Recursive
|
||||||
|
|
||||||
let gen_definition ((_, lexbuf) as lexbuf_with_name) auto l error =
|
let gen_definition ((_, lexbuf) as lexbuf_with_name) l error =
|
||||||
let loc = default_loc in
|
let loc = default_loc in
|
||||||
|
let brs = Array.of_list l in
|
||||||
|
let auto = Sedlex.compile (Array.map fst brs) in
|
||||||
let cases =
|
let cases =
|
||||||
List.mapi (fun i (_, e) -> case ~lhs:(pint ~loc i) ~guard:None ~rhs:e) l
|
Array.to_list
|
||||||
|
(Array.mapi
|
||||||
|
(fun i (_, e) -> case ~lhs:(pint ~loc i) ~guard:None ~rhs:e)
|
||||||
|
brs)
|
||||||
in
|
in
|
||||||
let states = Array.mapi (gen_state lexbuf_with_name auto) auto in
|
let states = Array.mapi (gen_state lexbuf_with_name auto) auto in
|
||||||
let states = List.flatten (Array.to_list states) in
|
let states = List.flatten (Array.to_list states) in
|
||||||
|
|
@ -278,46 +285,17 @@ let codepoint i =
|
||||||
failwith (Printf.sprintf "Invalid Unicode code point: %i" i);
|
failwith (Printf.sprintf "Invalid Unicode code point: %i" i);
|
||||||
i
|
i
|
||||||
|
|
||||||
let char c = Cset.singleton (Char.code c)
|
let regexp_for_char c = Sedlex.chars (Cset.singleton (Char.code c))
|
||||||
let uchar c = Cset.singleton (Uchar.to_int c)
|
|
||||||
|
|
||||||
let err loc fmt =
|
let regexp_for_string s =
|
||||||
Printf.ksprintf
|
let rec aux n =
|
||||||
(fun s ->
|
if n = String.length s then Sedlex.eps
|
||||||
raise (Location.Error (Location.Error.createf ~loc "Sedlex: %s" s)))
|
else Sedlex.seq (regexp_for_char s.[n]) (aux (succ n))
|
||||||
fmt
|
in
|
||||||
|
aux 0
|
||||||
|
|
||||||
type encoding = Utf8 | Latin1 | Ascii
|
let err loc s =
|
||||||
|
raise (Location.Error (Location.Error.createf ~loc "Sedlex: %s" s))
|
||||||
let string_of_encoding = function
|
|
||||||
| Utf8 -> "UTF-8"
|
|
||||||
| Latin1 -> "Latin-1"
|
|
||||||
| Ascii -> "ASCII"
|
|
||||||
|
|
||||||
let rev_csets_of_string ~loc ~encoding s =
|
|
||||||
match encoding with
|
|
||||||
| Utf8 ->
|
|
||||||
Utf8.fold
|
|
||||||
~f:(fun acc _ x ->
|
|
||||||
match x with
|
|
||||||
| `Malformed _ ->
|
|
||||||
err loc "Malformed %s string" (string_of_encoding encoding)
|
|
||||||
| `Uchar c -> uchar c :: acc)
|
|
||||||
[] s
|
|
||||||
| Latin1 ->
|
|
||||||
let l = ref [] in
|
|
||||||
for i = 0 to String.length s - 1 do
|
|
||||||
l := char s.[i] :: !l
|
|
||||||
done;
|
|
||||||
!l
|
|
||||||
| Ascii ->
|
|
||||||
let l = ref [] in
|
|
||||||
for i = 0 to String.length s - 1 do
|
|
||||||
match s.[i] with
|
|
||||||
| '\x00' .. '\x7F' as c -> l := char c :: !l
|
|
||||||
| _ -> err loc "Malformed %s string" (string_of_encoding encoding)
|
|
||||||
done;
|
|
||||||
!l
|
|
||||||
|
|
||||||
let rec repeat r = function
|
let rec repeat r = function
|
||||||
| 0, 0 -> Sedlex.eps
|
| 0, 0 -> Sedlex.eps
|
||||||
|
|
@ -325,41 +303,32 @@ let rec repeat r = function
|
||||||
| n, m -> Sedlex.seq r (repeat r (n - 1, m - 1))
|
| n, m -> Sedlex.seq r (repeat r (n - 1, m - 1))
|
||||||
|
|
||||||
let regexp_of_pattern env =
|
let regexp_of_pattern env =
|
||||||
let rec char_pair_op func name ~encoding ~loc tuple =
|
let rec char_pair_op func name p tuple =
|
||||||
(* Construct something like Sub(a,b) *)
|
(* Construct something like Sub(a,b) *)
|
||||||
match tuple with
|
match tuple with
|
||||||
| Some { ppat_desc = Ppat_tuple [p0; p1]; _ } ->
|
| Some { ppat_desc = Ppat_tuple [p0; p1] } -> begin
|
||||||
begin match func (aux ~encoding p0) (aux ~encoding p1) with
|
match func (aux p0) (aux p1) with
|
||||||
| Some r -> r
|
| Some r -> r
|
||||||
| None ->
|
| None ->
|
||||||
err loc
|
err p.ppat_loc @@ "the " ^ name
|
||||||
"the %s operator can only applied to single-character length \
|
^ " operator can only applied to single-character length \
|
||||||
regexps"
|
regexps"
|
||||||
name
|
|
||||||
end
|
end
|
||||||
| _ ->
|
| _ ->
|
||||||
err loc "the %s operator requires two arguments, like %s(a,b)" name
|
err p.ppat_loc @@ "the " ^ name
|
||||||
name
|
^ " operator requires two arguments, like " ^ name ^ "(a,b)"
|
||||||
and aux ~encoding p =
|
and aux p =
|
||||||
(* interpret one pattern node *)
|
(* interpret one pattern node *)
|
||||||
match p.ppat_desc with
|
match p.ppat_desc with
|
||||||
| Ppat_or (p1, p2) -> Sedlex.alt (aux ~encoding p1) (aux ~encoding p2)
|
| Ppat_or (p1, p2) -> Sedlex.alt (aux p1) (aux p2)
|
||||||
| Ppat_tuple (p :: pl) ->
|
| Ppat_tuple (p :: pl) ->
|
||||||
List.fold_left
|
List.fold_left (fun r p -> Sedlex.seq r (aux p)) (aux p) pl
|
||||||
(fun r p -> Sedlex.seq r (aux ~encoding p))
|
| Ppat_construct ({ txt = Lident "Star" }, Some (_, p)) ->
|
||||||
(aux ~encoding p) pl
|
Sedlex.rep (aux p)
|
||||||
| Ppat_construct ({ txt = Lident "Star"; _ }, Some (_, p)) ->
|
| Ppat_construct ({ txt = Lident "Plus" }, Some (_, p)) ->
|
||||||
Sedlex.rep (aux ~encoding p)
|
Sedlex.plus (aux p)
|
||||||
| Ppat_construct ({ txt = Lident "Plus"; _ }, Some (_, p)) ->
|
|
||||||
Sedlex.plus (aux ~encoding p)
|
|
||||||
| Ppat_construct ({ txt = Lident "Utf8"; _ }, Some (_, p)) ->
|
|
||||||
aux ~encoding:Utf8 p
|
|
||||||
| Ppat_construct ({ txt = Lident "Latin1"; _ }, Some (_, p)) ->
|
|
||||||
aux ~encoding:Latin1 p
|
|
||||||
| Ppat_construct ({ txt = Lident "Ascii"; _ }, Some (_, p)) ->
|
|
||||||
aux ~encoding:Ascii p
|
|
||||||
| Ppat_construct
|
| Ppat_construct
|
||||||
( { txt = Lident "Rep"; _ },
|
( { txt = Lident "Rep" },
|
||||||
Some
|
Some
|
||||||
( _,
|
( _,
|
||||||
{
|
{
|
||||||
|
|
@ -370,28 +339,26 @@ let regexp_of_pattern env =
|
||||||
{
|
{
|
||||||
ppat_desc =
|
ppat_desc =
|
||||||
Ppat_constant (i1 as i2) | Ppat_interval (i1, i2);
|
Ppat_constant (i1 as i2) | Ppat_interval (i1, i2);
|
||||||
_;
|
|
||||||
};
|
};
|
||||||
];
|
];
|
||||||
_;
|
} ) ) -> begin
|
||||||
} ) ) ->
|
match (i1, i2) with
|
||||||
begin match (i1, i2) with
|
|
||||||
| Pconst_integer (i1, _), Pconst_integer (i2, _) ->
|
| Pconst_integer (i1, _), Pconst_integer (i2, _) ->
|
||||||
let i1 = int_of_string i1 in
|
let i1 = int_of_string i1 in
|
||||||
let i2 = int_of_string i2 in
|
let i2 = int_of_string i2 in
|
||||||
if 0 <= i1 && i1 <= i2 then repeat (aux ~encoding p0) (i1, i2)
|
if 0 <= i1 && i1 <= i2 then repeat (aux p0) (i1, i2)
|
||||||
else err p.ppat_loc "Invalid range for Rep operator"
|
else err p.ppat_loc "Invalid range for Rep operator"
|
||||||
| _ ->
|
| _ ->
|
||||||
err p.ppat_loc "Rep must take an integer constant or interval"
|
err p.ppat_loc "Rep must take an integer constant or interval"
|
||||||
end
|
end
|
||||||
| Ppat_construct ({ txt = Lident "Rep"; _ }, _) ->
|
| Ppat_construct ({ txt = Lident "Rep" }, _) ->
|
||||||
err p.ppat_loc "the Rep operator takes 2 arguments"
|
err p.ppat_loc "the Rep operator takes 2 arguments"
|
||||||
| Ppat_construct ({ txt = Lident "Opt"; _ }, Some (_, p)) ->
|
| Ppat_construct ({ txt = Lident "Opt" }, Some (_, p)) ->
|
||||||
Sedlex.alt Sedlex.eps (aux ~encoding p)
|
Sedlex.alt Sedlex.eps (aux p)
|
||||||
| Ppat_construct ({ txt = Lident "Compl"; _ }, arg) ->
|
| Ppat_construct ({ txt = Lident "Compl" }, arg) -> begin
|
||||||
begin match arg with
|
match arg with
|
||||||
| Some (_, p0) ->
|
| Some (_, p0) -> begin
|
||||||
begin match Sedlex.compl (aux ~encoding p0) with
|
match Sedlex.compl (aux p0) with
|
||||||
| Some r -> r
|
| Some r -> r
|
||||||
| None ->
|
| None ->
|
||||||
err p.ppat_loc
|
err p.ppat_loc
|
||||||
|
|
@ -400,40 +367,30 @@ let regexp_of_pattern env =
|
||||||
end
|
end
|
||||||
| _ -> err p.ppat_loc "the Compl operator requires an argument"
|
| _ -> err p.ppat_loc "the Compl operator requires an argument"
|
||||||
end
|
end
|
||||||
| Ppat_construct ({ txt = Lident "Sub"; _ }, arg) ->
|
| Ppat_construct ({ txt = Lident "Sub" }, arg) ->
|
||||||
char_pair_op ~encoding Sedlex.subtract "Sub" ~loc:p.ppat_loc
|
char_pair_op Sedlex.subtract "Sub" p
|
||||||
(Option.map (fun (_, arg) -> arg) arg)
|
(Option.map (fun (_, arg) -> arg) arg)
|
||||||
| Ppat_construct ({ txt = Lident "Intersect"; _ }, arg) ->
|
| Ppat_construct ({ txt = Lident "Intersect" }, arg) ->
|
||||||
char_pair_op ~encoding Sedlex.intersection "Intersect" ~loc:p.ppat_loc
|
char_pair_op Sedlex.intersection "Intersect" p
|
||||||
(Option.map (fun (_, arg) -> arg) arg)
|
(Option.map (fun (_, arg) -> arg) arg)
|
||||||
| Ppat_construct ({ txt = Lident "Chars"; _ }, arg) -> (
|
| Ppat_construct ({ txt = Lident "Chars" }, arg) -> (
|
||||||
let const =
|
let const =
|
||||||
match arg with
|
match arg with
|
||||||
| Some (_, { ppat_desc = Ppat_constant const; _ }) -> Some const
|
| Some (_, { ppat_desc = Ppat_constant const }) -> Some const
|
||||||
| _ -> None
|
| _ -> None
|
||||||
in
|
in
|
||||||
match const with
|
match const with
|
||||||
| Some (Pconst_string (s, _, _)) ->
|
| Some (Pconst_string (s, _, _)) ->
|
||||||
let l = rev_csets_of_string ~loc:p.ppat_loc ~encoding s in
|
let c = ref Cset.empty in
|
||||||
let chars = List.fold_left Cset.union Cset.empty l in
|
for i = 0 to String.length s - 1 do
|
||||||
Sedlex.chars chars
|
c := Cset.union !c (Cset.singleton (Char.code s.[i]))
|
||||||
|
done;
|
||||||
|
Sedlex.chars !c
|
||||||
| _ ->
|
| _ ->
|
||||||
err p.ppat_loc "the Chars operator requires a string argument")
|
err p.ppat_loc "the Chars operator requires a string argument")
|
||||||
| Ppat_interval (i_start, i_end) ->
|
| Ppat_interval (i_start, i_end) -> begin
|
||||||
begin match (i_start, i_end) with
|
match (i_start, i_end) with
|
||||||
| Pconst_char c1, Pconst_char c2 ->
|
| Pconst_char c1, Pconst_char c2 ->
|
||||||
let valid =
|
|
||||||
match encoding with
|
|
||||||
(* utf8 char interval can only match ascii because
|
|
||||||
of the OCaml lexer. *)
|
|
||||||
| Ascii | Utf8 -> (
|
|
||||||
function '\x00' .. '\x7f' -> true | _ -> false)
|
|
||||||
| Latin1 -> ( function _ -> true)
|
|
||||||
in
|
|
||||||
if not (valid c1 && valid c2) then
|
|
||||||
err p.ppat_loc
|
|
||||||
"this pattern is not a valid %s interval regexp"
|
|
||||||
(string_of_encoding encoding);
|
|
||||||
Sedlex.chars (Cset.interval (Char.code c1) (Char.code c2))
|
Sedlex.chars (Cset.interval (Char.code c1) (Char.code c2))
|
||||||
| Pconst_integer (i1, _), Pconst_integer (i2, _) ->
|
| Pconst_integer (i1, _), Pconst_integer (i2, _) ->
|
||||||
Sedlex.chars
|
Sedlex.chars
|
||||||
|
|
@ -442,65 +399,22 @@ let regexp_of_pattern env =
|
||||||
(codepoint (int_of_string i2)))
|
(codepoint (int_of_string i2)))
|
||||||
| _ -> err p.ppat_loc "this pattern is not a valid interval regexp"
|
| _ -> err p.ppat_loc "this pattern is not a valid interval regexp"
|
||||||
end
|
end
|
||||||
| Ppat_constant const ->
|
| Ppat_constant const -> begin
|
||||||
begin match const with
|
match const with
|
||||||
| Pconst_string (s, _, _) ->
|
| Pconst_string (s, _, _) -> regexp_for_string s
|
||||||
let rev_l = rev_csets_of_string s ~loc:p.ppat_loc ~encoding in
|
| Pconst_char c -> regexp_for_char c
|
||||||
List.fold_left
|
|
||||||
(fun acc cset -> Sedlex.seq (Sedlex.chars cset) acc)
|
|
||||||
Sedlex.eps rev_l
|
|
||||||
| Pconst_char c -> Sedlex.chars (char c)
|
|
||||||
| Pconst_integer (i, _) ->
|
| Pconst_integer (i, _) ->
|
||||||
Sedlex.chars (Cset.singleton (codepoint (int_of_string i)))
|
Sedlex.chars (Cset.singleton (codepoint (int_of_string i)))
|
||||||
| _ -> err p.ppat_loc "this pattern is not a valid regexp"
|
| _ -> err p.ppat_loc "this pattern is not a valid regexp"
|
||||||
end
|
end
|
||||||
| Ppat_var { txt = x; _ } ->
|
| Ppat_var { txt = x } -> begin
|
||||||
begin try StringMap.find x env
|
try StringMap.find x env
|
||||||
with Not_found -> err p.ppat_loc "unbound regexp %s" x
|
with Not_found ->
|
||||||
|
err p.ppat_loc (Printf.sprintf "unbound regexp %s" x)
|
||||||
end
|
end
|
||||||
| _ -> err p.ppat_loc "this pattern is not a valid regexp"
|
| _ -> err p.ppat_loc "this pattern is not a valid regexp"
|
||||||
in
|
in
|
||||||
aux ~encoding:Ascii
|
aux
|
||||||
|
|
||||||
let handle_sedlex_match ~env ~map_rhs match_expr =
|
|
||||||
let lexbuf =
|
|
||||||
match match_expr with
|
|
||||||
| { pexp_desc = Pexp_match (lexbuf, _); _ } -> (
|
|
||||||
match lexbuf with
|
|
||||||
| { pexp_desc = Pexp_ident { txt = Lident txt; _ }; _ } ->
|
|
||||||
(txt, lexbuf)
|
|
||||||
| _ ->
|
|
||||||
err lexbuf.pexp_loc
|
|
||||||
"the matched expression must be a single identifier")
|
|
||||||
| _ ->
|
|
||||||
err match_expr.pexp_loc
|
|
||||||
"the %%sedlex extension is only recognized on match expressions"
|
|
||||||
in
|
|
||||||
let cases =
|
|
||||||
match match_expr with
|
|
||||||
| { pexp_desc = Pexp_match (_, cases); _ } -> cases
|
|
||||||
| _ -> assert false
|
|
||||||
in
|
|
||||||
let cases = List.rev cases in
|
|
||||||
let error =
|
|
||||||
match List.hd cases with
|
|
||||||
| { pc_lhs = [%pat? _]; pc_rhs = e; pc_guard = None } -> map_rhs e
|
|
||||||
| { pc_lhs = p; _ } ->
|
|
||||||
err p.ppat_loc "the last branch must be a catch-all error case"
|
|
||||||
in
|
|
||||||
let cases = List.rev (List.tl cases) in
|
|
||||||
let cases =
|
|
||||||
List.map
|
|
||||||
(function
|
|
||||||
| { pc_lhs = p; pc_rhs = e; pc_guard = None } ->
|
|
||||||
(regexp_of_pattern env p, map_rhs e)
|
|
||||||
| { pc_guard = Some e; _ } ->
|
|
||||||
err e.pexp_loc "'when' guards are not supported")
|
|
||||||
cases
|
|
||||||
in
|
|
||||||
let brs = Array.of_list cases in
|
|
||||||
let auto = Sedlex.compile (Array.map fst brs) in
|
|
||||||
(gen_definition lexbuf auto cases error, auto)
|
|
||||||
|
|
||||||
let previous = ref []
|
let previous = ref []
|
||||||
let regexps = ref []
|
let regexps = ref []
|
||||||
|
|
@ -516,18 +430,44 @@ let mapper =
|
||||||
|
|
||||||
method! expression e =
|
method! expression e =
|
||||||
match e with
|
match e with
|
||||||
| [%expr [%sedlex [%e? { pexp_desc = Pexp_match _; _ } as match_expr]]]
|
| [%expr [%sedlex [%e? { pexp_desc = Pexp_match (lexbuf, cases) }]]] ->
|
||||||
->
|
let lexbuf =
|
||||||
fst (handle_sedlex_match ~env ~map_rhs:this#expression match_expr)
|
match lexbuf with
|
||||||
|
| { pexp_desc = Pexp_ident { txt = Lident txt } } ->
|
||||||
|
(txt, lexbuf)
|
||||||
|
| _ ->
|
||||||
|
err lexbuf.pexp_loc
|
||||||
|
"the matched expression must be a single identifier"
|
||||||
|
in
|
||||||
|
let cases = List.rev cases in
|
||||||
|
let error =
|
||||||
|
match List.hd cases with
|
||||||
|
| { pc_lhs = [%pat? _]; pc_rhs = e; pc_guard = None } ->
|
||||||
|
this#expression e
|
||||||
|
| { pc_lhs = p } ->
|
||||||
|
err p.ppat_loc
|
||||||
|
"the last branch must be a catch-all error case"
|
||||||
|
in
|
||||||
|
let cases = List.rev (List.tl cases) in
|
||||||
|
let cases =
|
||||||
|
List.map
|
||||||
|
(function
|
||||||
|
| { pc_lhs = p; pc_rhs = e; pc_guard = None } ->
|
||||||
|
(regexp_of_pattern env p, this#expression e)
|
||||||
|
| { pc_guard = Some e } ->
|
||||||
|
err e.pexp_loc "'when' guards are not supported")
|
||||||
|
cases
|
||||||
|
in
|
||||||
|
gen_definition lexbuf cases error
|
||||||
| [%expr
|
| [%expr
|
||||||
let [%p? { ppat_desc = Ppat_var { txt = name; _ }; _ }] =
|
let [%p? { ppat_desc = Ppat_var { txt = name } }] =
|
||||||
[%sedlex.regexp? [%p? p]]
|
[%sedlex.regexp? [%p? p]]
|
||||||
in
|
in
|
||||||
[%e? body]] ->
|
[%e? body]] ->
|
||||||
(this#define_regexp name p)#expression body
|
(this#define_regexp name p)#expression body
|
||||||
| [%expr [%sedlex [%e? _]]] ->
|
| [%expr [%sedlex [%e? _]]] ->
|
||||||
err e.pexp_loc
|
err e.pexp_loc
|
||||||
"the %%sedlex extension is only recognized on match expressions"
|
"the %sedlex extension is only recognized on match expressions"
|
||||||
| _ -> super#expression e
|
| _ -> super#expression e
|
||||||
|
|
||||||
val toplevel = true
|
val toplevel = true
|
||||||
|
|
@ -540,7 +480,7 @@ let mapper =
|
||||||
(List.map
|
(List.map
|
||||||
(function
|
(function
|
||||||
| [%stri
|
| [%stri
|
||||||
let [%p? { ppat_desc = Ppat_var { txt = name; _ }; _ }] =
|
let [%p? { ppat_desc = Ppat_var { txt = name } }] =
|
||||||
[%sedlex.regexp? [%p? p]]] as i ->
|
[%sedlex.regexp? [%p? p]]] as i ->
|
||||||
regexps := i :: !regexps;
|
regexps := i :: !regexps;
|
||||||
mapper := !mapper#define_regexp name p;
|
mapper := !mapper#define_regexp name p;
|
||||||
|
|
@ -565,7 +505,7 @@ let mapper =
|
||||||
let pre_handler cookies =
|
let pre_handler cookies =
|
||||||
previous :=
|
previous :=
|
||||||
match Driver.Cookies.get cookies "sedlex.regexps" Ast_pattern.__ with
|
match Driver.Cookies.get cookies "sedlex.regexps" Ast_pattern.__ with
|
||||||
| Some { pexp_desc = Pexp_extension (_, PStr l); _ } -> l
|
| Some { pexp_desc = Pexp_extension (_, PStr l) } -> l
|
||||||
| Some _ -> assert false
|
| Some _ -> assert false
|
||||||
| None -> []
|
| None -> []
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
|
(* The package sedlex is released under the terms of an MIT-like license. *)
|
||||||
|
(* See the attached LICENSE file. *)
|
||||||
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
||||||
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
|
|
||||||
|
|
||||||
module Cset = Sedlex_cset
|
module Cset = Sedlex_cset
|
||||||
|
|
||||||
|
|
@ -24,7 +25,7 @@ let new_node () =
|
||||||
let seq r1 r2 succ = r1 (r2 succ)
|
let seq r1 r2 succ = r1 (r2 succ)
|
||||||
|
|
||||||
let is_chars final = function
|
let is_chars final = function
|
||||||
| { eps = []; trans = [(c, f)]; _ } when f == final -> Some c
|
| { eps = []; trans = [(c, f)] } when f == final -> Some c
|
||||||
| _ -> None
|
| _ -> None
|
||||||
|
|
||||||
let chars c succ =
|
let chars c succ =
|
||||||
|
|
@ -116,9 +117,6 @@ let transition (state : state) =
|
||||||
Array.sort (fun (c1, _) (c2, _) -> compare c1 c2) t;
|
Array.sort (fun (c1, _) (c2, _) -> compare c1 c2) t;
|
||||||
t
|
t
|
||||||
|
|
||||||
type dfa_state = { trans : (Cset.t * int) array; finals : bool array }
|
|
||||||
type dfa = dfa_state array
|
|
||||||
|
|
||||||
let compile rs =
|
let compile rs =
|
||||||
let rs = Array.map compile_re rs in
|
let rs = Array.map compile_re rs in
|
||||||
let counter = ref 0 in
|
let counter = ref 0 in
|
||||||
|
|
@ -133,7 +131,7 @@ let compile rs =
|
||||||
let trans = transition state in
|
let trans = transition state in
|
||||||
let trans = Array.map (fun (p, t) -> (p, aux t)) trans in
|
let trans = Array.map (fun (p, t) -> (p, aux t)) trans in
|
||||||
let finals = Array.map (fun (_, f) -> List.memq f state) rs in
|
let finals = Array.map (fun (_, f) -> List.memq f state) rs in
|
||||||
Hashtbl.add states_def i { trans; finals };
|
Hashtbl.add states_def i (trans, finals);
|
||||||
i
|
i
|
||||||
in
|
in
|
||||||
let init = ref [] in
|
let init = ref [] in
|
||||||
|
|
@ -141,56 +139,3 @@ let compile rs =
|
||||||
let i = aux !init in
|
let i = aux !init in
|
||||||
assert (i = 0);
|
assert (i = 0);
|
||||||
Array.init !counter (Hashtbl.find states_def)
|
Array.init !counter (Hashtbl.find states_def)
|
||||||
|
|
||||||
let cset_to_label cset =
|
|
||||||
let escape_dot c =
|
|
||||||
match c with
|
|
||||||
| '"' -> "\\\""
|
|
||||||
| '\\' -> "\\\\"
|
|
||||||
| '<' -> "\\<"
|
|
||||||
| '>' -> "\\>"
|
|
||||||
| _ -> String.make 1 c
|
|
||||||
in
|
|
||||||
let format_interval (lo, hi) =
|
|
||||||
if lo = -1 && hi = -1 then "EOF"
|
|
||||||
else if lo = hi then
|
|
||||||
if lo >= 32 && lo <= 126 then "'" ^ escape_dot (Char.chr lo) ^ "'"
|
|
||||||
else Printf.sprintf "U+%04X" lo
|
|
||||||
else if lo >= 32 && lo <= 126 && hi >= 32 && hi <= 126 then
|
|
||||||
"'" ^ escape_dot (Char.chr lo) ^ "'-'" ^ escape_dot (Char.chr hi) ^ "'"
|
|
||||||
else Printf.sprintf "U+%04X-U+%04X" lo hi
|
|
||||||
in
|
|
||||||
String.concat ", "
|
|
||||||
(List.map format_interval (cset : Cset.t :> (int * int) list))
|
|
||||||
|
|
||||||
let dfa_to_dot dfa =
|
|
||||||
let buf = Buffer.create 1024 in
|
|
||||||
let bprintf = Printf.bprintf in
|
|
||||||
bprintf buf "digraph {\n";
|
|
||||||
bprintf buf " rankdir=LR;\n";
|
|
||||||
bprintf buf " node [shape=circle];\n\n";
|
|
||||||
bprintf buf " _start [shape=point];\n";
|
|
||||||
bprintf buf " _start -> state0;\n\n";
|
|
||||||
Array.iteri
|
|
||||||
(fun i { trans; finals } ->
|
|
||||||
let accepted =
|
|
||||||
let acc = ref [] in
|
|
||||||
for r = Array.length finals - 1 downto 0 do
|
|
||||||
if finals.(r) then acc := r :: !acc
|
|
||||||
done;
|
|
||||||
!acc
|
|
||||||
in
|
|
||||||
(match accepted with
|
|
||||||
| [] -> bprintf buf " state%d [label=\"%d\"];\n" i i
|
|
||||||
| rules ->
|
|
||||||
bprintf buf
|
|
||||||
" state%d [label=\"%d\\n[rule %s]\", shape=doublecircle];\n" i i
|
|
||||||
(String.concat "," (List.map string_of_int rules)));
|
|
||||||
Array.iter
|
|
||||||
(fun (cset, target) ->
|
|
||||||
let label = cset_to_label cset in
|
|
||||||
bprintf buf " state%d -> state%d [label=\"%s\"];\n" i target label)
|
|
||||||
trans)
|
|
||||||
dfa;
|
|
||||||
bprintf buf "}\n";
|
|
||||||
Buffer.contents buf
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
|
(* The package sedlex is released under the terms of an MIT-like license. *)
|
||||||
|
(* See the attached LICENSE file. *)
|
||||||
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
||||||
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
|
|
||||||
|
|
||||||
type regexp
|
type regexp
|
||||||
|
|
||||||
|
|
@ -21,8 +22,4 @@ val intersection : regexp -> regexp -> regexp option
|
||||||
(* If each argument is a single [chars] regexp, returns a regexp
|
(* If each argument is a single [chars] regexp, returns a regexp
|
||||||
which matches the intersection set. Otherwise returns [None]. *)
|
which matches the intersection set. Otherwise returns [None]. *)
|
||||||
|
|
||||||
type dfa_state = { trans : (Sedlex_cset.t * int) array; finals : bool array }
|
val compile : regexp array -> ((Sedlex_cset.t * int) array * bool array) array
|
||||||
type dfa = dfa_state array
|
|
||||||
|
|
||||||
val compile : regexp array -> dfa
|
|
||||||
val dfa_to_dot : dfa -> string
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
|
(* The package sedlex is released under the terms of an MIT-like license. *)
|
||||||
|
(* See the attached LICENSE file. *)
|
||||||
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
(* Copyright 2005, 2013 by Alain Frisch and LexiFi. *)
|
||||||
(* Copyright 2026 by xenia <xenia@awoo.systems> *)
|
|
||||||
|
|
||||||
include Sedlex_utils.Cset
|
include Sedlex_utils.Cset
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,49 +0,0 @@
|
||||||
open Sedlexing
|
|
||||||
|
|
||||||
let unsafe_byte s j = Char.code (String.unsafe_get s j)
|
|
||||||
let malformed s j l = `Malformed (String.sub s j l)
|
|
||||||
|
|
||||||
let r_utf_8 s j l =
|
|
||||||
(* assert (0 <= j && 0 <= l && j + l <= String.length s); *)
|
|
||||||
let uchar c = `Uchar (Uchar.unsafe_of_int c) in
|
|
||||||
match l with
|
|
||||||
| 1 -> uchar (unsafe_byte s j)
|
|
||||||
| 2 -> (
|
|
||||||
let b0 = unsafe_byte s j in
|
|
||||||
let b1 = unsafe_byte s (j + 1) in
|
|
||||||
match Utf8.Helper.check_two b0 b1 with
|
|
||||||
| i -> uchar i
|
|
||||||
| exception MalFormed -> malformed s j l)
|
|
||||||
| 3 -> (
|
|
||||||
let b0 = unsafe_byte s j in
|
|
||||||
let b1 = unsafe_byte s (j + 1) in
|
|
||||||
let b2 = unsafe_byte s (j + 2) in
|
|
||||||
match Utf8.Helper.check_three b0 b1 b2 with
|
|
||||||
| i -> uchar i
|
|
||||||
| exception MalFormed -> malformed s j l)
|
|
||||||
| 4 -> (
|
|
||||||
let b0 = unsafe_byte s j in
|
|
||||||
let b1 = unsafe_byte s (j + 1) in
|
|
||||||
let b2 = unsafe_byte s (j + 2) in
|
|
||||||
let b3 = unsafe_byte s (j + 3) in
|
|
||||||
match Utf8.Helper.check_four b0 b1 b2 b3 with
|
|
||||||
| i -> uchar i
|
|
||||||
| exception MalFormed -> malformed s j l)
|
|
||||||
| _ -> assert false
|
|
||||||
|
|
||||||
let fold ~f acc s =
|
|
||||||
let rec loop acc f s i last =
|
|
||||||
if i > last then acc
|
|
||||||
else (
|
|
||||||
match Utf8.Helper.width (String.unsafe_get s i) with
|
|
||||||
| exception MalFormed ->
|
|
||||||
loop (f acc i (malformed s i 1)) f s (i + 1) last
|
|
||||||
| need ->
|
|
||||||
let rem = last - i + 1 in
|
|
||||||
if rem < need then f acc i (malformed s i rem)
|
|
||||||
else loop (f acc i (r_utf_8 s i need)) f s (i + need) last)
|
|
||||||
in
|
|
||||||
let pos = 0 in
|
|
||||||
let len = String.length s in
|
|
||||||
let last = pos + len - 1 in
|
|
||||||
loop acc f s pos last
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
val fold :
|
|
||||||
f:('a -> int -> [> `Malformed of string | `Uchar of Uchar.t ] -> 'a) ->
|
|
||||||
'a ->
|
|
||||||
string ->
|
|
||||||
'a
|
|
||||||
Binary file not shown.
|
|
@ -1,8 +0,0 @@
|
||||||
(library
|
|
||||||
(name sedlex_gen_test)
|
|
||||||
(libraries noslop-sedlex)
|
|
||||||
(inline_tests)
|
|
||||||
(enabled_if
|
|
||||||
(>= %{ocaml_version} 4.14))
|
|
||||||
(preprocess
|
|
||||||
(pps ppx_sedlex_test ppx_expect)))
|
|
||||||
|
|
@ -1,120 +0,0 @@
|
||||||
let%expect_test "simple string match" =
|
|
||||||
(match%sedlex_test buf with "ab" | "de" -> () | _ -> ());
|
|
||||||
[%expect
|
|
||||||
{|
|
|
||||||
DOT:
|
|
||||||
digraph {
|
|
||||||
rankdir=LR;
|
|
||||||
node [shape=circle];
|
|
||||||
|
|
||||||
_start [shape=point];
|
|
||||||
_start -> state0;
|
|
||||||
|
|
||||||
state0 [label="0"];
|
|
||||||
state0 -> state1 [label="'a'"];
|
|
||||||
state0 -> state3 [label="'d'"];
|
|
||||||
state1 [label="1"];
|
|
||||||
state1 -> state2 [label="'b'"];
|
|
||||||
state2 [label="2\n[rule 0]", shape=doublecircle];
|
|
||||||
state3 [label="3"];
|
|
||||||
state3 -> state2 [label="'e'"];
|
|
||||||
}
|
|
||||||
CODE:
|
|
||||||
let rec __sedlex_state_0 buf =
|
|
||||||
match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
|
|
||||||
| 0 -> __sedlex_state_1 buf
|
|
||||||
| 1 -> __sedlex_state_3 buf
|
|
||||||
| _ -> Sedlexing.backtrack buf
|
|
||||||
and __sedlex_state_1 buf =
|
|
||||||
match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with
|
|
||||||
| 0 -> 0
|
|
||||||
| _ -> Sedlexing.backtrack buf
|
|
||||||
and __sedlex_state_3 buf =
|
|
||||||
match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with
|
|
||||||
| 0 -> 0
|
|
||||||
| _ -> Sedlexing.backtrack buf in
|
|
||||||
Sedlexing.start buf; (match __sedlex_state_0 buf with | 0 -> () | _ -> ())
|
|
||||||
|}]
|
|
||||||
|
|
||||||
let%expect_test "character class" =
|
|
||||||
(match%sedlex_test buf with Plus 'a' .. 'z' -> () | _ -> ());
|
|
||||||
[%expect
|
|
||||||
{|
|
|
||||||
DOT:
|
|
||||||
digraph {
|
|
||||||
rankdir=LR;
|
|
||||||
node [shape=circle];
|
|
||||||
|
|
||||||
_start [shape=point];
|
|
||||||
_start -> state0;
|
|
||||||
|
|
||||||
state0 [label="0"];
|
|
||||||
state0 -> state1 [label="'a'-'z'"];
|
|
||||||
state1 [label="1\n[rule 0]", shape=doublecircle];
|
|
||||||
state1 -> state1 [label="'a'-'z'"];
|
|
||||||
}
|
|
||||||
CODE:
|
|
||||||
let rec __sedlex_state_0 buf =
|
|
||||||
match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
|
|
||||||
| 0 -> __sedlex_state_1 buf
|
|
||||||
| _ -> Sedlexing.backtrack buf
|
|
||||||
and __sedlex_state_1 buf =
|
|
||||||
Sedlexing.mark buf 0;
|
|
||||||
(match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
|
|
||||||
| 0 -> __sedlex_state_1 buf
|
|
||||||
| _ -> Sedlexing.backtrack buf) in
|
|
||||||
Sedlexing.start buf; (match __sedlex_state_0 buf with | 0 -> () | _ -> ())
|
|
||||||
|}]
|
|
||||||
|
|
||||||
let%expect_test "multi-rule" =
|
|
||||||
(match%sedlex_test buf with
|
|
||||||
| "ab" -> ()
|
|
||||||
| "de" -> ()
|
|
||||||
| Plus '0' .. '9' -> ()
|
|
||||||
| _ -> ());
|
|
||||||
[%expect
|
|
||||||
{|
|
|
||||||
DOT:
|
|
||||||
digraph {
|
|
||||||
rankdir=LR;
|
|
||||||
node [shape=circle];
|
|
||||||
|
|
||||||
_start [shape=point];
|
|
||||||
_start -> state0;
|
|
||||||
|
|
||||||
state0 [label="0"];
|
|
||||||
state0 -> state1 [label="'0'-'9'"];
|
|
||||||
state0 -> state2 [label="'a'"];
|
|
||||||
state0 -> state4 [label="'d'"];
|
|
||||||
state1 [label="1\n[rule 2]", shape=doublecircle];
|
|
||||||
state1 -> state1 [label="'0'-'9'"];
|
|
||||||
state2 [label="2"];
|
|
||||||
state2 -> state3 [label="'b'"];
|
|
||||||
state3 [label="3\n[rule 0]", shape=doublecircle];
|
|
||||||
state4 [label="4"];
|
|
||||||
state4 -> state5 [label="'e'"];
|
|
||||||
state5 [label="5\n[rule 1]", shape=doublecircle];
|
|
||||||
}
|
|
||||||
CODE:
|
|
||||||
let rec __sedlex_state_0 buf =
|
|
||||||
match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with
|
|
||||||
| 0 -> __sedlex_state_1 buf
|
|
||||||
| 1 -> __sedlex_state_2 buf
|
|
||||||
| 2 -> __sedlex_state_4 buf
|
|
||||||
| _ -> Sedlexing.backtrack buf
|
|
||||||
and __sedlex_state_1 buf =
|
|
||||||
Sedlexing.mark buf 2;
|
|
||||||
(match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with
|
|
||||||
| 0 -> __sedlex_state_1 buf
|
|
||||||
| _ -> Sedlexing.backtrack buf)
|
|
||||||
and __sedlex_state_2 buf =
|
|
||||||
match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with
|
|
||||||
| 0 -> 0
|
|
||||||
| _ -> Sedlexing.backtrack buf
|
|
||||||
and __sedlex_state_4 buf =
|
|
||||||
match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with
|
|
||||||
| 0 -> 1
|
|
||||||
| _ -> Sedlexing.backtrack buf in
|
|
||||||
Sedlexing.start buf;
|
|
||||||
(match __sedlex_state_0 buf with | 0 -> () | 1 -> () | 2 -> () | _ -> ())
|
|
||||||
|}]
|
|
||||||
|
|
@ -1,9 +1,8 @@
|
||||||
(library
|
(library
|
||||||
(name sedlex_test)
|
(name sedlex_test)
|
||||||
(libraries noslop-sedlex)
|
(libraries sedlex)
|
||||||
(inline_tests
|
(inline_tests)
|
||||||
(deps UTF-8-test.txt))
|
|
||||||
(enabled_if
|
(enabled_if
|
||||||
(>= %{ocaml_version} 4.14))
|
(>= %{ocaml_version} 4.14))
|
||||||
(preprocess
|
(preprocess
|
||||||
(pps noslop-sedlex.ppx ppx_expect)))
|
(pps sedlex.ppx ppx_expect)))
|
||||||
|
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
(library
|
|
||||||
(name ppx_sedlex_test)
|
|
||||||
(kind ppx_rewriter)
|
|
||||||
(libraries ppxlib noslop-sedlex.ppx)
|
|
||||||
(preprocess
|
|
||||||
(pps ppxlib.metaquot)))
|
|
||||||
|
|
@ -1,36 +0,0 @@
|
||||||
open Ppxlib
|
|
||||||
module P = Sedlex_ppx.Ppx_sedlex
|
|
||||||
module S = Sedlex_ppx.Sedlex
|
|
||||||
|
|
||||||
let reset_state () =
|
|
||||||
P.partition_counter := 0;
|
|
||||||
P.table_counter := 0;
|
|
||||||
Hashtbl.clear P.partitions;
|
|
||||||
Hashtbl.clear P.tables
|
|
||||||
|
|
||||||
let clear_tables () =
|
|
||||||
Hashtbl.clear P.partitions;
|
|
||||||
Hashtbl.clear P.tables
|
|
||||||
|
|
||||||
let expand ~ctxt:_ expr =
|
|
||||||
reset_state ();
|
|
||||||
let loc = Location.none in
|
|
||||||
let code_expr, auto =
|
|
||||||
P.handle_sedlex_match ~env:P.builtin_regexps ~map_rhs:Fun.id expr
|
|
||||||
in
|
|
||||||
let code_str = Pprintast.string_of_expression code_expr in
|
|
||||||
let dot_str = S.dfa_to_dot auto in
|
|
||||||
clear_tables ();
|
|
||||||
[%expr
|
|
||||||
print_string "DOT:\n";
|
|
||||||
print_string [%e Ast_builder.Default.estring ~loc dot_str];
|
|
||||||
print_string "CODE:\n";
|
|
||||||
print_string [%e Ast_builder.Default.estring ~loc code_str];
|
|
||||||
print_newline ()]
|
|
||||||
|
|
||||||
let ext =
|
|
||||||
Extension.V3.declare "sedlex_test" Extension.Context.expression
|
|
||||||
Ast_pattern.(single_expr_payload __)
|
|
||||||
expand
|
|
||||||
|
|
||||||
let () = Driver.register_transformation "sedlex_test" ~extensions:[ext]
|
|
||||||
Binary file not shown.
33
test/utf8.ml
33
test/utf8.ml
|
|
@ -1,33 +0,0 @@
|
||||||
open Printf
|
|
||||||
|
|
||||||
let next_tok buf =
|
|
||||||
let open Sedlexing.Utf8 in
|
|
||||||
match%sedlex buf with
|
|
||||||
| "a", Utf8 (Chars "+-×÷") -> sprintf "with Chars: %s" (lexeme buf)
|
|
||||||
| "b", Utf8 ("+" | "-" | "×" | "÷") ->
|
|
||||||
sprintf "with or_pattern: %s" (lexeme buf)
|
|
||||||
| Latin1 "\xc0", Utf8 "À", Ascii (Utf8 (Latin1 (Utf8 (Chars "À")))) ->
|
|
||||||
sprintf "mixed encoding: %s" (lexeme buf)
|
|
||||||
| Ascii (Star '\x00' .. '\x7f') -> sprintf "only ascii: %s" (lexeme buf)
|
|
||||||
| Utf8 (Star '\x00' .. '\x7f') ->
|
|
||||||
assert false
|
|
||||||
(* utf8 char interval can only match ascii because of the OCaml lexer. The regexp above should match instead *)
|
|
||||||
| Latin1 (Star '\x00' .. '\xff') -> sprintf "only latin1: %s" (lexeme buf)
|
|
||||||
| _ -> failwith (sprintf "Unexpected character: %s" (lexeme buf))
|
|
||||||
|
|
||||||
let%expect_test _ =
|
|
||||||
Sedlexing.Utf8.from_string "a+" |> next_tok |> print_string;
|
|
||||||
[%expect {| with Chars: a+ |}];
|
|
||||||
Sedlexing.Utf8.from_string "a÷" |> next_tok |> print_string;
|
|
||||||
[%expect {| with Chars: a÷ |}];
|
|
||||||
Sedlexing.Utf8.from_string "b+" |> next_tok |> print_string;
|
|
||||||
[%expect {| with or_pattern: b+ |}];
|
|
||||||
Sedlexing.Utf8.from_string "b÷" |> next_tok |> print_string;
|
|
||||||
[%expect {| with or_pattern: b÷ |}];
|
|
||||||
Sedlexing.Utf8.from_string "ÀÀÀ" |> next_tok |> print_string;
|
|
||||||
[%expect {| mixed encoding: ÀÀÀ |}];
|
|
||||||
Sedlexing.Utf8.from_string "az\x7f"
|
|
||||||
|> next_tok |> String.escaped |> print_string;
|
|
||||||
[%expect {| only ascii: az\127 |}];
|
|
||||||
Sedlexing.Utf8.from_string "az\u{c0}" |> next_tok |> print_string;
|
|
||||||
[%expect {| only latin1: azÀ |}]
|
|
||||||
Loading…
Reference in New Issue