.github/CODEOWNERS

+# These are the default owners for everything in the repo. They will
+# be requested for review when someone opens a pull request.
+* @alainfrisch @Drup @pmetzger

.gitignore

@@ -8,8 +8,13 @@
 *.cmxs
 *.cmxa
 *.exe
+*.install
+*.merlin
+.*sw*
+_build
 libdoc
 src/syntax/ppx_sedlex
+src/generator/data/*.txt
 examples/complement
 examples/tokenizer
 examples/subtraction

.travis.yml

+language: c
+sudo: required
+install: test -e .travis.opam.sh || wget https://raw.githubusercontent.com/ocaml/ocaml-ci-scripts/master/.travis-opam.sh
+script: bash -ex .travis-opam.sh
+env:
+  global:
+  - PACKAGE=sedlex
+  matrix:
+  - OCAML_VERSION=4.02
+  - OCAML_VERSION=4.03
+  - OCAML_VERSION=4.04
+  - OCAML_VERSION=4.05
+  - OCAML_VERSION=4.06
+  - OCAML_VERSION=4.07
+  - OCAML_VERSION=4.08
+os:
+  - linux
+  - osx

CHANGES

+2.1
+     * GPR#78: Auto-generate unicode data
+
+2.0
+     * GPR#70: Switch to dune, opam v2
+     * GPR#60: Breaking change: switch from int codepoints to Uchar.t
+       codepoints
+     * GPR#59: Track lexing position
+
+-------------------------------------------------------------------------------
+
 1.99.4
      * GPR#47: Switch to ocaml-migrate-parsetree (contributed by Adrien Guatto)
      * GPR#42: Added 'Rep' (repeat operator) (contributed by jpathy)
@@ -16,7 +27,7 @@
      * First version of sedlex.  The history below refers to ulex, the ancestor
        or sedlex implemented with Camlp4.
 
---------------------------------------------------------------------------
+-------------------------------------------------------------------------------
 
 1.1
      * Generate (more) globally unique identifiers to avoid conflicts when open'ing another module

META

-version = "1.99.4"
-description = "Runtime support for sedlex"
-archive(byte) = "sedlexing.cma"
-archive(native) = "sedlexing.cmxa"
-requires = "gen"
-ppx = "./ppx_sedlex --as-ppx"
-
-package "ppx" (
-  version = "1.99.4"
-  description = "ppx implementation of sedlex"
-  requires = "ocaml-migrate-parsetree ppx_tools_versioned gen"
-  archive(byte) = "sedlex.cma"
-  archive(native) = "sedlex.cmxa"
-)

Makefile

@@ -2,73 +2,26 @@
 # See the attached LICENSE file.
 # Copyright 2005, 2013 by Alain Frisch and LexiFi.
 
-include $(shell ocamlc -where)/Makefile.config
+INSTALL_ARGS := $(if $(PREFIX),--prefix $(PREFIX),)
 
-VERSION=1.99.2
-# Don't forget to change META file as well
+.PHONY: build install uninstall clean doc test all
 
-.PHONY: all opt clean test install package
-
-all:
-	(cd src/lib && $(MAKE) all doc)
-	(cd src/syntax && $(MAKE) all)
-
-opt:
-	(cd src/syntax && $(MAKE) opt)
-	(cd src/lib && $(MAKE) opt)
-
-clean:
-	rm -f *~ *.cm* *.a *.lib *.o *.obj
-	(cd src/lib && $(MAKE) clean)
-	(cd src/syntax && $(MAKE) clean)
-	(cd examples && $(MAKE) clean)
-	rm -rf libdoc
-
-test: clean all opt
-	cd examples && $(MAKE) test
-
-INSTALL=META src/syntax/sedlex.cma src/syntax/ppx_sedlex$(EXE) src/lib/sedlexing.cma src/lib/sedlexing.cmi
-
-INSTALL_OPT=src/syntax/sedlex.cmxs src/syntax/sedlex$(EXT_LIB) src/syntax/sedlex.cmxa src/syntax/ppx_sedlex.opt$(EXE) src/lib/sedlexing.cmx src/lib/sedlexing$(EXT_LIB) src/lib/sedlexing.cmxa
+build:
+	dune build @install
 
 install:
-	ocamlfind install sedlex $(INSTALL) $(INSTALL_OPT)
-
-install_byteonly:
-	ocamlfind install sedlex $(INSTALL)
+	dune install $(INSTALL_ARGS)
 
 uninstall:
-	ocamlfind remove sedlex
-
-PACKAGE = sedlex-$(VERSION)
-DISTRIB = \
-  CHANGES LICENSE META README.md Makefile \
-  examples/Makefile \
-  examples/tokenizer.ml \
-  src/lib/Makefile \
-  src/lib/sedlexing.ml \
-  src/lib/sedlexing.mli \
-  src/syntax/Makefile \
-  src/syntax/sedlex_cset.ml \
-  src/syntax/sedlex_cset.mli \
-  src/syntax/sedlex.ml \
-  src/syntax/sedlex.mli \
-  src/syntax/sedlex_ppx.ml \
-  src/syntax/unicode63.ml \
-  src/syntax/unicode63.mli
+	dune uninstall $(INSTALL_ARGS)
 
+clean:
+	dune clean
 
-package: clean
-	rm -rf sedlex.tar.gz
-	tar czf sedlex.tar.gz $(DISTRIB)
-	rm -Rf $(PACKAGE)
-	mkdir $(PACKAGE)
-	cd $(PACKAGE) && tar xzf ../sedlex.tar.gz
-	tar czf $(PACKAGE).tar.gz $(PACKAGE)
-	rm -Rf $(PACKAGE) sedlex.tar.gz
+doc:
+	dune build @doc
 
+test:
+	dune build @runtest
 
-TARGET=foo:bar/sedlex_dara
-upload:
-	scp $(PACKAGE).tar.gz README CHANGES $(TARGET)/
-	rsync -avz libdoc $(TARGET)
+all: build test doc

README.md

 # sedlex
 
+[![Build Status](https://travis-ci.com/ocaml-community/sedlex.svg?branch=master)](https://travis-ci.com/ocaml-community/sedlex)
+
 Unicode-friendly lexer generator for OCaml.
 
 This package is licensed by LexiFi under the terms of the MIT license.
 
-Contact: alain.frisch@lexifi.com
+sedlex was originally written by Alain Frisch
+<alain.frisch@lexifi.com> and is now maintained as part of the
+ocaml-community repositories on github.
+
+## API
+The API is documented [here](https://ocaml-community.github.io/sedlex).
 
 ## Overview
 
@@ -64,10 +71,10 @@ Guard expressions are not allowed.)
 
 where:
 - lexbuf is an arbitrary lowercase identifier, which must refer to
-  an existing value of type [Sedlexing.lexbuf].
+  an existing value of type `Sedlexing.lexbuf`.
 - the Ri are regular expressions (see below);
 - the ei and def are OCaml expressions (called actions) of the same type
-  (the type for the whole lexer definitioon).
+  (the type for the whole lexer definition).
 
 Unlike ocamllex, lexers work on stream of Unicode codepoints, not
 bytes.
@@ -172,11 +179,10 @@ The quick way:
 
 
 Otherwise, the first thing to do is to compile and install sedlex.
-You need a recent version of OCaml.
+You need a recent version of OCaml and [dune](https://dune.build/).
 
 ```
-  make all
-  make opt (* optional *)
+  make
 ```
 
 ### With findlib
@@ -184,7 +190,7 @@ You need a recent version of OCaml.
 If you have findlib, you can use it to install and use sedlex.
 The name of the findlib package is "sedlex".
 
-Installation (after "make all" and "make opt"):
+Installation (after "make"):
 
 ```
   make install
@@ -215,6 +221,9 @@ source file through -ppx rewriter ppx_sedlex. Moreover, you need to
 link the application with the runtime support library for sedlex
 (sedlexing.cma / sedlexing.cmxa).
 
+## Examples
+
+The `examples/` subdirectory contains several samples of sedlex in use.
 
 ## Contributors
 
@@ -223,3 +232,4 @@ link the application with the runtime support library for sedlex
 - Peter Zotov:
   - improvements to the build system
   - switched parts of ppx_sedlex to using concrete syntax (with ppx_metaquot)
+- Steffen Smolka: port to dune

debian/changelog

+ocaml-sedlex (2.1-1) unstable; urgency=medium
+
+  * New upstream version.
+  * Remove ppx_tools_versioned-521.patch, which is no longer needed.
+  * Bump Standards-Version to 4.4.1 (no changes).
+
+ -- Andy Li <andy@onthewings.net>  Sun, 03 Nov 2019 09:42:00 +0800
+
 ocaml-sedlex (1.99.4-2) unstable; urgency=medium
 
   * Add ppx_tools_versioned-521.patch that fixes compatibility with

debian/control

@@ -3,10 +3,11 @@ Section: ocaml
 Priority: optional
 Maintainer: Debian OCaml Maintainers <debian-ocaml-maint@lists.debian.org>
 Uploaders: Andy Li <andy@onthewings.net>
-Standards-Version: 4.3.0
+Standards-Version: 4.4.1
 Build-Depends:
  debhelper (>= 10~),
  ocaml-nox (>= 4.02.0),
+ dune,
  libgen-ocaml-dev,
  libppx-tools-versioned-ocaml-dev,
  ocaml-findlib,

debian/libsedlex-ocaml-dev.examples

-examples/Makefile
+examples/dune*
 examples/*.ml
\ No newline at end of file

debian/libsedlex-ocaml-dev.install.in

-@OCamlStdlibDir@/sedlex/*.cmi
-@OCamlStdlibDir@/sedlex/*.mli
-OPT: @OCamlStdlibDir@/sedlex/*.a
-@OCamlStdlibDir@/sedlex/ppx_sedlex
-OPT: @OCamlStdlibDir@/sedlex/ppx_sedlex.opt
-OPT: @OCamlStdlibDir@/sedlex/*.cmxa
-OPT: @OCamlStdlibDir@/sedlex/*.cmx
\ No newline at end of file
+@OCamlStdlibDir@/sedlex/dune*
+@OCamlStdlibDir@/sedlex/{,**/}*.cmi
+@OCamlStdlibDir@/sedlex/{,**/}*.cmt
+@OCamlStdlibDir@/sedlex/{,**/}*.cmti
+@OCamlStdlibDir@/sedlex/{,**/}*.ml
+@OCamlStdlibDir@/sedlex/{,**/}*.mli
+@OCamlStdlibDir@/sedlex/{,**/}*.exe
+OPT: @OCamlStdlibDir@/sedlex/{,**/}*.a
+OPT: @OCamlStdlibDir@/sedlex/{,**/}*.cmxa
+OPT: @OCamlStdlibDir@/sedlex/{,**/}*.cmx
\ No newline at end of file

debian/libsedlex-ocaml-dev.ocamldoc

 # Documentation generated by dh_ocamldoc
 -package gen
+-package ppx_tools_versioned
+
+# should use -lib when OCaml 4.08 arrives in unstable
+-lib Sedlex_ppx

debian/libsedlex-ocaml.install.in

 @OCamlStdlibDir@/sedlex/META
-@OCamlStdlibDir@/sedlex/*.cma
-DYN: @OCamlStdlibDir@/sedlex/*.cmxs
\ No newline at end of file
+@OCamlStdlibDir@/sedlex/opam
+@OCamlStdlibDir@/sedlex/{,**/}*.cma
+DYN: @OCamlStdlibDir@/sedlex/{,**/}*.cmxs
\ No newline at end of file

debian/not-installed

+usr/doc/*/LICENSE
+usr/doc/*/README.md
+usr/doc/*/CHANGES
\ No newline at end of file

debian/patches/ppx_tools_versioned-521.patch

-https://github.com/ocaml-community/sedlex/issues/64#issuecomment-433198249
---- a/src/syntax/Makefile
-+++ b/src/syntax/Makefile
-@@ -8,7 +8,7 @@
- 
- OCAMLC=ocamlfind ocamlc
- OCAMLOPT=ocamlfind ocamlopt
--PACKAGES=-package ppx_tools_versioned.metaquot_405 -package ocaml-migrate-parsetree
-+PACKAGES=-package ppx_tools_versioned.metaquot_405 -package ocaml-migrate-parsetree -package ppx_tools_versioned
- OCAMLFLAGS=$(PACKAGES) -linkall -w +A-4-9-42 -annot
- 
- all: ppx_sedlex$(EXE) sedlex.cma

debian/patches/series

-ppx_tools_versioned-521.patch

debian/rules

@@ -9,24 +9,18 @@ export OCAMLFIND_DESTDIR=$(DESTDIR)$(OCAML_STDLIB_DIR)
 
 override_dh_auto_configure:
 
-override_dh_auto_install:
-	mkdir -p '$(OCAMLFIND_DESTDIR)'
-ifeq ($(OCAML_HAVE_OCAMLOPT),yes)
-	dh_auto_install
-else
-	make install_byteonly DESTDIR=$(DESTDIR)
-endif
+override_dh_auto_build:
+	dune build -p sedlex --verbose
 
-	# ocaml-dangling-cmi
-	cp src/lib/sedlexing.mli $(OCAMLFIND_DESTDIR)/sedlex/
+override_dh_auto_install:
+	dune install --destdir=$(DESTDIR) --prefix=/usr --libdir=..$(OCAML_STDLIB_DIR) --verbose
 
 override_dh_auto_test:
-ifeq ($(OCAML_HAVE_OCAMLOPT),yes)
-	# the test target doesn't support parallel build
-	dh_auto_test --no-parallel
-else
-	# the test requires ocamlopt
-endif
+	dune build @runtest -p sedlex --verbose
+
+# -lib is not available in the ocamldoc version as of writing (4.05)
+override_dh_ocamldoc:
+	[ ! "$$(ocamldoc --version)" \< "4.08.0" ] && dh_ocamldoc || echo "skip doc generation"
 
 override_dh_missing:
 	dh_missing --fail-missing

debian/tests/control

 Tests: test-examples
-Depends: ocaml-nox, ocaml-findlib, @
+Depends: ocaml-nox, ocaml-findlib, dune, @
+Restrictions: allow-stderr

debian/tests/examples/Makefile

-# The package sedlex is released under the terms of an MIT-like license.
-# See the attached LICENSE file.
-# Copyright 2005, 2013 by Alain Frisch and LexiFi.
-
-include $(shell ocamlc -where)/Makefile.config
-
-.PHONY: test
-test: tokenizer$(EXE) complement$(EXE) subtraction$(EXE) repeat$(EXE)
-	./tokenizer$(EXE)
-	./complement$(EXE)
-	./subtraction$(EXE)
-	./repeat$(EXE)
-
-tokenizer$(EXE): tokenizer.ml
-	ocamlfind ocamlc -package gen -package sedlex -o tokenizer$(EXE) -linkpkg sedlexing.cma tokenizer.ml
-
-complement$(EXE): complement.ml
-	ocamlfind ocamlc -package gen -package sedlex -o complement$(EXE) -linkpkg sedlexing.cma complement.ml
-
-subtraction$(EXE): subtraction.ml
-	ocamlfind ocamlc -package gen -package sedlex -o subtraction$(EXE) -linkpkg sedlexing.cma subtraction.ml
-
-repeat$(EXE): repeat.ml
-	ocamlfind ocamlc -package gen -package sedlex -o repeat$(EXE) -linkpkg sedlexing.cma repeat.ml
-
-.PHONY: with_driver
-with_driver: tokenizer.ml
-	ocamlfind ocamlc -package gen -package sedlex -o tokenizer$(EXE) sedlexing.cma tokenizer.ml
-
-.PHONY: with_findlib
-with_findlib: tokenizer.ml
-	ocamlfind ocamlc -c -package sedlex -package gen tokenizer.ml
-	ocamlfind ocamlc -o tokenizer$(EXE) -linkpkg -package sedlex -package gen tokenizer.cmo
-
-clean:
-	rm -f *~ *.cm* *.a *.lib *.o *.obj tokenizer$(EXE) complement$(EXE) subtraction$(EXE) repeat$(EXE)

debian/tests/examples/complement.ml

-let ucase = [%sedlex.regexp? 'A'..'Z']
-let lcase = [%sedlex.regexp? 'a'..'z']
-
-let rec token buf =
-  match%sedlex buf with
-  | lcase -> print_char 'L';token buf
-  | Compl (ucase | lcase) -> print_char '?'; token buf
-  | ucase -> print_char 'U';token buf
-  | eof -> print_endline "."
-  | _ -> assert false
-
-let () =
-  let lexbuf = Sedlexing.Latin1.from_string "Abc::DefG" in
-  token lexbuf