pax_global_header00006660000000000000000000000064140666045600014521gustar00rootroot0000000000000052 comment=a4528e4553dad2d1146172a650da36ae5a23bec7 camlp-streams-5.0/000077500000000000000000000000001406660456000141355ustar00rootroot00000000000000camlp-streams-5.0/.gitignore000066400000000000000000000000341406660456000161220ustar00rootroot00000000000000_build camlpstreams.opam *~ camlp-streams-5.0/CHANGES.md000066400000000000000000000001561406660456000155310ustar00rootroot00000000000000# Release 5.0 (2021-06-29) - Initial import from OCaml 4.14 - Packaging improvements (#1) - Initial release camlp-streams-5.0/README.md000066400000000000000000000021411406660456000154120ustar00rootroot00000000000000# The Stream and Genlex libraries for use with Camlp4 and Camlp5 The `camlp-streams` package provides two library modules: - `Stream`: imperative streams, with in-place update and memoization of the latest element produced. - `Genlex`: a small parameterized lexical analyzer producing streams of tokens from streams of characters. The two modules are designed for use with [Camlp4](https://github.com/camlp4/camlp4/) and [Camlp5](https://github.com/camlp5/camlp5): - The stream patterns and stream expressions of Camlp4/Camlp5 consume and produce data of type `'a Stream.t`. - The `Genlex` tokenizer can be used as a simple lexical analyzer for Camlp4/Camlp5-generated parsers. The `Stream` module can also be used by hand-written recursive-descent parsers, but is not very convenient for this purpose. The `Stream` and `Genlex` modules have been part of the OCaml standard library for a long time, and have been distributed as part of the core OCaml system. They will be removed from the OCaml standard library at some future point, but will be maintained and distributed separately in this `camlp-streams` package. camlp-streams-5.0/camlp-streams.opam000066400000000000000000000032461406660456000175700ustar00rootroot00000000000000# This file is generated by dune, edit dune-project instead opam-version: "2.0" synopsis: "The Stream and Genlex libraries for use with Camlp4 and Camlp5" description: """ This package provides two library modules: - Stream: imperative streams, with in-place update and memoization of the latest element produced. - Genlex: a small parameterized lexical analyzer producing streams of tokens from streams of characters. The two modules are designed for use with Camlp4 and Camlp5: - The stream patterns and stream expressions of Camlp4/Camlp5 consume and produce data of type 'a Stream.t. - The Genlex tokenizer can be used as a simple lexical analyzer for Camlp4/Camlp5-generated parsers. The Stream module can also be used by hand-written recursive-descent parsers, but is not very convenient for this purpose. The Stream and Genlex modules have been part of the OCaml standard library for a long time, and have been distributed as part of the core OCaml system. They will be removed from the OCaml standard library at some future point, but will be maintained and distributed separately in this camlpstreams package. """ maintainer: [ "Florian Angeletti " "Xavier Leroy " ] authors: ["Daniel de Rauglaudre" "Xavier Leroy"] homepage: "https://github.com/ocaml/camlp-streams" bug-reports: "https://github.com/ocaml/camlp-streams/issues" depends: [ "dune" {>= "2.5"} "ocaml" {>= "4.05.0"} ] build: [ ["dune" "subst"] {pinned} [ "dune" "build" "-p" name "-j" jobs "@install" "@runtest" {with-test} "@doc" {with-doc} ] ] dev-repo: "git+https://github.com/ocaml/camlp-streams.git" camlp-streams-5.0/dune-project000066400000000000000000000026051406660456000164620ustar00rootroot00000000000000(lang dune 2.5) (generate_opam_files true) (formatting disabled) (name camlp-streams) (source (github ocaml/camlp-streams)) (authors "Daniel de Rauglaudre" "Xavier Leroy") (maintainers "Florian Angeletti " "Xavier Leroy ") (package (name camlp-streams) (synopsis "The Stream and Genlex libraries for use with Camlp4 and Camlp5") (description " This package provides two library modules: - Stream: imperative streams, with in-place update and memoization of the latest element produced. - Genlex: a small parameterized lexical analyzer producing streams of tokens from streams of characters. The two modules are designed for use with Camlp4 and Camlp5: - The stream patterns and stream expressions of Camlp4/Camlp5 consume and produce data of type 'a Stream.t. - The Genlex tokenizer can be used as a simple lexical analyzer for Camlp4/Camlp5-generated parsers. The Stream module can also be used by hand-written recursive-descent parsers, but is not very convenient for this purpose. The Stream and Genlex modules have been part of the OCaml standard library for a long time, and have been distributed as part of the core OCaml system. They will be removed from the OCaml standard library at some future point, but will be maintained and distributed separately in this camlpstreams package. ") (depends (ocaml (>= 4.05.0))) ) camlp-streams-5.0/src/000077500000000000000000000000001406660456000147245ustar00rootroot00000000000000camlp-streams-5.0/src/dune000066400000000000000000000001471406660456000156040ustar00rootroot00000000000000(library (name camlp_streams) (public_name camlp-streams) (wrapped false) (flags :standard -w -9)) camlp-streams-5.0/src/genlex.ml000066400000000000000000000177131406660456000165510ustar00rootroot00000000000000(**************************************************************************) (* *) (* OCaml *) (* *) (* Xavier Leroy, projet Cristal, INRIA Rocquencourt *) (* *) (* Copyright 1996 Institut National de Recherche en Informatique et *) (* en Automatique. *) (* *) (* All rights reserved. This file is distributed under the terms of *) (* the GNU Lesser General Public License version 2.1, with the *) (* special exception on linking described in the file LICENSE. *) (* *) (**************************************************************************) type token = Kwd of string | Ident of string | Int of int | Float of float | String of string | Char of char (* The string buffering machinery *) let initial_buffer = Bytes.create 32 let buffer = ref initial_buffer let bufpos = ref 0 let reset_buffer () = buffer := initial_buffer; bufpos := 0 let store c = if !bufpos >= Bytes.length !buffer then begin let newbuffer = Bytes.create (2 * !bufpos) in Bytes.blit !buffer 0 newbuffer 0 !bufpos; buffer := newbuffer end; Bytes.set !buffer !bufpos c; incr bufpos let get_string () = let s = Bytes.sub_string !buffer 0 !bufpos in buffer := initial_buffer; s (* The lexer *) let make_lexer keywords = let kwd_table = Hashtbl.create 17 in List.iter (fun s -> Hashtbl.add kwd_table s (Kwd s)) keywords; let ident_or_keyword id = try Hashtbl.find kwd_table id with Not_found -> Ident id and keyword_or_error c = let s = String.make 1 c in try Hashtbl.find kwd_table s with Not_found -> raise (Stream.Error ("Illegal character " ^ s)) in let rec next_token (strm__ : _ Stream.t) = match Stream.peek strm__ with Some (' ' | '\010' | '\013' | '\009' | '\026' | '\012') -> Stream.junk strm__; next_token strm__ | Some ('A'..'Z' | 'a'..'z' | '_' | '\192'..'\255' as c) -> Stream.junk strm__; let s = strm__ in reset_buffer (); store c; ident s | Some ('!' | '%' | '&' | '$' | '#' | '+' | '/' | ':' | '<' | '=' | '>' | '?' | '@' | '\\' | '~' | '^' | '|' | '*' as c) -> Stream.junk strm__; let s = strm__ in reset_buffer (); store c; ident2 s | Some ('0'..'9' as c) -> Stream.junk strm__; let s = strm__ in reset_buffer (); store c; number s | Some '\'' -> Stream.junk strm__; let c = try char strm__ with Stream.Failure -> raise (Stream.Error "") in begin match Stream.peek strm__ with Some '\'' -> Stream.junk strm__; Some (Char c) | _ -> raise (Stream.Error "") end | Some '\"' -> Stream.junk strm__; let s = strm__ in reset_buffer (); Some (String (string s)) | Some '-' -> Stream.junk strm__; neg_number strm__ | Some '(' -> Stream.junk strm__; maybe_comment strm__ | Some c -> Stream.junk strm__; Some (keyword_or_error c) | _ -> None and ident (strm__ : _ Stream.t) = match Stream.peek strm__ with Some ('A'..'Z' | 'a'..'z' | '\192'..'\255' | '0'..'9' | '_' | '\'' as c) -> Stream.junk strm__; let s = strm__ in store c; ident s | _ -> Some (ident_or_keyword (get_string ())) and ident2 (strm__ : _ Stream.t) = match Stream.peek strm__ with Some ('!' | '%' | '&' | '$' | '#' | '+' | '-' | '/' | ':' | '<' | '=' | '>' | '?' | '@' | '\\' | '~' | '^' | '|' | '*' as c) -> Stream.junk strm__; let s = strm__ in store c; ident2 s | _ -> Some (ident_or_keyword (get_string ())) and neg_number (strm__ : _ Stream.t) = match Stream.peek strm__ with Some ('0'..'9' as c) -> Stream.junk strm__; let s = strm__ in reset_buffer (); store '-'; store c; number s | _ -> let s = strm__ in reset_buffer (); store '-'; ident2 s and number (strm__ : _ Stream.t) = match Stream.peek strm__ with Some ('0'..'9' as c) -> Stream.junk strm__; let s = strm__ in store c; number s | Some '.' -> Stream.junk strm__; let s = strm__ in store '.'; decimal_part s | Some ('e' | 'E') -> Stream.junk strm__; let s = strm__ in store 'E'; exponent_part s | _ -> Some (Int (int_of_string (get_string ()))) and decimal_part (strm__ : _ Stream.t) = match Stream.peek strm__ with Some ('0'..'9' as c) -> Stream.junk strm__; let s = strm__ in store c; decimal_part s | Some ('e' | 'E') -> Stream.junk strm__; let s = strm__ in store 'E'; exponent_part s | _ -> Some (Float (float_of_string (get_string ()))) and exponent_part (strm__ : _ Stream.t) = match Stream.peek strm__ with Some ('+' | '-' as c) -> Stream.junk strm__; let s = strm__ in store c; end_exponent_part s | _ -> end_exponent_part strm__ and end_exponent_part (strm__ : _ Stream.t) = match Stream.peek strm__ with Some ('0'..'9' as c) -> Stream.junk strm__; let s = strm__ in store c; end_exponent_part s | _ -> Some (Float (float_of_string (get_string ()))) and string (strm__ : _ Stream.t) = match Stream.peek strm__ with Some '\"' -> Stream.junk strm__; get_string () | Some '\\' -> Stream.junk strm__; let c = try escape strm__ with Stream.Failure -> raise (Stream.Error "") in let s = strm__ in store c; string s | Some c -> Stream.junk strm__; let s = strm__ in store c; string s | _ -> raise Stream.Failure and char (strm__ : _ Stream.t) = match Stream.peek strm__ with Some '\\' -> Stream.junk strm__; begin try escape strm__ with Stream.Failure -> raise (Stream.Error "") end | Some c -> Stream.junk strm__; c | _ -> raise Stream.Failure and escape (strm__ : _ Stream.t) = match Stream.peek strm__ with Some 'n' -> Stream.junk strm__; '\n' | Some 'r' -> Stream.junk strm__; '\r' | Some 't' -> Stream.junk strm__; '\t' | Some ('0'..'9' as c1) -> Stream.junk strm__; begin match Stream.peek strm__ with Some ('0'..'9' as c2) -> Stream.junk strm__; begin match Stream.peek strm__ with Some ('0'..'9' as c3) -> Stream.junk strm__; Char.chr ((Char.code c1 - 48) * 100 + (Char.code c2 - 48) * 10 + (Char.code c3 - 48)) | _ -> raise (Stream.Error "") end | _ -> raise (Stream.Error "") end | Some c -> Stream.junk strm__; c | _ -> raise Stream.Failure and maybe_comment (strm__ : _ Stream.t) = match Stream.peek strm__ with Some '*' -> Stream.junk strm__; let s = strm__ in comment s; next_token s | _ -> Some (keyword_or_error '(') and comment (strm__ : _ Stream.t) = match Stream.peek strm__ with Some '(' -> Stream.junk strm__; maybe_nested_comment strm__ | Some '*' -> Stream.junk strm__; maybe_end_comment strm__ | Some _ -> Stream.junk strm__; comment strm__ | _ -> raise Stream.Failure and maybe_nested_comment (strm__ : _ Stream.t) = match Stream.peek strm__ with Some '*' -> Stream.junk strm__; let s = strm__ in comment s; comment s | Some _ -> Stream.junk strm__; comment strm__ | _ -> raise Stream.Failure and maybe_end_comment (strm__ : _ Stream.t) = match Stream.peek strm__ with Some ')' -> Stream.junk strm__; () | Some '*' -> Stream.junk strm__; maybe_end_comment strm__ | Some _ -> Stream.junk strm__; comment strm__ | _ -> raise Stream.Failure in fun input -> Stream.from (fun _count -> next_token input) camlp-streams-5.0/src/genlex.mli000066400000000000000000000066471406660456000167260ustar00rootroot00000000000000(**************************************************************************) (* *) (* OCaml *) (* *) (* Xavier Leroy, projet Cristal, INRIA Rocquencourt *) (* *) (* Copyright 1996 Institut National de Recherche en Informatique et *) (* en Automatique. *) (* *) (* All rights reserved. This file is distributed under the terms of *) (* the GNU Lesser General Public License version 2.1, with the *) (* special exception on linking described in the file LICENSE. *) (* *) (**************************************************************************) (** A generic lexical analyzer. This module implements a simple 'standard' lexical analyzer, presented as a function from character streams to token streams. It implements roughly the lexical conventions of OCaml, but is parameterized by the set of keywords of your language. Example: a lexer suitable for a desk calculator is obtained by {[ let lexer = make_lexer ["+"; "-"; "*"; "/"; "let"; "="; "("; ")"]]} The associated parser would be a function from [token stream] to, for instance, [int], and would have rules such as: {[ let rec parse_expr = parser | [< n1 = parse_atom; n2 = parse_remainder n1 >] -> n2 and parse_atom = parser | [< 'Int n >] -> n | [< 'Kwd "("; n = parse_expr; 'Kwd ")" >] -> n and parse_remainder n1 = parser | [< 'Kwd "+"; n2 = parse_expr >] -> n1 + n2 | [< >] -> n1 ]} One should notice that the use of the [parser] keyword and associated notation for streams are only available through camlp4 extensions. This means that one has to preprocess its sources {i e. g.} by using the ["-pp"] command-line switch of the compilers. *) (** The type of tokens. The lexical classes are: [Int] and [Float] for integer and floating-point numbers; [String] for string literals, enclosed in double quotes; [Char] for character literals, enclosed in single quotes; [Ident] for identifiers (either sequences of letters, digits, underscores and quotes, or sequences of 'operator characters' such as [+], [*], etc); and [Kwd] for keywords (either identifiers or single 'special characters' such as [(], [}], etc). *) type token = Kwd of string | Ident of string | Int of int | Float of float | String of string | Char of char val make_lexer : string list -> char Stream.t -> token Stream.t (** Construct the lexer function. The first argument is the list of keywords. An identifier [s] is returned as [Kwd s] if [s] belongs to this list, and as [Ident s] otherwise. A special character [s] is returned as [Kwd s] if [s] belongs to this list, and cause a lexical error (exception {!Stream.Error} with the offending lexeme as its parameter) otherwise. Blanks and newlines are skipped. Comments delimited by [(*] and [*)] are skipped as well, and can be nested. A {!Stream.Failure} exception is raised if end of stream is unexpectedly reached.*) camlp-streams-5.0/src/stream.ml000066400000000000000000000156361406660456000165640ustar00rootroot00000000000000(**************************************************************************) (* *) (* OCaml *) (* *) (* Daniel de Rauglaudre, projet Cristal, INRIA Rocquencourt *) (* *) (* Copyright 1997 Institut National de Recherche en Informatique et *) (* en Automatique. *) (* *) (* All rights reserved. This file is distributed under the terms of *) (* the GNU Lesser General Public License version 2.1, with the *) (* special exception on linking described in the file LICENSE. *) (* *) (**************************************************************************) type 'a t = 'a cell option and 'a cell = { mutable count : int; mutable data : 'a data } and 'a data = Sempty | Scons of 'a * 'a data | Sapp of 'a data * 'a data | Slazy of 'a data Lazy.t | Sgen of 'a gen | Sbuffio : buffio -> char data and 'a gen = { mutable curr : 'a option option; func : int -> 'a option } and buffio = { ic : in_channel; buff : bytes; mutable len : int; mutable ind : int } exception Failure exception Error of string let count = function | None -> 0 | Some { count } -> count let data = function | None -> Sempty | Some { data } -> data let fill_buff b = b.len <- input b.ic b.buff 0 (Bytes.length b.buff); b.ind <- 0 let rec get_data : type v. int -> v data -> v data = fun count d -> match d with (* Returns either Sempty or Scons(a, _) even when d is a generator or a buffer. In those cases, the item a is seen as extracted from the generator/buffer. The count parameter is used for calling `Sgen-functions'. *) Sempty | Scons (_, _) -> d | Sapp (d1, d2) -> begin match get_data count d1 with Scons (a, d11) -> Scons (a, Sapp (d11, d2)) | Sempty -> get_data count d2 | _ -> assert false end | Sgen {curr = Some None} -> Sempty | Sgen ({curr = Some(Some a)} as g) -> g.curr <- None; Scons(a, d) | Sgen g -> begin match g.func count with None -> g.curr <- Some(None); Sempty | Some a -> Scons(a, d) (* Warning: anyone using g thinks that an item has been read *) end | Sbuffio b -> if b.ind >= b.len then fill_buff b; if b.len == 0 then Sempty else let r = Bytes.unsafe_get b.buff b.ind in (* Warning: anyone using g thinks that an item has been read *) b.ind <- succ b.ind; Scons(r, d) | Slazy f -> get_data count (Lazy.force f) let rec peek_data : type v. v cell -> v option = fun s -> (* consult the first item of s *) match s.data with Sempty -> None | Scons (a, _) -> Some a | Sapp (_, _) -> begin match get_data s.count s.data with Scons(a, _) as d -> s.data <- d; Some a | Sempty -> None | _ -> assert false end | Slazy f -> s.data <- (Lazy.force f); peek_data s | Sgen {curr = Some a} -> a | Sgen g -> let x = g.func s.count in g.curr <- Some x; x | Sbuffio b -> if b.ind >= b.len then fill_buff b; if b.len == 0 then begin s.data <- Sempty; None end else Some (Bytes.unsafe_get b.buff b.ind) let peek = function | None -> None | Some s -> peek_data s let rec junk_data : type v. v cell -> unit = fun s -> match s.data with Scons (_, d) -> s.count <- (succ s.count); s.data <- d | Sgen ({curr = Some _} as g) -> s.count <- (succ s.count); g.curr <- None | Sbuffio b -> if b.ind >= b.len then fill_buff b; if b.len == 0 then s.data <- Sempty else (s.count <- (succ s.count); b.ind <- succ b.ind) | _ -> match peek_data s with None -> () | Some _ -> junk_data s let junk = function | None -> () | Some data -> junk_data data let rec nget_data n s = if n <= 0 then [], s.data, 0 else match peek_data s with Some a -> junk_data s; let (al, d, k) = nget_data (pred n) s in a :: al, Scons (a, d), succ k | None -> [], s.data, 0 let npeek_data n s = let (al, d, len) = nget_data n s in s.count <- (s.count - len); s.data <- d; al let npeek n = function | None -> [] | Some d -> npeek_data n d let next s = match peek s with Some a -> junk s; a | None -> raise Failure let empty s = match peek s with Some _ -> raise Failure | None -> () let iter f strm = let rec do_rec () = match peek strm with Some a -> junk strm; ignore(f a); do_rec () | None -> () in do_rec () (* Stream building functions *) let from f = Some {count = 0; data = Sgen {curr = None; func = f}} let of_list l = Some {count = 0; data = List.fold_right (fun x l -> Scons (x, l)) l Sempty} let of_string s = let count = ref 0 in from (fun _ -> (* We cannot use the index passed by the [from] function directly because it returns the current stream count, with absolutely no guarantee that it will start from 0. For example, in the case of [Stream.icons 'c' (Stream.from_string "ab")], the first access to the string will be made with count [1] already. *) let c = !count in if c < String.length s then (incr count; Some s.[c]) else None) let of_bytes s = let count = ref 0 in from (fun _ -> let c = !count in if c < Bytes.length s then (incr count; Some (Bytes.get s c)) else None) let of_channel ic = Some {count = 0; data = Sbuffio {ic = ic; buff = Bytes.create 4096; len = 0; ind = 0}} (* Stream expressions builders *) let iapp i s = Some {count = 0; data = Sapp (data i, data s)} let icons i s = Some {count = 0; data = Scons (i, data s)} let ising i = Some {count = 0; data = Scons (i, Sempty)} let lapp f s = Some {count = 0; data = Slazy (lazy(Sapp (data (f ()), data s)))} let lcons f s = Some {count = 0; data = Slazy (lazy(Scons (f (), data s)))} let lsing f = Some {count = 0; data = Slazy (lazy(Scons (f (), Sempty)))} let sempty = None let slazy f = Some {count = 0; data = Slazy (lazy(data (f ())))} (* For debugging use *) let rec dump : type v. (v -> unit) -> v t -> unit = fun f s -> print_string "{count = "; print_int (count s); print_string "; data = "; dump_data f (data s); print_string "}"; print_newline () and dump_data : type v. (v -> unit) -> v data -> unit = fun f -> function Sempty -> print_string "Sempty" | Scons (a, d) -> print_string "Scons ("; f a; print_string ", "; dump_data f d; print_string ")" | Sapp (d1, d2) -> print_string "Sapp ("; dump_data f d1; print_string ", "; dump_data f d2; print_string ")" | Slazy _ -> print_string "Slazy" | Sgen _ -> print_string "Sgen" | Sbuffio _ -> print_string "Sbuffio" camlp-streams-5.0/src/stream.mli000066400000000000000000000074211406660456000167260ustar00rootroot00000000000000(**************************************************************************) (* *) (* OCaml *) (* *) (* Daniel de Rauglaudre, projet Cristal, INRIA Rocquencourt *) (* *) (* Copyright 1997 Institut National de Recherche en Informatique et *) (* en Automatique. *) (* *) (* All rights reserved. This file is distributed under the terms of *) (* the GNU Lesser General Public License version 2.1, with the *) (* special exception on linking described in the file LICENSE. *) (* *) (**************************************************************************) (** Streams and parsers. *) type 'a t (** The type of streams holding values of type ['a]. *) exception Failure (** Raised by parsers when none of the first components of the stream patterns is accepted. *) exception Error of string (** Raised by parsers when the first component of a stream pattern is accepted, but one of the following components is rejected. *) (** {1 Stream builders} *) val from : (int -> 'a option) -> 'a t (** [Stream.from f] returns a stream built from the function [f]. To create a new stream element, the function [f] is called with the current stream count. The user function [f] must return either [Some ] for a value or [None] to specify the end of the stream. Do note that the indices passed to [f] may not start at [0] in the general case. For example, [[< '0; '1; Stream.from f >]] would call [f] the first time with count [2]. *) val of_list : 'a list -> 'a t (** Return the stream holding the elements of the list in the same order. *) val of_string : string -> char t (** Return the stream of the characters of the string parameter. *) val of_bytes : bytes -> char t (** Return the stream of the characters of the bytes parameter. @since 4.02.0 *) val of_channel : in_channel -> char t (** Return the stream of the characters read from the input channel. *) (** {1 Stream iterator} *) val iter : ('a -> unit) -> 'a t -> unit (** [Stream.iter f s] scans the whole stream s, applying function [f] in turn to each stream element encountered. *) (** {1 Predefined parsers} *) val next : 'a t -> 'a (** Return the first element of the stream and remove it from the stream. @raise Stream.Failure if the stream is empty. *) val empty : 'a t -> unit (** Return [()] if the stream is empty, else raise {!Stream.Failure}. *) (** {1 Useful functions} *) val peek : 'a t -> 'a option (** Return [Some] of "the first element" of the stream, or [None] if the stream is empty. *) val junk : 'a t -> unit (** Remove the first element of the stream, possibly unfreezing it before. *) val count : 'a t -> int (** Return the current count of the stream elements, i.e. the number of the stream elements discarded. *) val npeek : int -> 'a t -> 'a list (** [npeek n] returns the list of the [n] first elements of the stream, or all its remaining elements if less than [n] elements are available. *) (**/**) (* The following is for system use only. Do not call directly. *) val iapp : 'a t -> 'a t -> 'a t val icons : 'a -> 'a t -> 'a t val ising : 'a -> 'a t val lapp : (unit -> 'a t) -> 'a t -> 'a t val lcons : (unit -> 'a) -> 'a t -> 'a t val lsing : (unit -> 'a) -> 'a t val sempty : 'a t val slazy : (unit -> 'a t) -> 'a t val dump : ('a -> unit) -> 'a t -> unit