https://hal.archives-ouvertes.fr/hal-01897572
Tip revision: ad0c16675d221938530269610308cd5a2c142687 authored by Software Heritage on 17 October 2018, 13:20:37 UTC
hal: Deposit 205 in collection hal
hal: Deposit 205 in collection hal
Tip revision: ad0c166
pretokenizer.ml
(**************************************************************************)
(* -*- tuareg -*- *)
(* *)
(* Copyright (C) 2017,2018 Yann RĂ©gis-Gianas, Nicolas Jeannerod, *)
(* Ralf Treinen. *)
(* *)
(* This is free software: you can redistribute it and/or modify it *)
(* under the terms of the GNU General Public License, version 3. *)
(* *)
(* Additional terms apply, due to the reproduction of portions of *)
(* the POSIX standard. Please refer to the file COPYING for details. *)
(**************************************************************************)
let make (current : PrelexerState.t) lexbuf =
let pretokenizer = Prelexer.token current in
(**
The pretokenizer may produce several pretokens, we use an
intermediate queue to synchronize pretokens' consumption with
their production.
*)
let q = Queue.create () in
let push x = Queue.push x q in
let rec aux () =
try
Queue.take q
with Queue.Empty ->
List.iter (fun x -> Queue.push x q) (pretokenizer lexbuf);
aux ()
in
aux, push