
(* Tokenizer - Tom 7 *)
signature TOKENIZE =
sig

  exception ParseError of string

  (* returns Token.TT for "TT" *)
  val gettag : string -> Token.tag

  val tokenize : CharVector.vector -> Token.token list

  (* handles any input, for importing HTML files.
     Better use ToTreeLoose to fix it up, though. *)
  val tokenize_loose : CharVector.vector -> Token.token list

end
