
(* Tokens for "SML/NG" - Tom 7 *)
structure Token =
struct

    fun islegal #"\n" = true
      | islegal #"\r" = true
      | islegal #"\t" = true
      | islegal c =
	(ord c >= ord #" " andalso
	 ord c <= ord #"~")

    datatype attr = B | I

    val colors = "rgbcmykw"
    fun colorname c = str (CharVector.sub (colors, c))

    datatype tag = 
	EM | PL | S 
      | TT | U | Num of int
      | Clr of int
      | Att of attr

    datatype token = 
	Tag of tag 
      | Closetag of tag
      | Text of string

    fun tts EM = "EM"
      | tts PL = "PL"
      | tts S = "S"
      | tts TT = "TT"
      | tts U = "U"
      | tts (Num n) = Int.toString n
      | tts (Att B) = "B"
      | tts (Att I) = "I"
      | tts (Clr n) = colorname n


    fun ttsmls EM = "EM"
      | ttsmls PL = "PL"
      | ttsmls S = "S"
      | ttsmls TT = "TT"
      | ttsmls U = "U"
      | ttsmls (Num n) = "(Num " ^ Int.toString n ^ ")"
      | ttsmls (Att B) = "(Att B)"
      | ttsmls (Att I) = "(Att I)"
      | ttsmls (Clr n) = "(Clr " ^ Int.toString n ^ ")"

    (* size of <TT> *)
    fun tagsize t = size (tts t) + 2

    fun otagtostring t = "<" ^ tts t ^ ">"
    fun ctagtostring t = "</" ^ tts t ^ ">"

    val eqtag = op=

end