(*
 *  ENIAMtokenizer, a tokenizer for Polish
 *  Copyright (C) 2016 Wojciech Jaworski <wjaworski atSPAMfree mimuw dot edu dot pl>
 *  Copyright (C) 2016 Institute of Computer Science Polish Academy of Sciences
 *
 *  This library is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Lesser General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *)

open Printf
open ENIAMtokenizerTypes
open Xstd
open Xunicode

let string_of_interps interps =
      String.concat "|" (Xlist.map interps (fun interp ->
        (String.concat ":" (Xlist.map interp (fun interp2 ->
          (String.concat "." interp2))))))

let xml_of_interp interp =
  Xml.Element("interp",[],[Xml.PCData (string_of_interps [interp])])

let rec string_of_token = function
    SmallLetter orth -> sprintf "SmallLetter(%s)" orth
  | CapLetter(orth,lc) -> sprintf "CapLetter(%s,%s)" orth lc
  | AllSmall orth -> sprintf "AllSmall(%s)" orth
  | AllCap(orth,lc,lc2) -> sprintf "AllCap(%s,%s,%s)" orth lc lc2
  | FirstCap(orth,lc,cl,ll) -> sprintf "FirstCap(%s,%s,%s,%s)" orth lc cl ll
  | SomeCap orth -> sprintf "SomeCap(%s)" orth
  | RomanDig(v,t) -> sprintf "RomanDig(%s,%s)" v t
  | Interp orth -> sprintf "Interp(%s)" orth
  | Symbol orth  -> sprintf "Symbol(%s)" orth
  | Dig(v,t) -> sprintf "Dig(%s,%s)" v t
  | Other orth  -> sprintf "Other(%s)" orth
  | Lemma(lemma,cat,interps) -> sprintf "Lemma(%s,%s,%s)" lemma cat (string_of_interps interps)
  | Proper(lemma,cat,interps,senses) -> sprintf "Proper(%s,%s,%s,%s)" lemma cat (string_of_interps interps) (String.concat "|" senses)
(*  | Sense(lemma,cat,interps,senses) -> sprintf "Sense(%s,%s,%s,%s)" lemma cat (string_of_interps interps)
     (String.concat "|" (Xlist.map senses (fun (_,v,_) -> v)))*)
  | Compound(sense,l) -> sprintf "Compound(%s,[%s])" sense (String.concat ";" (Xlist.map l string_of_token))
  | Tokens(cat,l) -> sprintf "Tokens(%s,%s)" cat (String.concat ";" (Xlist.map l string_of_int))

let rec string_of_token_simple = function
    SmallLetter orth -> "SmallLetter"
  | CapLetter(orth,lc) -> "CapLetter"
  | AllSmall orth -> "AllSmall"
  | AllCap(orth,lc,lc2) -> "AllCap"
  | FirstCap(orth,lc,_,_) -> "FirstCap"
  | SomeCap orth -> "SomeCap"
  | RomanDig(v,t) -> "RomanDig"
  | Interp orth -> sprintf "Interp(%s)" orth
  | Symbol orth  -> sprintf "Symbol(%s)" orth
  | Dig(v,t) -> "Dig"
  | Other orth  -> sprintf "Other(%s)" orth
  | Lemma(lemma,cat,interp) -> "Lemma"
  | Proper(lemma,cat,interp,sense) -> "Proper"
(*   | Sense(lemma,cat,interp,sense) -> "Sense" *)
  | Compound(sense,l) -> sprintf "Compound"
  | Tokens _ -> sprintf "Tokens"

let rec xml_of_token = function
    SmallLetter orth -> Xml.Element("SmallLetter",[],[Xml.PCData orth])
  | CapLetter(orth,lc) -> Xml.Element("CapLetter",["lc",lc],[Xml.PCData orth])
  | AllSmall orth -> Xml.Element("AllSmall",[],[Xml.PCData orth])
  | AllCap(orth,lc,lc2) -> Xml.Element("AllCap",["lc",lc;"lc2",lc2],[Xml.PCData orth])
  | FirstCap(orth,lc,cl,ll) -> Xml.Element("FirstCap",["lc",lc;"cl",cl;"ll",ll],[Xml.PCData orth])
  | SomeCap orth -> Xml.Element("SomeCap",[],[Xml.PCData orth])
  | RomanDig(v,t) -> Xml.Element("RomanDig",["t",t],[Xml.PCData v])
  | Interp orth -> Xml.Element("Interp",[],[Xml.PCData orth])
  | Symbol orth  -> Xml.Element("Symbol",[],[Xml.PCData orth])
  | Dig(v,t) -> Xml.Element("Dig",["t",t],[Xml.PCData v])
  | Other orth  -> Xml.Element("Other",[],[Xml.PCData orth])
  | Lemma(lemma,cat,interps) -> Xml.Element("Lemma",["lemma",lemma;"pos",cat],Xlist.map interps xml_of_interp)
  | Proper(lemma,cat,interps,senses) -> Xml.Element("Proper",["lemma",lemma;"pos",cat],
                                                    (Xlist.map interps xml_of_interp) @
                                                    (Xlist.map senses (fun s -> Xml.Element("sense",[],[Xml.PCData s]))))
  | Compound(sense,l) -> Xml.Element("Compound",["sense",sense], Xlist.map l xml_of_token)
  | Tokens(cat,l) -> Xml.Element("Tokens",["pos",cat],Xlist.map l (fun x -> Xml.Element("id",[],[Xml.PCData (string_of_int x)])))

let string_of_token_env t =
  sprintf "{orth=%s;beg=%d;len=%d;next=%d;token=%s;weight=%.2f;attrs=[%s]}" t.orth t.beg t.len t.next (string_of_token t.token) t.weight
    (String.concat ";" t.attrs)

let xml_of_token_env id t =
  let id_attr = if id < 0 then [] else ["id",string_of_int id] in
  Xml.Element("token_record",id_attr @ ["beg",string_of_int t.beg;"len",string_of_int t.len;"next",string_of_int t.next;"weight",string_of_float t.weight],[
      Xml.Element("orth",[],[Xml.PCData t.orth]);
      xml_of_token t.token;
      Xml.Element("attrs",[],Xlist.map t.attrs (fun attr -> Xml.Element("attr",[],[Xml.PCData attr])))])

let rec spaces i =
  if i = 0 then "" else "  " ^ spaces (i-1)

let rec string_of_tokens i = function
    Token t -> sprintf "%s%s" (spaces i) (string_of_token_env t)
  | Variant l -> sprintf "%sVariant[\n%s]" (spaces i) (String.concat ";\n" (Xlist.map l (string_of_tokens (i+1))))
  | Seq l -> sprintf "%sSeq[\n%s]" (spaces i) (String.concat ";\n" (Xlist.map l (string_of_tokens (i+1))))

let rec string_of_tokens_simple = function
    Token t -> string_of_token_simple t.token
  | Variant l -> sprintf "Variant[%s]" (String.concat ";" (Xlist.map l string_of_tokens_simple))
  | Seq l -> sprintf "Seq[%s]" (String.concat ";" (Xlist.map l string_of_tokens_simple))

let get_orth = function
    SmallLetter orth -> orth
  | CapLetter(orth,lc) -> orth
  | AllSmall orth -> orth
  | AllCap(orth,lc,lc2) -> orth
  | FirstCap(orth,lc,_,_) -> orth
  | SomeCap orth -> orth
  | Symbol orth  -> orth
  | Dig(v,_) -> v
  | Other orth  -> orth
  | Interp orth  -> orth
  | _ -> ""(*failwith "get_orth"*)

let rec get_lemma = function
    Interp orth -> orth
  | Lemma(lemma,cat,_) -> lemma
  | Proper(lemma,cat,_,_) -> lemma
  | _ -> ""

let rec get_pos = function
    Interp _ -> "interp"
  | Lemma(lemma,cat,_) -> cat
  | Proper(lemma,cat,_,_) -> cat
  | _ -> ""

let months = StringSet.of_list ["1"; "2"; "3"; "4"; "5"; "6"; "7"; "8"; "9"; "01"; "02"; "03"; "04"; "05"; "06"; "07"; "08"; "09"; "10"; "11"; "12"]
let hours = StringSet.of_list ["0"; "1"; "2"; "3"; "4"; "5"; "6"; "7"; "8"; "9"; "00"; "01"; "02"; "03"; "04"; "05"; "06"; "07"; "08"; "09";
   "10"; "11"; "12"; "13"; "14"; "15"; "16"; "17"; "18"; "19"; "20"; "21"; "22"; "23"; "24"]
let days = StringSet.of_list ["1"; "2"; "3"; "4"; "5"; "6"; "7"; "8"; "9"; "01"; "02"; "03"; "04"; "05"; "06"; "07"; "08"; "09";
   "10"; "11"; "12"; "13"; "14"; "15"; "16"; "17"; "18"; "19"; "20"; "21"; "22"; "23"; "24"; "25"; "26"; "27"; "28"; "29"; "30"; "31"]
let romanmonths = StringSet.of_list ["I"; "II"; "III"; "IV"; "V"; "VI"; "VII"; "VIII"; "IX"; "X"; "XI"; "XII"]


let s_beg i = Token{empty_token_env with beg=i;len=1;next=i+1; token=Interp "<sentence>"}
let c_beg i = Token{empty_token_env with beg=i;len=1;next=i+1; token=Interp "<clause>"}

let dig_token orth i digs token =
  Token{empty_token_env with orth=orth;beg=i;len=Xlist.size digs * factor;next=i+Xlist.size digs * factor; token=token; attrs=["maybe cs"]}

let sc_dig_token orth i digs token =
  Seq[s_beg i;c_beg (i+1);Token{empty_token_env with orth=orth;beg=i+2;len=Xlist.size digs * factor - 2;next=i+Xlist.size digs * factor; token=token; attrs=["maybe cs"]}]

let dig_tokens orth poss_s_beg i digs v cat =
  if poss_s_beg then
    [dig_token orth i digs (Dig(v,cat));
     sc_dig_token orth i digs (Dig(v,cat))]
  else
    [dig_token orth i digs (Dig(v,cat))]

let merge_digits poss_s_beg i digs =
  let orth = String.concat "" digs in
  let t = dig_tokens orth poss_s_beg i digs in
  let v = try string_of_int (int_of_string orth) with _ -> failwith "merge_digits" in
  let variants =
    (t orth "dig") @
    [Token{empty_token_env with orth=orth;beg=i;len=Xlist.size digs * factor;next=i+Xlist.size digs * factor; token=Proper(orth,"obj-id",[[]],["obj-id"]); attrs=["maybe cs"]}] @
    (if digs = ["0"] || List.hd digs <> "0" then (t orth "intnum")(* @ (t orth "realnum")*) else []) @
    (if List.hd digs <> "0" then (t v "year") else []) @
    (if StringSet.mem months orth then (t v "month") else []) @
    (if StringSet.mem hours orth then (t v "hour") else []) @
    (if StringSet.mem days orth then (t v "day") else []) @
    (if Xlist.size digs = 2 && List.hd digs < "6" then (t v "minute") else []) @
    (if Xlist.size digs = 3 then (t orth "3dig") else []) @
    (if Xlist.size digs = 2 then (t orth "2dig") else []) @
    (if Xlist.size digs <= 3 && List.hd digs <> "0" then (t orth "pref3dig") else []) in
(*  let t = dig_token orth i digs in
  let sc_t = sc_dig_token orth i digs in
  let v = try int_of_string orth with _ -> failwith "merge_digits" in
  let variants =
    [t (Dig(v,"dig"));sc_t (Dig(v,"dig"))] @
    (if digs = ["0"] || List.hd digs <> "0" then [t (Dig(v,"intnum"));sc_t (Dig(v,"intnum"))] else []) @
    (if List.hd digs <> "0" then [t (Dig(v,"year"));sc_t (Dig(v,"year"))] else []) @
    (if StringSet.mem months orth then [t (Dig(v,"month"));sc_t (Dig(v,"month"))] else []) @
    (if StringSet.mem hours orth then [t (Dig(v,"hour"));sc_t (Dig(v,"hour"))] else []) @
    (if StringSet.mem days orth then [t (Dig(v,"day"));sc_t (Dig(v,"day"))] else []) @
    (if Xlist.size digs = 2 && List.hd digs < "6" then [t (Dig(v,"minute"));sc_t (Dig(v,"minute"))] else []) @
    (if Xlist.size digs = 3 then [t (Dig(v,"3dig"));sc_t (Dig(v,"3dig"))] else []) @
    (if Xlist.size digs <= 3 && List.hd digs <> "0" then [t (Dig(v,"pref3dig"));sc_t (Dig(v,"pref3dig"))] else []) in*)
  Variant variants

let recognize_roman_I v = function
    Capital("I",_) :: Capital("I",_) :: Capital("I",_) :: [] -> v+3,false
  | Capital("I",_) :: Capital("I",_) :: [] -> v+2,false
  | Capital("I",_) :: [] -> v+1,false
  | [] -> v,false
  | Capital("I",_) :: Capital("I",_) :: Capital("I",_) :: Small("w") :: [] -> v+3,true
  | Capital("I",_) :: Capital("I",_) :: Small("w") :: [] -> v+2,true
  | Capital("I",_) :: Small("w") :: [] -> v+1,true
  | Small("w") :: [] -> v,true
  | _ -> 0,false

let recognize_roman_V v = function
    Capital("I",_) :: ForeignCapital("V",_) :: [] -> v+4,false
  | ForeignCapital("V",_) :: l -> recognize_roman_I (v+5) l
  | Capital("I",_) :: ForeignCapital("X",_) :: [] -> v+9,false
  | Capital("I",_) :: ForeignCapital("V",_) :: Small("w") :: [] -> v+4,true
  | Capital("I",_) :: ForeignCapital("X",_) :: Small("w") :: [] -> v+9,true
  | l -> recognize_roman_I v l

let recognize_roman_X v = function
  | ForeignCapital("X",_) :: ForeignCapital("X",_) :: ForeignCapital("X",_) :: l -> recognize_roman_V (v+30) l
  | ForeignCapital("X",_) :: ForeignCapital("X",_) :: l -> recognize_roman_V (v+20) l
  | ForeignCapital("X",_) :: l -> recognize_roman_V (v+10) l
  | l -> recognize_roman_V v l

let recognize_roman_L v = function
    ForeignCapital("X",_) :: Capital("L",_) :: l -> recognize_roman_V (v+40) l
  | Capital("L",_) :: l -> recognize_roman_X (v+50) l
  | ForeignCapital("X",_) :: Capital("C",_) :: l -> recognize_roman_V (v+90) l
  | l -> recognize_roman_X v l

let recognize_roman_C v = function
  | Capital("C",_) :: Capital("C",_) :: Capital("C",_) :: l -> recognize_roman_L (v+300) l
  | Capital("C",_) :: Capital("C",_) :: l -> recognize_roman_L (v+200) l
  | Capital("C",_) :: l -> recognize_roman_L (v+100) l
  | l -> recognize_roman_L v l

let recognize_roman_D v = function
    Capital("C",_) :: Capital("D",_) :: l -> recognize_roman_L (v+400) l
  | Capital("D",_) :: l -> recognize_roman_C (v+500) l
  | Capital("C",_) :: Capital("M",_) :: l -> recognize_roman_L (v+900) l
  | l -> recognize_roman_C v l

let recognize_roman_M v = function
  | Capital("M",_) :: Capital("M",_) :: Capital("M",_) :: l -> recognize_roman_D (v+3000) l
  | Capital("M",_) :: Capital("M",_) :: l -> recognize_roman_D (v+2000) l
  | Capital("M",_) :: l -> recognize_roman_D (v+1000) l
  | l -> recognize_roman_D v l

let rec merge l =
  String.concat "" (Xlist.map l (function
      Capital(s,t) -> s
    | ForeignCapital(s,t) -> s
    | Small s -> s
    | ForeignSmall s -> s
    | _ -> failwith "merge"))

let lowercase_first = function
    [] -> []
  | Capital(s,t) :: l -> Small t :: l
  | ForeignCapital(s,t) :: l -> ForeignSmall t :: l
  | Small s :: l -> Small s :: l
  | ForeignSmall s :: l -> ForeignSmall s :: l
  | _ -> failwith "lowercase_first"

let rec lowercase_all = function
    [] -> []
  | Capital(s,t) :: l -> Small t :: lowercase_all l
  | ForeignCapital(s,t) :: l -> ForeignSmall t :: lowercase_all l
  | Small s :: l -> Small s :: lowercase_all l
  | ForeignSmall s :: l -> ForeignSmall s :: lowercase_all l
  | _ -> failwith "lowercase_all"

let lowercase_rest = function
    [] -> []
  | x :: l -> x :: lowercase_all l

let first_capital = function
    Capital _ :: _ -> true
  | ForeignCapital _ :: _ -> true
  | Small _ :: _ -> false
  | ForeignSmall _ :: _ -> false
  | _ -> failwith "first_capital"

let rec all_capital = function
    Capital _ :: l -> all_capital l
  | ForeignCapital _ :: l -> all_capital l
  | Small _ :: l -> false
  | ForeignSmall _ :: l -> false
  | [] -> true
  | _ -> failwith "first_capital"

let rec all_small = function
    Capital _ :: l -> false
  | ForeignCapital _ :: l -> false
  | Small _ :: l -> all_small l
  | ForeignSmall _ :: l -> all_small l
  | [] -> true
  | _ -> failwith "first_capital"

let rest_capital = function
    [] -> failwith "rest_capital"
  | _ :: l -> all_capital l

let rest_small = function
    [] -> failwith "rest_small"
  | _ :: l -> all_small l

let get_first_cap = function
  | Capital(s,t) :: l -> s
  | ForeignCapital(s,t) :: l -> s
  | _ -> failwith "get_first_cap"

let get_first_lower = function
  | Capital(s,t) :: l -> t
  | ForeignCapital(s,t) :: l -> t
  | _ -> failwith "get_first_lower"

let cs_weight = -1.
let sc_cap_weight = -0.3

let is_add_attr_token = function
    SmallLetter _ -> true
  | CapLetter _ -> true
  | AllSmall _ -> true
  | AllCap _ -> true
  | FirstCap _ -> true
  | SomeCap _ -> true
  | _ -> false

let rec add_attr s = function
    Token t -> if is_add_attr_token t.token then Token{t with attrs=s :: t.attrs} else Token t
  | Variant l -> Variant(Xlist.map l (add_attr s))
  | Seq l -> Seq(Xlist.map l (add_attr s))

let recognize_stem poss_s_beg has_sufix i letters =
  let orth = merge letters in
  let t = {empty_token_env with orth=orth;beg=i;len=Xlist.size letters * factor;next=i+Xlist.size letters * factor} in
  let t = if poss_s_beg then
    if Xlist.size letters = 1 then
      if first_capital letters then Variant[
          Token{t with token=SmallLetter(merge (lowercase_first letters)); weight=cs_weight; attrs="cs" :: t.attrs};
          Seq[s_beg i;c_beg (i+1);Token{t with beg=t.beg+2; len=t.len-2; token=SmallLetter(merge (lowercase_first letters)); attrs="maybe cs" :: t.attrs}];
          Token{t with token=CapLetter(orth,merge (lowercase_first letters)); attrs="maybe cs" :: t.attrs};
          Seq[s_beg i;c_beg (i+1);Token{t with beg=t.beg+2; len=t.len-2; token=CapLetter(orth,merge (lowercase_first letters)); weight=sc_cap_weight; attrs="maybe cs" :: t.attrs}]]
      else Token{t with token=SmallLetter orth}
    else
      if first_capital letters then
        if rest_small letters then Variant[
          Seq[s_beg i;c_beg (i+1);Token{t with beg=t.beg+2; len=t.len-2; token=AllSmall(merge (lowercase_first letters))}];
          Token{t with token=FirstCap(orth,merge (lowercase_first letters),get_first_cap letters,get_first_lower letters)};
          Seq[s_beg i;c_beg (i+1);Token{t with beg=t.beg+2; len=t.len-2; token=FirstCap(orth,merge (lowercase_first letters),get_first_cap letters,get_first_lower letters); weight=sc_cap_weight}]]
        else if rest_capital letters then Variant([
          Token{t with token=AllSmall(merge (lowercase_all letters)); weight=cs_weight; attrs="cs" :: t.attrs};
          Seq[s_beg i;c_beg (i+1);Token{t with beg=t.beg+2; len=t.len-2; token=AllSmall(merge (lowercase_all letters)); weight=cs_weight; attrs="cs" :: t.attrs}];
          Token{t with token=FirstCap(merge (lowercase_rest letters),merge (lowercase_all letters),get_first_cap letters,get_first_lower letters); weight=cs_weight; attrs="cs" :: t.attrs};
          Seq[s_beg i;c_beg (i+1);Token{t with beg=t.beg+2; len=t.len-2; token=FirstCap(merge (lowercase_rest letters),merge (lowercase_all letters),get_first_cap letters,get_first_lower letters); weight=cs_weight+.sc_cap_weight; attrs="cs" :: t.attrs}]] @
          (if has_sufix then [] else [
           Token{t with token=AllCap(orth,merge (lowercase_rest letters),merge (lowercase_all letters)); attrs="maybe cs" :: t.attrs};
           Seq[s_beg i;c_beg (i+1);Token{t with beg=t.beg+2; len=t.len-2; token=AllCap(orth,merge (lowercase_rest letters),merge (lowercase_all letters)); attrs="maybe cs" :: t.attrs}]]))
        else Token{t with token=SomeCap orth}
      else
        if rest_small letters then Token{t with token=AllSmall orth}
        else Token{t with token=SomeCap orth}
  else
    if Xlist.size letters = 1 then
      if first_capital letters then Variant[
          Token{t with token=SmallLetter orth; weight=cs_weight; attrs="cs" :: t.attrs};
          Token{t with token=CapLetter(orth,merge (lowercase_first letters)); attrs="maybe cs" :: t.attrs}]
      else Token{t with token=SmallLetter orth}
    else
      if first_capital letters then
        if rest_small letters then
          Token{t with token=FirstCap(orth,merge (lowercase_first letters),get_first_cap letters,get_first_lower letters)}
        else if rest_capital letters then Variant([
          Token{t with token=AllSmall(merge (lowercase_all letters)); weight=cs_weight; attrs="cs" :: t.attrs};
          Token{t with token=FirstCap(merge (lowercase_rest letters),merge (lowercase_all letters),get_first_cap letters,get_first_lower letters); weight=cs_weight; attrs="cs" :: t.attrs}] @
          (if has_sufix then [] else [
           Token{t with token=AllCap(orth,merge (lowercase_rest letters),merge (lowercase_all letters)); attrs="maybe cs" :: t.attrs}]))
        else Token{t with token=SomeCap orth}
      else
        if rest_small letters then Token{t with token=AllSmall orth}
        else Token{t with token=SomeCap orth} in
  if has_sufix then add_attr "required validated lemmatization" t else t

let parse_postags s =
  List.map (fun s ->
    match List.map (fun t -> Str.split (Str.regexp "\\.") t) (Str.split (Str.regexp ":") s) with
      [pos] :: tags -> pos, tags
    | _ -> failwith ("parse_postags: " ^ s)) (Str.split (Str.regexp "|") s)

let make_lemma (lemma,interp) =
  match parse_postags interp with
    [pos,tags] -> Lemma(lemma,pos,[tags])
  | _ -> failwith "make_lemma"

let merge_attrs l =
(*   print_endline (String.concat " " (Xlist.map l (fun token -> "[" ^ token.orth ^ " " ^ String.concat ";" token.attrs ^ "]"))); *)
  let len = Xlist.size l in
  let attrs = Xlist.fold l StringQMap.empty (fun attrs token ->
    Xlist.fold token.attrs attrs StringQMap.add) in
  let n_cs = try StringQMap.find attrs "cs" with Not_found -> 0 in
  let n_maybe_cs = try StringQMap.find attrs "maybe cs" with Not_found -> 0 in
  let new_attrs =
    (if n_cs > 0 then
      if n_cs + n_maybe_cs = len then ["cs"] else raise Not_found
    else
      if n_maybe_cs = len then ["maybe cs"] else []) @
    (StringQMap.fold attrs [] (fun attrs attr _ -> if attr = "cs" || attr = "maybe cs" then attrs else attr :: attrs)) in
(*   print_endline (String.concat " " new_attrs); *)
  new_attrs

let suffix_lemmata = Xlist.fold [
  "em",make_lemma ("być","aglt:sg:pri:imperf:wok");
  "eś",make_lemma ("być","aglt:sg:sec:imperf:wok");
  "eście",make_lemma ("być","aglt:pl:sec:imperf:wok");
  "eśmy",make_lemma ("być","aglt:pl:pri:imperf:wok");
  "m",make_lemma ("być","aglt:sg:pri:imperf:nwok");
  "ś",make_lemma ("być","aglt:sg:sec:imperf:nwok");
  "ście",make_lemma ("być","aglt:pl:sec:imperf:nwok");
  "śmy",make_lemma ("być","aglt:pl:pri:imperf:nwok");
  "by",make_lemma ("by","qub");
  ] StringMap.empty (fun map (suf,lemma) -> StringMap.add map suf lemma)

let recognize_suffix i letters =
  let orth = merge letters in
  let t = {empty_token_env with orth=orth;beg=i;len=Xlist.size letters * factor;next=i+Xlist.size letters * factor} in
  if all_capital letters then Token{t with token=StringMap.find suffix_lemmata (merge (lowercase_all letters)); weight=cs_weight; attrs="cs" :: t.attrs}
  else if all_small letters then Token{t with token=StringMap.find suffix_lemmata orth}
  else raise Not_found

let recognize_romandig poss_s_beg i letters =
  let roman,w = recognize_roman_M 0 letters in
  if roman > 0 then
    let letters,w = if w then let l = List.rev letters in List.rev (List.tl l), [List.hd l] else letters,[] in
    let orth = merge letters in
    let roman = string_of_int roman in
    let t = {empty_token_env with orth=orth;beg=i;len=Xlist.size letters * factor;next=i+Xlist.size letters * factor} in
    let w = if w = [] then [] else
      let beg = i + Xlist.size letters * factor in
      [Variant[Token{empty_token_env with orth=merge w; beg=beg; len=factor; next=beg+factor; token=SmallLetter(merge w)};
               Token{empty_token_env with orth=merge w; beg=beg; len=factor; next=beg+factor; token=make_lemma ("wiek","subst:sg:_:m3")}]] in
    if StringSet.mem romanmonths orth then [
      Seq(Token{t with token=RomanDig(roman,"roman"); attrs="maybe cs" :: t.attrs}::w);
      Seq(Token{t with token=RomanDig(roman,"month"); attrs="maybe cs" :: t.attrs}::w)] @
      (if poss_s_beg then [
        Seq([s_beg i;c_beg (i+1);Token{t with beg=t.beg+2; len=t.len-2; token=RomanDig(roman,"roman"); attrs="maybe cs" :: t.attrs}]@w);
        Seq([s_beg i;c_beg (i+1);Token{t with beg=t.beg+2; len=t.len-2; token=RomanDig(roman,"month"); attrs="maybe cs" :: t.attrs}]@w);
      ] else [])
    else [
      Seq(Token{t with token=RomanDig(roman,"roman"); attrs="maybe cs" :: t.attrs}::w)] @
      (if poss_s_beg then [
      Seq([s_beg i;c_beg (i+1);Token{t with beg=t.beg+2; len=t.len-2; token=RomanDig(roman,"roman"); attrs="maybe cs" :: t.attrs}]@w);
      ] else [])
  else []

let sufixes1 = Xlist.map [
  ["m"];
  ["e"; "m"];
  ["ś"];
  ["e"; "ś"];
  ["ś"; "m"; "y"];
  ["e"; "ś"; "m"; "y"];
  ["ś"; "c"; "i"; "e"];
  ["e"; "ś"; "c"; "i"; "e"];
  ] List.rev

let sufixes2 = Xlist.map [
  ["b"; "y"];
  ] List.rev

let rec find_suffix rev = function
    _, [] -> raise Not_found
  | [], l -> rev, l
  | s :: pat, Capital(c,t) :: l -> if s = t then find_suffix (Capital(c,t) :: rev) (pat,l) else raise Not_found
  | s :: pat, Small t :: l -> if s = t then find_suffix (Small t :: rev) (pat,l) else raise Not_found
  | _,_ -> raise Not_found

let find_suffixes2 sufixes letters sufs =
  Xlist.fold sufixes [] (fun l suf ->
    try
      let suf,rev_stem = find_suffix [] (suf,letters) in
      (rev_stem,suf :: sufs) :: l
    with Not_found -> l)

let find_suffixes i letters =
  let letters = List.rev letters in
  let l = (letters,[]) :: find_suffixes2 sufixes1 letters [] in
  let l = Xlist.fold l l (fun l (letters,sufs) ->
    (find_suffixes2 sufixes2 letters sufs) @ l) in
  Xlist.map l (fun (rev_stem, sufs) ->
    List.rev (fst (Xlist.fold (List.rev rev_stem :: sufs) ([],i) (fun (seq,i) letters ->
      (letters,i) :: seq, i + factor * Xlist.size letters))))

let merge_letters poss_s_beg i letters =
  let l = find_suffixes i letters in
  let roman = recognize_romandig poss_s_beg i letters in
  let variants = Xlist.fold l roman (fun variants -> function
      [] -> failwith "merge_letters"
    | [stem,i] -> (recognize_stem poss_s_beg false i stem) :: variants
    | (stem,i) :: suffixes ->
        (try (Seq((recognize_stem poss_s_beg true i stem) :: Xlist.map suffixes (fun (suf,i) -> recognize_suffix i suf))) :: variants
         with Not_found -> variants)) in
  Variant variants

let rec group_digits rev = function
    [] -> List.rev rev, []
  | Digit s :: l -> group_digits (s :: rev) l
  | x :: l -> List.rev rev, x :: l

let rec group_letters rev = function
    [] -> List.rev rev, []
  | Capital(s,t) :: l -> group_letters ((Capital(s,t)) :: rev) l
  | ForeignCapital(s,t) :: l -> group_letters ((ForeignCapital(s,t)) :: rev) l
  | Small s :: l -> group_letters ((Small s) :: rev) l
  | ForeignSmall s :: l -> group_letters ((ForeignSmall s) :: rev) l
  | x :: l -> List.rev rev, x :: l

let rec group_others rev = function
    [] -> List.rev rev, []
  | Other(s,_) :: l -> group_others (s :: rev) l
  | x :: l -> List.rev rev, x :: l

let create_sign_token poss_s_beg i signs l token =
  let orth = String.concat "" (Xlist.map signs (function Sign s -> s | _ -> failwith "create_sign_token")) in
  let len = Xlist.size signs * factor in
  Token{empty_token_env with orth=orth;beg=i;len=len;next=i+len;token=token; attrs=["maybe cs"]},i+len,l,poss_s_beg

let create_empty_sign_token i signs =
  let orth = String.concat "" (Xlist.map signs (function Sign s -> s | _ -> failwith "create_sign_token")) in
  let len = Xlist.size signs * factor in
  {empty_token_env with orth=orth;beg=i;len=len;next=i+len; attrs=["maybe cs"]},i+len

let create_sentence_seq i signs l lemma =
  let orth = String.concat "" (Xlist.map signs (function Sign s -> s | _ -> failwith "create_sign_token")) in
  let len = Xlist.size signs * factor in
  Seq[Token{empty_token_env with beg=i;len=20;next=i+20;token=Interp "</clause>"};
      Token{empty_token_env with orth=orth;beg=i+20;len=len-30;next=i+len-10;token=make_lemma (lemma,"sinterj")};
      Token{empty_token_env with beg=i+len-10;len=10;next=i+len;token=Interp "</sentence>"}]

let create_sentence_seq_hapl i signs l lemma =
  let orth = String.concat "" (Xlist.map signs (function Sign s -> s | _ -> failwith "create_sign_token")) in
  let len = Xlist.size signs * factor in
  Seq[Token{empty_token_env with beg=i;len=10;next=i+10;token=Symbol "."; attrs=["maybe cs"]};
      Token{empty_token_env with beg=i+10;len=10;next=i+20;token=Interp "</clause>"};
      Token{empty_token_env with orth=orth;beg=i+20;len=len-30;next=i+len-10;token=make_lemma (lemma,"sinterj")};
      Token{empty_token_env with beg=i+len-10;len=10;next=i+len;token=Interp "</sentence>"}]

let create_sentence_seq_q i signs l lemma =
  let orth = String.concat "" (Xlist.map signs (function Sign s -> s | _ -> failwith "create_sign_token")) in
  let len = Xlist.size signs * factor in
  Seq[Token{empty_token_env with beg=i;len=20;next=i+20;token=Interp "?"};
      Token{empty_token_env with beg=i+20;len=10;next=i+30;token=Interp "</clause>"};
      Token{empty_token_env with orth=orth;beg=i+30;len=len-40;next=i+len-10;token=make_lemma (lemma,"sinterj")};
      Token{empty_token_env with beg=i+len-10;len=10;next=i+len;token=Interp "</sentence>"}]

let create_sentence_seq_hapl_q i signs l lemma =
  let orth = String.concat "" (Xlist.map signs (function Sign s -> s | _ -> failwith "create_sign_token")) in
  let len = Xlist.size signs * factor in
  Seq[Token{empty_token_env with beg=i;len=10;next=i+10;token=Symbol "."; attrs=["maybe cs"]};
      Token{empty_token_env with beg=i+10;len=10;next=i+20;token=Interp "?"};
      Token{empty_token_env with beg=i+20;len=10;next=i+30;token=Interp "</clause>"};
      Token{empty_token_env with orth=orth;beg=i+30;len=len-40;next=i+len-10;token=make_lemma (lemma,"sinterj")};
      Token{empty_token_env with beg=i+len-10;len=10;next=i+len;token=Interp "</sentence>"}]

let create_or_beg i signs l poss_s_beg =
  let orth = String.concat "" (Xlist.map signs (function Sign s -> s | _ -> failwith "create_sign_token")) in
  let len = Xlist.size signs * factor in
  Variant[
    Token{empty_token_env with orth=orth;beg=i;len=len;next=i+len;token=Symbol "-"; attrs=["maybe cs"]};
    Token{empty_token_env with orth=orth;beg=i;len=len;next=i+len;token=Interp "-"; attrs=["maybe cs"]}; (* hyphen *)
    Token{empty_token_env with orth=orth;beg=i;len=len;next=i+len;token=Interp "<or>"};
    (* Seq[Token{empty_token_env with   beg=i;       len=20;    next=i+20;    token=Interp "<sentence>"};
        Token{empty_token_env with orth=orth;beg=i+20;    len=len-20;next=i+len;   token=Interp "<or>"}]; *)
    Seq[Token{empty_token_env with   beg=i;       len=21;    next=i+21;    token=Interp "</clause>"};
        Token{empty_token_env with   beg=i+21;    len=20;    next=i+41;    token=Interp "</sentence>"};
        Token{empty_token_env with orth=orth;beg=i+41;    len=len-59;next=i+len-20;token=Interp "</or>"};
        Token{empty_token_env with   beg=i+len-20;len=20;    next=i+len;   token=Interp "<clause>"}];
    Seq[Token{empty_token_env with orth=orth;beg=i;       len=len-22;next=i+len-22;token=Interp "</or>"};
        Token{empty_token_env with   beg=i+len-22;len=22;    next=i+len;   token=Interp "<clause>"}];
    ],i+len,l,poss_s_beg

let create_or_beg2 i signs l poss_s_beg =
  let orth = String.concat "" (Xlist.map signs (function Sign s -> s | _ -> failwith "create_sign_token")) in
  let len = Xlist.size signs * factor in
  Variant[
    Token{empty_token_env with orth=orth;beg=i;len=len;next=i+len;token=Interp "<or>"};
    (* Seq[Token{empty_token_env with   beg=i;       len=20;    next=i+20;    token=Interp "<sentence>"};
        Token{empty_token_env with orth=orth;beg=i+20;    len=len-20;next=i+len;   token=Interp "<or>"}]; *)
    Seq[Token{empty_token_env with   beg=i;       len=21;    next=i+21;    token=Interp "</clause>"};
        Token{empty_token_env with   beg=i+21;    len=20;    next=i+41;    token=Interp "</sentence>"};
        Token{empty_token_env with orth=orth;beg=i+41;    len=len-59;next=i+len-20;token=Interp "</or>"};
        Token{empty_token_env with   beg=i+len-20;len=20;    next=i+len;   token=Interp "<clause>"}];
    Seq[Token{empty_token_env with orth=orth;beg=i;       len=len-22;next=i+len-22;token=Interp "</or>"};
        Token{empty_token_env with   beg=i+len-22;len=22;    next=i+len;   token=Interp "<clause>"}];
    ],i+len,l,poss_s_beg

let is_dot_sentence_end_marker = function
    [] -> true
  | [Sign " "] -> true
  | [Sign "﻿"] -> true
  | [Sign " "] -> true
  | [Sign "\""] -> true
  | [Sign "»"] -> true
  | [Sign "”"] -> true
  | _ -> false

let not_dot_sentence_end_marker = function
    Sign " " :: Small _ :: _ -> true
  | Sign "﻿" :: Small _ :: _ -> true
  | Sign " " :: Small _ :: _ -> true
  | Sign "," :: _ -> true
  | Sign ":" :: _ -> true
  | Sign "?" :: _ -> true
  | Sign "!" :: _ -> true
  | Small _ :: _ -> true
  | ForeignSmall _ :: _ -> true
  | Capital _ :: _ -> true
  | ForeignCapital _ :: _ -> true
  | Digit _ :: _ -> true
  | _ -> false

let is_comma_digit_marker = function
    Digit _ :: l -> true
  | _ -> false

let is_colon_sentence_end_marker = function
    [] -> true
  | [Sign " "] -> true
  | [Sign "﻿"] -> true
  | [Sign " "] -> true
  | _ -> false

let is_colon_symbol = function
    Digit _ :: _ -> true
  | Sign "/" :: _ -> true
  | _ -> false

let is_multidot_sentence_end_marker = function
    [] -> true
  | [Sign " "] -> true
  | [Sign "﻿"] -> true
  | [Sign " "] -> true
  | [Sign "\""] -> true
  | [Sign "»"] -> true
  | [Sign "”"] -> true
(*  | "\"" :: l -> true
  | "»" :: l -> true
  | "”" :: l -> true
  | "“" :: l -> true
  | " " :: "-" :: l -> true
  | " " :: "–" :: l -> true
  | " " :: "—" :: l -> true
  | ")" :: l -> true
  | "]" :: l -> true*)
  | _ -> false

let create_quot_digit_token i signs l =
  let t,i2 = create_empty_sign_token i signs in
  Variant[
    Seq[Token{empty_token_env with beg=i;len=20;next=i+20;token=Interp "</clause>"};
        Token{empty_token_env with orth=".";beg=i+20;len=factor-20;next=i+factor;token=Interp "</sentence>"};
        Token{t with beg=t.beg+factor; next=t.next+factor;token=Interp "”s"}];
    Seq[Token{t with token=Interp "”"};
        Token{empty_token_env with beg=i2;len=20;next=i2+20;token=Interp "</clause>"};
        Token{empty_token_env with orth=".";beg=i2+20;len=factor-20;next=i2+factor;token=Interp "</sentence>"}];
    ],i2+factor,l,true

let rec recognize_sign_group poss_s_beg i = function
  | (Sign " ") :: l -> create_sign_token poss_s_beg i [Sign " "] l (Symbol " ")
  | (Sign "﻿") :: l -> create_sign_token poss_s_beg i [Sign "﻿"] l (Symbol " ")
  | (Sign " ") :: l -> create_sign_token poss_s_beg i [Sign " "] l (Symbol " ")
  | (Sign "\"") :: (Sign ".") :: l -> create_quot_digit_token i [Sign "\""] l
  | (Sign "\"") :: l ->
      let t,i = create_empty_sign_token i [Sign "\""] in
      Variant[Token{t with token=Interp "„"};Token{t with token=Interp "„s"};Token{t with token=Interp "”"};Token{t with token=Interp "”s"}],i,l,poss_s_beg
  | (Sign "˝") :: (Sign ".") :: l -> create_quot_digit_token i [Sign "˝"] l
  | (Sign "˝") :: l ->
      let t,i = create_empty_sign_token i [Sign "˝"] in
      Variant[Token{t with token=Interp "„"};Token{t with token=Interp "„s"};Token{t with token=Interp "”"};Token{t with token=Interp "”s"}],i,l,poss_s_beg
  | (Sign "„") :: l ->
      let t,i = create_empty_sign_token i [Sign "„"] in
      Variant[Token{t with token=Interp "„"};Token{t with token=Interp "„s"}],i,l,poss_s_beg
  | (Sign "”") :: (Sign ".") :: l -> create_quot_digit_token i [Sign "”"] l
  | (Sign "”") :: l ->
      let t,i = create_empty_sign_token i [Sign "”"] in
      Variant[Token{t with token=Interp "”"};Token{t with token=Interp "”s"}],i,l,poss_s_beg
  | (Sign "“") :: (Sign ".") :: l -> create_quot_digit_token i [Sign "“"] l
  | (Sign "“") :: l ->
      let t,i = create_empty_sign_token i [Sign "“"] in
      Variant[Token{t with token=Interp "„"};Token{t with token=Interp "„s"};Token{t with token=Interp "”"};Token{t with token=Interp "”s"}],i,l,poss_s_beg
  | (Sign ",") :: (Sign ",") :: l ->
      let t,i = create_empty_sign_token i [Sign ",";Sign ","] in
      Variant[Token{t with token=Interp "„"};Token{t with token=Interp "„s"}],i,l,poss_s_beg
  | (Sign ",") :: l ->
      let t,i2 = create_empty_sign_token i [Sign ","] in
      if is_comma_digit_marker l then
        Token{t with token=Symbol ","},i2,l,false
      else
        Variant[Token{t with token=Interp ","};
                Seq[Token{empty_token_env with orth=",";beg=i;len=factor/2;next=i+factor/2;token=Interp "</clause>"};
                    Token{empty_token_env with beg=i+factor/2;len=factor-(factor/2);next=i+factor;token=Interp "<clause>"}]],i2,l,false
  | (Sign "(") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign "(") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ")") :: []) l (make_lemma ("(…)","sinterj"))
  | (Sign "(") :: (Sign "?") :: (Sign "!") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign "(") :: (Sign "?") :: (Sign "!") :: (Sign ")") :: []) l (make_lemma ("(?!)","sinterj"))
  | (Sign "(") :: (Sign ".") :: (Sign ".") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign "(") :: (Sign ".") :: (Sign ".") :: (Sign ")") :: []) l (make_lemma ("(…)","sinterj"))
  | (Sign "(") :: (Sign "+") :: (Sign "+") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign "(") :: (Sign "+") :: (Sign "+") :: (Sign ")") :: []) l (make_lemma ("(++)","sinterj"))
  | (Sign "(") :: (Sign "-") :: (Sign "-") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign "(") :: (Sign "-") :: (Sign "-") :: (Sign ")") :: []) l (make_lemma ("(--)","symbol"))
  | (Sign "(") :: (Sign "…") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign "(") :: (Sign "…") :: (Sign ")") :: []) l (make_lemma ("(…)","sinterj"))
  | (Sign "(") :: (Sign "?") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign "(") :: (Sign "?") :: (Sign ")") :: []) l (make_lemma ("(?)","sinterj"))
  | (Sign "(") :: (Sign "+") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign "(") :: (Sign "+") :: (Sign ")") :: []) l (make_lemma ("(+)","symbol"))
  | (Sign "(") :: (Sign "!") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign "(") :: (Sign "!") :: (Sign ")") :: []) l (make_lemma ("(!)","sinterj"))
  | (Sign "(") :: (Sign "-") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign "(") :: (Sign "-") :: (Sign ")") :: []) l (make_lemma ("(-)","symbol"))
  | (Sign "(") :: (Sign "*") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign "(") :: (Sign "*") :: (Sign ")") :: []) l (make_lemma ("(*)","symbol"))
  | (Sign "(") :: l -> create_sign_token poss_s_beg i [Sign "("] l (Interp "(")
  | (Sign ":") :: (Sign "(") :: (Sign "(") :: (Sign "(") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign "(") :: (Sign "(") :: (Sign "(") :: []) l (make_lemma (":(((","sinterj"))
  | (Sign ":") :: (Sign "(") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign "(") :: []) l (make_lemma (":(","sinterj"))
  | (Sign ":") :: (Sign "-") :: (Sign "(") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign "-") :: (Sign "(") :: []) l (make_lemma (":-(","sinterj"))
  | (Sign ";") :: (Sign "(") :: (Sign "(") :: l -> create_sign_token poss_s_beg i ((Sign ";") :: (Sign "(") :: (Sign "(") :: []) l (make_lemma (";((","sinterj"))
  | (Sign ";") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ";") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: []) l (make_lemma (";-))))","sinterj"))
  | (Sign ":") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: []) l (make_lemma (":-))))","sinterj"))
  | (Sign ":") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: []) l (make_lemma (":-)))","sinterj"))
  | (Sign ";") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ";") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: []) l (make_lemma (";-)))","sinterj"))
  | (Sign ";") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ";") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: []) l (make_lemma (";)))","sinterj"))
  | (Sign ":") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: []) l (make_lemma (":-))","sinterj"))
  | (Sign ";") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ";") :: (Sign "-") :: (Sign ")") :: (Sign ")") :: []) l (make_lemma (";-))","sinterj"))
  | (Sign ":") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign ")") :: (Sign ")") :: (Sign ")") :: []) l (make_lemma (":)))","sinterj"))
  | (Sign ":") :: (Sign ")") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign ")") :: (Sign ")") :: []) l (make_lemma (":))","sinterj"))
  | (Sign ";") :: (Sign ")") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ";") :: (Sign ")") :: (Sign ")") :: []) l (make_lemma (";))","sinterj"))
  | (Sign ";") :: (Sign "-") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ";") :: (Sign "-") :: (Sign ")") :: []) l (make_lemma (";-)","sinterj"))
  | (Sign ":") :: (Sign "|") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign "|") :: []) l (make_lemma (":|","sinterj"))
  | (Sign ":") :: (Sign "\\") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign "\\") :: []) l (make_lemma (":\\","sinterj"))
  | (Sign ":") :: (Sign "-") :: (Sign "/") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign "-") :: (Sign "/") :: []) l (make_lemma (":-/","sinterj"))
  | (Sign ":") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ":") :: (Sign ")") :: []) l (make_lemma (":)","sinterj"))
  | (Sign ";") :: (Sign ")") :: l -> create_sign_token poss_s_beg i ((Sign ";") :: (Sign ")") :: []) l (make_lemma (";)","sinterj"))
  | (Sign ")") :: l -> create_sign_token poss_s_beg i [Sign ")"] l (Interp ")")
  | (Sign "[") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign "]") :: l -> create_sign_token poss_s_beg i ((Sign "[") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign "]") :: []) l (make_lemma ("(…)","sinterj"))
  | (Sign "[") :: (Sign ".") :: (Sign ".") :: (Sign "]") :: l -> create_sign_token poss_s_beg i ((Sign "[") :: (Sign ".") :: (Sign ".") :: (Sign "]") :: []) l (make_lemma ("(…)","sinterj"))
  | (Sign "[") :: (Sign "+") :: (Sign "]") :: l -> create_sign_token poss_s_beg i ((Sign "[") :: (Sign "+") :: (Sign "]") :: []) l (make_lemma ("[+]","symbol"))
  | (Sign "[") :: (Sign "-") :: (Sign "]") :: l -> create_sign_token poss_s_beg i ((Sign "[") :: (Sign "-") :: (Sign "]") :: []) l (make_lemma ("[-]","symbol"))
  | (Sign "[") :: (Sign "?") :: (Sign "]") :: l -> create_sign_token poss_s_beg i ((Sign "[") :: (Sign "?") :: (Sign "]") :: []) l (make_lemma ("[?]","sinterj"))
  | (Sign ":") :: (Sign "]") :: l ->
      let t,i2 = create_empty_sign_token i [Sign ":";Sign "]"] in
      Variant[Token{t with token=make_lemma (":]","sinterj")};
              Seq[Token{empty_token_env with orth=":";beg=i;len=factor;next=i+factor;token=Interp ":"; attrs=["maybe cs"]};
                  Token{empty_token_env with orth="]";beg=i+factor;len=factor;next=i+2*factor;token=Interp "]"; attrs=["maybe cs"]}]],i2,l,false
  | (Sign ";") :: (Sign "]") :: l -> create_sign_token poss_s_beg i ((Sign ";") :: (Sign "]") :: []) l (make_lemma (";]","sinterj"))
  | (Sign "]") :: l -> create_sign_token poss_s_beg i [Sign "]"] l (Interp "]")
  | (Sign "[") :: l -> create_sign_token poss_s_beg i [Sign "["] l (Interp "[")
  | (Sign ":") :: l ->
      if is_colon_symbol l then
        Token{empty_token_env with orth=":";beg=i;len=factor;next=i+factor;token=Symbol ":"; attrs=["maybe cs"]},i+factor,l,false
      else
        Variant[
          Seq[Token{empty_token_env with beg=i;len=11;next=i+11;token=Interp "</clause>"}; (* wyliczenie*)
              Token{empty_token_env with orth=":";beg=i+11;len=factor-11;next=i+factor;token=Interp "<clause>"}];
          Seq[Token{empty_token_env with beg=i;len=10;next=i+10;token=Interp "</clause>"};
            Token{empty_token_env with orth=":";beg=i+10;len=factor-30;next=i+factor-20;token=Interp ":"}; (* mowa zależna, koniec zdania *)
            Token{empty_token_env with beg=i+factor-20;len=20;next=i+factor;token=Interp "</sentence>"}];
          Seq[Token{empty_token_env with beg=i;len=10;next=i+10;token=Interp "</clause>"};
            Token{empty_token_env with orth=":";beg=i+10;len=factor-40;next=i+factor-30;token=Interp ":"}; (* po ':' zdanie z małej litery *)
            Token{empty_token_env with beg=i+factor-30;len=10;next=i+factor-20;token=Interp "</sentence>"};
            Token{empty_token_env with beg=i+factor-20;len=10;next=i+factor-10;token=Interp "<sentence>"};
            Token{empty_token_env with beg=i+factor-10;len=10;next=i+factor;token=Interp "<clause>"}];
         Seq[Token{empty_token_env with beg=i;len=10;next=i+10;token=Interp "</clause>"};
            Token{empty_token_env with beg=i+10;len=10;next=i+20;token=Interp "</sentence>"};
            Token{empty_token_env with orth=":";beg=i+20;len=factor-20;next=i+factor;token=Interp ":s"}]; (* speaker *)
          ],i+factor,l,true
(*      if is_colon_sentence_end_marker l then
        Seq[Token{empty_token_env with beg=i;len=10;next=i+10;token=Interp "</clause>"};
            Token{empty_token_env with orth=":";beg=i+10;len=10;next=i+20;token=Interp ":"};
            Token{empty_token_env with beg=i+20;len=factor-20;next=i+factor;token=Interp "</sentence>"}],i+factor,l,true
      else
      else
        Seq[Token{empty_token_env with beg=i;len=10;next=i+10;token=Interp "</clause>"};
            Token{empty_token_env with orth=":";beg=i+10;len=10;next=i+20;token=Interp ""};
            Token{empty_token_env with beg=i+20;len=factor-20;next=i+factor;token=Interp "<clause>"}],i+factor,l,false*)
  | (Sign "'") :: (Sign "'") :: (Sign ".") :: l -> create_quot_digit_token i [Sign "'";Sign "'"] l
  | (Sign "'") :: (Sign "'") :: l ->
      let t,i = create_empty_sign_token i [Sign "”"] in
      Variant[Token{t with token=Interp "”"};Token{t with token=Interp "”s"}],i,l,poss_s_beg
  | (Sign "'") :: l -> create_sign_token poss_s_beg i [Sign "'"] l (Symbol "’")
  | (Sign "’") :: (Sign "’") :: (Sign ".") :: l -> create_quot_digit_token i [Sign "’";Sign "’"] l
  | (Sign "’") :: (Sign "’") :: l ->
      let t,i = create_empty_sign_token i [Sign "”"] in
      Variant[Token{t with token=Interp "”"};Token{t with token=Interp "”s"}],i,l,poss_s_beg
  | (Sign "’") :: l -> create_sign_token poss_s_beg i [Sign "’"] l (Symbol "’")
  | (Sign ";") :: (Sign "*") :: l -> create_sign_token poss_s_beg i ((Sign ";") :: (Sign "*") :: []) l (make_lemma (";*","sinterj"))
  | (Sign ";") :: l ->
        Variant[Seq[Token{empty_token_env with beg=i;len=20;next=i+20;token=Interp "</clause>"};
                    Token{empty_token_env with orth=";";beg=i+20;len=20;next=i+40;token=Interp "</sentence>"};
                    Token{empty_token_env with beg=i+40;len=20;next=i+60;token=Interp "<sentence>"};
                    Token{empty_token_env with beg=i+60;len=factor-60;next=i+factor;token=Interp "<clause>"}];
                Token{empty_token_env with orth=";";beg=i;len=factor;next=i+factor;token=Interp ";"; attrs=["maybe cs"]}],i+factor,l,false
  | (Sign "?") :: (Sign "!") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: l ->
        create_sentence_seq_q i ((Sign "?") :: (Sign "!") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "?!...",i+5*factor,l,true
  | (Sign "?") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: l ->
        create_sentence_seq_q i ((Sign "?") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "?...",i+4*factor,l,true
  | (Sign "?") :: (Sign "?") :: (Sign "?") :: (Sign "?") :: l ->
        create_sentence_seq_q i ((Sign "?") :: (Sign "?") :: (Sign "?") :: (Sign "?") :: []) l "????",i+4*factor,l,true
  | (Sign "?") :: (Sign "!") :: (Sign "!") :: (Sign "!") :: l ->
        create_sentence_seq_q i ((Sign "?") :: (Sign "!") :: (Sign "!") :: (Sign "!") :: []) l "?!!!",i+4*factor,l,true
  | (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign "?") :: l ->
        Variant[create_sentence_seq_hapl_q i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign "?") :: []) l "…?";
                create_sentence_seq_q i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign "?") :: []) l "…?"],i+4*factor,l,true
  | (Sign "?") :: (Sign "!") :: (Sign "?") :: l ->
        create_sentence_seq_q i ((Sign "?") :: (Sign "!") :: (Sign "?") :: []) l "?!?",i+3*factor,l,true
  | (Sign "?") :: (Sign "?") :: (Sign "?") :: l ->
        create_sentence_seq_q i ((Sign "?") :: (Sign "?") :: (Sign "?") :: []) l "???",i+3*factor,l,true
  | (Sign "?") :: (Sign "!") :: l ->
        create_sentence_seq_q i ((Sign "?") :: (Sign "!") :: []) l "?!",i+2*factor,l,true
  | (Sign "?") :: (Sign "?") :: l ->
        create_sentence_seq_q i ((Sign "?") :: (Sign "?") :: []) l "??",i+2*factor,l,true
(*   | (Sign "?") :: (Sign ".") :: l -> *)
  | (Sign "!") :: (Sign "?") :: l ->
        create_sentence_seq_q i ((Sign "!") :: (Sign "?") :: []) l "!?",i+2*factor,l,true
  | (Sign "?") :: (Sign "…") :: l ->
        create_sentence_seq_q i ((Sign "?") :: (Sign "…") :: []) l "?…",i+2*factor,l,true
  | (Sign "…") :: (Sign "?") :: l ->
        Variant[create_sentence_seq_hapl_q i ((Sign "…") :: (Sign "?") :: []) l "…?";
                create_sentence_seq_q i ((Sign "…") :: (Sign "?") :: []) l "…?"],i+2*factor,l,true
  | (Sign "?") :: l ->
        create_sentence_seq_q i ((Sign "?") :: []) l "?",i+factor,l,true
  | (Sign "!") :: (Sign "!") :: (Sign "!") :: (Sign "!") :: l ->
        create_sentence_seq i ((Sign "!") :: (Sign "!") :: (Sign "!") :: (Sign "!") :: []) l "!!!!",i+4*factor,l,true
  | (Sign "!") :: (Sign "!") :: (Sign "!") :: l ->
        create_sentence_seq i ((Sign "!") :: (Sign "!") :: (Sign "!") :: []) l "!!!",i+3*factor,l,true
  | (Sign "!") :: (Sign "!") :: l ->
        create_sentence_seq i ((Sign "!") :: (Sign "!") :: []) l "!!",i+2*factor,l,true
  | (Sign "!") :: l ->
        create_sentence_seq i ((Sign "!") :: []) l "!",i+factor,l,true
  | (Sign "…") :: l ->
      if is_multidot_sentence_end_marker l then
        Variant[create_sentence_seq_hapl i ((Sign "…") :: []) l "…";
                create_sentence_seq i ((Sign "…") :: []) l "…"],i+factor,l,true
      else
        Variant[create_sentence_seq_hapl i ((Sign "…") :: []) l "…";
                create_sentence_seq i ((Sign "…") :: []) l "…";
                Token{empty_token_env with orth="…";beg=i;len=factor;next=i+factor;token=make_lemma ("…","sinterj"); attrs=["maybe cs"]}],i+factor,l,true
  | (Sign "/") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign "/") :: l -> create_sign_token poss_s_beg i ((Sign "/") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign "/") :: []) l (make_lemma ("(…)","sinterj"))
  | (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: l -> (* Różne natęrzenia wielokropka i wypunktowania *)
        Variant[create_sentence_seq_hapl i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "……";
                create_sentence_seq i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "……"],i+8*factor,l,true
  | (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: l ->
        Variant[create_sentence_seq_hapl i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "……";
                create_sentence_seq i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "……"],i+7*factor,l,true
  | (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: l ->
        Variant[create_sentence_seq_hapl i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "……";
                create_sentence_seq i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "……"],i+6*factor,l,true
  | (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: l ->
        Variant[create_sentence_seq_hapl i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "……";
                create_sentence_seq i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "……"],i+5*factor,l,true
  | (Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: l ->
        Variant[create_sentence_seq_hapl i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "……";
                create_sentence_seq i ((Sign ".") :: (Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "……"],i+4*factor,l,true
  | (Sign ".") :: (Sign ".") :: (Sign ".") :: l ->
      if is_multidot_sentence_end_marker l then
        Variant[create_sentence_seq_hapl i ((Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "…";
                create_sentence_seq i ((Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "…"],i+3*factor,l,true
      else
        Variant[create_sentence_seq_hapl i ((Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "…";
                create_sentence_seq i ((Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "…";
                Token{empty_token_env with orth="...";beg=i;len=3*factor;next=i+3*factor;token=make_lemma ("…","sinterj"); attrs=["maybe cs"]}],i+3*factor,l,true
  | (Sign ".") :: (Sign ".") :: l ->
      if is_multidot_sentence_end_marker l then
        Variant[create_sentence_seq_hapl i ((Sign ".") :: (Sign ".") :: []) l "…";
                create_sentence_seq i ((Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "…"],i+2*factor,l,true
      else
        Variant[create_sentence_seq_hapl i ((Sign ".") :: (Sign ".") :: []) l "…";
                create_sentence_seq i ((Sign ".") :: (Sign ".") :: (Sign ".") :: []) l "…";
                Token{empty_token_env with orth="..";beg=i;len=2*factor;next=i+2*factor;token=make_lemma ("…","sinterj"); attrs=["maybe cs"]}],i+2*factor,l,true
  | (Sign ".") :: l ->
      if is_dot_sentence_end_marker l then
        Variant[Seq[Token{empty_token_env with beg=i;len=10;next=i+10;token=Symbol "."; attrs=["maybe cs"]};
                    Token{empty_token_env with beg=i+10;len=10;next=i+20;token=Interp "</clause>"};
                    Token{empty_token_env with orth=".";beg=i+20;len=factor-20;next=i+factor;token=Interp "</sentence>"}];
                Seq[Token{empty_token_env with beg=i;len=20;next=i+20;token=Interp "</clause>"};
                    Token{empty_token_env with orth=".";beg=i+20;len=factor-20;next=i+factor;token=Interp "</sentence>"}]],i+factor,l,true
      else if not_dot_sentence_end_marker l then
        Token{empty_token_env with orth=".";beg=i;len=factor;next=i+factor;token=Symbol "."; attrs=["maybe cs"]},i+factor,l,false
      else
        Variant[Seq[Token{empty_token_env with beg=i;len=10;next=i+10;token=Symbol "."; attrs=["maybe cs"]};
                    Token{empty_token_env with beg=i+10;len=10;next=i+20;token=Interp "</clause>"};
                    Token{empty_token_env with orth=".";beg=i+20;len=factor-20;next=i+factor;token=Interp "</sentence>"}];
                Seq[Token{empty_token_env with beg=i;len=20;next=i+20;token=Interp "</clause>"};
                    Token{empty_token_env with orth=".";beg=i+20;len=factor-20;next=i+factor;token=Interp "</sentence>"}];
                Token{empty_token_env with orth=".";beg=i;len=factor;next=i+factor;token=Symbol "."; attrs=["maybe cs"]}],i+factor,l,true
  | (Sign "*") :: (Sign "*") :: (Sign "*") :: (Sign "*") :: (Sign "*") :: l -> create_sign_token poss_s_beg i [Sign "*";Sign "*";Sign "*";Sign "*";Sign "*"] l (Interp "*****") (* zastępniki liter *)
  | (Sign "*") :: (Sign "*") :: (Sign "*") :: (Sign "*") :: l -> create_sign_token poss_s_beg i [Sign "*";Sign "*";Sign "*";Sign "*"] l (Interp "****")
  | (Sign "*") :: (Sign "*") :: (Sign "*") :: l -> create_sign_token poss_s_beg i [Sign "*";Sign "*";Sign "*"] l (Interp "***")
  | (Sign "*") :: (Sign "*") :: l -> create_sign_token poss_s_beg i [Sign "*";Sign "*"] l (Interp "**")
  | (Sign "*") :: l -> (* Interp zastępnik liter i cudzysłów, symbol listy *)
      let t,i = create_empty_sign_token i [Sign "*"] in
      Variant[Token{t with token=Interp "*"};Token{t with token=Symbol "*"}],i,l,poss_s_beg
  | (Sign "+") :: l -> create_sign_token poss_s_beg i [Sign "+"] l (Symbol "+")
  | (Sign "«") :: l ->
      let t,i = create_empty_sign_token i [Sign "«"] in
      Variant[Token{t with token=Interp "«"};Token{t with token=Interp "«s"}],i,l,poss_s_beg
  | (Sign "»") :: l ->
      let t,i = create_empty_sign_token i [Sign "»"] in
      Variant[Token{t with token=Interp "»"};Token{t with token=Interp "»s"}],i,l,poss_s_beg
  | (Sign "<") :: (Sign "<") :: l -> create_sign_token poss_s_beg i [Sign "<";Sign "<"] l (Interp "«") (* prawy cudzysłów *)
  | (Sign "<") :: l -> (* prawy cudzysłów i element wzoru matematycznego *)
      let t,i = create_empty_sign_token i [Sign "<"] in
      Variant[Token{t with token=Interp "«"};Token{t with token=Symbol "<"}],i,l,poss_s_beg
  | (Sign ">") :: (Sign ">") :: l -> create_sign_token poss_s_beg i [Sign ">";Sign ">"] l (Interp "»") (* lewy cudzysłów *)
  | (Sign ">") :: l -> create_sign_token poss_s_beg i [Sign ">"] l (Symbol ">")
  | (Sign "-") :: (Sign "-") :: (Sign "-") :: l -> create_or_beg2 i [Sign "-";Sign "-";Sign "-"] l poss_s_beg
  | (Sign "-") :: (Sign "-") :: l -> create_or_beg2 i [Sign "-";Sign "-"] l poss_s_beg
  | (Sign "-") :: l -> create_or_beg i [Sign "-"] l poss_s_beg
  | (Sign "‐") :: l -> create_or_beg i [Sign "‐"] l poss_s_beg
  | (Sign "‑") :: l -> create_or_beg i [Sign "‑"] l poss_s_beg
  | (Sign "‒") :: l -> create_or_beg i [Sign "‒"] l poss_s_beg
  | (Sign "−") :: l -> create_or_beg i [Sign "−"] l poss_s_beg
  | (Sign "–") :: l -> create_or_beg i [Sign "–"] l poss_s_beg
  | (Sign "—") :: l -> create_or_beg i [Sign "—"] l poss_s_beg
  | (Sign "‘") :: l -> create_sign_token poss_s_beg i [Sign "‘"] l (Interp "‘")
  | (Sign "´") :: l -> create_sign_token poss_s_beg i [Sign "´"] l (Symbol "’")
  | (Sign "`") :: (Sign "`") :: l ->
      let t,i = create_empty_sign_token i [Sign "`";Sign "`"] in
      Variant[Token{t with token=Interp "„"};Token{t with token=Interp "„s"}],i,l,poss_s_beg
  | (Sign "`") :: l -> create_sign_token poss_s_beg i [Sign "`"] l (Symbol "’")
  | (Sign "·") :: l -> create_sign_token poss_s_beg i [Sign "·"] l (Interp "·")
  | (Sign "•") :: l -> create_sign_token poss_s_beg i [Sign "•"] l (Interp "•")
  | (Sign "¨") :: l -> create_sign_token poss_s_beg i [Sign "¨"] l (Interp "¨")
  | (Sign "~") :: l ->
      let t,i = create_empty_sign_token i [Sign "~"] in
      Variant[Token{t with token=Symbol "~"};Token{t with token=make_lemma ("około","prep:gen")}],i,l,false
  | (Sign "{") :: l ->
      let t,i = create_empty_sign_token i [Sign "{"] in
      Variant[Token{t with token=Symbol "{"};Token{t with token=Interp "{"}],i,l,poss_s_beg
  | (Sign "}") :: l ->
      let t,i = create_empty_sign_token i [Sign "}"] in
      Variant[Token{t with token=Symbol "}"};Token{t with token=Interp "}"}],i,l,poss_s_beg
  | (Sign "#") :: l -> create_sign_token poss_s_beg i [Sign ""] l (Symbol "")
  | (Sign "^") :: (Sign "^") :: l -> create_sign_token poss_s_beg i [Sign "^";Sign "^"] l (make_lemma ("^^","sinterj"))
  | (Sign "^") :: l -> create_sign_token poss_s_beg i [Sign "^"] l (Symbol "^")
  | (Sign "|") :: l -> create_sign_token poss_s_beg i [Sign "|"] l (Symbol "|")
  | (Sign "&") :: l -> create_sign_token poss_s_beg i [Sign "&"] l (Symbol "&")
  | (Sign "=") :: l -> create_sign_token poss_s_beg i [Sign "="] l (Symbol "=")
  | (Sign "/") :: l ->
      let t,i = create_empty_sign_token i [Sign "/"] in
      Variant[Token{t with token=Symbol "/"};Token{t with token=make_lemma ("na","prep:acc")}],i,l,false
  | (Sign "_") :: l -> create_sign_token poss_s_beg i [Sign "_"] l (Symbol "_")
  | (Sign "@") :: l -> create_sign_token poss_s_beg i [Sign "@"] l (Symbol "@")
  | (Sign "×") :: l -> create_sign_token poss_s_beg i [Sign "×"] l (Symbol "×")
  | (Sign "%") :: l ->
      let t,i = create_empty_sign_token i [Sign "%"] in
      Variant[Token{t with token=Symbol "%"};Token{t with token=make_lemma ("procent","subst:_:_:m3")}],i,l,false
  | (Sign "$") :: l ->
      let t,i = create_empty_sign_token i [Sign "$"] in
      Variant[Token{t with token=Symbol "$"};Token{t with token=make_lemma ("dolar","subst:_:_:m2")}],i,l,false
  | (Sign "€") :: l -> create_sign_token poss_s_beg i [Sign "€"] l (make_lemma ("euro","subst:_:_:n2"))
  | (Sign "²") :: l -> create_sign_token poss_s_beg i [Sign "²"] l (Symbol "²")
  | (Sign "°") :: l -> create_sign_token poss_s_beg i [Sign "°"] l (make_lemma ("stopień","subst:_:_:m3"))
  | (Sign "§") :: l -> create_sign_token false i [Sign "§"] l (make_lemma ("paragraf","subst:_:_:m3"))
  | (Sign s) :: l -> print_endline ("recognize_sign_group: " ^ s); create_sign_token poss_s_beg i [Sign s] l (Symbol s)
  | l ->  failwith "recognize_sign_group"

(* FIXME: "„Szpak” frunie." trzeba przenie przenieść <sentence> przed „, ale zostawić po „s. *)

let rec group_chars poss_s_beg i rev = function
    [] -> List.rev ((Token{empty_token_env with beg=i;len=factor;next=i+factor;token=Interp "</query>"}) :: rev)
  | (Digit s) :: l -> let x,l = group_digits [] ((Digit s) :: l) in group_chars false (i + Xlist.size x * factor) ((merge_digits poss_s_beg i x) :: rev) l
  | (Sign s) :: l -> let x,i,l,poss_s_beg = recognize_sign_group poss_s_beg i ((Sign s) :: l) in group_chars poss_s_beg i (x :: rev) l
  | (Capital(s,t)) :: l -> let x,l = group_letters [] ((Capital(s,t)) :: l) in group_chars false (i + Xlist.size x * factor) ((merge_letters poss_s_beg i x) ::  rev) l
  | (ForeignCapital(s,t)) :: l -> let x,l = group_letters [] ((ForeignCapital(s,t)) :: l) in group_chars false (i + Xlist.size x * factor) ((merge_letters poss_s_beg i x) ::  rev) l
  | (Small s) :: l -> let x,l = group_letters [] ((Small s) :: l) in group_chars false (i + Xlist.size x * factor) ((merge_letters poss_s_beg i x) ::  rev) l
  | (ForeignSmall s) :: l -> let x,l = group_letters [] ((ForeignSmall s) :: l) in group_chars false (i + Xlist.size x * factor) ((merge_letters poss_s_beg i x) ::  rev) l
  | (Other(s,x)) :: l ->
        let x,l = group_others [] ((Other(s,x)) :: l) in
        group_chars false (i + Xlist.size x * factor)
          ((Token{empty_token_env with orth=String.concat "" x;beg=i;len=Xlist.size x * factor;next=i+factor;token=Other(String.concat "" x)}) ::  rev) l

let tokenize l =
  (Token{empty_token_env with beg=0;len=factor;next=factor;token=Interp "<query>"}) :: (group_chars true factor [] l)
