Wojciech Jaworski / ENIAM

(*
 *  ENIAMtokenizer, a tokenizer for Polish
 *  Copyright (C) 2016 Wojciech Jaworski <wjaworski atSPAMfree mimuw dot edu dot pl>
 *  Copyright (C) 2016 Institute of Computer Science Polish Academy of Sciences
 *
 *  This library is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Lesser General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *)

open Xstd
open Printf
open ENIAMtokenizerTypes

let rec flatten_tokens rev_variants = function
  | [] -> rev_variants
  | Token t :: l -> flatten_tokens (Xlist.map rev_variants (fun rev_variant -> Token t :: rev_variant)) l
  | Seq seq :: l -> flatten_tokens rev_variants (seq @ l)
  | Variant variants :: l -> flatten_tokens (List.flatten (Xlist.map variants (fun variant -> flatten_tokens rev_variants [variant]))) l

let rec normalize_tokens rev = function
    [] -> List.rev rev
  | Token t :: l -> normalize_tokens (Token t :: rev) l
  | Seq seq :: l -> normalize_tokens rev (seq @ l)
  | Variant[t] :: l -> normalize_tokens rev (t :: l)
  | Variant variants :: l ->
      let variants = flatten_tokens [[]] [Variant variants] in
      let variants = Xlist.map variants (fun rev_seq ->
        match List.rev rev_seq with
          [] -> failwith "normalize_tokens"
        | [t] -> t
        | seq -> Seq seq) in
      let t = match variants with
          [] -> failwith "normalize_tokens"
        | [t] -> t
        | variants -> Variant variants in
      normalize_tokens (t :: rev) l

let concat_orths l =
  String.concat "" (Xlist.map l (fun t -> t.orth))

let concat_orths2 l =
  String.concat "" (Xlist.map l (fun t -> ENIAMtokens.get_orth t.token))

let concat_intnum = function
    [{token=Dig(v4,_)};_;{token=Dig(v3,_)};_;{token=Dig(v2,_)};_;{token=Dig(v1,_)}] -> v4^v3^v2^v1
  | [{token=Dig(v3,_)};_;{token=Dig(v2,_)};_;{token=Dig(v1,_)}] -> v3^v2^v1
  | [{token=Dig(v2,_)};_;{token=Dig(v1,_)}] -> v2^v1
  | [{token=Dig(v1,_)}] -> v1
  | _ -> failwith "concat_intnum"

let dig_value t =
  match t.token with
    Dig(v,_) -> v
  | _ -> failwith "dig_value"

let rec make_tys_rec n = function
    Dig(v,"intnum") -> Dig(v ^ String.make n '0',"intnum")
  | Dig(v,"realnum") ->
     (* print_endline ("make_tys_rec: v='" ^ v ^ "'"); *)
     let a,b = match Xstring.split "," v with [a;b] -> a,b | [a] -> a,"" | _ -> failwith "make_tys_rec" in
     (* print_endline ("make_tys_rec: '" ^ a ^ "' '" ^ b ^ "'"); *)
     let a = if a = "0" then "" else if a = "-0" then "-" else a in
     if String.length b > n then Dig(a ^ String.sub b 0 n ^ "," ^ String.sub b n (String.length b - n),"realnum") else
     Dig(a ^ b ^ String.make (n-String.length b) '0',"intnum")
  | Compound("intnum-interval",[x;y]) -> Compound("intnum-interval",[make_tys_rec n x; make_tys_rec n y])
  | Compound("realnum-interval",[x;y]) -> Compound("realnum-interval",[make_tys_rec n x; make_tys_rec n y])
  | _ -> failwith "make_tys_rec"

let make_tys n t =
  make_tys_rec n t.token

let digit_patterns1 = [ (* FIXME: problem z nadmiarowymi interpretacjami - trzeba uwzględnić w preprocesingu brak spacji - albo w dezambiguacji *)
  [D "dig"; S "."; D "dig"; S "."; D "dig"; S "."; D "dig"; S "."; D "dig"], (fun tokens -> Proper(concat_orths tokens,"obj-id",[[]],["obj-id"]));
  [D "dig"; S "."; D "dig"; S "."; D "dig"; S "."; D "dig"], (fun tokens -> Proper(concat_orths tokens,"obj-id",[[]],["obj-id"]));
  [D "dig"; S "."; D "dig"; S "."; D "dig"], (fun tokens -> Proper(concat_orths tokens,"obj-id",[[]],["obj-id"]));
  [D "dig"; S "."; D "dig"], (fun tokens -> Proper(concat_orths tokens,"obj-id",[[]],["obj-id"]));
(*   [D "dig"], "obj-id"; *)
  [D "pref3dig"; S "."; D "3dig"; S "."; D "3dig"; S "."; D "3dig"], (fun tokens -> Dig(concat_intnum tokens,"intnum"));
  [D "pref3dig"; S "."; D "3dig"; S "."; D "3dig"], (fun tokens -> Dig(concat_intnum tokens,"intnum"));
  [D "pref3dig"; S "."; D "3dig"], (fun tokens -> Dig(concat_intnum tokens,"intnum"));
  [D "pref3dig"; S " "; D "3dig"; S " "; D "3dig"; S " "; D "3dig"], (fun tokens -> Dig(concat_intnum tokens,"intnum"));
  [D "pref3dig"; S " "; D "3dig"; S " "; D "3dig"], (fun tokens -> Dig(concat_intnum tokens,"intnum"));
  [D "pref3dig"; S " "; D "3dig"], (fun tokens -> Dig(concat_intnum tokens,"intnum"));
  (* [D "intnum"; S "."], (function [token;_] -> Dig(concat_intnum [token],"ordnum") | _ -> failwith "digit_patterns1"); *)
  [D "day"; S "."; D "month"; S "."; D "year"], (function [day;_;month;_;year] -> Compound("date",[day.token;month.token;year.token]) | _ -> failwith "digit_patterns2");
  [D "day"; S "."; RD "month"; S "."; D "year"], (function [day;_;month;_;year] -> Compound("date",[day.token;month.token;year.token]) | _ -> failwith "digit_patterns3");
  [D "day"; S " "; RD "month"; S " "; D "year"], (function [day;_;month;_;year] -> Compound("date",[day.token;month.token;year.token]) | _ -> failwith "digit_patterns3");
  [D "day"; S "."; D "month"; S "."; D "2dig"], (function [day;_;month;_;year] -> Compound("date",[day.token;month.token;year.token]) | _ -> failwith "digit_patterns2");
  [D "day"; S "."; RD "month"; S "."; D "2dig"], (function [day;_;month;_;year] -> Compound("date",[day.token;month.token;year.token]) | _ -> failwith "digit_patterns3");
  [D "day"; S "."; D "month"; S "."], (function [day;_;month;_] -> Compound("day-month",[day.token;month.token]) | _ -> failwith "digit_patterns4");
  [D "hour"; S "."; D "minute"], (function [hour;_;minute] -> Compound("hour-minute",[hour.token;minute.token]) | _ -> failwith "digit_patterns5");
  [D "hour"; S ":"; D "minute"], (function [hour;_;minute] -> Compound("hour-minute",[hour.token;minute.token]) | _ -> failwith "digit_patterns6");
  [D "intnum"; S ":"; D "intnum"], (function [x;_;y] -> Compound("match-result",[x.token;y.token]) | _ -> failwith "digit_patterns7");
  [D "2dig"; S "-"; D "3dig"], (fun tokens -> Proper(concat_orths tokens,"postal-code",[[]],["postal-code"]));
  [D "3dig"; S "-"; D "3dig"; S "-"; D "3dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [D "3dig"; S " "; D "3dig"; S " "; D "3dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [D "3dig"; S "-"; D "2dig"; S "-"; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [D "3dig"; S " "; D "2dig"; S " "; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [D "2dig"; S "-"; D "2dig"; S "-"; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [D "2dig"; S " "; D "2dig"; S " "; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [D "3dig"; S "-"; D "3dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [D "3dig"; S " "; D "3dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [O "0"; S "-"; D "2dig"; O " "; D "3dig"; S "-"; D "2dig"; S "-"; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [O "0"; S "-"; D "2dig"; O " "; D "2dig"; S "-"; D "2dig"; S "-"; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [S "("; O "0"; S "-"; D "2dig"; S ")"; O " "; D "3dig"; S "-"; D "2dig"; S "-"; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [S "("; O "0"; S "-"; D "2dig"; S ")"; O " "; D "2dig"; S "-"; D "2dig"; S "-"; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [S "("; D "3dig"; S ")"; O " "; D "3dig"; S "-"; D "2dig"; S "-"; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [S "("; D "3dig"; S ")"; O " "; D "2dig"; S "-"; D "2dig"; S "-"; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [S "("; D "3dig"; S ")"; O " "; D "3dig"; S " "; D "2dig"; S " "; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  [S "("; D "3dig"; S ")"; O " "; D "2dig"; S " "; D "2dig"; S " "; D "2dig"], (fun tokens -> Proper(concat_orths tokens,"phone-number",[[]],["phone-number"]));
  ] (* bez 1 i *2 *3 *4 mamy rec *) (* w morfeuszu zawsze num:pl?*)

let digit_patterns2 = [
  [D "intnum"; S ","; D "dig"], (function [x;_;y] ->  Dig(dig_value x ^ "," ^ dig_value y,"realnum") | _ -> failwith "digit_patterns8");
(*  [S "-"; D "intnum"; S ","; D "dig"], (function [_;x;_;y] ->  Dig("-" ^ dig_value x ^ "," ^ dig_value y,"realnum") | _ -> failwith "digit_patterns9");
  [S "-"; D "intnum"], (function [_;x] ->  Dig("-" ^ dig_value x,"realnum") | _ -> failwith "digit_patterns10");*)
  [S "’"; D "2dig"], (function [_;x] -> Dig("’" ^ dig_value x,"year") | _ -> failwith "digit_patterns12");
(*   [D "intnum"], "realnum"; *)
  ]

let compose_latek_lemma t interp =
  ENIAMtokens.make_lemma (ENIAMtokens.get_orth t.token ^ "-latek", interp)

let compose_latka_lemma t interp =
  ENIAMtokens.make_lemma (ENIAMtokens.get_orth t.token ^ "-latka", interp)

let compose_latek_int_lemma t t2 interp =
  ENIAMtokens.make_lemma (ENIAMtokens.get_orth t.token ^ "-" ^ ENIAMtokens.get_orth t2.token ^ "-latek", interp)

let compose_latka_int_lemma t t2 interp =
  ENIAMtokens.make_lemma (ENIAMtokens.get_orth t.token ^ "-" ^ ENIAMtokens.get_orth t2.token ^ "-latka", interp)

let compose_lecie_lemma t interp =
  ENIAMtokens.make_lemma (ENIAMtokens.get_orth t.token ^ "-lecie", interp)

let compose_ordnum_lemma t interp =
  ENIAMtokens.make_lemma (ENIAMtokens.get_orth t.token ^ ".", interp)

let digit_patterns3 = [
  [S "-"; D "intnum"], (function [_;x] ->  Dig("-" ^ dig_value x,"intnum") | _ -> failwith "digit_patterns10");
  [S "-"; D "realnum"], (function [_;x] ->  Dig("-" ^ dig_value x,"realnum") | _ -> failwith "digit_patterns10");
  [D "intnum"; S "-"; D "intnum"], (function [x;_;y] -> Compound("intnum-interval",[x.token;y.token]) | _ -> failwith "digit_patterns11");
  [D "realnum"; S "-"; D "realnum"], (function [x;_;y] -> Compound("realnum-interval",[x.token;y.token]) | _ -> failwith "digit_patterns12"); (* FIXME: konflikt z liczbami ujemnymi *)
  [D "intnum"; S "-"; D "realnum"], (function [x;_;y] -> Compound("realnum-interval",[x.token;y.token]) | _ -> failwith "digit_patterns12"); (* FIXME: konflikt z liczbami ujemnymi *)
  [D "realnum"; S "-"; D "intnum"], (function [x;_;y] -> Compound("realnum-interval",[x.token;y.token]) | _ -> failwith "digit_patterns12"); (* FIXME: konflikt z liczbami ujemnymi *)
  [C "date"; S "-"; C "date"], (function [x;_;y] -> Compound("date-interval",[x.token;y.token]) | _ -> failwith "digit_patterns13");
  [C "day-month"; S "-"; C "day-month"], (function [x;_;y] -> Compound("day-month-interval",[x.token;y.token]) | _ -> failwith "digit_patterns14");
  [D "day"; S "-"; D "day"], (function [x;_;y] -> Compound("day-interval",[x.token;y.token]) | _ -> failwith "digit_patterns15");
  [D "month"; S "-"; D "month"], (function [x;_;y] -> Compound("month-interval",[x.token;y.token]) | _ -> failwith "digit_patterns16");
  [RD "month"; S "-"; RD "month"], (function [x;_;y] -> Compound("month-interval",[x.token;y.token]) | _ -> failwith "digit_patterns17");
  [D "year"; S "-"; D "year"], (function [x;_;y] -> Compound("year-interval",[x.token;y.token]) | _ -> failwith "digit_patterns16");
  [D "year"; S "-"; D "2dig"], (function [x;_;y] -> Compound("year-interval",[x.token;y.token]) | _ -> failwith "digit_patterns16");
  [C "hour-minute"; S "-"; C "hour-minute"], (function [x;_;y] -> Compound("hour-minute-interval",[x.token;y.token]) | _ -> failwith "digit_patterns18");
  [D "hour"; S "-"; D "hour"], (function [x;_;y] -> Compound("hour-interval",[x.token;y.token]) | _ -> failwith "digit_patterns19");
  [D "minute"; S "-"; D "minute"], (function [x;_;y] -> Compound("minute-interval",[x.token;y.token]) | _ -> failwith "digit_patterns20");
  [RD "roman"; S "-"; RD "roman"], (function [x;_;y] -> Compound("roman-interval",[x.token;y.token]) | _ -> failwith "digit_patterns21");
  [D "intnum"; S " "; S "-"; S " "; D "intnum"], (function [x;_;_;_;y] -> Compound("intnum-interval",[x.token;y.token]) | _ -> failwith "digit_patterns11");
  [D "realnum"; S " "; S "-"; S " "; D "realnum"], (function [x;_;_;_;y] -> Compound("realnum-interval",[x.token;y.token]) | _ -> failwith "digit_patterns12"); (* FIXME: konflikt z liczbami ujemnymi *)
  [D "intnum"; S " "; S "-"; S " "; D "realnum"], (function [x;_;_;_;y] -> Compound("realnum-interval",[x.token;y.token]) | _ -> failwith "digit_patterns12"); (* FIXME: konflikt z liczbami ujemnymi *)
  [D "realnum"; S " "; S "-"; S " "; D "intnum"], (function [x;_;_;_;y] -> Compound("realnum-interval",[x.token;y.token]) | _ -> failwith "digit_patterns12"); (* FIXME: konflikt z liczbami ujemnymi *)
  [C "date"; S " "; S "-"; S " "; C "date"], (function [x;_;_;_;y] -> Compound("date-interval",[x.token;y.token]) | _ -> failwith "digit_patterns13");
  [C "day-month"; S " "; S "-"; S " "; C "day-month"], (function [x;_;_;_;y] -> Compound("day-month-interval",[x.token;y.token]) | _ -> failwith "digit_patterns14");
  [D "day"; S " "; S "-"; S " "; D "day"], (function [x;_;_;_;y] -> Compound("day-interval",[x.token;y.token]) | _ -> failwith "digit_patterns15");
  [D "month"; S " "; S "-"; S " "; D "month"], (function [x;_;_;_;y] -> Compound("month-interval",[x.token;y.token]) | _ -> failwith "digit_patterns16");
  [RD "month"; S " "; S "-"; S " "; RD "month"], (function [x;_;_;_;y] -> Compound("month-interval",[x.token;y.token]) | _ -> failwith "digit_patterns17");
  [D "year"; S " "; S "-"; S " "; D "year"], (function [x;_;_;_;y] -> Compound("year-interval",[x.token;y.token]) | _ -> failwith "digit_patterns16");
  [D "year"; S " "; S "-"; S " "; D "2dig"], (function [x;_;_;_;y] -> Compound("year-interval",[x.token;y.token]) | _ -> failwith "digit_patterns16");
  [C "hour-minute"; S " "; S "-"; S " "; C "hour-minute"], (function [x;_;_;_;y] -> Compound("hour-minute-interval",[x.token;y.token]) | _ -> failwith "digit_patterns18");
  [D "hour"; S " "; S "-"; S " "; D "hour"], (function [x;_;_;_;y] -> Compound("hour-interval",[x.token;y.token]) | _ -> failwith "digit_patterns19");
  [D "minute"; S " "; S "-"; S " "; D "minute"], (function [x;_;_;_;y] -> Compound("minute-interval",[x.token;y.token]) | _ -> failwith "digit_patterns20");
  [RD "roman"; S " "; S "-"; S " "; RD "roman"], (function [x;_;_;_;y] -> Compound("roman-interval",[x.token;y.token]) | _ -> failwith "digit_patterns21");
  [D "intnum"; S "-"; O "latek"], (function [x;_;_] -> compose_latek_lemma x "subst:sg:nom:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latka"], (function [x;_;_] -> compose_latek_lemma x "subst:sg:gen.acc:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latkowi"], (function [x;_;_] -> compose_latek_lemma x "subst:sg:dat:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latkiem"], (function [x;_;_] -> compose_latek_lemma x "subst:sg:inst:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latku"], (function [x;_;_] -> compose_latek_lemma x "subst:sg:loc.voc:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latkowie"], (function [x;_;_] -> compose_latek_lemma x "subst:pl:nom.voc:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latków"], (function [x;_;_] -> compose_latek_lemma x "subst:pl:gen.acc:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latkom"], (function [x;_;_] -> compose_latek_lemma x "subst:pl:dat:m1.f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latkami"], (function [x;_;_] -> compose_latek_lemma x "subst:pl:inst:m1.f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latkach"], (function [x;_;_] -> compose_latek_lemma x "subst:pl:loc:m1.f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latka"], (function [x;_;_] -> compose_latka_lemma x "subst:sg:nom:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latki"], (function [x;_;_] -> compose_latka_lemma x "subst:sg:gen:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latce"], (function [x;_;_] -> compose_latka_lemma x "subst:sg:dat.loc:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latkę"], (function [x;_;_] -> compose_latka_lemma x "subst:sg:acc:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latką"], (function [x;_;_] -> compose_latka_lemma x "subst:sg:inst:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latko"], (function [x;_;_] -> compose_latka_lemma x "subst:sg:voc:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latki"], (function [x;_;_] -> compose_latka_lemma x "subst:pl:nom.acc.voc:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "latek"], (function [x;_;_] -> compose_latka_lemma x "subst:pl:gen:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latek"], (function [x;_;y;_;_] -> compose_latek_int_lemma x y "subst:sg:nom:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latka"], (function [x;_;y;_;_] -> compose_latek_int_lemma x y "subst:sg:gen.acc:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latkowi"], (function [x;_;y;_;_] -> compose_latek_int_lemma x y "subst:sg:dat:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latkiem"], (function [x;_;y;_;_] -> compose_latek_int_lemma x y "subst:sg:inst:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latku"], (function [x;_;y;_;_] -> compose_latek_int_lemma x y "subst:sg:loc.voc:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latkowie"], (function [x;_;y;_;_] -> compose_latek_int_lemma x y "subst:pl:nom.voc:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latków"], (function [x;_;y;_;_] -> compose_latek_int_lemma x y "subst:pl:gen.acc:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latkom"], (function [x;_;y;_;_] -> compose_latek_int_lemma x y "subst:pl:dat:m1.f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latkami"], (function [x;_;y;_;_] -> compose_latek_int_lemma x y "subst:pl:inst:m1.f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latkach"], (function [x;_;y;_;_] -> compose_latek_int_lemma x y "subst:pl:loc:m1.f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latka"], (function [x;_;y;_;_] -> compose_latka_int_lemma x y "subst:sg:nom:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latki"], (function [x;_;y;_;_] -> compose_latka_int_lemma x y "subst:sg:gen:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latce"], (function [x;_;y;_;_] -> compose_latka_int_lemma x y "subst:sg:dat.loc:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latkę"], (function [x;_;y;_;_] -> compose_latka_int_lemma x y "subst:sg:acc:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latką"], (function [x;_;y;_;_] -> compose_latka_int_lemma x y "subst:sg:inst:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latko"], (function [x;_;y;_;_] -> compose_latka_int_lemma x y "subst:sg:voc:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latki"], (function [x;_;y;_;_] -> compose_latka_int_lemma x y "subst:pl:nom.acc.voc:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; D "intnum"; S "-"; O "latek"], (function [x;_;y;_;_] -> compose_latka_int_lemma x y "subst:pl:gen:f" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "lecie"], (function [x;_;_] -> compose_lecie_lemma x "subst:sg:nom.acc.voc:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "lecia"], (function [x;_;_] -> compose_lecie_lemma x "subst:sg:gen:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "leciu"], (function [x;_;_] -> compose_lecie_lemma x "subst:sg:dat.loc:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "leciem"], (function [x;_;_] -> compose_lecie_lemma x "subst:sg:inst:m1" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "szy"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:m1.m2.m3:pos|adj:sg:acc:m3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "sze"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.acc.voc:n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "sza"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "szego"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen:m1.m2.m3.n1.n2:pos|adj:sg:acc:m1.m2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "szemu"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:dat:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "szej"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen.dat.loc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "szym"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:inst.loc:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "szą"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:acc.inst:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "si"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.voc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "sze"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.acc.voc:m2.m3.n1.n2.f.p2.p3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "szych"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:gen.loc:_:pos|adj:pl:acc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "szym"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:dat:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "szymi"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:inst:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "szo"], (function [x;_;_] -> compose_ordnum_lemma x "adja" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "gi"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:m1.m2.m3:pos|adj:sg:acc:m3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "gie"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.acc.voc:n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ga"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "giego"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen:m1.m2.m3.n1.n2:pos|adj:sg:acc:m1.m2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "giemu"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:dat:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "giej"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen.dat.loc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "gim"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:inst.loc:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "gą"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:acc.inst:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "dzy"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.voc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "gie"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.acc.voc:m2.m3.n1.n2.f.p2.p3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "gich"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:gen.loc:_:pos|adj:pl:acc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "gim"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:dat:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "gimi"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:inst:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "go"], (function [x;_;_] -> compose_ordnum_lemma x "adja" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ci"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:m1.m2.m3:pos|adj:sg:acc:m3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "cie"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.acc.voc:n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "cia"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ciego"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen:m1.m2.m3.n1.n2:pos|adj:sg:acc:m1.m2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ciemu"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:dat:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ciej"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen.dat.loc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "cim"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:inst.loc:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "cią"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:acc.inst:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ci"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.voc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "cie"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.acc.voc:m2.m3.n1.n2.f.p2.p3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "cich"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:gen.loc:_:pos|adj:pl:acc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "cim"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:dat:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "cimi"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:inst:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "cio"], (function [x;_;_] -> compose_ordnum_lemma x "adja" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ty"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:m1.m2.m3:pos|adj:sg:acc:m3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "te"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.acc.voc:n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ta"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "tego"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen:m1.m2.m3.n1.n2:pos|adj:sg:acc:m1.m2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "temu"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:dat:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "tej"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen.dat.loc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "tym"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:inst.loc:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "tą"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:acc.inst:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ci"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.voc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "te"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.acc.voc:m2.m3.n1.n2.f.p2.p3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "tych"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:gen.loc:_:pos|adj:pl:acc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "tym"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:dat:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "tymi"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:inst:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "to"], (function [x;_;_] -> compose_ordnum_lemma x "adja" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "my"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:m1.m2.m3:pos|adj:sg:acc:m3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "me"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.acc.voc:n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ma"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "mego"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen:m1.m2.m3.n1.n2:pos|adj:sg:acc:m1.m2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "memu"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:dat:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "mej"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen.dat.loc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "mym"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:inst.loc:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "mą"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:acc.inst:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "mi"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.voc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "me"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.acc.voc:m2.m3.n1.n2.f.p2.p3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "mych"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:gen.loc:_:pos|adj:pl:acc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "mym"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:dat:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "mymi"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:inst:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "mo"], (function [x;_;_] -> compose_ordnum_lemma x "adja" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "y"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:m1.m2.m3:pos|adj:sg:acc:m3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "i"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:m1.m2.m3:pos|adj:sg:acc:m3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "e"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.acc.voc:n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "a"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:nom.voc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ego"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen:m1.m2.m3.n1.n2:pos|adj:sg:acc:m1.m2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "go"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen:m1.m2.m3.n1.n2:pos|adj:sg:acc:m1.m2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "emu"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:dat:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "mu"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:dat:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ej"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:gen.dat.loc:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ym"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:inst.loc:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "im"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:inst.loc:m1.m2.m3.n1.n2:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ą"], (function [x;_;_] -> compose_ordnum_lemma x "adj:sg:acc.inst:f:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "i"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.voc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "y"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.voc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "e"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:nom.acc.voc:m2.m3.n1.n2.f.p2.p3:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ych"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:gen.loc:_:pos|adj:pl:acc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ich"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:gen.loc:_:pos|adj:pl:acc:m1.p1:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ym"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:dat:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "im"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:dat:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "ymi"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:inst:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "imi"], (function [x;_;_] -> compose_ordnum_lemma x "adj:pl:inst:_:pos" | _ -> failwith "digit_patterns22");
  [D "intnum"; S "-"; O "o"], (function [x;_;_] -> compose_ordnum_lemma x "adja" | _ -> failwith "digit_patterns22");
  ]

let digit_patterns4 = [
  [D "intnum"; S "-"; O "tys"; S "."], (function [x;_;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "intnum"; S " "; O "tys"; S "."], (function [x;_;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "intnum"; O "tys"; S "."], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S "-"; O "tys"; S "."], (function [x;_;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S " "; O "tys"; S "."], (function [x;_;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "realnum"; O "tys"; S "."], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S "-"; O "tys"; S "."], (function [x;_;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S " "; O "tys"; S "."], (function [x;_;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; O "tys"; S "."], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S "-"; O "tys"; S "."], (function [x;_;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S " "; O "tys"; S "."], (function [x;_;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; O "tys"; S "."], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "intnum"; S "-"; O "mln"; S "."], (function [x;_;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "intnum"; S " "; O "mln"; S "."], (function [x;_;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "intnum"; O "mln"; S "."], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S "-"; O "mln"; S "."], (function [x;_;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S " "; O "mln"; S "."], (function [x;_;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "realnum"; O "mln"; S "."], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S "-"; O "mln"; S "."], (function [x;_;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S " "; O "mln"; S "."], (function [x;_;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; O "mln"; S "."], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S "-"; O "mln"; S "."], (function [x;_;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S " "; O "mln"; S "."], (function [x;_;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; O "mln"; S "."], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "intnum"; S "-"; O "mld"; S "."], (function [x;_;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [D "intnum"; S " "; O "mld"; S "."], (function [x;_;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [D "intnum"; O "mld"; S "."], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S "-"; O "mld"; S "."], (function [x;_;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S " "; O "mld"; S "."], (function [x;_;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [D "realnum"; O "mld"; S "."], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S "-"; O "mld"; S "."], (function [x;_;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S " "; O "mld"; S "."], (function [x;_;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; O "mld"; S "."], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S "-"; O "mld"; S "."], (function [x;_;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S " "; O "mld"; S "."], (function [x;_;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; O "mld"; S "."], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [D "intnum"; S "-"; O "tys"], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "intnum"; S " "; O "tys"], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "intnum"; O "tys"], (function [x;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S "-"; O "tys"], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S " "; O "tys"], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "realnum"; O "tys"], (function [x;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S "-"; O "tys"], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S " "; O "tys"], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; O "tys"], (function [x;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S "-"; O "tys"], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S " "; O "tys"], (function [x;_;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; O "tys"], (function [x;_] -> make_tys 3 x | _ -> failwith "digit_patterns8");
  [D "intnum"; S "-"; O "mln"], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "intnum"; S " "; O "mln"], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "intnum"; O "mln"], (function [x;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S "-"; O "mln"], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S " "; O "mln"], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "realnum"; O "mln"], (function [x;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S "-"; O "mln"], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S " "; O "mln"], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; O "mln"], (function [x;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S "-"; O "mln"], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S " "; O "mln"], (function [x;_;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; O "mln"], (function [x;_] -> make_tys 6 x | _ -> failwith "digit_patterns8");
  [D "intnum"; S "-"; O "mld"], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [D "intnum"; S " "; O "mld"], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [D "intnum"; O "mld"], (function [x;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S "-"; O "mld"], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [D "realnum"; S " "; O "mld"], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [D "realnum"; O "mld"], (function [x;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S "-"; O "mld"], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; S " "; O "mld"], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "intnum-interval"; O "mld"], (function [x;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S "-"; O "mld"], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; S " "; O "mld"], (function [x;_;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  [C "realnum-interval"; O "mld"], (function [x;_] -> make_tys 9 x | _ -> failwith "digit_patterns8");
  ]

let url_patterns1 = [
  [L; S "."; O "pl"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; O "pl"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; O "pl"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; O "pl"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; L; S "."; O "pl"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "-"; L; S "."; O "pl"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; O "uk"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; O "uk"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; O "uk"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; O "uk"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; L; S "."; O "uk"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "-"; L; S "."; O "uk"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; O "cz"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; O "cz"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; O "cz"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; O "cz"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; L; S "."; O "cz"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "-"; L; S "."; O "cz"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; O "eu"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; O "eu"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; O "eu"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; O "eu"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; L; S "."; O "eu"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "-"; L; S "."; O "eu"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; O "org"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; O "org"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; O "org"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; O "org"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; L; S "."; O "org"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "-"; L; S "."; O "org"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; O "com"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; O "com"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; O "com"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; O "com"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; L; S "."; O "com"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "-"; L; S "."; O "com"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; O "net"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; O "net"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; O "net"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; O "net"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; L; S "."; O "net"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "-"; L; S "."; O "net"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; O "gov"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; O "gov"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; O "gov"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; O "gov"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "."; L; S "."; L; S "."; L; S "."; O "gov"], (function l -> Dig(concat_orths2 l,"url"));
  [L; S "."; L; S "-"; L; S "."; O "gov"], (function l -> Dig(concat_orths2 l,"url"));
  ]

let url_patterns2 = [
  [L; S "@"; D "url"], (function l -> Dig(concat_orths2 l,"email"));
  [O "http"; S ":"; S "/"; S "/"; D "url"], (function l -> Dig(concat_orths2 l,"url"));
  ]

let url_patterns3 = [
  [D "url"; S "/"], (function l -> Dig(concat_orths2 l,"url"));
  [D "url"; S "/"; L], (function l -> Dig(concat_orths2 l,"url"));
  [D "url"; S "/"; L; S "."; L], (function l -> Dig(concat_orths2 l,"url"));
]

let html_patterns = [
  [S "<"; L; S ">"], (function l -> Dig(concat_orths2 l,"html-tag"));
  [S "<"; S "/"; L; S ">"], (function l -> Dig(concat_orths2 l,"html-tag"));
]

type matching = {
  prefix: tokens list;
  matched: token_env list;
  suffix: tokens list;
  pattern: pat list;
  command: token_env list -> token;
  command_abr: token_env list -> tokens list;
  }

let execute_command matching =
  let l = List.rev matching.matched in
  let len = Xlist.fold l 0 (fun len t -> t.len + len) in
  Seq((List.rev matching.prefix) @ [Token{empty_token_env with
    orth=concat_orths l;
    beg=(List.hd l).beg;
    len=len;
    next=(List.hd l).beg+len;
    token=matching.command l;
    weight=0.; (* FIXME: dodać wagi do konkretnych reguł i uwzględnić wagi maczowanych tokenów *)
    attrs=ENIAMtokens.merge_attrs l}] @ matching.suffix)

let execute_abr_command matching =
  let l = List.rev matching.matched in
  Seq((List.rev matching.prefix) @ (matching.command_abr l) @ matching.suffix)

let match_token = function
    D cat, Dig(_,cat2) -> cat = cat2
  | C s, Compound(s2,_) -> s = s2
  | S s, Symbol s2 -> s = s2
  | RD cat, RomanDig(_,cat2) -> cat = cat2
  | O pat, Dig(s,"dig") -> pat = s
  | O pat, Symbol s -> pat = s
  | O pat, SmallLetter orth -> pat = orth
  | O pat, CapLetter(orth,lc) -> pat = orth
  | O pat, AllSmall orth -> pat = orth
  | O pat, AllCap(orth,lc,lc2) -> pat = orth
  | O pat, FirstCap(orth,lc,_,_) -> pat = orth
  | O pat, SomeCap orth -> pat = orth
  | L, SmallLetter _ -> true
  | L, CapLetter _ -> true
  | L, AllSmall _ -> true
  | L, AllCap _ -> true
  | L, FirstCap _ -> true
  | L, SomeCap _ -> true
  | CL, CapLetter _ -> true
  | CL, AllCap _ -> true
  | CL, SomeCap _ -> true
  | I pat, Interp s -> pat = s
  | _ -> false

let rec find_first_token matching pat = function
    Token t -> if match_token (pat,t.token) then [{matching with matched = t :: matching.matched}] else []
  | Seq l -> Xlist.map (find_first_token matching pat (List.hd (List.rev l))) (fun matching -> {matching with prefix = matching.prefix @ (List.tl (List.rev l))})
  | Variant l -> List.flatten (Xlist.map l (find_first_token matching pat))

let rec find_middle_token matching pat = function
    Token t -> if match_token (pat,t.token) then [{matching with matched = t :: matching.matched}] else []
  | Seq _ -> []
  | Variant l -> List.flatten (Xlist.map l (find_middle_token matching pat))

let rec find_last_token matching pat = function
    Token t -> if match_token (pat,t.token) then [{matching with matched = t :: matching.matched}] else []
  | Seq l -> Xlist.map (find_last_token matching pat (List.hd l)) (fun matching -> {matching with suffix = matching.suffix @ (List.tl l)})
  | Variant l -> List.flatten (Xlist.map l (find_last_token matching pat))

let rec find_pattern_tail matchings = function
    [] -> raise Not_found
  | token :: l ->
      let found,finished = Xlist.fold matchings ([],[]) (fun (found,finished) matching ->
        match matching.pattern with
          [pat] -> found, (find_last_token {matching with pattern=[]} pat token) @ finished
        | pat :: pattern -> (find_middle_token {matching with pattern=pattern} pat token) @ found, finished
        | _ -> failwith "find_pattern: ni") in
      (try
        if found = [] then raise Not_found else
        find_pattern_tail found l
      with Not_found ->
        let finished = List.flatten (Xlist.map finished (fun matching -> try [execute_command matching] with Not_found -> [])) in
        if finished = [] then raise Not_found else Variant finished,l)

(* wzorce nie mogą mieć długości 1 *)
let rec find_pattern matchings rev = function
    token :: l ->
      let found = Xlist.fold matchings [] (fun found matching ->
        match matching.pattern with
          pat :: pattern -> (find_first_token {matching with pattern=pattern} pat token) @ found
        | [] -> failwith "find_pattern: empty pattern") in
      if found = [] then find_pattern matchings (token :: rev) l else
      (try
        let token,l = find_pattern_tail found l in
        find_pattern matchings (token :: rev) l
      with Not_found -> find_pattern matchings (token :: rev) l)
  | [] -> List.rev rev

let find_patterns patterns tokens =
  find_pattern (Xlist.map patterns (fun (pattern,command) ->
    {prefix=[]; matched=[]; suffix=[]; pattern=pattern; command=command; command_abr=(fun _ -> [])})) [] tokens

let rec find_abr_pattern_tail matchings = function
    [] -> raise Not_found
  | token :: l ->
      (* print_endline ("find_abr_pattern_tail 1: " ^ ENIAMtokens.string_of_tokens 0 token); *)
      let found,finished = Xlist.fold matchings ([],[]) (fun (found,finished) matching ->
        match matching.pattern with
          [pat] -> (*print_endline ("find_abr_pattern_tail 2a:");*) (find_last_token {matching with pattern=[]} pat token) @ found, finished
        | pat :: pattern -> (*print_endline ("find_abr_pattern_tail 2b:");*) (find_middle_token {matching with pattern=pattern} pat token) @ found, finished
        | [] -> (*print_endline "find_abr_pattern_tail 2c: []";*) found, matching :: finished) in
      (try
        (* print_endline "find_abr_pattern_tail 3"; *)
        if found = [] then raise Not_found else
        find_abr_pattern_tail found l
      with Not_found ->
        (* print_endline "find_abr_pattern_tail 4"; *)
        let finished = List.flatten (Xlist.map finished (fun matching -> try [execute_abr_command matching] with Not_found -> [])) in
        if finished = [] then raise Not_found else Variant finished,token :: l)

let rec find_abr_pattern matchings rev = function
    token :: l ->
      (* print_endline ("find_abr_pattern 1: " ^ ENIAMtokens.string_of_tokens 0 token); *)
      let found = Xlist.fold matchings [] (fun found matching ->
        match matching.pattern with
          pat :: pattern -> (find_first_token {matching with pattern=pattern} pat token) @ found
        | [] -> failwith "find_abr_pattern: empty pattern") in
      if found = [] then find_abr_pattern matchings (token :: rev) l else
      (try
        let token,l = find_abr_pattern_tail found l in
        find_abr_pattern matchings (token :: rev) l
      with Not_found -> find_abr_pattern matchings (token :: rev) l)
  | [] -> List.rev rev

let find_abr_patterns patterns tokens =
  (* Xlist.iter tokens (fun token -> print_endline ("A " ^ ENIAMtokens.string_of_tokens 0 token)); *)
  let tokens = find_abr_pattern (Xlist.map patterns (fun (pattern,command) ->
    {prefix=[]; matched=[]; suffix=[]; pattern=pattern; command=(fun _ -> Symbol ""); command_abr=command})) [] tokens in
  (* Xlist.iter tokens (fun token -> print_endline ("B " ^ ENIAMtokens.string_of_tokens 0 token)); *)
  tokens


exception PatternFound

let query_beg_patterns = [
  [I "<query>";I "<sentence>"];
  [I "<query>";I "„s";I "<sentence>"];
  [I "<query>";I "<or>";I "<sentence>"];
  ]

let query_end_patterns = [
  [I "</sentence>";I "</query>"];
  [I "</sentence>";I "”s";I "</query>"];
  ]

let find_beg_pattern pattern tokens =
  try
    let _ = find_pattern_tail [{prefix=[]; matched=[]; suffix=[];
         pattern=pattern; command=(fun _ -> raise PatternFound);
         command_abr=(fun _ -> [])}] tokens in false
  with PatternFound -> true | Not_found -> false

let replace_beg_pattern pattern command tokens =
  try
    let t,l = find_abr_pattern_tail [{prefix=[]; matched=[]; suffix=[];
         pattern=pattern; command=(fun _ -> Symbol "");
         command_abr=command}] tokens in
    t :: l
  with Not_found -> failwith "replace_beg_pattern"

(* let s_beg i = {empty_token_env with beg=i;len=1;next=i+1; token=Interp "<sentence>"}
let c_beg i = {empty_token_env with beg=i;len=1;next=i+1; token=Interp "<clause>"} *)
let s_end i = Token{empty_token_env with beg=i;len=1;next=i+1; token=Interp "</sentence>"}
let c_end i = Token{empty_token_env with beg=i;len=1;next=i+1; token=Interp "</clause>"}

let add_sentence_beg = function
    [q;t] -> let next=t.next in [Token q;Token{t with len=t.len-2;next=next-2};ENIAMtokens.s_beg (next-2);ENIAMtokens.c_beg (next-1)]
  | [q] -> let next=q.next in [Token{q with len=q.len-2;next=next-2};ENIAMtokens.s_beg (next-2);ENIAMtokens.c_beg (next-1)]
  | _ -> failwith "add_sentence_beg"

let add_sentence_end = function
    [q;t] -> let beg=t.beg in [Token q;Token{t with len=t.len-2;beg=beg+2};s_end (beg+1);c_end beg]
  | [q] -> let beg=q.beg in [Token{q with len=q.len-2;beg=beg+2};s_end (beg+1);c_end beg]
  | _ -> failwith "add_sentence_end"

let rec revert_tokens = function
    Token t -> Token t
  | Seq l -> Seq(Xlist.rev_map l revert_tokens)
  | Variant l -> Variant(Xlist.map l revert_tokens)

let manage_query_boundaries tokens =
  (* let b =
    try
      let _ = find_pattern_tail (Xlist.map query_beg_patterns (fun pattern ->
        {prefix=[]; matched=[]; suffix=[];
         pattern=pattern; command=(fun _ -> raise PatternFound);
         command_abr=(fun _ -> [])})) tokens in false
    with PatternFound -> true | Not_found -> false in
  (if b then print_endline "sentence beg found" else print_endline "sentence beg not found"); *)
  let tokens =
    if find_beg_pattern [I "<query>";I "„s"] tokens then
      if find_beg_pattern [I "<query>";I "„s";I "<sentence>"] tokens then tokens else
      replace_beg_pattern [I "<query>";I "„s"] add_sentence_beg tokens else
    if find_beg_pattern [I "<query>";I "<or>"] tokens then
      if find_beg_pattern [I "<query>";I "<or>";I "<sentence>"] tokens then tokens else
      replace_beg_pattern [I "<query>";I "<or>"] add_sentence_beg tokens else
    if find_beg_pattern [I "<query>";I "(s";I "<sentence>"] tokens then tokens else
    if find_beg_pattern [I "<query>";I "<sentence>"] tokens then tokens else
    replace_beg_pattern [I "<query>"] add_sentence_beg tokens in
  (* let b =
    try
      let _ = find_pattern (Xlist.map query_end_patterns (fun pattern ->
        {prefix=[]; matched=[]; suffix=[];
         pattern=pattern; command=(fun _ -> raise PatternFound);
         command_abr=(fun _ -> [])})) [] tokens in false
    with PatternFound -> true in
  (if b then print_endline "sentence end found" else print_endline "sentence end not found"); *)
  let tokens = Xlist.rev_map tokens revert_tokens in
  let tokens =
    if find_beg_pattern [I "</query>";I "</sentence>"] tokens then tokens else
    if find_beg_pattern [I "</query>";I "”s";I "</sentence>"] tokens then tokens else
    if find_beg_pattern [I "</query>";I "”s"] tokens then
      replace_beg_pattern [I "</query>";I "”s"] add_sentence_end tokens else
    if find_beg_pattern [I "</query>";I ")s";I "</sentence>"] tokens then tokens else
    replace_beg_pattern [I "</query>"] add_sentence_end tokens in
  let tokens = Xlist.rev_map tokens revert_tokens in
  tokens


let find_replacement_patterns tokens =
  let tokens = find_patterns digit_patterns1 tokens in
  let tokens = normalize_tokens [] tokens in
  let tokens = find_patterns digit_patterns2 tokens in
  let tokens = normalize_tokens [] tokens in
  let tokens = find_patterns digit_patterns3 tokens in
  let tokens = normalize_tokens [] tokens in
  let tokens = find_patterns digit_patterns4 tokens in
  let tokens = normalize_tokens [] tokens in
  let tokens = find_patterns ENIAMacronyms.acronym_patterns tokens in
  let tokens = normalize_tokens [] tokens in
  let tokens = find_patterns !ENIAMacronyms.mte_patterns tokens in
  let tokens = normalize_tokens [] tokens in
(*   Xlist.iter tokens (fun t -> print_endline (ENIAMtokens.string_of_tokens 0 t)); *)
  let tokens = find_patterns ENIAMacronyms.name_patterns tokens in
(*   Xlist.iter tokens (fun t -> print_endline (ENIAMtokens.string_of_tokens 0 t)); *)
  let tokens = normalize_tokens [] tokens in
  let tokens = find_patterns url_patterns1 tokens in
  let tokens = normalize_tokens [] tokens in
  let tokens = find_patterns url_patterns2 tokens in
  let tokens = normalize_tokens [] tokens in
  let tokens = find_patterns url_patterns3 tokens in
  let tokens = normalize_tokens [] tokens in
  let tokens = find_patterns html_patterns tokens in
  let tokens = normalize_tokens [] tokens in
  (*   Xlist.iter tokens (fun t -> print_endline (ENIAMtokens.string_of_tokens 0 t)); *)
  tokens

let rec set_next_id n = function
    Token t -> Token{t with next=n}
  | Seq l ->
      (match List.rev l with
        t :: l -> Seq(List.rev ((set_next_id n t) :: l))
      | [] -> failwith "set_next_id n")
  | Variant l -> Variant(Xlist.map l (set_next_id n))

let rec remove_spaces rev = function
    [] -> List.rev rev
  | x :: Token{token=Symbol " "; next=n} ::  l -> remove_spaces ((set_next_id n x) :: rev) l
  | Token{token=Symbol " "} :: l -> remove_spaces rev l
  | x :: l -> remove_spaces (x :: rev) l