Analiza różnic w lematyzacji

Wojciech Jaworski
1 parent be209dd2
Showing 4 changed files with 421 additions and 50 deletions
NKJP2/validateMorphology.ml
NKJP2/validateTokenizer.ml
morphology/doc/model2.pdf
morphology/resources/alt_supplement.tab
@@ -26,10 +26,10 @@ let rec has_brev = function
   | _ :: l -> has_brev l
   | [] -> false
  
-let rec get_ntoken = function
-    (Disamb(nlemma,ncat,ninterp) : attr) :: _ -> nlemma,ncat,ninterp
-  | _ :: l -> get_ntoken l
-  | [] -> raise Not_found
+let rec get_brev = function
+    BrevLemma s :: _ -> s
+  | _ :: l -> get_brev l
+  | [] -> failwith "get_brev"
  
 let rec add_ntoken stats = function
     Token t ->
@@ -91,7 +91,7 @@ let lemmatize_string s =
       | Token ({token=FirstCap _} as t) -> t :: l
       | Token ({token=AllCap _} as t) -> t :: l
       | Token ({token=CapLetter _} as t) -> t :: l
-      | Token ({token=RomanDig _}) -> (*print_endline ("lemmatize_string: " ^ s);*) (*t ::*) l
+      | Token ({token=RomanDig _} as t) -> (*print_endline ("lemmatize_string: " ^ s);*) t :: l
       | Token ({token=Dig _} as t) -> (*print_endline ("lemmatize_string: " ^ s);*) t :: l
       | Token ({token=Proper _} as t) -> t :: l
       | Seq[Token {token=AllSmall _};Token {token=Lemma _}] -> l
@@ -125,34 +125,52 @@ let lemmatize_string s =
  
 let get_cat_interp = function
     "subst","subst",[n;c;["m1"]],[_;_;["m1"]] -> "subst",[n;c;["m1"]]
+  | "subst","subst",[n;c;["m1"]],[_;_;["m1"];col] -> "subst",[n;c;["m1"];col]
   | "subst","subst",[n;c;["m2"]],[_;_;["m2"]] -> "subst",[n;c;["m2"]]
   | "subst","subst",[n;c;["m3"]],[_;_;["m3"]] -> "subst",[n;c;["m3"]]
-  | "subst","subst",[n;c;["n1";"n2"]],[_;_;["n1"]] -> "subst",[n;c;["n1"]]
-  | "subst","subst",[n;c;["n1";"n2"]],[_;_;["n2"]] -> "subst",[n;c;["n2"]]
+  | "subst","subst",[n;c;["n"]],[_;_;["n"];col] -> "subst",[n;c;["n"];col]
   | "subst","subst",[n;c;["f"]],[_;_;["f"]] -> "subst",[n;c;["f"]]
-  | "subst","subst",[n;c;["n1";"n2";"p2";"p3"]],[_;_;["n1"]] -> "subst",[n;c;["n1"]]
-  | "subst","subst",[n;c;["n1";"n2";"p2";"p3"]],[_;_;["n2"]] -> "subst",[n;c;["n2"]]
-  | "subst","subst",[n;c;["n1";"n2";"p2";"p3"]],[_;_;["p2"]] -> "subst",[n;c;["p2"]]
-  | "subst","subst",[n;c;["n1";"n2";"p2";"p3"]],[_;_;["p3"]] -> "subst",[n;c;["p3"]]
-  | "subst","subst",[n;c;["m1";"p1"]],[_;_;["m1"]] -> "subst",[n;c;["m1"]]
-  | "subst","subst",[n;c;["m1";"p1"]],[_;_;["p1"]] -> "subst",[n;c;["p1"]]
+  | "subst","subst",[n;c;g],[_;_;_] -> "subst",[n;c;g]
+  | "subst","subst",[n;c;g],[_;_;_;_] -> "subst",[n;c;g]
+  | "subst","adj",[n;c;g],_ -> "subst",[n;c;g]
   | "depr","subst",[["pl"];["nom"];["m2"]],[["sg"];["nom"];["m1"]] -> "depr",[["pl"];["nom"];["m2"]]
+  | "depr","subst",[["pl"];["acc"];["m2"]],[["sg"];["nom"];["m1"]] -> "depr",[["pl"];["acc"];["m2"]]
   | "ppron3","ppron3",ninterp,[["sg"];["nom"];["m1";"m2";"m3"];["ter"];_;_] -> "ppron3",ninterp
-  | "ppron12","ppron12",ninterp,[_;["nom"];_;_] -> "ppron3",ninterp
-  | "numcol","num",ninterp,_ -> "num",ninterp (* FIXME: wiele wpisów przejdzie *)
-  | "num","num",ninterp,_ -> "num",ninterp (* FIXME: wiele wpisów przejdzie *)
+  | "ppron12","ppron12",ninterp,[_;["nom"];_;_] -> "ppron12",ninterp
+  | "numcol","num",ninterp,_ -> "num",ninterp
+  | "num","num",ninterp,_ -> "num",ninterp (* na tym etapie nie da się skorygować błędów *)
+  (* | "num","num",[["pl"];c;g;["rec"]],[["sg";"pl"];["nom";"gen";"acc"];["m1";"m2";"m3";"f";"n"];["rec"]] -> "num",[["pl"];c;g;["rec"]]
+  | "num","num",[["pl"];c;["m2"];["rec"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"f";"n"];["rec"];col] -> "num",[["pl"];c;["m2"];["rec"]]
+  | "num","num",[["pl"];c;["m3"];["rec"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"f";"n"];["rec"];col] -> "num",[["pl"];c;["m3"];["rec"]]
+  | "num","num",[["pl"];c;["f"];["rec"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"f";"n"];["rec"];col] -> "num",[["pl"];c;["f"];["rec"]]
+  | "num","num",[["pl"];c;["n"];["rec"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"f";"n"];["rec"];col] -> "num",[["pl"];c;["n"];["rec"];col]
+  | "num","num",[["pl"];c;["m1"];["congr"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"f";"n"];["congr"];col] -> "num",[["pl"];c;["m1"];["congr"]]
+  | "num","num",[["pl"];c;["m2"];["congr"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"f";"n"];["congr"];col] -> "num",[["pl"];c;["m2"];["congr"]]
+  | "num","num",[["pl"];c;["m3"];["congr"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"f";"n"];["congr"];col] -> "num",[["pl"];c;["m3"];["congr"]]
+  | "num","num",[["pl"];c;["f"];["congr"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"f";"n"];["congr"];col] -> "num",[["pl"];c;["f"];["congr"]]
+  | "num","num",[["pl"];c;["n"];["congr"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"f";"n"];["congr"];col] -> "num",[["pl"];c;["n"];["congr"];col]
+  | "num","num",[["pl"];c;["m2"];["congr"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"n"];["congr"];col] -> "num",[["pl"];c;["m2"];["congr"]]
+  | "num","num",[["pl"];c;["m3"];["congr"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"n"];["congr"];col] -> "num",[["pl"];c;["m3"];["congr"]]
+  | "num","num",[["pl"];c;["n"];["congr"]],[["pl"];["nom";"acc";"voc"];["m2";"m3";"n"];["congr"];col] -> "num",[["pl"];c;["n"];["congr"];col] *)
   | "siebie","siebie",[[c]],[["acc";"gen"]] -> "siebie",[[c]]
   | "adj","adj",ninterp,[["sg"];["nom";"voc"];["m1";"m2";"m3"];["pos"]] -> "adj",ninterp
+  | "adj","adj",ninterp,[["sg";"pl"];["nom";"gen";"dat";"acc";"inst";"loc";"voc"];["m1";"m2";"m3";"f";"n"];["pos"]] -> "adj",ninterp
   | "adja","adj",ninterp,[["sg"];["nom";"voc"];["m1";"m2";"m3"];["pos"]] -> "adja",ninterp
   | "adjc","adj",ninterp,[["sg"];["nom";"voc"];["m1";"m2";"m3"];["pos"]] -> "adjc",ninterp
   | "adjp","adj",ninterp,[["sg"];["nom";"voc"];["m1";"m2";"m3"];["pos"]] -> "adjp",ninterp
+  | "adj","adj",ninterp,[["sg"];["nom"];["m1";"m2";"m3"];["pos"]] -> "adj",ninterp
   | "adv","adv",[[g]],[["pos"]] -> "adv",[[g]]
-  | "adv","adv",ninterp,interp -> if ninterp = interp then "adv",ninterp else raise Not_found
+  | "adv","adv",[],[["pos"]] -> "adv",[["pos"]]
+  | "adv",_,ninterp,_ -> "adv",ninterp
+  | "comp","comp",ninterp,interp -> if ninterp = interp then "comp",ninterp else raise Not_found
+  | "conj","conj",ninterp,interp -> if ninterp = interp then "conj",ninterp else raise Not_found
+  | "conj",_,ninterp,_ -> "conj",ninterp
+  | "prep","prep",[c1;w],[c2;_] -> if c1 = c2 then "prep",[c1;w] else raise Not_found
   | "prep","prep",ninterp,interp -> if ninterp = interp then "prep",ninterp else raise Not_found
   | "qub","qub",ninterp,interp -> if ninterp = interp then "qub",ninterp else raise Not_found
-  | "conj","conj",ninterp,interp -> if ninterp = interp then "conj",ninterp else raise Not_found
-  | "comp","comp",ninterp,interp -> if ninterp = interp then "comp",ninterp else raise Not_found
+  | "qub",_,ninterp,_ -> "qub",ninterp
   | "interj","interj",ninterp,interp -> if ninterp = interp then "interj",ninterp else raise Not_found
+  | "interj",_,ninterp,_ -> "interj",ninterp
   | "burk","burk",ninterp,interp -> if ninterp = interp then "burk",ninterp else raise Not_found
   | "pred","pred",ninterp,interp -> if ninterp = interp then "pred",ninterp else raise Not_found
   | "fin","inf",[n;p;["imperf"]],[["imperf";"perf"]] -> "fin",[n;p;["imperf"]]
@@ -163,6 +181,8 @@ let get_cat_interp = function
   | "impt","inf",[n;p;["imperf"]],[["imperf"]] -> "impt",[n;p;["imperf"]]
   | "impt","inf",[n;p;["perf"]],[["imperf";"perf"]] -> "impt",[n;p;["perf"]]
   | "impt","inf",[n;p;["perf"]],[["perf"]] -> "impt",[n;p;["perf"]]
+  | "bedzie","inf",[n;p;["imperf"]],[["imperf"]] -> "bedzie",[n;p;["imperf"]]
+  | "aglt","inf",[n;p;["imperf"];w],[["imperf"]] -> "aglt",[n;p;["imperf"];w]
   | "inf","inf",[["imperf"]],[["imperf";"perf"]] -> "inf",[["imperf"]]
   | "inf","inf",[["imperf"]],[["imperf"]] -> "inf",[["imperf"]]
   | "inf","inf",[["perf"]],[["imperf";"perf"]] -> "inf",[["perf"]]
@@ -175,7 +195,7 @@ let get_cat_interp = function
   | "praet","inf",[n;g;["imperf"];a],[["imperf"]] -> "praet",[n;g;["imperf"];a]
   | "praet","inf",[n;g;["perf"];a],[["imperf";"perf"]] -> "praet",[n;g;["perf"];a]
   | "praet","inf",[n;g;["perf"];a],[["perf"]] -> "praet",[n;g;["perf"];a]
-  | "winien","inf",[n;g;["imperf"]],[["imperf"]] -> "winien",[n;g;["imperf"]]
+  | "winien","winien",[n;g;["imperf"]],[_;_;["imperf"]] -> "winien",[n;g;["imperf"]]
   | "ppas","inf",[n;c;g;["imperf"];a],[["imperf";"perf"]] -> "ppas",[n;c;g;["imperf"];a]
   | "ppas","inf",[n;c;g;["imperf"];a],[["imperf"]] -> "ppas",[n;c;g;["imperf"];a]
   | "ppas","inf",[n;c;g;["perf"];a],[["imperf";"perf"]] -> "ppas",[n;c;g;["perf"];a]
@@ -202,6 +222,16 @@ let get_cat_interp = function
   | "imps","inf",[["perf"]],[["perf"]] -> "imps",[["perf"]]
   | _ -> raise Not_found
  
+let get_lemma_cat_interp = function
+    nlemma,lemma,"adj","ppas",[n;c;g;["pos"]],[["sg"];["nom";"voc"];["m1";"m2";"m3"];a;aff] -> lemma,"ppas",[n;c;g;a;aff]
+  | nlemma,lemma,"adja","adja",[],[] -> lemma,"adja",[]
+  | nlemma,lemma,"subst","subst",[["pl"];c;g],[["pl"];["nom";"voc"];_] -> lemma,"subst",[["pl"];c;g]
+  (* | "5","5","adj","dig",ninterp,[] -> "piąty","adj",ninterp
+  | "6","6","adj","dig",ninterp,[] -> "szósty","adj",ninterp *)
+  (* | "adj","ppas",ninterp,interp -> print_endline (ENIAMtokens.string_of_interps [ninterp] ^ " " ^ ENIAMtokens.string_of_interps [interp]); raise Not_found *)
+  | _ -> raise Not_found
+
+
 let correct_nlemma = function
     "letnia  " -> "letnia"
   | "10minutowy" -> "minutowy"
@@ -233,7 +263,7 @@ let correct_nlemma = function
   | "16-latek" -> raise Not_found
   | s -> s
  
-let process_ntoken stats nlemma ncat ninterp =
+let process_ntoken stats q nlemma ncat ninterp =
   try
     let nlemma = correct_nlemma nlemma in
     let nl = lemmatize_string nlemma in
@@ -241,44 +271,291 @@ let process_ntoken stats nlemma ncat ninterp =
         {token=Lemma(lemma,cat,interp)} ->
           Xlist.fold interp nl (fun nl interp ->
             try
-              let cat,interp = get_cat_interp (ncat,cat,ninterp,interp) in
-              if lemma = nlemma then (Lemma(lemma,cat,[interp])) :: nl else nl
+              if lemma = nlemma then
+                let cat,interp = get_cat_interp (ncat,cat,ninterp,interp) in
+                (Lemma(lemma,cat,[interp])) :: nl else
+              let lemma,cat,interp = get_lemma_cat_interp (nlemma,lemma,ncat,cat,ninterp,interp) in
+              (Lemma(lemma,cat,[interp])) :: nl
+            with Not_found -> nl)
+      | {token=Dig(_,"dig")} -> nl (* FIXME: todo *)
+            (* (try
+              let lemma,cat,interp = get_lemma_cat_interp (nlemma,lemma,ncat,cat,ninterp,interp) in
+              (Lemma(lemma,cat,[interp])) :: nl
+            with Not_found -> nl) *)
+      | {token=RomanDig(_,"roman")} ->
+          if ncat = "adj" then (Lemma(nlemma,ncat,[ninterp])) :: nl else nl
+      | {token=Proper(lemma,cat,interp,_)} -> (*print_endline ("P " ^ nlemma);*) nl (* FIXME: todo *)
+      | _ -> nl) in
+    if nl2 = [] then StringQMap.add_val stats (ncat ^ " " ^ ENIAMtokens.string_of_token (Lemma(nlemma,ncat,[ninterp])) ^ ": " ^ String.concat " " (Xlist.map nl (fun t -> ENIAMtokens.string_of_token t.token))) q
+    else StringQMap.add_val stats "lemmatized" q
+  with Not_found -> StringQMap.add_val stats "incorrect" q
+
+let process_ntoken2 stats q name id_div orth beg paragraph nlemma ncat ninterp =
+  try
+    let nlemma = correct_nlemma nlemma in
+    let nl = lemmatize_string nlemma in
+    let nl2 = Xlist.fold nl [] (fun nl -> function
+        {token=Lemma(lemma,cat,interp)} ->
+          Xlist.fold interp nl (fun nl interp ->
+            try
+              if lemma = nlemma then
+                let cat,interp = get_cat_interp (ncat,cat,ninterp,interp) in
+                (Lemma(lemma,cat,[interp])) :: nl else
+              let lemma,cat,interp = get_lemma_cat_interp (nlemma,lemma,ncat,cat,ninterp,interp) in
+              (Lemma(lemma,cat,[interp])) :: nl
             with Not_found -> nl)
       | {token=Dig _} -> nl (* FIXME: todo *)
+      | {token=RomanDig(_,"roman")} ->
+          if ncat = "adj" then (Lemma(nlemma,ncat,[ninterp])) :: nl else nl
       | {token=Proper(lemma,cat,interp,_)} -> nl (* FIXME: todo *)
       | _ -> nl) in
-    if nl2 = [] then StringQMap.add stats (ncat ^ " " ^ ENIAMtokens.string_of_token (Lemma(nlemma,ncat,[ninterp])) ^ ": " ^ String.concat " " (Xlist.map nl (fun t -> ENIAMtokens.string_of_token t.token)))
-    else StringQMap.add stats "lemmatized"
-  with Not_found -> StringQMap.add stats "incorrect"
+    if nl2 = [] then
+      StringQMap.add_val stats (ncat ^ " " ^ ENIAMtokens.string_of_token (Lemma(nlemma,ncat,[ninterp])) ^ ": " ^ String.concat " " (Xlist.map nl (fun t -> ENIAMtokens.string_of_token t.token))
+        ^ "\n" ^ name ^ " " ^ string_of_int id_div ^ " " ^ string_of_int beg ^ " " ^ orth ^ "\n" ^ paragraph) q
+    else StringQMap.add_val stats "lemmatized" q
+  with Not_found -> StringQMap.add_val stats "incorrect" q
+
+let validate_ntoken stats q (nlemma,ncat,ninterp) =
+  process_ntoken stats q nlemma ncat ninterp
  
-let validate_ntoken stats (nlemma,ncat,ninterp) =
-  process_ntoken stats nlemma ncat ninterp
+let rec validate_ntoken_token name id_div paragraph stats = function
+    Token t ->
+      (try
+        let nlemma,ncat,ninterp = get_ntoken t.attrs in
+        process_ntoken2 stats 1 name id_div t.orth t.beg paragraph nlemma ncat ninterp
+        (* print_endline (nlemma ^ "\t" ^ ncat ^ "\t" ^ ENIAMtokens.string_of_interps [ninterp]);
+        Printf.printf "%s\t%d\t%s\t%d\n" name id_div t.orth t.beg;
+        print_endline paragraph;
+        stats *)
+      with Not_found -> stats)
+  | Seq l -> Xlist.fold l stats (validate_ntoken_token name id_div paragraph)
+  | Variant l -> Xlist.fold l stats (validate_ntoken_token name id_div paragraph)
  
-let match_lemmatize stats t =
-  if has_brev t.attrs then StringQMap.add stats "brev" else
+let validate_ntoken_entry stats name typ channel entries =
+  prerr_endline name;
+  Xlist.fold entries stats (fun stats (id_div,has_ne,paragraphs) ->
+    Xlist.fold paragraphs stats (fun stats (paragraph,sentences) ->
+      let paragraph,tokens = annotate name sentences in
+      Xlist.fold tokens stats (validate_ntoken_token name id_div paragraph)))
+
+let rec subset_list = function
+    [],[] -> true
+  | [x] :: l1, y :: l2 -> if Xlist.mem y x then subset_list (l1,l2) else false
+  | _ -> false
+
+let match_cat_interp = function
+  | "subst","subst",[nn;nc;ng],[n;c;g;col] -> if subset_list ([nn;nc;ng],[n;c;g]) then "subst",[nn;nc;ng;col] else raise Not_found
+(*  | "numcol","num",ninterp,_ -> "num",ninterp*)
+  | "num","num",[nn;nc;["n"];na],[n;c;g;a;col] -> if subset_list ([nn;nc;["n"];na],[n;c;g;a]) then "num",[nn;nc;["n"];na;col] else raise Not_found
+  | "num","num",[nn;nc;ng;na],[n;c;g;a;col] -> if subset_list ([nn;nc;ng;na],[n;c;g;a]) then "num",[nn;nc;ng;na] else raise Not_found
+  | "adv","adv",[],[["pos"]] -> "adv",[["pos"]]
+  | _ -> raise Not_found
+
+let match_cat_interp_substgender = function
+    "subst","subst",[nn;nc;ng],[n;c;_] -> if subset_list ([nn;nc],[n;c]) then "subst",[nn;nc;ng] else raise Not_found
+  | "subst","subst",[nn;nc;ng],[n;c;_;_] -> if subset_list ([nn;nc],[n;c]) then "subst",[nn;nc;ng] else raise Not_found
+  | _ -> raise Not_found
+
+exception HasBrev
+exception NoNtoken
+exception LemmaNotMatched of string * string * string list list * token_env list
+exception MultipleLemmaMatched of string * string * string list list * token list
+
+let rec sort_uniq_rec rev = function
+    [] -> rev
+  | x :: y :: l -> if x = y then sort_uniq_rec rev (y :: l) else sort_uniq_rec (x :: rev) (y :: l)
+  | [x] -> x :: rev
+
+let sort_uniq l =
+  match sort_uniq_rec [] (Xlist.sort l compare) with
+    [Lemma(lemma1,"subst",[[n1;c1;["n"];["ncol"]]]);Lemma(lemma2,"subst",[[n2;c2;["n"];["col"]]])] as l ->
+       if lemma1 = lemma2 && n1 = n2 && c1 = c2 then [Lemma(lemma1,"subst",[[n1;c1;["n"];["ncol";"col"]]])] else l
+  | [Lemma("kląsknięcie","subst",[[["pl"];c1;["n"];["pt"]]]);Lemma("kląsknięcie","subst",[[["pl"];c2;["n"];["ncol"]]])] as l ->
+       if c1 = c2 then [Lemma("kląsknięcie","subst",[[["pl"];c1;["n"];["pt"]]])] else l
+  | [Lemma("wybrażenie","subst",[[["pl"];c1;["n"];["pt"]]]);Lemma("wybrażenie","subst",[[["pl"];c2;["n"];["ncol"]]])] as l ->
+       if c1 = c2 then [Lemma("wybrażenie","subst",[[["pl"];c1;["n"];["pt"]]])] else l
+  | [Lemma(lemma1,"subst",[[["pl"];c1;["n"];["pt"]]]);Lemma(lemma2,"subst",[[["pl"];c2;["n"];["ncol"]]])] as l ->
+       (* print_endline lemma1; *)
+       if lemma1 = lemma2 && c1 = c2 then [Lemma(lemma1,"subst",[[["pl"];c1;["n"];["pt"]]])] else l
+  | l -> (*print_endline (String.concat " " (Xlist.map l (fun t -> ENIAMtokens.string_of_token t)));*) l
+
+type t = TokenMatched | TokenLowercase | TokenBrev | TokenSubstGender | TokenDeviated
+
+let match_lemmatize_simple t nlemma ncat ninterp =
+  let l1 = ENIAMpaths.lemmatize_token t in
+  let l2 = Xlist.fold l1 [] (fun l -> function
+        {token=Lemma(lemma,cat,interp)} ->
+          Xlist.fold interp l (fun l interp ->
+            try
+              if lemma = nlemma && cat = ncat && subset_list (ninterp,interp) then (Lemma(nlemma,ncat,[ninterp])) :: l else
+              if lemma = nlemma then
+                let cat,interp = match_cat_interp (ncat,cat,ninterp,interp) in
+                (Lemma(lemma,cat,[interp])) :: l else l
+            with Not_found -> l)
+      | {token=Dig _} -> l (* FIXME: todo *)
+      | {token=RomanDig(_,"roman")} ->
+          if ncat = "adj" then (Lemma(nlemma,ncat,[ninterp])) :: l else l
+      | {token=Proper(lemma,cat,interp,_)} -> l (* FIXME: todo *)
+      | _ -> l) in
+  match sort_uniq l2 with
+    [] -> raise (LemmaNotMatched(nlemma,ncat,ninterp,l1))
+  | [t] -> t, TokenMatched
+  | _ -> raise (MultipleLemmaMatched(nlemma,ncat,ninterp,l2))
+
+let match_lemmatize_lowercase t nlemma ncat ninterp =
+  let t = match t.token with
+    | FirstCap(s,lower,cl,ll) -> {t with token=AllSmall lower}
+    | CapLetter(s,lower) -> {t with token=SmallLetter lower}
+    | AllCap(_,a,b) -> {t with token=FirstCap(a,b,"","")} (* FIXME: to powinno być zdezambiguowane *)
+    | _ -> t in
   let l = ENIAMpaths.lemmatize_token t in
-  try
-    let nlemma,ncat,ninterp = get_ntoken t.attrs in
+  let l2 = Xlist.fold l [] (fun l -> function
+      {token=Lemma(lemma,cat,interp)} ->
+          Xlist.fold interp l (fun l interp ->
+            try
+          if lemma = nlemma && cat = ncat && subset_list (ninterp,interp) then (Lemma(nlemma,ncat,[ninterp])) :: l else
+              if lemma = nlemma then
+                let cat,interp = match_cat_interp (ncat,cat,ninterp,interp) in
+                (Lemma(lemma,cat,[interp])) :: l else l
+            with Not_found -> l)
+    | {token=Dig _} -> l (* FIXME: todo *)
+    | {token=RomanDig(_,"roman")} ->
+          if ncat = "adj" then (Lemma(nlemma,ncat,[ninterp])) :: l else l
+    | {token=Proper(lemma,cat,interp,_)} -> l (* FIXME: todo *)
+    | _ -> l) in
+  match sort_uniq l2 with
+    [] -> raise (LemmaNotMatched(nlemma,ncat,ninterp,l))
+  | [t] -> t, TokenLowercase
+  | _ -> raise (MultipleLemmaMatched(nlemma,ncat,ninterp,l2))
+
+let match_lemmatize_substgender t nlemma ncat ninterp =
+  let l1 = ENIAMpaths.lemmatize_token t in
+  let l2 = Xlist.fold l1 [] (fun l -> function
+        {token=Lemma(lemma,cat,interp)} ->
+          Xlist.fold interp l (fun l interp ->
+            try
+              if lemma = nlemma then
+                let cat,interp = match_cat_interp_substgender (ncat,cat,ninterp,interp) in
+                (Lemma(lemma,cat,[interp])) :: l else l
+            with Not_found -> l)
+      | {token=Dig _} -> l (* FIXME: todo *)
+      | {token=RomanDig(_,"roman")} ->
+          if ncat = "adj" then (Lemma(nlemma,ncat,[ninterp])) :: l else l
+      | {token=Proper(lemma,cat,interp,_)} -> l (* FIXME: todo *)
+      | _ -> l) in
+  match sort_uniq l2 with
+    [] -> raise (LemmaNotMatched(nlemma,ncat,ninterp,l1))
+  | [t] -> t, TokenSubstGender
+  | _ -> raise (MultipleLemmaMatched(nlemma,ncat,ninterp,l2))
+
+let match_lemmatize_deviated t nlemma ncat ninterp =
+  let l1 = ENIAMpaths.lemmatize_token t in
+  let nlemma = try correct_nlemma nlemma with Not_found -> raise (LemmaNotMatched(nlemma,ncat,ninterp,l1)) in
+  let nl = lemmatize_string nlemma in
+  let nl2 = Xlist.fold nl [] (fun nl -> function
+        {token=Lemma(lemma,cat,interp)} ->
+          Xlist.fold interp nl (fun nl interp ->
+            try
+              let lemma,cat,interp = get_lemma_cat_interp (nlemma,lemma,ncat,cat,ninterp,interp) in
+              (lemma,cat,interp) :: nl
+            with Not_found -> nl)
+      | _ -> nl) in
+  let l2 = Xlist.fold nl2 [] (fun l (nlemma,ncat,ninterp) ->
+    Xlist.fold l1 l (fun l -> function
+        {token=Lemma(lemma,cat,interp)} ->
+          Xlist.fold interp l (fun l interp ->
+            try
+              if lemma = nlemma && cat = ncat && subset_list (ninterp,interp) then (Lemma(nlemma,ncat,[ninterp])) :: l else
+              if lemma = nlemma then
+                let cat,interp = match_cat_interp (ncat,cat,ninterp,interp) in
+                (Lemma(lemma,cat,[interp])) :: l else l
+            with Not_found -> l)
+      | _ -> l)) in
+  match sort_uniq l2 with
+    [] -> raise (LemmaNotMatched(nlemma,ncat,ninterp,l1))
+  | [t] -> t, TokenDeviated
+  | _ -> raise (MultipleLemmaMatched(nlemma,ncat,ninterp,l2))
+
+let rec match_lemmatize_rec t nlemma ncat ninterp f0 = function
+    f :: l ->
+       (try f t nlemma ncat ninterp
+        with LemmaNotMatched _ -> match_lemmatize_rec t nlemma ncat ninterp f0 l)
+  | [] -> f0 t nlemma ncat ninterp
+
+let match_lemmatize (*stats q name id_div paragraph*) t =
+  if has_brev t.attrs then raise HasBrev (*StringQMap.add_val stats "brev" q*)
+(*    let nlemma = get_brev t.attrs in
+      (let l = ENIAMpaths.lemmatize_token t in
+      let l2 = Xlist.fold l [] (fun l -> function
+            {token=Lemma(lemma,cat,interp)} ->
+              Xlist.fold interp l (fun l interp ->
+                try
+                  if lemma = nlemma then (Lemma(nlemma,cat,[interp])) :: l else l
+                with Not_found -> l)
+          (* | {token=Dig _} -> l (* FIXME: todo *)
+          | {token=RomanDig(_,"roman")} ->
+              if ncat = "adj" then (Lemma(nlemma,ncat,[ninterp])) :: l else l
+          | {token=Proper(lemma,cat,interp,_)} -> l (* FIXME: todo *) *)
+          | _ -> l) in
+      match sort_uniq l2 with
+        [] -> raise (LemmaNotMatched(nlemma,"BREV",[],l))
+      | [t] -> t, TokenBrev
+      | _ -> raise (MultipleLemmaMatched(nlemma,"BREV",[],l2)))*)
+  else
+  let nlemma,ncat,ninterp = try get_ntoken t.attrs with Not_found -> raise NoNtoken in
+  match_lemmatize_rec t nlemma ncat ninterp match_lemmatize_simple
+    [match_lemmatize_simple; match_lemmatize_lowercase; match_lemmatize_substgender; match_lemmatize_deviated]
+  (* let ninterp = if ncat = "adv" && ninterp = [] then [["pos"]] else ninterp in *)
+(*  let l1 = ENIAMpaths.lemmatize_token t in
+  let l2 = Xlist.fold l1 [] (fun l -> function
+        {token=Lemma(lemma,cat,interp)} ->
+          Xlist.fold interp l (fun l interp ->
+            try
+              if lemma = nlemma && cat = ncat && subset_list (ninterp,interp) then (Lemma(nlemma,ncat,[ninterp])) :: l else
+              if lemma = nlemma then
+                let cat,interp = match_cat_interp (ncat,cat,ninterp,interp) in
+                (Lemma(lemma,cat,[interp])) :: l else l
+            with Not_found -> l)
+      | {token=Dig _} -> l (* FIXME: todo *)
+      | {token=RomanDig(_,"roman")} ->
+          if ncat = "adj" then (Lemma(nlemma,ncat,[ninterp])) :: l else l
+      | {token=Proper(lemma,cat,interp,_)} -> l (* FIXME: todo *)
+      | _ -> l) in
+  match sort_uniq l2 with
+    [] -> (*raise (LemmaNotMatched(nlemma,ncat,ninterp,l))*)
+lowercase
+  | [t] -> t, TokenMatched
+  | _ -> raise (MultipleLemmaMatched(nlemma,ncat,ninterp,l2))*)
+
+(*  try
     let nlemma = correct_nlemma nlemma in
     let nl = lemmatize_string nlemma in
     let nl2 = Xlist.fold nl [] (fun nl -> function
         {token=Lemma(lemma,cat,interp)} ->
           Xlist.fold interp nl (fun nl interp ->
             try
-              let cat,interp = get_cat_interp (ncat,cat,ninterp,interp) in
-              if lemma = nlemma then (Lemma(lemma,cat,[interp])) :: nl else nl
+              if lemma = nlemma then
+                let cat,interp = get_cat_interp (ncat,cat,ninterp,interp) in
+                (Lemma(lemma,cat,[interp])) :: nl else
+              let lemma,cat,interp = get_lemma_cat_interp (nlemma,lemma,ncat,cat,ninterp,interp) in
+              (Lemma(lemma,cat,[interp])) :: nl
             with Not_found -> nl)
       | {token=Dig _} -> nl (* FIXME: todo *)
+      | {token=RomanDig(_,"roman")} ->
+          if ncat = "adj" then (Lemma(nlemma,ncat,[ninterp])) :: nl else nl
       | {token=Proper(lemma,cat,interp,_)} -> nl (* FIXME: todo *)
       | _ -> nl) in
-    if nl2 = [] then StringQMap.add stats (ENIAMtokens.string_of_token (Lemma(nlemma,ncat,[ninterp])) ^ ": " ^ String.concat " " (Xlist.map nl (fun t -> ENIAMtokens.string_of_token t.token)))
+    if nl2 = [] then
+      StringQMap.add_val stats (ncat ^ " " ^ ENIAMtokens.string_of_token (Lemma(nlemma,ncat,[ninterp])) ^ ": " ^ String.concat " " (Xlist.map nl (fun t -> ENIAMtokens.string_of_token t.token))
+        ^ "\n" ^ name ^ " " ^ string_of_int id_div ^ " " ^ string_of_int t.beg ^ " " ^ t.orth ^ "\n" ^ paragraph) q
+    (* if nl2 = [] then StringQMap.add_val stats (ENIAMtokens.string_of_token (Lemma(nlemma,ncat,[ninterp])) ^ ": " ^ String.concat " " (Xlist.map nl (fun t -> ENIAMtokens.string_of_token t.token))) q *)
     (* let l2 = Xlist.fold l [] (fun l2 t2 ->
       match t2.token with
         Lemma(lemma,cat,interp) -> if lemma = nlemma (*|| lemma = lowercase nlemma t.token*) then t2 :: l2 else l2
       (* | Proper(lemma,cat,interp,_) -> if lemma = nlemma || lemma = lowercase nlemma t.token then t2 :: l2 else l2 *)
       | _  -> l2) in
     if l2 = [] then StringQMap.add stats ("no lemma: " ^ t.orth ^ " " ^ nlemma) else *)
-    else StringQMap.add stats "lemmatized"
+    else StringQMap.add_val stats "lemmatized" q
 (*  let l3 = Xlist.fold l2 [] (fun l3 t ->
     match t.token with
       Lemma(lemma2,cat2,interp2) -> if cat = cat2 then t :: l3 else l3
@@ -308,12 +585,30 @@ let match_lemmatize stats t =
   | [{token=Lemma _};{token=SmallLetter _}] -> stats
   | [{token=Lemma _};{token=FirstCap _}] -> stats
   | l -> StringQMap.add stats ("multiple interp: " ^ t.orth ^ " " ^ lemma ^ " " ^ cat ^ "\n" ^ String.concat "\n" (Xlist.map l ENIAMtokens.string_of_token_env))*)
-  with Not_found -> StringQMap.add stats "no ntoken" (*("no ntoken for: " ^ t.orth ^ " " ^ ENIAMtokens.string_of_token t.token)*)
+  with Not_found -> StringQMap.add_val stats "no ntoken/incorrect" q
+  (* with Not_found -> StringQMap.add_val stats "no ntoken" q (*("no ntoken for: " ^ t.orth ^ " " ^ ENIAMtokens.string_of_token t.token)*) *)*)
  
-let rec validate_token stats = function
-    Token t -> match_lemmatize stats t
-  | Seq l -> Xlist.fold l stats validate_token
-  | Variant l -> Xlist.fold l stats validate_token
+let rec validate_token name id_div paragraph stats = function
+    Token t ->
+      (* if t.orth = "POWIŚLE" then Printf.printf "%s %d %s\n%s\n" name id_div paragraph (ENIAMtokens.string_of_token_env t); *)
+      (try let _,f = match_lemmatize (*stats 1 name id_div paragraph*) t in
+        match f with
+          TokenMatched -> StringQMap.add stats "validated"
+        | TokenLowercase -> StringQMap.add stats "validated as lowercase"
+        | TokenBrev -> StringQMap.add stats "validated abbreviation"
+        | TokenSubstGender -> StringQMap.add stats "validated substgender"
+        | TokenDeviated -> StringQMap.add stats "validated deviated"
+      with
+        HasBrev -> StringQMap.add stats ("has brev: " ^ t.orth (*^ " " ^ lemma ^ " " ^ cat ^ "\n"*))
+      (* | NoNtoken -> StringQMap.add stats ("no ntoken: " ^ t.orth (*^ " " ^ lemma ^ " " ^ cat ^ "\n"*)) *)
+      | NoNtoken -> StringQMap.add stats "no ntoken"
+      | LemmaNotMatched(nlemma,ncat,ninterp,l) ->
+          (* StringQMap.add stats (Printf.sprintf "lemma not matched: %s %s : %s \n%s" t.orth (ENIAMtokens.string_of_token (Lemma(nlemma,ncat,[ninterp]))) (String.concat " " (Xlist.map l (fun t -> ENIAMtokens.string_of_token t.token))) paragraph) *)
+          StringQMap.add stats (Printf.sprintf "%s %s %s %d %s\n#%s\n#%s" ncat t.orth name id_div (ENIAMtokens.string_of_token (Lemma(nlemma,ncat,[ninterp])))
+          (String.concat " " (Xlist.map l (fun t -> ENIAMtokens.string_of_token t.token))) paragraph)
+      | MultipleLemmaMatched(nlemma,ncat,ninterp,l) -> StringQMap.add stats (Printf.sprintf "multiple lemma matched: %s %s : %s" t.orth (ENIAMtokens.string_of_token (Lemma(nlemma,ncat,[ninterp]))) (String.concat " " (Xlist.map l (fun t -> ENIAMtokens.string_of_token t)))))
+  | Seq l -> Xlist.fold l stats (validate_token name id_div paragraph)
+  | Variant l -> Xlist.fold l stats (validate_token name id_div paragraph)
  
 let validate_morphology stats name typ channel entries =
   prerr_endline name;
@@ -323,7 +618,7 @@ let validate_morphology stats name typ channel entries =
       (* print_endline paragraph; *)
       (*let s = "W Specjalnym Ośrodku Szkolno-Wychowawczym" in
       if String.length paragraph >= String.length s && String.sub paragraph 0 (String.length s) = s then*)
-        Xlist.fold tokens stats validate_token
+        Xlist.fold tokens stats (validate_token name id_div paragraph)
       (*else stats*)))
  
 let ntokens_filename = "results/ntokens.tab"
@@ -333,7 +628,7 @@ let parse_ninterp s =
  
 let fold_ntokens ntokens_filename s f =
   File.fold_tab ntokens_filename s (fun s -> function
-      [_;nlemma;ncat;ninterp] -> f s (nlemma,ncat,parse_ninterp ninterp)
+      [q;nlemma;ncat;ninterp] -> f s (int_of_string q) (nlemma,ncat,parse_ninterp ninterp)
     | l -> failwith ("fold_ntokens: " ^ String.concat "\t" l))
  
 let selection = StringSet.of_list [(*"Rzeczpospolita";"200-4-000014";"040-2-000007";"120-2-900126";"120-2-910000001";"120-2-910000002";"120-4-900005";
@@ -371,12 +666,15 @@ let _ =
     create_ntoken_list stats name typ channel entries) in *)
   (* let stats = ENIAM_NKJP.fold ENIAM_NKJP.nkjp_path StringQMap.empty (fun stats (name,typ,channel,entries) ->
     create_ntoken_list stats name typ channel entries) in *)
-  let stats = fold_ntokens ntokens_filename StringQMap.empty validate_ntoken in
+  (* let stats = fold_ntokens ntokens_filename StringQMap.empty validate_ntoken in *)
+  (* let stats = ENIAM_NKJP.fold_selected ENIAM_NKJP.nkjp_path selection [] [] StringQMap.empty (fun stats (name,typ,channel,entries) ->
+    validate_ntoken_entry stats name typ channel entries) in *)
   (* let stats = ENIAM_NKJP.fold_selected ENIAM_NKJP.nkjp_path selection [] [] StringQMap.empty (fun stats (name,typ,channel,entries) ->
     validate_morphology stats name typ channel entries) in *)
-  (* let stats = ENIAM_NKJP.fold ENIAM_NKJP.nkjp_path StringQMap.empty (fun stats (name,typ,channel,entries) ->
-    validate_morphology stats name typ channel entries) in *)
+  let stats = ENIAM_NKJP.fold ENIAM_NKJP.nkjp_path StringQMap.empty (fun stats (name,typ,channel,entries) ->
+    validate_morphology stats name typ channel entries) in
   let stats = StringQMap.fold stats [] (fun stats k v -> (v,k) :: stats) in
   Xlist.iter (Xlist.sort stats compare) (fun (v,k) -> Printf.printf "%d\t%s\n" v k);
+  flush stdout;
   ignore(Sys.command "mpg123 \"../../Inne/gong/gong_00m_30s.mp3\"");
   ()
@@ -609,7 +609,7 @@ let rec annotate_paragraph name paragraph l = function
       with Not_found -> (try
         let m,ets,l = annotate_apply_rules (et :: ets) l rules in
         m :: annotate_paragraph name paragraph l ets
-      with Not_found -> failwith "annotate_paragraph 1")))
+      with Not_found -> (*print_endline ("annotate_paragraph 1: " ^ (string_of_vtoken et));*)failwith "annotate_paragraph 1")))
   | [] -> if l = [] then [] else failwith "annotate_paragraph 2"
  
 let validate_segmentation stats name typ channel entries =
@@ -713,12 +713,15 @@ let transform_nkjp_interp cat interp1 =
   | "prep" | "adv" | "fin" | "inf" | "imps" | "pcon" | "bedzie" | "impt" | "siebie" | "aglt" | "pant" | "brev" | "qub" -> interp
   | _ -> print_endline ("transform_nkjp_interp: " ^ cat ^ " " ^ String.concat ":" interp1); interp
  
+let transform_nkjp_interp_simple cat interp1 =
+  Xlist.map interp1 (fun s -> [s])
+
 let merge_token = function
     t,[] -> Token t
   | t,[{ncat="brev"} as n] -> set_sent n.nsent {t with attrs=BrevLemma n.nlemma :: t.attrs}
   | t,[n] ->
       if n.nlemma = "+/-" then set_sent n.nsent t else
-      if is_lemmatizable t.token then set_sent n.nsent {t with attrs=Disamb(n.nlemma,n.ncat,transform_nkjp_interp n.ncat n.ninterp) :: t.attrs}
+      if is_lemmatizable t.token then set_sent n.nsent {t with attrs=Disamb(n.nlemma,n.ncat,transform_nkjp_interp_simple n.ncat n.ninterp) :: t.attrs}
       else set_sent n.nsent t
   | _ -> failwith "merge_token"
  
@@ -732,7 +735,7 @@ let merge_letni l seq =
   match List.rev seq with
     last :: l ->
       let attrs = if n.nsent=SentEnd || n.nsent=SentBegEnd then (SentEnd : attr) :: last.attrs else last.attrs in
-      Seq(Xlist.rev_map ({last with attrs=Disamb(lemma,n.ncat,transform_nkjp_interp n.ncat n.ninterp) :: attrs} :: l) (fun t -> Token t))
+      Seq(Xlist.rev_map ({last with attrs=Disamb(lemma,n.ncat,transform_nkjp_interp_simple n.ncat n.ninterp) :: attrs} :: l) (fun t -> Token t))
   | _ -> failwith "merge_letni"
  
 let blabla_orths = StringSet.of_list ["8.12"; "9.11"; "1.1"; "1.2"]
@@ -751,7 +754,7 @@ let merge_paragraph name = function
   | AR("brev",variants,l) -> Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map (set_sent_list ets l) (fun t -> Token t))))
   | AR("both-correct",variants,l) -> Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map (set_sent_list ets l) (fun t -> Token t))))
   | AR("eniam-correct",variants,l) -> Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map (set_sent_list ets l) (fun t -> Token t))))
-  | AR("nkjp-correct",variants,l) -> Seq(Xlist.map l (fun n -> set_sent n.nsent {empty_token_env with orth=n.north; token=Lemma(n.nlemma,n.ncat,[transform_nkjp_interp n.ncat n.ninterp])})) (* FIXME: ustalenie beg len next *)
+  | AR("nkjp-correct",variants,l) -> Seq(Xlist.map l (fun n -> set_sent n.nsent {empty_token_env with orth=n.north; token=Lemma(n.nlemma,n.ncat,[transform_nkjp_interp_simple n.ncat n.ninterp])})) (* FIXME: ustalenie beg len next *)
   | t -> failwith ("merge_paragraph: " ^ string_of_atoken t)
  
 let test_annotate name typ channel entries =
@@ -783,6 +786,60 @@ let test_annotate name typ channel entries =
       (* print_endline (String.concat "\n" (Xlist.map m string_of_atoken))); *)
       ()))
  
+type cap = Capital | Small | Sign
+
+let classify_cap s =
+   match Xunicode.classified_chars_of_utf8_string s with
+        Xunicode.Capital _ :: _ -> Capital
+      | Xunicode.ForeignCapital _ :: _ -> Capital
+      | Xunicode.Small _ :: _ -> Small
+      | Xunicode.ForeignSmall _ :: _ -> Small
+      | _ -> Sign
+
+let rec get_ntoken = function
+    (Disamb(nlemma,ncat,ninterp) : attr) :: _ -> nlemma,ncat,ninterp
+  | _ :: l -> get_ntoken l
+  | [] -> raise Not_found
+
+let rec disambiguate_capitalics = function
+    Token t ->
+      (try
+        let nlemma,ncat,ninterp = get_ntoken t.attrs in
+        let c = match t.token, classify_cap nlemma with
+          ENIAMtokenizerTypes.SmallLetter _, Small -> true
+        | ENIAMtokenizerTypes.CapLetter _, Capital -> true
+        | ENIAMtokenizerTypes.AllSmall _ , Small-> true
+        | ENIAMtokenizerTypes.AllCap _, Capital -> true
+        (* | ENIAMtokenizerTypes.AllCap _, Small -> true *)
+        | ENIAMtokenizerTypes.FirstCap _, Capital -> true
+        | ENIAMtokenizerTypes.SomeCap _, Capital -> true
+        | ENIAMtokenizerTypes.SomeCap _, Small -> true
+        | ENIAMtokenizerTypes.RomanDig _, Capital -> true
+        | ENIAMtokenizerTypes.Interp _, _ -> true
+        | ENIAMtokenizerTypes.Symbol _, _ -> true
+        | ENIAMtokenizerTypes.Dig _, _ -> true
+        | ENIAMtokenizerTypes.Other _, _ -> true
+        | ENIAMtokenizerTypes.Lemma _, _ -> true
+        | ENIAMtokenizerTypes.Proper _, _ -> true
+        | ENIAMtokenizerTypes.Compound _, _ -> true
+        | ENIAMtokenizerTypes.Tokens _, _ -> true
+        | _ -> false in
+        Token t, c
+        (* let nc = classify_cap nlemma in
+        let no = classify_cap t.orth in
+        if no = nc then Token t,true else Token t,false *)
+      with Not_found -> Token t,true)
+  | Seq l ->
+      let l,c = Xlist.fold l ([],true) (fun (l,c) t ->
+        let t,d = disambiguate_capitalics t in
+        t :: l, c && d) in
+      Seq(List.rev l), c
+  | Variant l ->
+      let l2 = Xlist.fold l [] (fun l t ->
+        let t,d = disambiguate_capitalics t in
+        if d then t :: l else l) in
+      if l2 = [] then Variant l,false else Variant l2,true
+
 let annotate name sentences =
   let tokens = flatten_sentences sentences in
   let tokens = simple_allign "" "" [] tokens in
@@ -793,8 +850,21 @@ let annotate name sentences =
   let eniam_tokens = annotate_variants_par eniam_tokens in
   let m = annotate_paragraph name paragraph tokens eniam_tokens in
   let m = List.rev (Xlist.rev_map m (merge_paragraph name)) in
+  let m = List.rev (Xlist.fold m [] (fun m t ->
+    let t,_ = disambiguate_capitalics t in
+    t :: m)) in
   paragraph, m
  
+let test_disambiguate_capitalics stats name typ channel entries =
+  prerr_endline name;
+  Xlist.fold entries stats (fun stats (id_div,has_ne,paragraphs) ->
+    Xlist.fold paragraphs stats (fun stats (paragraph,sentences) ->
+      let paragraph,tokens = annotate name sentences in
+      Xlist.fold tokens stats (fun stats t ->
+        let _,c = disambiguate_capitalics t in
+        if c then stats else StringQMap.add stats (Printf.sprintf "%s %s" (ENIAMtokens.string_of_tokens 0 t) paragraph))))
+
+
 let selection = StringSet.of_list [(*"Rzeczpospolita";"200-4-000014";"040-2-000007";"120-2-900126";"120-2-910000001";"120-2-910000002";"120-4-900005";
 "620-3-010001110";"620-3-010001449";"620-3-010001622";"620-3-010001727";
 "620-3-010001731";"620-3-010001741";"620-3-010001854";"711-3-010000051";"711-3-010000056";
@@ -839,6 +909,8 @@ let _ =
     test_annotate name typ channel entries); *)
   (* ENIAM_NKJP.fold ENIAM_NKJP.nkjp_path () (fun () (name,typ,channel,entries) ->
     test_annotate name typ channel entries); *)
+  (* let stats = ENIAM_NKJP.fold_selected ENIAM_NKJP.nkjp_path selection [] [] StringQMap.empty (fun stats (name,typ,channel,entries) ->
+    test_disambiguate_capitalics stats name typ channel entries) in *)
   (* let stats = StringQMap.fold stats [] (fun stats k v -> (v,k) :: stats) in
   Xlist.iter (Xlist.sort stats compare) (fun (v,k) -> Printf.printf "%d\t%s\n" v k); *)
   (* ignore(Sys.command "mpg123 \"../../Inne/gong/gong_00m_30s.mp3\""); *)
@@ -2,4 +2,5 @@ się	się	qub
 siebie	siebie	siebie:acc.gen
 sobie	siebie	siebie:dat.loc
 sobą	siebie	siebie:inst
+to	to	pred