Początek walidacji lematyzacji

Wojciech Jaworski
1 parent 644020fc
Showing 8 changed files with 347 additions and 59 deletions
NKJP2/data/eniam-correct.tab
NKJP2/makefile
NKJP2/validateMorphology.ml
NKJP2/validateTokenizer.ml
documentation/motto.txt
subsyntax/ENIAMsubsyntax.ml
tokenizer/ENIAMtokenizerTypes.ml
tokenizer/ENIAMtokens.ml
@@ -33,3 +33,4 @@ rowecki@wp.pl	rowecki @ wp . pl
 28-29	28 -29
 Praca ˝	Praca˝
 marzycielem -	marzycielem-
+:))))))	:) ) ) ) ) )
@@ -6,7 +6,7 @@ OCAMLFLAGS=$(INCLUDES) -g
 OCAMLOPTFLAGS=$(INCLUDES) unix.cmxa xml-light.cmxa str.cmxa nums.cmxa zip.cmxa bz2.cmxa xlib.cmxa eniam-tokenizer.cmxa eniam-morphology.cmxa eniam-subsyntax.cmxa
 INSTALLDIR=`ocamlc -where`/eniam
  
-SOURCES=ENIAM_NKJP.ml validateTokenizer.ml #validateMorphology.ml #validateSubsyntax.ml
+SOURCES=ENIAM_NKJP.ml validateTokenizer.ml validateMorphology.ml #validateSubsyntax.ml
  
 all: $(SOURCES)
 	$(OCAMLOPT) -o test $(OCAMLOPTFLAGS) $^
@@ -21,7 +21,33 @@ open ENIAMtokenizerTypes
 open Xstd
 open ValidateTokenizer
  
-let rec select_interp = function (* przejście z m1 do m1.p1 *)
+let rec has_brev = function
+    BrevLemma _ :: _ -> true
+  | _ :: l -> has_brev l
+  | [] -> false
+
+let rec get_ntoken = function
+    (Disamb(nlemma,ncat,ninterp) : attr) :: _ -> nlemma,ncat,ninterp
+  | _ :: l -> get_ntoken l
+  | [] -> raise Not_found
+
+let rec add_ntoken stats = function
+    Token t ->
+      (try
+        let nlemma,ncat,ninterp = get_ntoken t.attrs in
+        StringQMap.add stats (nlemma ^ "\t" ^ ncat ^ "\t" ^ ENIAMtokens.string_of_interps [ninterp])
+      with Not_found -> stats)
+  | Seq l -> Xlist.fold l stats add_ntoken
+  | Variant l -> Xlist.fold l stats add_ntoken
+
+let create_ntoken_list stats name typ channel entries =
+  prerr_endline name;
+  Xlist.fold entries stats (fun stats (id_div,has_ne,paragraphs) ->
+    Xlist.fold paragraphs stats (fun stats (paragraph,sentences) ->
+      let paragraph,tokens = annotate name sentences in
+      Xlist.fold tokens stats add_ntoken))
+
+(* let rec select_interp = function (* przejście z m1 do m1.p1 *)
     "n" :: l,["n1"] :: ll -> ["n1"] :: (select_interp (l,ll))
   | "n" :: l,["n2"] :: ll -> ["n2"] :: (select_interp (l,ll))
   | "n" :: l,["p2"] :: ll -> ["p2"] :: (select_interp (l,ll))
@@ -39,7 +65,7 @@ let rec select_interp = function (* przejście z m1 do m1.p1 *)
   | "n" :: l,["_"] :: ll -> ["n1";"n2";"p2";"p3"] :: (select_interp (l,ll))
   | a :: l,al :: ll -> if Xlist.mem al a then [a] :: (select_interp (l,ll)) else raise Not_found
   | [],[] -> []
-  | _ -> raise Not_found
+  | _ -> raise Not_found *)
  
 let lowercase s = function
     AllSmall _ -> s
@@ -50,17 +76,210 @@ let lowercase s = function
       else failwith ("lowercase: " ^ s ^ " " ^ c)
   | t -> failwith ("lowercase: " ^ ENIAMtokens.string_of_token t)
  
-let match_lemmatize stats t lemma cat interp =
-  if cat = "brev" then stats else
-  if t.token = Symbol "." then stats else
+let lemmatize_string s =
+  let l = Xunicode.classified_chars_of_utf8_string s in
+  let l = ENIAMtokens.tokenize l in
+  let l = ENIAMpatterns.normalize_tokens [] l in
+  let l = match l with
+      [Token {token=Interp "<query>"};Variant l;Token {token=Interp "</query>"}] -> l
+    | [Token {token=Interp "<query>"};t;Token {token=Interp "</query>"}] -> [t]
+    | _ -> failwith ("lemmatize_string 1: " ^ s ^ " " ^ String.concat " " (Xlist.map l (fun t -> ENIAMtokens.string_of_tokens_simple t))) in
+  let l = Xlist.fold l [] (fun l -> function
+        Token ({token=AllSmall _} as t) -> t :: l
+      | Token ({token=SmallLetter _} as t) -> t :: l
+      | Token ({token=SomeCap _} as t) -> t :: l
+      | Token ({token=FirstCap _} as t) -> t :: l
+      | Token ({token=AllCap _} as t) -> t :: l
+      | Token ({token=CapLetter _} as t) -> t :: l
+      | Token ({token=RomanDig _}) -> (*print_endline ("lemmatize_string: " ^ s);*) (*t ::*) l
+      | Token ({token=Dig _} as t) -> (*print_endline ("lemmatize_string: " ^ s);*) t :: l
+      | Token ({token=Proper _} as t) -> t :: l
+      | Seq[Token {token=AllSmall _};Token {token=Lemma _}] -> l
+      | Seq[Token {token=SmallLetter _};Token {token=Lemma _}] -> l
+      | Seq[Token {token=FirstCap _};Token {token=Lemma _}] -> l
+      | Seq[Token {token=CapLetter _};Token {token=Lemma _}] -> l
+      | Seq[Token {token=SomeCap _};Token {token=Lemma _}] -> l
+      | Seq[Token {token=AllSmall _};Token {token=Lemma _};Token {token=Lemma _}] -> l
+      | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=AllSmall _}] -> l
+      | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=SmallLetter _}] -> l
+      | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=FirstCap _}] -> l
+      | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=AllCap _}] -> l
+      | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=CapLetter _}] -> l
+      | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=RomanDig _}] -> l
+      | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=Dig _}] -> l
+      | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=AllSmall _};Token {token=Lemma _}] -> l
+      | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=FirstCap _};Token {token=Lemma _}] -> l
+      | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=SmallLetter _};Token {token=Lemma _}] -> l
+      | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=CapLetter _};Token {token=Lemma _}] -> l
+      (* | Seq[Token {token=Interp "<sentence>"};Token {token=Interp "<clause>"};Token {token=AllSmall _};Token {token=Lemma _};Token {token=Lemma _}] -> l *)
+      | t -> failwith ("lemmatize_string 3: " ^ ENIAMtokens.string_of_tokens_simple t)) in
+  if l = [] then failwith "lemmatize_string 3" else
+  List.flatten (Xlist.map l ENIAMpaths.lemmatize_token)
+  (* match l with
+    [] -> failwith "lemmatize_string 2"
+  | [t] -> t
+  | _ -> Xlist.iter l (fun t -> print_endline (ENIAMtokens.string_of_tokens_simple t)); failwith "lemmatize_string 3" *)
+    (* Xlist.iter l (fun t -> print_endline (ENIAMtokens.string_of_tokens_simple t));
+    print_endline "";
+    Token empty_token_env *)
+
+let get_cat_interp = function
+    "subst","subst",[n;c;["m1"]],[_;_;["m1"]] -> "subst",[n;c;["m1"]]
+  | "subst","subst",[n;c;["m2"]],[_;_;["m2"]] -> "subst",[n;c;["m2"]]
+  | "subst","subst",[n;c;["m3"]],[_;_;["m3"]] -> "subst",[n;c;["m3"]]
+  | "subst","subst",[n;c;["n1";"n2"]],[_;_;["n1"]] -> "subst",[n;c;["n1"]]
+  | "subst","subst",[n;c;["n1";"n2"]],[_;_;["n2"]] -> "subst",[n;c;["n2"]]
+  | "subst","subst",[n;c;["f"]],[_;_;["f"]] -> "subst",[n;c;["f"]]
+  | "subst","subst",[n;c;["n1";"n2";"p2";"p3"]],[_;_;["n1"]] -> "subst",[n;c;["n1"]]
+  | "subst","subst",[n;c;["n1";"n2";"p2";"p3"]],[_;_;["n2"]] -> "subst",[n;c;["n2"]]
+  | "subst","subst",[n;c;["n1";"n2";"p2";"p3"]],[_;_;["p2"]] -> "subst",[n;c;["p2"]]
+  | "subst","subst",[n;c;["n1";"n2";"p2";"p3"]],[_;_;["p3"]] -> "subst",[n;c;["p3"]]
+  | "subst","subst",[n;c;["m1";"p1"]],[_;_;["m1"]] -> "subst",[n;c;["m1"]]
+  | "subst","subst",[n;c;["m1";"p1"]],[_;_;["p1"]] -> "subst",[n;c;["p1"]]
+  | "depr","subst",[["pl"];["nom"];["m2"]],[["sg"];["nom"];["m1"]] -> "depr",[["pl"];["nom"];["m2"]]
+  | "ppron3","ppron3",ninterp,[["sg"];["nom"];["m1";"m2";"m3"];["ter"];_;_] -> "ppron3",ninterp
+  | "ppron12","ppron12",ninterp,[_;["nom"];_;_] -> "ppron3",ninterp
+  | "numcol","num",ninterp,_ -> "num",ninterp (* FIXME: wiele wpisów przejdzie *)
+  | "num","num",ninterp,_ -> "num",ninterp (* FIXME: wiele wpisów przejdzie *)
+  | "siebie","siebie",[[c]],[["acc";"gen"]] -> "siebie",[[c]]
+  | "adj","adj",ninterp,[["sg"];["nom";"voc"];["m1";"m2";"m3"];["pos"]] -> "adj",ninterp
+  | "adja","adj",ninterp,[["sg"];["nom";"voc"];["m1";"m2";"m3"];["pos"]] -> "adja",ninterp
+  | "adjc","adj",ninterp,[["sg"];["nom";"voc"];["m1";"m2";"m3"];["pos"]] -> "adjc",ninterp
+  | "adjp","adj",ninterp,[["sg"];["nom";"voc"];["m1";"m2";"m3"];["pos"]] -> "adjp",ninterp
+  | "adv","adv",[[g]],[["pos"]] -> "adv",[[g]]
+  | "adv","adv",ninterp,interp -> if ninterp = interp then "adv",ninterp else raise Not_found
+  | "prep","prep",ninterp,interp -> if ninterp = interp then "prep",ninterp else raise Not_found
+  | "qub","qub",ninterp,interp -> if ninterp = interp then "qub",ninterp else raise Not_found
+  | "conj","conj",ninterp,interp -> if ninterp = interp then "conj",ninterp else raise Not_found
+  | "comp","comp",ninterp,interp -> if ninterp = interp then "comp",ninterp else raise Not_found
+  | "interj","interj",ninterp,interp -> if ninterp = interp then "interj",ninterp else raise Not_found
+  | "burk","burk",ninterp,interp -> if ninterp = interp then "burk",ninterp else raise Not_found
+  | "pred","pred",ninterp,interp -> if ninterp = interp then "pred",ninterp else raise Not_found
+  | "fin","inf",[n;p;["imperf"]],[["imperf";"perf"]] -> "fin",[n;p;["imperf"]]
+  | "fin","inf",[n;p;["imperf"]],[["imperf"]] -> "fin",[n;p;["imperf"]]
+  | "fin","inf",[n;p;["perf"]],[["imperf";"perf"]] -> "fin",[n;p;["perf"]]
+  | "fin","inf",[n;p;["perf"]],[["perf"]] -> "fin",[n;p;["perf"]]
+  | "impt","inf",[n;p;["imperf"]],[["imperf";"perf"]] -> "impt",[n;p;["imperf"]]
+  | "impt","inf",[n;p;["imperf"]],[["imperf"]] -> "impt",[n;p;["imperf"]]
+  | "impt","inf",[n;p;["perf"]],[["imperf";"perf"]] -> "impt",[n;p;["perf"]]
+  | "impt","inf",[n;p;["perf"]],[["perf"]] -> "impt",[n;p;["perf"]]
+  | "inf","inf",[["imperf"]],[["imperf";"perf"]] -> "inf",[["imperf"]]
+  | "inf","inf",[["imperf"]],[["imperf"]] -> "inf",[["imperf"]]
+  | "inf","inf",[["perf"]],[["imperf";"perf"]] -> "inf",[["perf"]]
+  | "inf","inf",[["perf"]],[["perf"]] -> "inf",[["perf"]]
+  | "praet","inf",[n;g;["imperf"]],[["imperf";"perf"]] -> "praet",[n;g;["imperf"]]
+  | "praet","inf",[n;g;["imperf"]],[["imperf"]] -> "praet",[n;g;["imperf"]]
+  | "praet","inf",[n;g;["perf"]],[["imperf";"perf"]] -> "praet",[n;g;["perf"]]
+  | "praet","inf",[n;g;["perf"]],[["perf"]] -> "praet",[n;g;["perf"]]
+  | "praet","inf",[n;g;["imperf"];a],[["imperf";"perf"]] -> "praet",[n;g;["imperf"];a]
+  | "praet","inf",[n;g;["imperf"];a],[["imperf"]] -> "praet",[n;g;["imperf"];a]
+  | "praet","inf",[n;g;["perf"];a],[["imperf";"perf"]] -> "praet",[n;g;["perf"];a]
+  | "praet","inf",[n;g;["perf"];a],[["perf"]] -> "praet",[n;g;["perf"];a]
+  | "winien","inf",[n;g;["imperf"]],[["imperf"]] -> "winien",[n;g;["imperf"]]
+  | "ppas","inf",[n;c;g;["imperf"];a],[["imperf";"perf"]] -> "ppas",[n;c;g;["imperf"];a]
+  | "ppas","inf",[n;c;g;["imperf"];a],[["imperf"]] -> "ppas",[n;c;g;["imperf"];a]
+  | "ppas","inf",[n;c;g;["perf"];a],[["imperf";"perf"]] -> "ppas",[n;c;g;["perf"];a]
+  | "ppas","inf",[n;c;g;["perf"];a],[["perf"]] -> "ppas",[n;c;g;["perf"];a]
+  | "pact","inf",[n;c;g;["imperf"];a],[["imperf";"perf"]] -> "pact",[n;c;g;["imperf"];a]
+  | "pact","inf",[n;c;g;["imperf"];a],[["imperf"]] -> "pact",[n;c;g;["imperf"];a]
+  | "pact","inf",[n;c;g;["perf"];a],[["imperf";"perf"]] -> "pact",[n;c;g;["perf"];a]
+  | "pact","inf",[n;c;g;["perf"];a],[["perf"]] -> "pact",[n;c;g;["perf"];a]
+  | "pant","inf",[["imperf"]],[["imperf";"perf"]] -> "pant",[["imperf"]]
+  | "pant","inf",[["imperf"]],[["imperf"]] -> "pant",[["imperf"]]
+  | "pant","inf",[["perf"]],[["imperf";"perf"]] -> "pant",[["perf"]]
+  | "pant","inf",[["perf"]],[["perf"]] -> "pant",[["perf"]]
+  | "pcon","inf",[["imperf"]],[["imperf";"perf"]] -> "pcon",[["imperf"]]
+  | "pcon","inf",[["imperf"]],[["imperf"]] -> "pcon",[["imperf"]]
+  | "pcon","inf",[["perf"]],[["imperf";"perf"]] -> "pcon",[["perf"]]
+  | "pcon","inf",[["perf"]],[["perf"]] -> "pcon",[["perf"]]
+  | "ger","inf",[n;c;g;["imperf"];a],[["imperf";"perf"]] -> "ger",[n;c;g;["imperf"];a]
+  | "ger","inf",[n;c;g;["imperf"];a],[["imperf"]] -> "ger",[n;c;g;["imperf"];a]
+  | "ger","inf",[n;c;g;["perf"];a],[["imperf";"perf"]] -> "ger",[n;c;g;["perf"];a]
+  | "ger","inf",[n;c;g;["perf"];a],[["perf"]] -> "ger",[n;c;g;["perf"];a]
+  | "imps","inf",[["imperf"]],[["imperf";"perf"]] -> "imps",[["imperf"]]
+  | "imps","inf",[["imperf"]],[["imperf"]] -> "imps",[["imperf"]]
+  | "imps","inf",[["perf"]],[["imperf";"perf"]] -> "imps",[["perf"]]
+  | "imps","inf",[["perf"]],[["perf"]] -> "imps",[["perf"]]
+  | _ -> raise Not_found
+
+let correct_nlemma = function
+    "letnia  " -> "letnia"
+  | "10minutowy" -> "minutowy"
+  | "23-letni" -> "letni"
+  | "40--letni" -> "letni"
+  | "5minutowy" -> "minutowy"
+  | "10-ta" -> (*"10."*)raise Not_found
+  | "10-tej" -> (*"10."*)raise Not_found
+  | "13-letni" -> "letni"
+  | "itineraryjny " -> "itineraryjny"
+  | "Składowy " -> "Składowy"
+  | "tak " -> "tak"
+  | "letni " -> "letni"
+  | "Kaznodziey'a" -> raise Not_found
+  | "Naczelna Rada Łowiecka" -> raise Not_found
+  | "PR-owy" -> raise Not_found
+  | "starać się" -> raise Not_found
+  | "vis-à-vis" -> raise Not_found
+  | "Ewangelia wg św. Jana" -> raise Not_found
+  | "`a" -> raise Not_found
+  | "6-piętrowy" -> "piętrowy"
+  | "6-letni" -> "letni"
+  | "5—lampowy" -> "lampowy"
+  | "4-piętrowy" -> "piętrowy"
+  | "3-centymetrowy" -> "centymetrowy"
+  | "34-letni" -> "letni"
+  | "18-ka" -> (*"18"*)raise Not_found
+  | "185-osobowy" -> "osobowy"
+  | "16-latek" -> raise Not_found
+  | s -> s
+
+let process_ntoken stats nlemma ncat ninterp =
+  try
+    let nlemma = correct_nlemma nlemma in
+    let nl = lemmatize_string nlemma in
+    let nl2 = Xlist.fold nl [] (fun nl -> function
+        {token=Lemma(lemma,cat,interp)} ->
+          Xlist.fold interp nl (fun nl interp ->
+            try
+              let cat,interp = get_cat_interp (ncat,cat,ninterp,interp) in
+              if lemma = nlemma then (Lemma(lemma,cat,[interp])) :: nl else nl
+            with Not_found -> nl)
+      | {token=Dig _} -> nl (* FIXME: todo *)
+      | {token=Proper(lemma,cat,interp,_)} -> nl (* FIXME: todo *)
+      | _ -> nl) in
+    if nl2 = [] then StringQMap.add stats (ncat ^ " " ^ ENIAMtokens.string_of_token (Lemma(nlemma,ncat,[ninterp])) ^ ": " ^ String.concat " " (Xlist.map nl (fun t -> ENIAMtokens.string_of_token t.token)))
+    else StringQMap.add stats "lemmatized"
+  with Not_found -> StringQMap.add stats "incorrect"
+
+let validate_ntoken stats (nlemma,ncat,ninterp) =
+  process_ntoken stats nlemma ncat ninterp
+
+let match_lemmatize stats t =
+  if has_brev t.attrs then StringQMap.add stats "brev" else
   let l = ENIAMpaths.lemmatize_token t in
-  let l2 = Xlist.fold l [] (fun l2 t2 ->
-    match t2.token with
-      Lemma(lemma2,cat2,interp2) -> if lemma = lemma2 || lemma = lowercase lemma2 t.token then t2 :: l2 else l2
-    | Proper(lemma2,cat2,interp2,_) -> if lemma = lemma2 || lemma = lowercase lemma2 t.token then t2 :: l2 else l2
-    | _  -> t2 :: l2) in
-  if l2 = [] then StringQMap.add stats ("no lemma: " ^ t.orth ^ " " ^ lemma) else
-  let l3 = Xlist.fold l2 [] (fun l3 t ->
+  try
+    let nlemma,ncat,ninterp = get_ntoken t.attrs in
+    let nlemma = correct_nlemma nlemma in
+    let nl = lemmatize_string nlemma in
+    let nl2 = Xlist.fold nl [] (fun nl -> function
+        {token=Lemma(lemma,cat,interp)} ->
+          Xlist.fold interp nl (fun nl interp ->
+            try
+              let cat,interp = get_cat_interp (ncat,cat,ninterp,interp) in
+              if lemma = nlemma then (Lemma(lemma,cat,[interp])) :: nl else nl
+            with Not_found -> nl)
+      | {token=Dig _} -> nl (* FIXME: todo *)
+      | {token=Proper(lemma,cat,interp,_)} -> nl (* FIXME: todo *)
+      | _ -> nl) in
+    if nl2 = [] then StringQMap.add stats (ENIAMtokens.string_of_token (Lemma(nlemma,ncat,[ninterp])) ^ ": " ^ String.concat " " (Xlist.map nl (fun t -> ENIAMtokens.string_of_token t.token)))
+    (* let l2 = Xlist.fold l [] (fun l2 t2 ->
+      match t2.token with
+        Lemma(lemma,cat,interp) -> if lemma = nlemma (*|| lemma = lowercase nlemma t.token*) then t2 :: l2 else l2
+      (* | Proper(lemma,cat,interp,_) -> if lemma = nlemma || lemma = lowercase nlemma t.token then t2 :: l2 else l2 *)
+      | _  -> l2) in
+    if l2 = [] then StringQMap.add stats ("no lemma: " ^ t.orth ^ " " ^ nlemma) else *)
+    else StringQMap.add stats "lemmatized"
+(*  let l3 = Xlist.fold l2 [] (fun l3 t ->
     match t.token with
       Lemma(lemma2,cat2,interp2) -> if cat = cat2 then t :: l3 else l3
     | Proper(lemma2,cat2,interp2,_) -> if cat = cat2 then t :: l3 else l3
@@ -88,27 +307,13 @@ let match_lemmatize stats t lemma cat interp =
   | [{token=Lemma _};{token=AllSmall _}] -> stats
   | [{token=Lemma _};{token=SmallLetter _}] -> stats
   | [{token=Lemma _};{token=FirstCap _}] -> stats
-  | l -> StringQMap.add stats ("multiple interp: " ^ t.orth ^ " " ^ lemma ^ " " ^ cat ^ "\n" ^ String.concat "\n" (Xlist.map l ENIAMtokens.string_of_token_env))
-
-let is_lemmatizable = function
-  | AllSmall _ -> true
-  | SmallLetter _ -> true
-  | FirstCap _ -> true
-  | AllCap _ -> true
-  | CapLetter _ -> true
-  | SomeCap _ -> true
-  | t -> false
-
-let validate_token stats = function
-    AT(t,[sent,orth,lemma,"brev",interp]) -> StringQMap.add stats "brev"
-  | AT(t,l(*[sent,orth,lemma,cat,interp]*)) ->
-      if is_lemmatizable t.token then
-        StringQMap.add stats "lemmatizable" else StringQMap.add stats "non lemmatizable"
-         (*match_lemmatize stats t lemma cat interp*)
-  (* | AT(_,l) as t -> StringQMap.add stats ("validate_token: " ^ string_of_atoken t)*)
-  | AV(tl,l) as t -> StringQMap.add stats ("validate_token: " ^ string_of_atoken t)
-  | AR(stat,tl,l) as t -> StringQMap.add stats ("validate_token: " ^ string_of_atoken t)
-  (* | _ -> StringQMap.add stats "validate_token: ni" *)
+  | l -> StringQMap.add stats ("multiple interp: " ^ t.orth ^ " " ^ lemma ^ " " ^ cat ^ "\n" ^ String.concat "\n" (Xlist.map l ENIAMtokens.string_of_token_env))*)
+  with Not_found -> StringQMap.add stats "no ntoken" (*("no ntoken for: " ^ t.orth ^ " " ^ ENIAMtokens.string_of_token t.token)*)
+
+let rec validate_token stats = function
+    Token t -> match_lemmatize stats t
+  | Seq l -> Xlist.fold l stats validate_token
+  | Variant l -> Xlist.fold l stats validate_token
  
 let validate_morphology stats name typ channel entries =
   prerr_endline name;
@@ -121,18 +326,28 @@ let validate_morphology stats name typ channel entries =
         Xlist.fold tokens stats validate_token
       (*else stats*)))
  
+let ntokens_filename = "results/ntokens.tab"
+
+let parse_ninterp s =
+  Xlist.map (Xstring.split ":" s) (fun s -> Xstring.split "\\." s)
+
+let fold_ntokens ntokens_filename s f =
+  File.fold_tab ntokens_filename s (fun s -> function
+      [_;nlemma;ncat;ninterp] -> f s (nlemma,ncat,parse_ninterp ninterp)
+    | l -> failwith ("fold_ntokens: " ^ String.concat "\t" l))
+
 let selection = StringSet.of_list [(*"Rzeczpospolita";"200-4-000014";"040-2-000007";"120-2-900126";"120-2-910000001";"120-2-910000002";"120-4-900005";
 "620-3-010001110";"620-3-010001449";"620-3-010001622";"620-3-010001727";
 "620-3-010001731";"620-3-010001741";"620-3-010001854";"711-3-010000051";"711-3-010000056";
 "711-3-010000079";"720-3-010000217";"720-3-010000335";"720-3-010000341";"forumowisko.pl_18535";"forumowisko.pl_424";"";"";"";"";"";"";"";"";"";"";"";"";"";"";"";"";"";*)
-  (*"040-2-000001";"040-2-000007";"040-4-000000103";"120-2-000003";"120-2-000007";"120-2-000009";"120-2-000010";"120-2-900017";"120-2-900041";"120-2-900044";"120-2-900083";
+  (* "040-2-000001";"040-2-000007";"040-4-000000103";"120-2-000003";"120-2-000007";"120-2-000009";"120-2-000010";"120-2-900017";"120-2-900041";"120-2-900044";"120-2-900083";
   "120-2-900092";"120-2-900094";"120-2-900123";"120-2-910000011";"120-4-900000001";"120-4-900008";"120-4-900010";"130-3-900001";"130-3-910001";"130-5-000000267";
   "130-5-000000406";"130-5-000000817";"130-5-000001188";"130-5-000001274";"130-5-000001338";"130-5-000001628";"130-5-000001742";"200-1-000011";"200-1-000026";"200-2-000078";
   "200-2-000173";"200-2-000175";"200-4-000000307";"200-4-000000316";"310-2-000007";"320-2-000000094";"320-2-000034";"320-2-000064";"320-3-000226";"330-2-000000030";
   "330-2-000000033";"330-2-000000200";"330-2-000000213";"330-2-000003";"330-2-000013";"620-3-010000057";"620-3-010000838";"620-3-010001103";"620-3-010001107";"620-3-010001108";
   "620-3-010001109";"620-3-010001125";"620-3-010001274";"620-3-010001448";"620-3-010001732";"620-3-010001772";"711-3-010000021";"712-1-900003";"712-1-900004";"720-3-000071";
   "720-3-010000323";"DP1999";"DP2002";"DP2003";"EkspressWieczorny";"forumowisko.pl_20218";"forumowisko.pl_42911";"forumowisko.pl_724";"GazetaGoleniowska";"GazetaTczewska";
-  "NIE";"SuperExpress";"TrybunaSlaska";*)
+  "NIE";"SuperExpress";"TrybunaSlaska"; *)
   (* "120-2-000009";"120-2-000010";"120-2-000012";"120-2-900019";"120-2-900041";"120-2-900044";"120-2-900092";"120-2-900123";"120-2-910000011";"120-4-900000001";"120-4-900001";
   "120-4-900008";"130-3-900001";"130-5-000000267";"130-5-000000817";"130-5-000001188";"130-5-000001274";"130-5-000001628";"130-5-000001635";"130-5-000001742";"200-1-000011";
   "200-2-000078";"200-2-000181";"200-4-000000314";"200-4-000026";"200-4-000059";"310-2-000007";"320-2-000000087";"320-2-000000094";"320-2-000034";"330-2-000013";"620-3-010000057";
@@ -152,10 +367,15 @@ let selection = StringSet.of_list [(*&quot;Rzeczpospolita&quot;;&quot;200-4-000014&quot;;&quot;040-2-0000
 let _ =
   ENIAMtokenizer.initialize ();
   ENIAMinflexion.initialize ();
-  let stats = ENIAM_NKJP.fold_selected ENIAM_NKJP.nkjp_path selection [] [] StringQMap.empty (fun stats (name,typ,channel,entries) ->
-    validate_morphology stats name typ channel entries) in
+  (* let stats = ENIAM_NKJP.fold_selected ENIAM_NKJP.nkjp_path selection [] [] StringQMap.empty (fun stats (name,typ,channel,entries) ->
+    create_ntoken_list stats name typ channel entries) in *)
+  (* let stats = ENIAM_NKJP.fold ENIAM_NKJP.nkjp_path StringQMap.empty (fun stats (name,typ,channel,entries) ->
+    create_ntoken_list stats name typ channel entries) in *)
+  let stats = fold_ntokens ntokens_filename StringQMap.empty validate_ntoken in
+  (* let stats = ENIAM_NKJP.fold_selected ENIAM_NKJP.nkjp_path selection [] [] StringQMap.empty (fun stats (name,typ,channel,entries) ->
+    validate_morphology stats name typ channel entries) in *)
   (* let stats = ENIAM_NKJP.fold ENIAM_NKJP.nkjp_path StringQMap.empty (fun stats (name,typ,channel,entries) ->
-    validate_segmentation stats name typ channel entries) in *)
+    validate_morphology stats name typ channel entries) in *)
   let stats = StringQMap.fold stats [] (fun stats k v -> (v,k) :: stats) in
   Xlist.iter (Xlist.sort stats compare) (fun (v,k) -> Printf.printf "%d\t%s\n" v k);
   ()
@@ -589,9 +589,9 @@ let rec match_and_combine name paragraph stats l = function
       with Not_found ->
         let e_tokens,n_tokens,ets,l = combine "" "" [] [] (et :: ets) l in
         (* let stats = StringQMap.add stats (string_of_eniam_token_orths e_tokens ^ "\t" ^ string_of_nkjp_token_orths n_tokens ^ "\t" ^ name) in *)
-        (* let stats = StringQMap.add stats (string_of_eniam_token_orths e_tokens ^ "\t" ^ string_of_nkjp_token_orths n_tokens ^ "\t" ^ paragraph) in *)
+        let stats = StringQMap.add stats (string_of_eniam_token_orths e_tokens ^ "\t" ^ string_of_nkjp_token_orths n_tokens ^ "\t" ^ paragraph) in
         (* let stats = StringQMap.add stats ("[\"" ^ string_of_eniam_token_orths2 e_tokens ^ "\"],[\"" ^ string_of_nkjp_token_orths2 n_tokens ^ "\"];" ^ "\t" ^ name) in *)
-        let stats = StringQMap.add stats ("[\"" ^ string_of_eniam_token_orths2 e_tokens ^ "\"],[\"" ^ string_of_nkjp_token_orths2 n_tokens ^ "\"];" ^ "\t" ^ paragraph) in
+        (* let stats = StringQMap.add stats ("[\"" ^ string_of_eniam_token_orths2 e_tokens ^ "\"],[\"" ^ string_of_nkjp_token_orths2 n_tokens ^ "\"];" ^ "\t" ^ paragraph) in *)
         match_and_combine name paragraph stats l ets)))
   | [] -> if l = [] then stats else StringQMap.add stats ("match_and_combine: " ^ name ^ "\t" ^ string_of_nkjp_token_orths l ^ "\t" ^ paragraph)
  
@@ -667,6 +667,16 @@ let set_sent sent t =
   | SentBegEnd  -> Token {t with attrs=SentBegEnd :: t.attrs}
   | Space -> failwith "set_sent"
  
+let set_sent_list ets l = (* FIXME: todo *)
+  (* print_endline (String.concat " " (Xlist.map l (fun n ->
+    match n.nsent with
+      SentBeg -> "B"
+    | SentEnd -> "E"
+    | Inside -> "I"
+    | SentBegEnd -> "BE"
+    | Space -> "S"))); *)
+  ets
+
 let rec allign rev = function
     {orth=""} as t :: ets,nts -> allign ((t,[]) :: rev) (ets,nts)
   | [{orth="."} as x;{orth="''"} as y],[{north="''"};{north="."}] -> List.rev rev @ [x,[];y,[]]
@@ -679,26 +689,69 @@ let rec allign rev = function
   | [],[] -> List.rev rev
   | _ -> failwith "allign 3"
  
+let transform_nkjp_interp cat interp1 =
+  if interp1 = [] then [] else
+  let interp = Xlist.map interp1 (fun s -> [s]) in
+  match cat with
+    "subst" | "ppron12" | "ppron3" | "ppas" | "pact" | "adj" | "num" | "depr" | "numcol" ->
+       (match interp with
+         ["sg"] :: case :: ["n"] :: l -> ["sg"] :: case :: ["n1";"n2"] :: l
+       | ["pl"] :: case :: ["n"] :: l -> ["pl"] :: case :: ["n1";"n2";"p2";"p3"] :: l
+       | ["pl"] :: case :: ["m1"] :: l -> ["pl"] :: case :: ["m1";"p1"] :: l
+       | l -> l)
+  | "ger" ->
+       (match interp with
+         num :: case :: ["n"] :: l -> num :: case :: ["n2"] :: l
+       | l -> l)
+  | "praet" | "winien" ->
+       (match interp with
+         ["sg"] :: ["n"] :: l -> ["sg"] :: ["n1";"n2"] :: l
+       | ["pl"] :: ["n"] :: l -> ["pl"] :: ["n1";"n2";"p2";"p3"] :: l
+       | ["pl"] :: ["m1"] :: l -> ["pl"] :: ["m1";"p1"] :: l
+       | l -> l)
+  | "prep" | "adv" | "fin" | "inf" | "imps" | "pcon" | "bedzie" | "impt" | "siebie" | "aglt" | "pant" | "brev" | "qub" -> interp
+  | _ -> print_endline ("transform_nkjp_interp: " ^ cat ^ " " ^ String.concat ":" interp1); interp
+
 let merge_token = function
     t,[] -> Token t
   | t,[{ncat="brev"} as n] -> set_sent n.nsent {t with attrs=BrevLemma n.nlemma :: t.attrs}
   | t,[n] ->
-      if is_lemmatizable t.token then set_sent n.nsent {t with attrs=Disamb(n.nlemma,n.ncat,n.ninterp) :: t.attrs}
+      if n.nlemma = "+/-" then set_sent n.nsent t else
+      if is_lemmatizable t.token then set_sent n.nsent {t with attrs=Disamb(n.nlemma,n.ncat,transform_nkjp_interp n.ncat n.ninterp) :: t.attrs}
       else set_sent n.nsent t
   | _ -> failwith "merge_token"
  
-let transform_nkjp_interp = function
-  | l -> (*print_endline ("transform_nkjp_interp: " ^ String.concat ":" l);*) Xlist.map l (fun s -> [s])
+let merge_letni l seq =
+  if l = [] then failwith "merge_letni" else
+  let n = List.hd (List.rev l) in
+  let lemma = List.hd (List.rev (Xstring.split "-" n.nlemma)) in
+  let seq = match seq with
+      first :: l -> if n.nsent=SentBeg || n.nsent=SentBegEnd then {first with attrs=SentBeg :: first.attrs} :: l else first :: l
+    | _ -> failwith "merge_letni" in
+  match List.rev seq with
+    last :: l ->
+      let attrs = if n.nsent=SentEnd || n.nsent=SentBegEnd then (SentEnd : attr) :: last.attrs else last.attrs in
+      Seq(Xlist.rev_map ({last with attrs=Disamb(lemma,n.ncat,transform_nkjp_interp n.ncat n.ninterp) :: attrs} :: l) (fun t -> Token t))
+  | _ -> failwith "merge_letni"
+
+let blabla_orths = StringSet.of_list ["8.12"; "9.11"; "1.1"; "1.2"]
+
+let is_blabla = function
+    [{north=s};{north="."}] -> StringSet.mem blabla_orths s (*then (print_endline ("blabla: " ^ s); true) else false*)
+  | _ -> false
  
 let merge_paragraph name = function
     AT(t,l) -> merge_token (t,l)
-  | AV(variants,l) as t -> (*print_endline (string_of_atoken t);*) Variant(Xlist.rev_map variants (fun ets ->
-      Seq(Xlist.map (allign [] (ets,l)) merge_token)))
-  | AR("tys",variants,l) -> Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map ets (fun t -> Token t))))
-  | AR("both-correct",variants,l) -> Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map ets (fun t -> Token t))))
-  | AR("eniam-correct",variants,l) -> Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map ets (fun t -> Token t))))
-  | AR("nkjp-correct",variants,l) -> Seq(Xlist.map l (fun n -> set_sent n.nsent {empty_token_env with orth=n.north; token=Lemma(n.nlemma,n.ncat,[transform_nkjp_interp n.ninterp])}))
-  | t -> (*print_endline (string_of_atoken t);*) Token empty_token_env
+  | AV(variants,l) ->
+      if is_blabla l then Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map (set_sent_list ets l) (fun t -> Token t)))) else
+      Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map (allign [] (ets,l)) merge_token)))
+  | AR("tys",variants,l) -> Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map (set_sent_list ets l) (fun t -> Token t))))
+  | AR("letni",variants,l) -> Variant(Xlist.rev_map variants (merge_letni l)) (*in print_endline (ENIAMtokens.string_of_tokens 0 t); t*)
+  | AR("brev",variants,l) -> Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map (set_sent_list ets l) (fun t -> Token t))))
+  | AR("both-correct",variants,l) -> Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map (set_sent_list ets l) (fun t -> Token t))))
+  | AR("eniam-correct",variants,l) -> Variant(Xlist.rev_map variants (fun ets -> Seq(Xlist.map (set_sent_list ets l) (fun t -> Token t))))
+  | AR("nkjp-correct",variants,l) -> Seq(Xlist.map l (fun n -> set_sent n.nsent {empty_token_env with orth=n.north; token=Lemma(n.nlemma,n.ncat,[transform_nkjp_interp n.ncat n.ninterp])})) (* FIXME: ustalenie beg len next *)
+  | t -> failwith ("merge_paragraph: " ^ string_of_atoken t)
  
 let test_annotate name typ channel entries =
   (* if name = "620-3-010001854" then prerr_endline "620-3-010001854 omited" else ( *)
@@ -723,7 +776,7 @@ let test_annotate name typ channel entries =
       (* print_endline "test_annotate 2"; *)
       let m = annotate_paragraph name paragraph tokens eniam_tokens in
       (* print_endline "test_annotate 3"; *)
-      (* check_annotation paragraph m; *)
+      check_annotation paragraph m;
       let _ = List.rev (Xlist.rev_map m (merge_paragraph name)) in
       ());
       (* print_endline (String.concat "\n" (Xlist.map m string_of_atoken))); *)
@@ -738,20 +791,21 @@ let annotate name sentences =
   let eniam_tokens = convert_eniam_tokens [] eniam_tokens in
   let eniam_tokens = annotate_variants_par eniam_tokens in
   let m = annotate_paragraph name paragraph tokens eniam_tokens in
+  let m = List.rev (Xlist.rev_map m (merge_paragraph name)) in
   paragraph, m
  
 let selection = StringSet.of_list [(*"Rzeczpospolita";"200-4-000014";"040-2-000007";"120-2-900126";"120-2-910000001";"120-2-910000002";"120-4-900005";
 "620-3-010001110";"620-3-010001449";"620-3-010001622";"620-3-010001727";
 "620-3-010001731";"620-3-010001741";"620-3-010001854";"711-3-010000051";"711-3-010000056";
 "711-3-010000079";"720-3-010000217";"720-3-010000335";"720-3-010000341";"forumowisko.pl_18535";"forumowisko.pl_424";"";"";"";"";"";"";"";"";"";"";"";"";"";"";"";"";"";*)
-  (* "040-2-000001";"040-2-000007";"040-4-000000103";"120-2-000003";"120-2-000007";"120-2-000009";"120-2-000010";"120-2-900017";"120-2-900041";"120-2-900044";"120-2-900083";
+  "040-2-000001";"040-2-000007";"040-4-000000103";"120-2-000003";"120-2-000007";"120-2-000009";"120-2-000010";"120-2-900017";"120-2-900041";"120-2-900044";"120-2-900083";
   "120-2-900092";"120-2-900094";"120-2-900123";"120-2-910000011";"120-4-900000001";"120-4-900008";"120-4-900010";"130-3-900001";"130-3-910001";"130-5-000000267";
   "130-5-000000406";"130-5-000000817";"130-5-000001188";"130-5-000001274";"130-5-000001338";"130-5-000001628";"130-5-000001742";"200-1-000011";"200-1-000026";"200-2-000078";
   "200-2-000173";"200-2-000175";"200-4-000000307";"200-4-000000316";"310-2-000007";"320-2-000000094";"320-2-000034";"320-2-000064";"320-3-000226";"330-2-000000030";
   "330-2-000000033";"330-2-000000200";"330-2-000000213";"330-2-000003";"330-2-000013";"620-3-010000057";"620-3-010000838";"620-3-010001103";"620-3-010001107";"620-3-010001108";
   "620-3-010001109";"620-3-010001125";"620-3-010001274";"620-3-010001448";"620-3-010001732";"620-3-010001772";"711-3-010000021";"712-1-900003";"712-1-900004";"720-3-000071";
   "720-3-010000323";"DP1999";"DP2002";"DP2003";"EkspressWieczorny";"forumowisko.pl_20218";"forumowisko.pl_42911";"forumowisko.pl_724";"GazetaGoleniowska";"GazetaTczewska";
-  "NIE";"SuperExpress";"TrybunaSlaska"; *)
+  "NIE";"SuperExpress";"TrybunaSlaska";
   (* "120-2-000009";"120-2-000010";"120-2-000012";"120-2-900019";"120-2-900041";"120-2-900044";"120-2-900092";"120-2-900123";"120-2-910000011";"120-4-900000001";"120-4-900001";
   "120-4-900008";"130-3-900001";"130-5-000000267";"130-5-000000817";"130-5-000001188";"130-5-000001274";"130-5-000001628";"130-5-000001635";"130-5-000001742";"200-1-000011";
   "200-2-000078";"200-2-000181";"200-4-000000314";"200-4-000026";"200-4-000059";"310-2-000007";"320-2-000000087";"320-2-000000094";"320-2-000034";"330-2-000013";"620-3-010000057";
@@ -764,7 +818,7 @@ let selection = StringSet.of_list [(*&quot;Rzeczpospolita&quot;;&quot;200-4-000014&quot;;&quot;040-2-0000
   "KurierKwidzynski";"NIE";"Rzeczpospolita";"TrybunaSlaska" *)
   (* "110-4-000000102";"120-2-000006";"120-2-900032";"130-5-000000507";"130-5-000001156";
   "620-3-010000835";"GazetaGoleniowska";"KurierKwidzynski";"NIE";"Rzeczpospolita"; *)
-  (*"110-4-000000102";"KurierKwidzynski";*)(*"620-3-010001496;"*)(*"130-5-000001341";*)(*"620-3-010001854"*)"620-3-010001106"
+  (*"110-4-000000102";"KurierKwidzynski";*)(*"620-3-010001496;"*)(*"130-5-000001341";*)(*"620-3-010001854"*)(*"620-3-010001106"*)
 ]
  
 let _ =
@@ -779,8 +833,8 @@ let _ =
     validate_segmentation stats name typ channel entries) in *)
   (* ENIAM_NKJP.fold_selected ENIAM_NKJP.nkjp_path selection [] [] () (fun () (name,typ,channel,entries) ->
     test_annotate name typ channel entries); *)
-  ENIAM_NKJP.fold ENIAM_NKJP.nkjp_path () (fun () (name,typ,channel,entries) ->
-    test_annotate name typ channel entries);
+  (* ENIAM_NKJP.fold ENIAM_NKJP.nkjp_path () (fun () (name,typ,channel,entries) ->
+    test_annotate name typ channel entries); *)
   (* let stats = StringQMap.fold stats [] (fun stats k v -> (v,k) :: stats) in
   Xlist.iter (Xlist.sort stats compare) (fun (v,k) -> Printf.printf "%d\t%s\n" v k); *)
   ()
+Jeśli wyznajesz jakąkolwiek religię, 
+żeby się oświecić, 
+musisz ją porzucić.
+Jeśli nie wierzysz w Boga,
+żeby się oświecić, 
+musisz dostrzec boskość przenikającą rzeczywistość.
+
+IAAM
@@ -241,6 +241,11 @@ let rec calculate_quality q = function
   | NotValProper :: l -> calculate_quality (q-1) l
   | LemmLowercase :: l -> calculate_quality q l
   | Roman :: l -> calculate_quality q l
+  | SentBeg :: l -> calculate_quality q l
+  | SentBegEnd :: l -> calculate_quality q l
+  | SentEnd :: l -> calculate_quality q l
+  | BrevLemma _ :: l -> calculate_quality q l
+  | Disamb _ :: l -> calculate_quality q l
   | [] -> q
  
 let select_tokens2 paths =
@@ -44,7 +44,7 @@ type attr =
     CS | MaybeCS | ReqValLemm | MWE | LemmNotVal | TokNotFound | NotValProper | LemmLowercase | Roman
   | SentBeg | SentEnd | SentBegEnd
   | BrevLemma of string
-  | Disamb of string * string * string list
+  | Disamb of string * string * string list list
  
 (* Tekst reprezentuję jako zbiór obiektów typu token_record zawierających
    informacje o poszczególnych tokenach *)
@@ -100,7 +100,7 @@ let string_of_attr = function
   | SentEnd -> "NKJP sentence end"
   | SentBegEnd -> "NKJP sentence begin-end"
   | BrevLemma s -> "NKJP brev lemma: " ^ s
-  | Disamb(lemma,cat,interp) -> "NKJP disamb: " ^ lemma ^ ":" ^ cat ^ ":" ^ String.concat ":" interp
+  | Disamb(lemma,cat,interp) -> "NKJP disamb: " ^ lemma ^ ":" ^ cat ^ ":" ^ String.concat ":" (Xlist.map interp (String.concat "."))
  
 let string_of_token_env t =
   sprintf "{orth=%s;beg=%d;len=%d;next=%d;token=%s;weight=%.2f;attrs=[%s]}" t.orth t.beg t.len t.next (string_of_token t.token) t.weight