algorytm selekcji lematów

Wojciech Jaworski
1 parent 74683cab
Showing 59 changed files with 266391 additions and 138 deletions
LCGlexicon/ENIAM_LCGlexicon.ml
LCGlexicon/ENIAM_LCGlexiconTypes.ml
LCGlexicon/ENIAMcategoriesPL.ml
LCGlexicon/resources/lexicon-pl.dic
NKJP2/validateMorphology.ml
corpora/CONLL2.ml
lexSemantics/ENIAMvalence.ml
lexSemantics/ENIAMwalRenderer.ml
lexSemantics/ENIAMwalStringOf.ml
lexSemantics/ENIAMwalTypes.ml
morphology/ENIAMinflexion.ml
morphology/ENIAMmorphologyTypes.ml
morphology/generate.ml
morphology/resources/lemmata.tab
semantics/ENIAMdisambiguation.ml
semantics/ENIAMsemGraph.ml
semantics/ENIAMsemTypes.ml
subsyntax/ENIAM_MWE.ml
subsyntax/ENIAMconcraft.ml
subsyntax/ENIAMpaths.ml
@@ -380,3 +380,9 @@ let create_entries rules id orth cats valence lex_entries =
         let rules = make_term id orth rules in
         (* print_endline "create_entries 5"; *)
         rules @ l)
+
+let initialize () =
+  ENIAMcategoriesPL.initialize ();
+  let filenames = [rules_filename; user_lexicon_filename] @ Xlist.map (!theories_paths) (fun path -> path ^ "/lexicon.dic") in
+  rules := make_rules_list false filenames;
+  dep_rules := make_rules_list true filenames
@@ -17,6 +17,8 @@
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *)
+open Xstd
+
 type categories = {lemma: string; pos: string; pos2: string;
                    cat: string; coerced: string list; roles: string list; snode: string list; phrase: string list;
                    numbers: string list; cases: string list; genders: string list; persons: string list;
@@ -126,3 +128,7 @@ let subst_time_lexemes_filename = resource_path ^ &quot;/LCGlexicon/subst_time.dat&quot;
 let adv_modes_filename = resource_path ^ "/Walenty/adv_modes.tab"
 let num_nsems_filename = resource_path ^ "/LCGlexicon/num.tab"
+
+let rules = ref (StringMap.empty : (entry list StringMap.t * entry list) StringMap.t)
+let dep_rules = ref (StringMap.empty : (entry list StringMap.t * entry list) StringMap.t)
+let theories_paths = ref ([] : string list)
@@ -29,7 +29,7 @@ let all_persons = [&quot;pri&quot;;&quot;sec&quot;;&quot;ter&quot;]
 let all_phrases = [
   "np";"adjp";"advp";"infp";"ip";
-  "prepnp";"cp";"ncp";"prepncp";"padvp";"colonp";"mp";"intp";
+  "prepnp";"cp";"ncp";"prepncp";"padvp";"colonp";"mp";"intp";"admod";
   "adja";"prepadjp";"comparp";"xp";"xpnom";"xpgen";"symbol";"fixed";
   "s";"<root>";"<sentence>";"<paragraph>";(*"";"";"";"";"";"";"";"";*)]
@@ -36,9 +36,6 @@ measure_weight=1
 @LEXICON
-pos=ordnum,phrase=adjp:
-  adjp*number*case*gender*grad*coerced*role*node;
-
 #oznaczenia godzin i minut
 pos=hour-minute|hour,phrase=np:
   QUANT[number=sg,case=nom&gen&dat&acc&inst&loc,gender=f,person=ter,role=0]
@@ -211,24 +208,6 @@ pos=subst,case=gen,nsyn=pronoun,cat!=Measure|imię|nazwisko,phrase=np:
    {distant-schema}{\np*unumber*case*ugender*uperson*Measure*Measure*concept,
   schema};
-# liczebniki
-#pos=num|intnum|realnum|intnum-interval|realnum-interval,nsem=count:
-#  QUANT[role=0]
-#  num*number*case*gender*person*acm*nsem*role*node; # FIXME: jak usunięcie Phrase ProNG wpływa na pokrycie?
-#pos=num|intnum|realnum|intnum-interval|realnum-interval,nsem=mass:
-#  QUANT[role=0]
-#  num*number*case*gender*person*acm*nsem*role*node; # FIXME: jak usunięcie Phrase ProNG wpływa na pokrycie?
-pos=num|intnum|realnum|intnum-interval|realnum-interval,nsem=count,phrase=np:
-  QUANT[cat=Number,role=Count] 
-  num*number*case*gender*person*acm*nsem*role*node|(1+fixed*T*OpAdNum*Mod*concept);
-pos=num|intnum|realnum|intnum-interval|realnum-interval,nsem=mass,phrase=np:
-  QUANT[cat=Amount,role=Amount] 
-  num*number*case*gender*person*acm*nsem*role*node|(1+fixed*T*OpAdNum*Mod*concept);
-
-lemma=jeden,pos=adj,grad=pos,phrase=np:
-  QUANT[person=all_persons,acm=congr,nsem=count,role=Count]
-  num*number*case*gender*person*acm*nsem*role*node;
-
 # pojemniki
 pos=subst,cat=Measure:
   np*number*case*gender*person*cat*role*node
@@ -240,6 +219,8 @@ pos=subst,case=gen,cat=Measure:
    {distant-schema}{\num*number*case*gender*person*rec*count*T*concept}
    {schema}{local-schema}: measure_weight; # UWAGA: number "sg" i gender "n", żeby uzgadniać z podmiotem czasownika
+pos=subst,phrase=xp:
+  xp*coerced*role*node{distant-schema}{schema}{local-schema};
 #frazy przymiotnikowe
 pos=adj|adjc|adjp,phrase=adjp:			
@@ -284,15 +265,15 @@ lemma=po,pos=prep,phrase=prepadjp:
   prepadjp*lemma*case*cat*role*node
    {/adjp*T*case*T*T*cat*CORE*node+adjp*sg*dat*m1*T*cat*CORE*node};
-lemma=za|zbyt|niezbyt,pos=prep,phrase=adjp:
-  QUANT[cat=0,number=all_numbers,case=all_cases,gender=all_genders,grad=pos]
-  adjp*number*case*gender*grad*cat*role*node
-   {/adjp*number*case*gender*grad*cat*Arg*node};
+#lemma=za|zbyt|niezbyt,pos=x,phrase=adjp:
+#  QUANT[cat=0,coerced=0,number=all_numbers,case=all_cases,gender=all_genders,grad=pos]
+#  adjp*number*case*gender*grad*cat*role*node
+#   {/adjp*number*case*gender*grad*cat*Arg*node};
-lemma=jak,pos=x,phrase=adjp:
-  QUANT[cat=0,coerced=0,number=all_numbers,case=all_cases,gender=all_genders]
-  adjp*number*case*gender*sup*cat*role*node
-   {/adjp*number*case*gender*sup*cat*Arg*node};
+#lemma=jak,pos=x,phrase=adjp:
+#  QUANT[cat=0,coerced=0,number=all_numbers,case=all_cases,gender=all_genders]
+#  adjp*number*case*gender*sup*cat*role*node
+#   {/adjp*number*case*gender*sup*cat*Arg*node};
 pos=compar: 
   QUANT[cat=0] 
@@ -304,6 +285,22 @@ pos=compar:
   comparp*lemma*case*cat*role*node
     /xp*cat*CORE*node;
+#modyfikatory przyimków i przysłówków
+lemma=za,pos=qub,phrase=admod:
+  QUANT[grad=pos]
+  admod*grad*cat*role*node{distant-schema}{schema}{local-schema};
+
+lemma=bardzo|zbyt|niezbyt,pos=adv,phrase=admod:
+  QUANT[grad=pos]
+  admod*grad*cat*role*node{distant-schema}{schema}{local-schema};
+
+lemma=nieco|trochę,pos=adv,phrase=admod:
+  QUANT[grad=pos&com]
+  admod*grad*cat*role*node{distant-schema}{schema}{local-schema};
+
+lemma=jak,pos=adv,phrase=admod:
+  QUANT[grad=sup]
+  admod*grad*cat*role*node{distant-schema}{schema}{local-schema};
 # czasowniki
 pos=ger,phrase=np:  
@@ -35,7 +35,7 @@ let rec add_ntoken stats = function
     Token t ->
       (try
         let nlemma,ncat,ninterp = get_ntoken t.attrs in
-        StringQMap.add stats (nlemma ^ "\t" ^ ncat ^ "\t" ^ ENIAMtokens.string_of_interps [ninterp])
+        StringQMap.add stats (nlemma ^ "\t" ^ ncat ^ "\t" ^ ENIAMtagset.render [ninterp])
       with Not_found -> stats)
   | Seq l -> Xlist.fold l stats add_ntoken
   | Variant l -> Xlist.fold l stats add_ntoken
@@ -228,7 +228,7 @@ let get_lemma_cat_interp = function
   | nlemma,lemma,"subst","subst",[["pl"];c;g],[["pl"];["nom";"voc"];_] -> lemma,"subst",[["pl"];c;g]
   (* | "5","5","adj","dig",ninterp,[] -> "piąty","adj",ninterp
   | "6","6","adj","dig",ninterp,[] -> "szósty","adj",ninterp *)
-  (* | "adj","ppas",ninterp,interp -> print_endline (ENIAMtokens.string_of_interps [ninterp] ^ " " ^ ENIAMtokens.string_of_interps [interp]); raise Not_found *)
+  (* | "adj","ppas",ninterp,interp -> print_endline (ENIAMtagset.render [ninterp] ^ " " ^ ENIAMtagset.render [interp]); raise Not_found *)
   | _ -> raise Not_found
@@ -323,7 +323,7 @@ let rec validate_ntoken_token name id_div paragraph stats = function
       (try
         let nlemma,ncat,ninterp = get_ntoken t.attrs in
         process_ntoken2 stats 1 name id_div t.orth t.beg paragraph nlemma ncat ninterp
-        (* print_endline (nlemma ^ "\t" ^ ncat ^ "\t" ^ ENIAMtokens.string_of_interps [ninterp]);
+        (* print_endline (nlemma ^ "\t" ^ ncat ^ "\t" ^ ENIAMtagset.render [ninterp]);
         Printf.printf "%s\t%d\t%s\t%d\n" name id_div t.orth t.beg;
         print_endline paragraph;
         stats *)
@@ -46,7 +46,7 @@ let load_token beg compound in_channel =
           [a;b] -> (try int_of_string b - int_of_string a with _ -> failwith "load_token: interval id")
         | _ -> failwith "load_token: interval id" in
       raise (Interval_id len) in
-    let pos,tags = match ENIAMtokens.parse_postags interp with [x] -> x | _ -> failwith "n_token" in
+    let pos,tags = match ENIAMtagset.parse interp with [x] -> x | _ -> failwith "n_token" in
     {empty_token_env with orth = orth; beg=beg; len=len; next=next;
       token = Lemma(lemma,pos,[tags])}, next, id, sl, sem in
   let line = input_line in_channel in
@@ -139,7 +139,7 @@ let get_tagset corpus =
     Int.fold 1 (ExtArray.size tokens - 1) qmap (fun qmap i ->
       let t = ExtArray.get tokens i in
       match t.token with
-        Lemma(lemma,cat,interp) -> StringQMap.add qmap (cat ^ ":" ^ ENIAMtokens.string_of_interps interp)
+        Lemma(lemma,cat,interp) -> StringQMap.add qmap (cat ^ ":" ^ ENIAMtagset.render interp)
       | _ -> failwith "get_tagset"))
 let numbers = StringSet.of_list ["sg";"pl"]
@@ -315,9 +315,9 @@ let convert_tagset corpus =
   | _,"discourse:emo",_ -> "discourse:emo"
   | (Lemma(lemma1,"subst",[[_] :: [c1] :: _]) as s),"case",(Lemma(lemma2,"prep",[[c2] :: _]) as t) ->
       if c1 = c2 then "subst" ^ " -> case -> " ^ "prep" else ENIAMtokens.string_of_token s ^ " -> " ^ "case" ^ " -> " ^ ENIAMtokens.string_of_token t
-  | Lemma(lemma1,cat1,interp1),"case",Lemma(lemma2,"adv",interp2) -> cat1 ^ ":" ^ ENIAMtokens.string_of_interps interp1 ^ " -> case -> " ^ lemma2 ^ ":" ^ "adv" ^ ":" ^ ENIAMtokens.string_of_interps interp2
+  | Lemma(lemma1,cat1,interp1),"case",Lemma(lemma2,"adv",interp2) -> cat1 ^ ":" ^ ENIAMtagset.render interp1 ^ " -> case -> " ^ lemma2 ^ ":" ^ "adv" ^ ":" ^ ENIAMtagset.render interp2
   | Lemma(lemma1,cat1,interp1),label,Lemma(lemma2,cat2,interp2) ->
-      cat1 ^ ":" ^ ENIAMtokens.string_of_interps interp1 ^ " -> " ^ label ^ " -> " ^ cat2 ^ ":" ^ ENIAMtokens.string_of_interps interp2
+      cat1 ^ ":" ^ ENIAMtagset.render interp1 ^ " -> " ^ label ^ " -> " ^ cat2 ^ ":" ^ ENIAMtagset.render interp2
   | s,label,t -> ENIAMtokens.string_of_token s ^ " -> " ^ label ^ " -> " ^ ENIAMtokens.string_of_token t
 let list_dependencies corpus =
@@ -348,12 +348,12 @@ let string_of_sem sem =
   if sem = "" then "" else "[" ^ sem ^ "]"
 let string_of_lci d =
-  let interp = ENIAMtokens.string_of_interps d.interp in
+  let interp = ENIAMtagset.render d.interp in
   if interp = "" then Printf.sprintf "%s,%s" d.lemma d.cat
   else Printf.sprintf "%s,%s:%s" d.lemma d.cat interp
 let string_of_phrase (phrase,interp) =
-  let interp = ENIAMtokens.string_of_interps interp in
+  let interp = ENIAMtagset.render interp in
   if interp = "" then phrase
   else Printf.sprintf "%s:%s" phrase interp
@@ -585,9 +585,9 @@ let rec flatten_coordination is_coord ulabel usem = function
 let string_of_dependency2 is_coord (lemma1,cat1,interp1) label sem (lemma2,cat2,interp2) =
   (if is_coord then "COORD " else "") ^
-  lemma1 ^ ":" ^ cat1 ^ ":" ^ ENIAMtokens.string_of_interps interp1 ^
+  lemma1 ^ ":" ^ cat1 ^ ":" ^ ENIAMtagset.render interp1 ^
   " -> " ^ label ^ (if sem = "" then "" else "["^sem^"]") ^ " -> "
-  (*^ lemma2 ^ ":"*) ^ cat2 ^ ":" ^ ENIAMtokens.string_of_interps interp2
+  (*^ lemma2 ^ ":"*) ^ cat2 ^ ":" ^ ENIAMtagset.render interp2
 type sel = Any | Value of string list | Agr of string
 type coord = Coord | Gen
@@ -914,34 +914,34 @@ let rec split_tree forest = function
 (* let rec rules_of_tree2 = function
     Dep({sons=[]} as d) ->
-      d.lemma ^ ":" ^ d.cat ^ ":" ^ ENIAMtokens.string_of_interps d.interp
+      d.lemma ^ ":" ^ d.cat ^ ":" ^ ENIAMtagset.render d.interp
   | Dep({sons=[Dep d2]} as d) ->
-      "[ " ^ d.lemma ^ ":" ^ d.cat ^ ":" ^ ENIAMtokens.string_of_interps d.interp ^
+      "[ " ^ d.lemma ^ ":" ^ d.cat ^ ":" ^ ENIAMtagset.render d.interp ^
       " -> " ^ d2.label ^ " -> " ^ rules_of_tree2 (Dep d2) ^ " ]"
   | Dep({sons=[Dep d2;Dep d3]} as d) ->
-      "[ " ^ d.lemma ^ ":" ^ d.cat ^ ":" ^ ENIAMtokens.string_of_interps d.interp ^
+      "[ " ^ d.lemma ^ ":" ^ d.cat ^ ":" ^ ENIAMtagset.render d.interp ^
       " -> " ^ d2.label ^ " -> " ^ rules_of_tree2 (Dep d2) ^ " | " ^ d3.label ^ " -> " ^ rules_of_tree2 (Dep d3) ^ " ]"
   | _ -> failwith "rules_of_tree2" *)
 (* let rec rules_of_tree2 = function
     Dep({sons=[]} as d) ->
-      "_:" ^ d.cat ^ ":" ^ ENIAMtokens.string_of_interps d.interp
+      "_:" ^ d.cat ^ ":" ^ ENIAMtagset.render d.interp
   | Dep({sons=[Dep d2]} as d) ->
-      "[ _:" ^ d.cat ^ ":" ^ ENIAMtokens.string_of_interps d.interp ^
+      "[ _:" ^ d.cat ^ ":" ^ ENIAMtagset.render d.interp ^
       " -> " ^ d2.label ^ " -> " ^ rules_of_tree2 (Dep d2) ^ " ]"
   | Dep({sons=[Dep d2;Dep d3]} as d) ->
-      "[ _:" ^ d.cat ^ ":" ^ ENIAMtokens.string_of_interps d.interp ^
+      "[ _:" ^ d.cat ^ ":" ^ ENIAMtagset.render d.interp ^
       " -> " ^ d2.label ^ " -> " ^ rules_of_tree2 (Dep d2) ^ " | " ^ d3.label ^ " -> " ^ rules_of_tree2 (Dep d3) ^ " ]"
   | _ -> failwith "rules_of_tree2" *)
 let rec rules_of_tree2 = function
     Dep({sons=[]} as d) ->
-      "_:" ^ d.cat (*^ ":" ^ ENIAMtokens.string_of_interps d.interp*)
+      "_:" ^ d.cat (*^ ":" ^ ENIAMtagset.render d.interp*)
   | Dep({sons=[Dep d2]} as d) ->
-      "[ _:" ^ d.cat ^ (*":" ^ ENIAMtokens.string_of_interps d.interp ^*)
+      "[ _:" ^ d.cat ^ (*":" ^ ENIAMtagset.render d.interp ^*)
       " -> " ^ d2.label ^ " -> " ^ rules_of_tree2 (Dep d2) ^ " ]"
   | Dep({sons=[Dep d2;Dep d3]} as d) ->
-      "[ _:" ^ d.cat ^ (*":" ^ ENIAMtokens.string_of_interps d.interp ^*)
+      "[ _:" ^ d.cat ^ (*":" ^ ENIAMtagset.render d.interp ^*)
       " -> " ^ d2.label ^ " -> " ^ rules_of_tree2 (Dep d2) ^ " | " ^ d3.label ^ " -> " ^ rules_of_tree2 (Dep d3) ^ " ]"
   | _ -> failwith "rules_of_tree2"
@@ -59,6 +59,7 @@ let transform_phrase pos lemma = function
   | XP as morf -> [morf]
   | AdjA as morf -> [morf]
   | PadvP as morf -> [morf]
+  | AdMod _ as morf -> [morf]
   | Qub as morf -> [morf]
   | FixedP _ as morf -> [morf]
   | SymbolP as morf -> [morf]
@@ -270,6 +270,8 @@ let render_phrase_cat cat role node = function
     | Measure(AllUAgr) -> Tensor[Atom "measure"; AVar "unumber"; AVar "ucase"; AVar "ugender"; AVar "uperson"] *)
     | Or -> Tensor[Atom "or"; Atom cat; Atom role; Atom node]
     | Qub -> Tensor[Atom "qub"; Atom cat; Atom role; Atom node]
+    | AdMod(GradAgr) -> Tensor[Atom "admod"; AVar "grad"; Atom cat; Atom role; Atom node]
+    | AdMod(Grad grad) -> Tensor[Atom "admod"; Atom grad; Atom cat; Atom role; Atom node]
     (* | Inclusion -> Tensor[Atom "inclusion"]
     | Adja -> Tensor[Atom "adja"]
     | Aglt -> Tensor[Atom "aglt"; AVar "number"; AVar "person"]
@@ -86,6 +86,7 @@ let gender = function
 let grad = function
     Grad s -> s
+  | GradAgr -> "agr"
   | GradUndef -> "_"
 (* let psem = function
@@ -154,6 +155,7 @@ let rec phrase = function
   (* | Num(c,a) -> "num(" ^ case c ^ "," ^ acm a ^ ")" *)
   | Or -> "or"
   | Qub -> "qub"
+  | AdMod g -> "admod(" ^ grad g ^ ")"
   | Inclusion -> "inclusion"
   | Pro -> "pro"
   | ProNG -> "prong"
@@ -29,7 +29,7 @@ type comp = Comp of string | Zeby | Gdy | CompUndef
 type comp_type = Int | Rel | CompTypeUndef (*| CompTypeAgr*)
 type number = Number of string | NumberUndef | NumberAgr
 type gender = Gender of string | GenderUndef | GenderAgr | Genders of string list
-type grad = Grad of string | GradUndef
+type grad = Grad of string | GradUndef | GradAgr
 (* type psem = Psem | Pnosem *)
 (* type refl = (*ReflEmpty |*) ReflTrue | ReflFalse | ReflUndef *)
 (* type acm = Acm of string | AcmUndef *)
@@ -87,6 +87,7 @@ type phrase =
   (* | Refl
   | Recip *)
   | Qub
+  | AdMod of grad
   | Inclusion
   | Pro
   | ProNG
@@ -108,13 +108,15 @@ let alt = ref (StringMap.empty : (bool * t) list StringMap.t)
 let stems = ref (StringMap.empty : (string * string * StringSet.t) list StringMap.t)
 let rules = ref ([] : (StringMap.key * ENIAMmorphologyRules.CharTrees.t) list)
 let wyglos = ref ([] : (StringMap.key * ENIAMmorphologyRules.CharTrees.t) list)
+let lemmata = ref StringSet.empty
 let initialize () =
   alt := prepare_alt StringMap.empty alt_filename;
   alt := prepare_alt !alt alt_supplement_filename;
   stems := load_stems stem_filename;
   rules := prepare_rules rules_filename;
-  wyglos := prepare_wyglos wyglos_filename
+  wyglos := prepare_wyglos wyglos_filename;
+  lemmata := StringSet.of_list (File.load_lines lemmata_filename)
 (* let initialize () =
   alt := prepare_alt StringMap.empty "resources/alt.tab";
@@ -65,5 +65,6 @@ let alt_filename = resource_path ^ &quot;/morphology/alt.tab&quot;
 let stem_filename = resource_path ^ "/morphology/stem.tab"
 let rules_filename = resource_path ^ "/morphology/freq_rules.tab"
 let wyglos_filename = resource_path ^ "/morphology/wyglos.tab"
+let lemmata_filename = resource_path ^ "/morphology/lemmata.tab"
 let alt_supplement_filename = resource_path ^ "/morphology/alt_supplement.tab"
+open Xstd
+
 let nlp_resources_path = "../../NLP resources/"
 let sgjp_path = nlp_resources_path ^ "SGJP/"
 let sgjp_filename = "sgjp-20170730.tab.gz"
@@ -35,9 +37,18 @@ let generate_alt rules_filename path filename out_filename =
   let dict = Dict.remove_validated_forms dict in
   Dict.print out_filename dict
+let generate_lemmata path filename out_filename =
+  let dict = Dict.load_tab (path ^ filename) in
+  let lemmata = Xlist.fold dict StringSet.empty (fun set e ->
+    StringSet.add set (Stem.simplify_lemma e.ENIAMmorphologyTypes.lemma)) in
+  File.file_out out_filename (fun file ->
+    StringSet.iter lemmata (Printf.fprintf file "%s\n"))
+
+
 let _ =
-  Dict.generate_rule_frequencies_list interp_compound_rule_trees sources "resources/freq_rules.tab";
+  (* Dict.generate_rule_frequencies_list interp_compound_rule_trees sources "resources/freq_rules.tab";
   generate_alt "resources/freq_rules.tab" sgjp_path sgjp_filename "resources/alt.tab";
   Dict.generate_stem_dict "resources/freq_rules.tab" sgjp_path sgjp_filename "resources/stem.tab";
-  Dict.generate_wyglos sgjp_path sgjp_filename "resources/wyglos.tab";
+  Dict.generate_wyglos sgjp_path sgjp_filename "resources/wyglos.tab"; *)
+  generate_lemmata sgjp_path sgjp_filename "resources/lemmata.tab";
   ()