Wojciech Jaworski / ENIAM

open Xstd
open Printf
open Types

(* let nexus_path = "/home/yacheu/Dokumenty/Badania/Jezyk i Umysl/Przetwarzanie Języka Naturalnego/zasoby/"
let toshiba_ub_path = "/home/wjaworski/Dokumenty/zasoby/"

let get_host_name () =
  let chan = Unix.open_process_in "uname -n" in
  input_line chan

let zasoby_path =
  match get_host_name () with
    "nexus" -> nexus_path
  | "toshiba-UB" -> toshiba_ub_path
(*   | "mozart" -> "." *)
  | s -> failwith ("unknown host: " ^ s) *)

let nlp_resources_path = "../../NLP resources/"
let sgjp_path = nlp_resources_path ^ "SGJP/"
let results_path = "results/"
let lu_path = "../morphology2/plWordnet/"

let lematy_nkjp_filename = "lematy_NKJP1M_freq.tab"

let sgjp_filename2015 = "sgjp-20151020.tab.gz"
let polimorf_filename2015 = "polimorf-20151020.tab.gz"
let sgjp_filename201605 = "sgjp-20160508.tab.gz"
let polimorf_filename201605 = "polimorf-20160508.tab.gz"
let sgjp_filename201607 = "sgjp-20160724.tab.gz"
let polimorf_filename201607 = "polimorf-20160724.tab.gz"
let sgjp_filename = "sgjp-20170730.tab.gz"
let polimorf_filename = "polimorf-20170402.tab.gz"

let adv_sgjp_filename = "adv_sgjp-20170730.tab"
let adj_sgjp_filename = "adj_sgjp-20170730.tab"
let noun_sgjp_filename = "noun_sgjp-20170730.tab"
let verb_sgjp_filename = "verb_sgjp-20170730.tab"
let adv_polimorf_filename = "adv_polimorf-20170402.tab"
let adj_polimorf_filename = "adj_polimorf-20170402.tab"
let noun_polimorf_filename = "noun_polimorf-20170402.tab"
let verb_polimorf_filename = "verb_polimorf-20170402.tab"

(* Test wczytywania słowników i liczenie częstości *)
let _ =
  (* print_endline "sgjp_filename2015";
  let _ = Dict.load_tab (sgjp_path ^ sgjp_filename2015) in
  print_endline "polimorf_filename2015";
  let _ = Dict.load_tab (sgjp_path ^ polimorf_filename2015) in
  print_endline "sgjp_filename201605";
  let _ = Dict.load_tab (sgjp_path ^ sgjp_filename201605) in
  print_endline "polimorf_filename201605";
  let _ = Dict.load_tab (sgjp_path ^ polimorf_filename201605) in
  print_endline "sgjp_filename201607";
  let _ = Dict.load_tab (sgjp_path ^ sgjp_filename201607) in
  print_endline "polimorf_filename201607";
  let _ = Dict.load_tab (sgjp_path ^ polimorf_filename201607) in
  print_endline "sgjp_filename";
  let _ = Dict.load_tab (sgjp_path ^ sgjp_filename) in
  print_endline "polimorf_filename";
  let _ = Dict.load_tab (sgjp_path ^ polimorf_filename) in
  print_endline "adv_sgjp_filename";
  let _ = Dict.load_tab (results_path ^ adv_sgjp_filename) in
  print_endline "adj_sgjp_filename";
  let _ = Dict.load_tab (results_path ^ adj_sgjp_filename) in
  print_endline "noun_sgjp_filename";
  let _ = Dict.load_tab (results_path ^ noun_sgjp_filename) in
  print_endline "verb_sgjp_filename";
  let _ = Dict.load_tab (results_path ^ verb_sgjp_filename) in
  print_endline "sgjp_filename2015";
  let dict = Dict.load_tab_full (sgjp_path ^ sgjp_filename2015) in
  Dict.print_quantities "results/proper-type-sgjp-20151020.txt" Dict.proper_type_selector dict;
  Dict.print_quantities "results/genre-sgjp-20151020.txt" Dict.genre_selector dict;
  Dict.print_quantities "results/interp-sgjp-20151020.txt" Dict.interp_selector dict;
  print_endline "polimorf_filename2015";
  let dict = Dict.load_tab_full (sgjp_path ^ polimorf_filename2015) in
  Dict.print_quantities "results/proper-type-polimorf-20151020.txt" Dict.proper_type_selector dict;
  Dict.print_quantities "results/genre-polimorf-20151020.txt" Dict.genre_selector dict;
  Dict.print_quantities "results/interp-polimorf-20151020.txt" Dict.interp_selector dict;
  print_endline "sgjp_filename201605";
  let dict = Dict.load_tab_full (sgjp_path ^ sgjp_filename201605) in
  Dict.print_quantities "results/proper-type-sgjp-20160508.txt" Dict.proper_type_selector dict;
  Dict.print_quantities "results/genre-sgjp-20160508.txt" Dict.genre_selector dict;
  Dict.print_quantities "results/interp-sgjp-20160508.txt" Dict.interp_selector dict;
  print_endline "polimorf_filename201605";
  let dict = Dict.load_tab_full (sgjp_path ^ polimorf_filename201605) in
  Dict.print_quantities "results/proper-type-polimorf-20160508.txt" Dict.proper_type_selector dict;
  Dict.print_quantities "results/genre-polimorf-20160508.txt" Dict.genre_selector dict;
  Dict.print_quantities "results/interp-polimorf-20160508.txt" Dict.interp_selector dict;
  print_endline "sgjp_filename201607";
  let dict = Dict.load_tab_full (sgjp_path ^ sgjp_filename201607) in
  Dict.print_quantities "results/proper-type-sgjp-20160724.txt" Dict.proper_type_selector dict;
  Dict.print_quantities "results/genre-sgjp-20160724.txt" Dict.genre_selector dict;
  Dict.print_quantities "results/interp-sgjp-20160724.txt" Dict.interp_selector dict;
  print_endline "polimorf_filename201607";
  let dict = Dict.load_tab_full (sgjp_path ^ polimorf_filename201607) in
  Dict.print_quantities "results/proper-type-polimorf-20160724.txt" Dict.proper_type_selector dict;
  Dict.print_quantities "results/genre-polimorf-20160724.txt" Dict.genre_selector dict;
  Dict.print_quantities "results/interp-polimorf-20160724.txt" Dict.interp_selector dict;
  print_endline "sgjp_filename";
  let dict = Dict.load_tab_full (sgjp_path ^ sgjp_filename) in
  Dict.print_quantities "results/proper-type-sgjp-20170730.txt" Dict.proper_type_selector dict;
  Dict.print_quantities "results/genre-sgjp-20170730.txt" Dict.genre_selector dict;
  Dict.print_quantities "results/interp-sgjp-20170730.txt" Dict.interp_selector dict;
  print_endline "polimorf_filename";
  let dict = Dict.load_tab_full (sgjp_path ^ polimorf_filename) in
  Dict.print_quantities "results/proper-type-polimorf-20170402.txt" Dict.proper_type_selector dict;
  Dict.print_quantities "results/genre-polimorf-20170402.txt" Dict.genre_selector dict;
  Dict.print_quantities "results/interp-polimorf-20170402.txt" Dict.interp_selector dict;
  print_endline "adv_sgjp_filename";
  let dict = Dict.load_tab_full (results_path ^ adv_sgjp_filename) in
  print_endline "adj_sgjp_filename";
  let dict = Dict.load_tab_full (results_path ^ adj_sgjp_filename) in
  print_endline "noun_sgjp_filename";
  let dict = Dict.load_tab_full (results_path ^ noun_sgjp_filename) in
  print_endline "verb_sgjp_filename";
  let dict = Dict.load_tab_full (results_path ^ verb_sgjp_filename) in
  print_endline "lematy_nkjp_filename";
  let dict = Dict.load_freq_tab (zasoby_path ^ lematy_nkjp_filename) in
  Dict.print_quantities "results/interp-lematy_nkjp.txt" Dict.interp_selector dict;
  Dict.print_quantities "results/freq-lematy_nkjp.txt" Dict.freq_selector dict;*)
  ()

(* Porównanie wersji słowników *)
let _ =
  (* Dict.compare_dicts_full (sgjp_path ^ sgjp_filename2015) (sgjp_path ^ sgjp_filename201605) "results/comparition_sgjp1_full.out"; *)
  (* Dict.compare_dicts_full (sgjp_path ^ sgjp_filename201605) (sgjp_path ^ sgjp_filename) "results/comparition_sgjp2_full.out"; *)
  (* Dict.compare_dicts_full (sgjp_path ^ polimorf_filename2015) (sgjp_path ^ polimorf_filename201605) "results/comparition_polimorf1_full.out"; *)
  (* Dict.compare_dicts_full (sgjp_path ^ polimorf_filename201605) (sgjp_path ^ polimorf_filename) "results/comparition_polimorf2_full.out"; *)
  ()

(* Podział słownika *)
let _ =
  (* Dict.split_dict sgjp_path sgjp_filename results_path; *)
  (* Dict.split_dict sgjp_path sgjp_filename201607 results_path;  *)
  (* Dict.split_dict sgjp_path sgjp_filename201605 results_path; *)
  (* Dict.split_dict sgjp_path polimorf_filename results_path; *)
  (* Dict.split_language "data/obce_langs.tab" sgjp_path sgjp_filename results_path; *)
  ()

(* Usunięcie form z prefixami *)
let _ =
  (* Dict.remove_prefix_forms results_path "sgjp_selected.tab" "results/verb_sgjp_no_pref.tab"; *)
  (* Dict.remove_prefix_forms results_path verb_sgjp_filename "results/verb_sgjp_no_pref.tab"; *)
  ()

let test_process_interps path filename =
  let dict = Dict.load_tab_full (path ^ filename) in
  let dict = Dict.merge_entries dict in
  let _ = Dict.process_interps dict in
  ()

(* Test mapowania interpretacji *)
let _ =
  (* test_process_interps results_path adv_sgjp_filename; *)
  (* test_process_interps results_path adj_sgjp_filename; *)
  (* test_process_interps results_path verb_sgjp_filename; *)
  (* test_process_interps results_path noun_sgjp_filename; *)
  ()


let obce = StringSet.of_list (File.load_lines "../morphology/data/obce.tab")

let find_not_fonetic_parsed ex path filename =
  let dict = Dict.load_tab_full (path ^ filename) in
  let dict = Dict.remove_exceptional_lemmata_gen ex dict in
  Xlist.iter dict (fun e ->
    Xlist.iter e.forms (fun f ->
      if Fonetics.translate false Fonetics.rules f.orth = [] then printf "%s\n%!" e.lemma))

(* Test translacji fonetycznej *)
let _ =
  (* find_not_fonetic_parsed obce results_path adv_sgjp_filename; *)
  (* find_not_fonetic_parsed obce results_path adj_sgjp_filename; *)
  (* find_not_fonetic_parsed obce results_path noun_sgjp_filename; *)
  (* find_not_fonetic_parsed obce results_path verb_sgjp_filename; *)
  (* find_not_fonetic_parsed obce sgjp_path sgjp_filename; *)
  (* find_not_fonetic_parsed obce sgjp_path polimorf_filename; *)
  ()

let test_fonetics path filename =
  let dict = Dict.load_tab_full (path ^ filename) in
  let dict = Dict.remove_exceptional_lemmata_gen obce dict in
  let _ = Dict.fonetic_translation dict in
  ()

(* Test translacji fonetycznej *)
let _ =
  (* test_fonetics results_path adv_sgjp_filename; *)
  (* test_fonetics results_path adj_sgjp_filename; *)
  (* test_fonetics results_path noun_sgjp_filename; *)
  (* test_fonetics results_path verb_sgjp_filename; *)
  ()

let _ =
  (* Fonetics.latex_of_rules "data/fonetics_pl.dic"; *)
  (* Rules.latex_of_alternations "../morphology/data/alternations.dic"; *)
  (* Rules.latex_of_schemata (); *)
  (* Rules.latex_of_interp_rules (); *)
  ()

(* Wypisanie nieodmiennych *)
(*let _ =
  (* let dict = Dict.load_tab_full (sgjp_path ^ sgjp_filename) in
  let dict = Dict.merge_entries dict in
  let dict = Dict.process_interps dict in
  let dict = Dict.mark_ndm dict in
  Dict.print_ndm "results/ndm-sgjp.tab" dict; *)
  ()*)

let check_stem_generation path filename =
  let dict = Dict.load_tab_full (path ^ filename) in
  let dict = Dict.merge_entries dict in
  let dict = Dict.process_interps dict in
  (* let dict = Dict.mark_ndm dict in *)
  let dict = Dict.remove_exceptional_lemmata dict in
  let dict = Dict.find_kolwiek_suffixes dict in
  (* let dict = Dict.fonetic_translation dict in *)
  let _ = Dict.generate_stem dict in
  ()

(* Sprawdzenie działania stemowania *)
let _ =
  (* check_stem_generation sgjp_path sgjp_filename; *)
(*   check_stem_generation results_path adj_sgjp_filename;  *)
  (* check_stem_generation results_path noun_sgjp_filename; *)
  (* check_stem_generation results_path noun_polimorf_filename; *)
  ()

(* let _ =  print_endline "1" *)
let compound_rules = Rules.make_compound_rules ()
(* let _ =  print_endline "2" *)
let compound_rule_trees = Rules.make_compound_rule_trees compound_rules
(* let _ =  print_endline "3" *)
let interp_compound_rule_trees = Rules.make_interp_compound_rule_trees compound_rules
(* let _ =  print_endline "4" *)

(* let _ = Rules.CharTrees.print_rules "results/rules_tree.txt" compound_rule_trees *)

let find_not_validated_forms rules path filename out_filename =
  let dict = Dict.load_tab (path ^ filename) in
  let dict = Dict.fonetic_translation dict in
  let dict = Dict.phon_validate rules dict in
  let dict = Dict.remove_validated_forms dict in
  Dict.print out_filename dict

let find_not_validated_entries rules path filename out_filename =
  let dict = Dict.load_tab (path ^ filename) in
  let dict = Dict.merge_entries dict in
  let dict = Dict.process_interps dict in
  (* let dict = Dict.mark_ndm dict in
  let dict = Dict.remove_ndm dict in *)
  let dict = Dict.remove_exceptional_lemmata dict in
  let dict = Dict.fonetic_translation dict in
  let dict = Dict.phon_validate rules dict in
  let dict = Dict.remove_validated_entries dict in
  Dict.print out_filename dict

(* Wypisanie niezwalidowanych form *)
let _ =
  (* find_not_validated_forms compound_rule_trees results_path adj_sgjp_filename "results/not_validated_adj.tab";  *)
  (* find_not_validated_entries compound_rule_trees results_path adj_sgjp_filename "results/not_validated_adj.tab"; *)
  (* find_not_validated_entries compound_rule_trees results_path adj_polimorf_filename "results/not_validated_p_adj.tab"; *)
  (* find_not_validated_entries compound_rule_trees "results/" "not_validated_adj.tab" "results/not_validated_adj2.tab"; *)
  (* find_not_validated_entries compound_rule_trees results_path adv_sgjp_filename "results/not_validated_adv.tab"; *)
  (* find_not_validated_entries compound_rule_trees results_path adv_polimorf_filename "results/not_validated_p_adv.tab"; *)
  (* find_not_validated_entries compound_rule_trees results_path noun_sgjp_filename "results/not_validated_noun.tab"; *)
  (*find_not_validated_forms compound_rule_trees results_path noun_sgjp_filename "results/not_validated_noun2.tab";
  find_not_validated_entries compound_rule_trees results_path noun_polimorf_filename "results/not_validated_p_noun.tab";
  find_not_validated_forms compound_rule_trees results_path noun_polimorf_filename "results/not_validated_p_noun2.tab"; *)
  (* find_not_validated_entries compound_rule_trees results_path "not_validated_noun.tab" "results/not_validated_noun3.tab"; *)
  (* find_not_validated_forms compound_rule_trees results_path "not_validated_noun.tab" "results/not_validated_noun4.tab"; *)
  (* find_not_validated_entries compound_rule_trees results_path verb_sgjp_filename "results/not_validated_verb.tab"; *)
  (*find_not_validated_forms compound_rule_trees results_path verb_sgjp_filename "results/not_validated_verb2.tab";
  find_not_validated_entries compound_rule_trees results_path verb_polimorf_filename "results/not_validated_p_verb.tab";
  find_not_validated_forms compound_rule_trees results_path verb_polimorf_filename "results/not_validated_p_verb2.tab"; *)
  (* find_not_validated_forms compound_rule_trees results_path "sgjp_selected.tab" "results/not_validated_verb.tab"; *)
  (* find_not_validated_forms compound_rule_trees results_path "lang_en_sgjp-20170730.tab" "results/lang_en.tab"; *)
  (* find_not_validated_forms compound_rule_trees results_path "lang_fr_sgjp-20170730.tab" "results/lang_fr.tab"; *)
  find_not_validated_forms compound_rule_trees results_path "lang_de_sgjp-20170730.tab" "results/lang_de.tab";
  ()

let find_not_validated_lemmata rules path filename out_filename =
  let dict = Dict.load_tab (path ^ filename) in
  let dict = Dict.merge_entries dict in
  let dict = Dict.process_interps dict in
  let dict = Dict.remove_exceptional_lemmata dict in
  let dict = Dict.fonetic_translation dict in
  let dict = Dict.phon_validate rules dict in
  let dict = Dict.remove_validated_entries dict in
  Dict.print_lemmata out_filename dict

(* Wypisanie niezwalidowanych lematów *)
let _ =
  (* find_not_validated_lemmata compound_rule_trees results_path "not_validated_noun.tab" "results/not_validated_noun_lemma.tab"; *)
  (* find_not_validated_lemmata compound_rule_trees results_path "not_validated_verb.tab" "results/not_validated_verb_lemma.tab";  *)
  (* find_not_validated_lemmata compound_rule_trees results_path "sgjp_selected.tab" "results/not_validated_verb_lemma.tab";   *)
  ()

let find_validated_lemmata rules path filename out_filename =
  let dict = Dict.load_tab (path ^ filename) in
  let dict = Dict.merge_entries dict in
  let dict = Dict.process_interps dict in
  let dict = Dict.fonetic_translation dict in
  let dict = Dict.phon_validate rules dict in
  let dict = Dict.remove_not_validated_entries dict in
  Dict.print_lemmata out_filename dict

(* Wypisanie zwalidowanych lematów *)
let _ =
  (* find_validated_lemmata compound_rule_trees results_path adj_sgjp_filename "results/validated_adj.tab";
  find_validated_lemmata compound_rule_trees results_path noun_sgjp_filename "results/validated_noun.tab";
  find_validated_lemmata compound_rule_trees results_path verb_sgjp_filename "results/validated_verb.tab"; *)
  ()

let test_lemmatize lemma orth =
  printf "test_lemmatize: %s %s\n%!" lemma orth;
  let simple_lemma = Stem.simplify_lemma lemma in
  let phon_orths = Fonetics.translate(*_and_check*) true Fonetics.rules (*Fonetics.rev_rules*) orth in
  printf "phon_orths: \n  %s\n%!" (String.concat "\n  " (Xlist.map phon_orths Fonetics.string_of_phon));
  Xlist.iter phon_orths (fun phon_orth ->
    Xlist.iter (Rules.CharTrees.find compound_rule_trees phon_orth.phon) (fun (stem,rule) ->
      let candidate_lemmas = Fonetics.rev_translate2 true Fonetics.rev_rules (stem ^ rule.set) (phon_orth.mapping) in
      Xlist.iter candidate_lemmas (fun candidate_lemma ->
      if candidate_lemma = simple_lemma then printf "E" else printf " ";
      printf " %s %s %s %s\n%!" phon_orth.phon stem (string_of_rule rule) candidate_lemma)))

(* Sprawdzenie przebiegu lematyzacji *)
let _ =
  (* test_lemmatize "delegacja" "delegacyj"; *)
  (* test_lemmatize "delegacja" "delegacji"; *)
  (* test_lemmatize "stodoła" "stodole"; *)
  (* test_lemmatize "komedia" "komedii"; *)
  (* test_lemmatize "Zenobia" "Zenobij";
  test_lemmatize "aerografia" "aerografij";
  test_lemmatize "Oktawia" "Oktawij";
  test_lemmatize "Olimpia" "Olimpij"; *)
  (* test_lemmatize "Amati" "Amatiego"; *)
  (* test_lemmatize "plemię" "plemieniu"; *)
  (* test_lemmatize "Abisynia" "Abisynii";
  test_lemmatize "Abisynia" "Abisynią";
  test_lemmatize "Abisynia" "Abisynij"; *)
  (* test_lemmatize "kokosz" "kokosze"; *)
  (* test_lemmatize "Kempisti" "Kempisty"; *)
  (* test_lemmatize "witarianin" "witariany"; *)
  (* test_lemmatize "witarianin" "witarianom";
  test_lemmatize "" "";
  test_lemmatize "witarianin" "witarianów"; *)
  (* test_lemmatize "Czuwaszja" "Czuwaszyj"; *)
  (* test_lemmatize "donieść" "doniesiona"; *)
  (* test_lemmatize "zlodzić" "zlodzona"; *)
  (* test_lemmatize "ciąć" "tniemy";
  test_lemmatize "prać" "pierzemy";
  test_lemmatize "myć" "myjemy";
  test_lemmatize "obuć" "obujemy"; *)
  (* test_lemmatize "bać" "boimy";
  test_lemmatize "stać" "stoimy";
  test_lemmatize "szczać" "szczymy";
  test_lemmatize "amnestiować" "amnestiujemy"; *)
  (* test_lemmatize "powziąć" "poweźmiemy";
  test_lemmatize "dowiedzieć" "dowiemy";
  test_lemmatize "dobyć" "dobędziemy"; *)
  (* test_lemmatize "nadeżreć" "nadżarto";
  test_lemmatize "pleść" "pleciono"; *)
  (* test_lemmatize "głaskać" "głaszczmy";
  test_lemmatize "chlastać" "chlaszczmy";
  test_lemmatize "mamleć" "mamlajmy";
  test_lemmatize "ostać" "ostańmy";
  test_lemmatize "dopowiedzieć" "dopowiedzmy"; *)
  (* test_lemmatize "nazywać" "nazywoł";
  test_lemmatize "Danja" "Danji";
  test_lemmatize "Kolej" "Koleji";
  test_lemmatize "rozumiem" "rozumicie"; *)
  (* test_lemmatize "podesłać" "podścielemy"; *)
  (* test_lemmatize "izolować" "izolował"; *)
  (* test_lemmatize "IKEA" "IKE-ach";
  test_lemmatize "IKEA" "IKE-ę";
  test_lemmatize "IKEA" "IKE-i";
  test_lemmatize "ZUS" "ZUS-u";
  test_lemmatize "ZUS" "ZUS-ie";
  test_lemmatize "WAT" "WA-cie"; *)
  (* test_lemmatize "ICJ" "ICJ-ocie";
  test_lemmatize "ICJ" "ICJ-otach"; *)
  (* test_lemmatize "WAT" "Wacie";
  test_lemmatize "WAT" "WACIE";
  test_lemmatize "WAT" "WAcie"; *)
  (* test_lemmatize "BOŚ" "BOŚ-u"; *)
  (* test_lemmatize "upgrade" "upgrade’om"; *)
  (* test_lemmatize "software" "software’y"; *)
  (* test_lemmatize "spray" "sprayu";  *)
  (* test_lemmatize "unixowość" "unixowościach";   *)
  (* test_lemmatize "rolls-royce" "rolls-roysie"; *)
  (* test_lemmatize "Arrow" "Arrowa";*)
  (* test_lemmatize "Boy" "Boyowie"; *)
  (* test_lemmatize "Chomsky" "Chomskiego"; *)
  (* test_lemmatize "Bradbury" "Bradburych"; *)
  (* test_lemmatize "compact" "compakcie";
  test_lemmatize "Dixa" "Diksie"; *)
  (* test_lemmatize "developerski" "developersko"; *)
  (* test_lemmatize "Jessica" "Jessice"; *)
  (* test_lemmatize "Gurkha" "Gurce";
  test_lemmatize "Gurkha" "Gurkhi"; *)
  (* test_lemmatize "heavymetalowiec" "heavymetalowca"; *)
  (* test_lemmatize "niesoftware’owy" "niesoftware’owi"; *)
  (* test_lemmatize "Barrès" "Barrès’go"; *)
  (* test_lemmatize "Dolphy" "Dolphy’ego"; *)
  (* test_lemmatize "Bernoulli" "Bernoulliego"; *)
  (* test_lemmatize "Handke" "Handkami"; *)
  (* test_lemmatize "Radetzky" "Radetzky’ego"; *)
  (* test_lemmatize "" "";
  test_lemmatize "" "";
  test_lemmatize "" "";  *)
  ()

(* Generowanie reguł *)
let _ =
  (* Dict.generate_rules compound_rule_trees results_path adj_sgjp_filename "results/rules-odm-adj.txt"; *)
  (* Dict.generate_rules compound_rule_trees results_path noun_sgjp_filename "results/rules-odm-noun.txt"; *)
  (* Dict.generate_rules compound_rule_trees results_path adv_sgjp_filename "results/rules-adv.txt"; *)
  (* Dict.generate_rules compound_rule_trees results_path verb_sgjp_filename "results/rules-verb.txt"; *)
  (* Dict.generate_rules compound_rule_trees results_path "test.tab" "results/rules-test.txt"; *)
  (* Dict.generate_rules compound_rule_trees results_path "not_validated_noun.tab" "results/rules-nv-noun.txt"; *)
  (* Dict.generate_rules compound_rule_trees results_path "not_validated_p_noun.tab" "results/rules-nv-noun.txt"; *)
  (* Dict.generate_rules compound_rule_trees results_path "not_validated_verb.tab" "results/rules-nv-verb.txt"; *)
  (* Dict.generate_rules compound_rule_trees results_path "sgjp_selected.tab" "results/rules-verb.txt"; *)
  (* Dict.generate_rules compound_rule_trees results_path "verb_sgjp_no_pref.tab" "results/rules-verb.txt"; *)
  (* Dict.generate_rules compound_rule_trees results_path "selected_verb.tab" "results/rules-verb.txt"; *)
  (* Dict.generate_rules_lu compound_rule_trees 142 lu_path "results/rules-142_lu.txt"; *)
  (* Dict.generate_rules_lu compound_rule_trees 148 lu_path "results/rules-148_lu.txt"; *)
  (* Dict.generate_rules_lu compound_rule_trees 42 lu_path "results/rules-42_lu.txt"; *)
  ()

let find_interp_validated_lemmata interp_rules path filename out_filename =
  let dict = Dict.load_tab (path ^ filename) in
  let dict = Dict.merge_entries dict in
  let dict = Dict.process_interps dict in
  (* let dict = Dict.mark_ndm dict in (* FIXME: remove_ndm? *) *)
  (* let dict = Dict.remove_exceptional_lemmata dict in *)
  (* let dict = find_kolwiek_suffixes dict in *)
  (* let dict = generate_stem dict in *)
  let dict = Dict.fonetic_translation dict in
  let dict = Dict.validate_interp interp_rules dict in
  let dict = Dict.remove_not_validated_entries dict in
  Dict.print_lemmata out_filename dict

let find_not_interp_validated_lemmata interp_rules path filename out_filename =
  let dict = Dict.load_tab (path ^ filename) in
  let dict = Dict.merge_entries dict in
  let dict = Dict.process_interps dict in
  (* let dict = Dict.mark_ndm dict in (* FIXME: remove_ndm? *) *)
  (* let dict = Dict.remove_exceptional_lemmata dict in *)
  (* let dict = find_kolwiek_suffixes dict in *)
  (* let dict = generate_stem dict in *)
  let dict = Dict.fonetic_translation dict in
  let dict = Dict.validate_interp interp_rules dict in
  let dict = Dict.remove_validated_entries dict in
  Dict.print_lemmata out_filename dict

let find_not_interp_validated_entries interp_rules path filename out_filename =
  let dict = Dict.load_tab (path ^ filename) in
  let dict = Dict.merge_entries dict in
  let dict = Dict.process_interps dict in
  (* let dict = Dict.mark_ndm dict in
  let dict = Dict.remove_ndm dict in *)
  let dict = Dict.fonetic_translation dict in
  let dict = Dict.validate_interp interp_rules dict in
  let dict = Dict.remove_validated_entries dict in
  Dict.print out_filename dict

let find_not_interp_validated_forms interp_rules path filename out_filename =
  let dict = Dict.load_tab (path ^ filename) in
  let dict = Dict.merge_entries dict in
  let dict = Dict.process_interps dict in
  let dict = Dict.fonetic_translation dict in
  let dict = Dict.validate_interp interp_rules dict in
  let dict = Dict.remove_validated_forms dict in
  Dict.print out_filename dict

(* Wypisanie lematów ze zwalidowaną interpretacją *)
let _ =
  (* find_interp_validated_lemmata interp_compound_rule_trees results_path noun_sgjp_filename "results/interp_validated_noun.tab";
  find_not_interp_validated_lemmata interp_compound_rule_trees results_path noun_sgjp_filename "results/interp_not_validated_noun.tab";
  find_interp_validated_lemmata interp_compound_rule_trees results_path adj_sgjp_filename "results/interp_validated_adj.tab";
  find_not_interp_validated_lemmata interp_compound_rule_trees results_path adj_sgjp_filename "results/interp_not_validated_adj.tab";
  find_interp_validated_lemmata interp_compound_rule_trees results_path adv_sgjp_filename "results/interp_validated_adv.tab";
  find_not_interp_validated_lemmata interp_compound_rule_trees results_path adv_sgjp_filename "results/interp_not_validated_adv.tab"; *)
  (* find_interp_validated_lemmata interp_compound_rule_trees results_path verb_sgjp_filename "results/interp_validated_verb.tab";
  find_not_interp_validated_lemmata interp_compound_rule_trees results_path verb_sgjp_filename "results/interp_not_validated_verb.tab";
  find_interp_validated_lemmata interp_compound_rule_trees results_path "verb_sgjp_no_pref.tab" "results/interp_validated_no_pref_verb.tab";
  find_not_interp_validated_lemmata interp_compound_rule_trees results_path "verb_sgjp_no_pref.tab" "results/interp_not_validated_no_pref_verb.tab"; *)
  (* find_interp_validated_lemmata interp_compound_rule_trees results_path "sgjp_selected.tab" "results/interp_validated_verb.tab"; *)
  (* find_not_interp_validated_lemmata interp_compound_rule_trees results_path "sgjp_selected.tab" "results/interp_not_validated_verb.tab"; *)
  (* find_not_interp_validated_entries interp_compound_rule_trees results_path verb_sgjp_filename "results/selected_verb.tab"; *)
  (* find_not_interp_validated_entries interp_compound_rule_trees results_path "verb_sgjp_no_pref.tab" "results/selected_verb.tab"; *)
  (* find_not_interp_validated_forms interp_compound_rule_trees results_path "lang_en_sgjp-20170730.tab" "results/lang_en.tab"; *)
  ()

(* Generowanie reguł dla interpretacji *)
let _ =
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["pref";"cat";"flex";"lemma"] results_path adv_sgjp_filename "results/interp_rules_adv.tab"; *)
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["pref";"cat";"flex";"lemma"] results_path adj_sgjp_filename "results/interp_rules_adj.tab"; *)
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["cat";"flex";"lemma";"palat";"velar"] sgjp_path "sgjp_selected.tab" "results/interp_rules_s.tab"; *)
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["cat";"flex";"lemma"] sgjp_path "sgjp_selected.tab" "results/interp_rules_s2.tab"; *)
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["cat";"flex";"lemma";"palat";"velar";"con"] results_path noun_sgjp_filename "results/interp_rules_noun4.tab"; *)
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["cat";"flex";"lemma";"palat";"velar"] results_path noun_sgjp_filename "results/interp_rules_noun3.tab"; *)
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["cat";"flex";"lemma";"palat"] results_path noun_sgjp_filename "results/interp_rules_noun2.tab"; *)
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["cat";"flex";"lemma"] results_path noun_sgjp_filename "results/interp_rules_noun1.tab"; *)
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["pref";"cat";"flex";"flex2";"group";"lemma"] results_path "sgjp_selected.tab" "results/interp_rules_verb.tab"; *)
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["pref";"cat";"flex";"flex2";"group";"lemma"] results_path "verb_sgjp_no_pref.tab" "results/interp_rules_verb.tab"; *)
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["pref";"cat";"flex";"flex2";"group";"lemma"] results_path verb_sgjp_filename "results/interp_rules_verb.tab"; *)
  (* Dict.generate_interp_rules compound_rule_trees interp_compound_rule_trees ["pref";"cat";"flex";"flex2"] results_path verb_sgjp_filename "results/interp_rules_verb2.tab"; *)
  (* ignore (Sys.command "totem ~/Dokumenty/Inne/gong/gong_00m_30s.ogg"); *)
  (* ignore (Sys.command "totem ~/\"Muzyka/Era/Era 1998 Era/Era 02 Ameno (remix).ogg\""); *)
  ()

  (* Printexc.record_backtrace true;
  (try *)
  (* with e -> Printexc.print_backtrace stdout; print_endline (Printexc.to_string e));   *)

(* Generowanie złożonych reguł zaopatrzonych we frekwencje *)
let _ =
  (* Dict.generate_rule_frequencies interp_compound_rule_trees results_path adj_sgjp_filename "results/freq_rules-adj.tab"; *)
  (* Dict.generate_rule_frequencies interp_compound_rule_trees sgjp_path sgjp_filename "results/freq_rules.tab"; *)
  ()

let generate_alt rules_filename path filename out_filename =
  let rules = Rules.load_freq_rules rules_filename in
  let rules = Rules.CharTrees.create rules in
  let dict = Dict.load_tab (path ^ filename) in
  let dict = Dict.merge_entries dict in
  let dict = Dict.process_interps dict in
  let dict = Dict.remove_cat "cond" dict in
  (* let dict = Dict.mark_ndm dict in *)
  let dict = Dict.validate_interp rules dict in
  let dict = Dict.remove_validated_forms dict in
  Dict.print out_filename dict

(* Walidacja reguł zaopatrznych we frekwencje/generowanie listy wyjątków *)
let _ =
  (* generate_alt "results/freq_rules-adj.tab" results_path adj_sgjp_filename "results/alt-adj.tab"; *)
  (* generate_alt "results/freq_rules.tab" sgjp_path sgjp_filename "results/alt.tab"; *)
  ()

(* Generowanie stemów z regułami *)
let _ =
  (* Dict.generate_stem_dict "results/freq_rules-adj.tab" results_path adj_sgjp_filename "results/stem-adj.tab"; *)
  (* Dict.generate_stem_dict "results/freq_rules.tab" sgjp_path sgjp_filename "results/stem.tab"; *)
  ()

(**********************************************************************************)
(* Testy *)

let print_interpretations l =
  Xlist.iter (Xlist.sort l compare) (fun (lemma,interp,freq,tags) ->
    Printf.printf "%s\t%s\t%d\t%s\n" lemma interp freq (String.concat ";" tags))

let _ =
  (* let l = Inflexion.get_interpretations "życzliwą" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "żyźniejszego" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "zwiśli" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "najzieleńsza" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "najtandetniejsza" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "nieżelazny" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "któregokolwiek" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "większą" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "bordo" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "sexi" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "sexy" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "sepulkową" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "profesory" in
  print_interpretations l; *)
  (* let l = Inflexion.get_interpretations "chrobotnąwszy" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "chronografowi" in
  print_interpretations l;
  let l = Inflexion.get_interpretations "chowaniami" in
  print_interpretations l; *)
  (* let l = Inflexion.get_interpretations "Czechami" in
  print_interpretations l; *)
  (* let l = Inflexion.get_interpretations "Włoszech" in
  print_interpretations l; *)
  ()

(* let _ =
  let dict = Dict.load_tab (sgjp_path ^ sgjp_filename) in
  let dict = Dict.merge_entries dict in
  let dict = Dict.process_interps dict in
  let dict = Dict.remove_cat "cond" dict in
  Xlist.iter dict (fun entry ->
    let simple_lemma = Stem.simplify_lemma entry.lemma in
    Xlist.iter entry.forms (fun form ->
      let l = Inflexion.get_interpretations form.orth in
      let n = Xlist.fold l 0 (fun n (lemma,interp,freq,tags) ->
        if lemma = simple_lemma && interp = form.interp then n+1 else n) in
      if n <> 1 then printf "%d\t%s\t%s\t%s\n%!" n form.orth entry.lemma form.interp)) *)

(**********************************************************************************)
(**********************************************************************************)