łączenie na poziomie haseł

Wojciech Jaworski
1 parent 584f9657
Showing 1 changed file with 106 additions and 15 deletions
NKJP2/ENIAM_NKJP.ml
@@ -19,6 +19,34 @@
 open Xstd
+type id = {corref: string; prefix: string; suffix: string; numbers: int list}
+
+let empty_id = {corref = ""; prefix = ""; suffix = ""; numbers = []}
+
+let parse_id id =
+  (* if String.length s = 0 then empty_id else *)
+  if String.length id < 6 then failwith "parse_id: za krótkie id"  else
+  let corref,id = match Xstring.split "#" id with
+      [corref;id] -> corref,id
+    | [id] -> "",id
+    | _ -> failwith ("parse_id 1: " ^ id) in
+  let prefix,id = match Xstring.split "_" id with
+      [prefix;id] -> prefix,id
+    | _ -> failwith ("parse_id 2: " ^ id) in
+  let suffix,id = match Xstring.split "-" id with
+      [id;suffix] -> suffix,id
+    | _ -> failwith ("parse_id 3: " ^ id) in
+  let numbers =  try Xlist.map (Xstring.split "\\." id) int_of_string with _ -> failwith ("parse_id 4: " ^ id) in
+  {corref=corref; prefix=prefix; suffix=suffix; numbers=numbers}
+
+let process_header_type typ =
+  if Xstring.check_prefix "#typ_" typ then Xstring.cut_prefix "#typ_" typ
+  else failwith ("process_header_type: " ^ typ)
+
+let process_header_channel c =
+  if Xstring.check_prefix "#kanal_" c then Xstring.cut_prefix "#kanal_" c
+  else failwith ("process_header_channel: " ^ c)
+
 let load_header path name =
   match Xml.parse_file (path ^ name ^ "/header.xml") with
       Xml.Element("teiHeader",_,[Xml.Element("fileDesc",[],_);
@@ -26,7 +54,7 @@ let load_header path name =
                      Xml.Element("catRef",["scheme","#taxonomy-NKJP-type";"target",typ],[]);
                      Xml.Element("catRef",["scheme","#taxonomy-NKJP-channel";"target",channel],[])])]);
                      Xml.Element("revisionDesc",_,_)]) ->
-        typ,channel
+        process_header_type typ,process_header_channel channel
     | _ -> failwith "load_header"
 let get_folders path =
@@ -35,12 +63,12 @@ let get_folders path =
 let load_paragraph = function
     Xml.Element("ab",["n",_;"xml:id",id_ab],[Xml.PCData paragraph]) ->
-      id_ab,paragraph
+      parse_id id_ab,paragraph
   | xml -> failwith ("load_text_entry: " ^ Xml.to_string_fmt xml)
 let load_text_entry = function
     Xml.Element("div",["xml:id",id_div;"decls",_],paragraphs) ->
-      id_div,List.rev (Xlist.rev_map paragraphs load_paragraph)
+      parse_id id_div,List.rev (Xlist.rev_map paragraphs load_paragraph)
   | xml -> failwith ("load_text_entry: " ^ Xml.to_string_fmt xml)
 let load_text path name =
@@ -64,9 +92,9 @@ let remove_rejected rev = function
 let rec load_segm_token = function
     Xml.Element("seg",["corresp",corresp;"xml:id",id_seg],[]) ->
-      [corresp,false,id_seg]
+      [corresp,false,parse_id id_seg]
   | Xml.Element("seg",["corresp",corresp;"nkjp:nps","true";"xml:id",id_seg],[]) ->
-      [corresp,true,id_seg]
+      [corresp,true,parse_id id_seg]
   | Xml.Element("nkjp:paren",[],tokens) -> List.flatten (Xlist.map tokens load_segm_token)
   | Xml.Element("choice",[],alt) as xml ->
       let alt = Xlist.fold alt [] remove_rejected in
@@ -77,12 +105,12 @@ let rec load_segm_token = function
 let load_segm_sentence = function
     Xml.Element("s",["xml:id",id_s],tokens) ->
-      id_s,List.flatten (List.rev (Xlist.rev_map tokens load_segm_token))
+      parse_id id_s,List.flatten (List.rev (Xlist.rev_map tokens load_segm_token))
   | xml -> failwith ("load_segm_sentence: " ^ Xml.to_string_fmt xml)
 let load_segm_entry = function
     Xml.Element("p",["corresp",corresp;"xml:id",id_p],sentences) ->
-      corresp,id_p,List.rev (Xlist.rev_map sentences load_segm_sentence)
+      parse_id corresp,parse_id id_p,List.rev (Xlist.rev_map sentences load_segm_sentence)
   | xml -> failwith ("load_segm_entry: " ^ Xml.to_string_fmt xml)
 let load_segmentation path name =
@@ -104,22 +132,22 @@ let load_morph_token = function
     Xml.Element("seg",["corresp",corresp;"xml:id",id_seg],[Xml.Element("fs",["type","morph"],
       [Xml.Element("f",["name","orth"],[Xml.Element("string",[],[Xml.PCData orth])]);
        Xml.Element("f",["name","interps"],_);Xml.Element("f",["name","disamb"],[disamb])])]) ->
-      corresp,id_seg,orth,load_disamb disamb
+      parse_id corresp,parse_id id_seg,orth,load_disamb disamb
   | Xml.Element("seg",["corresp",corresp;"xml:id",id_seg],[Xml.Element("fs",["type","morph"],
       [Xml.Element("f",["name","orth"],[Xml.Element("string",[],[Xml.PCData orth])]);
        Xml.Element("f",["name","nps"],[Xml.Element("binary",["value","true"],[])]);
        Xml.Element("f",["name","interps"],_);Xml.Element("f",["name","disamb"],[disamb])])]) ->
-      corresp,id_seg,orth,load_disamb disamb
+      parse_id corresp,parse_id id_seg,orth,load_disamb disamb
   | xml -> failwith ("load_morph_token: " ^ Xml.to_string_fmt xml)
 let load_morph_sentence = function
     Xml.Element("s",["corresp",corresp;"xml:id",id_s],tokens) ->
-      corresp,id_s,List.rev (Xlist.rev_map tokens load_morph_token)
+      parse_id corresp,parse_id id_s,List.rev (Xlist.rev_map tokens load_morph_token)
   | xml -> failwith ("load_morph_sentence: " ^ Xml.to_string_fmt xml)
 let load_morph_entry = function
     Xml.Element("p",["corresp",corresp;"xml:id",id_p],sentences) ->
-      corresp,id_p,List.rev (Xlist.rev_map sentences load_morph_sentence)
+      parse_id corresp,parse_id id_p,List.rev (Xlist.rev_map sentences load_morph_sentence)
   | xml -> failwith ("load_morph_entry: " ^ Xml.to_string_fmt xml)
 let load_morphosyntax path name =
@@ -130,6 +158,16 @@ let load_morphosyntax path name =
         List.rev (Xlist.rev_map entries load_morph_entry)
     | _ -> failwith "load_morphosyntax"
+let rec merge_entries rev = function
+    ({corref=""; prefix="txt"; numbers=[id_div]; suffix="div"},paragraphs) :: text,
+    ({corref="text.xml"; prefix="txt"; numbers=[c_div]; suffix="div"},
+     {corref=""; prefix="segm"; numbers=[id_segm_p]; suffix="p"},segm_sentences) :: segmentation,
+    ({corref="ann_segmentation.xml"; prefix="segm"; numbers=[c_segm_p]; suffix="p"},
+     {corref=""; prefix="morph"; numbers=[id_morph_p]; suffix="p"},morph_sentences) :: morphosyntax ->
+        if id_div <> c_div || c_div <> id_segm_p || id_segm_p <> c_segm_p || c_segm_p <> id_morph_p then failwith "merge_entries 2" else
+        merge_entries ((id_div,paragraphs,segm_sentences,morph_sentences) :: rev) (text,segmentation,morphosyntax)
+  | [],[],[] -> List.rev rev
+  | _ -> failwith "merge_entries"
 let nkjp_path = "../../NLP resources/NKJP-PodkorpusMilionowy-1.2/"
@@ -137,13 +175,66 @@ let _ =
   let names = get_folders nkjp_path in
   Xlist.iter names (fun name ->
     print_endline name;
-    let header = load_text nkjp_path name in
-    (* let text = load_text nkjp_path name in *)
-    (* let segmentation = load_segmentation nkjp_path name in *)
-    (* let morphosyntax = load_morphosyntax nkjp_path name in *)
+    let typ,channel = load_header nkjp_path name in
+    (* print_endline typ; *)
+    (* print_endline channel; *)
+    (* print_endline (typ ^ "\t" ^ channel); *)
+    let text = load_text nkjp_path name in
+    let segmentation = load_segmentation nkjp_path name in
+    let morphosyntax = load_morphosyntax nkjp_path name in
+    let entries = merge_entries [] (text,segmentation,morphosyntax) in
     ())
 (*
+frekwencje typów:
+    127 fakt
+     56 inf-por
+    283 konwers
+      2 listy
+    376 lit
+      1 lit_poezja
+     80 media
+    175 nd
+    161 net_interakt
+    227 net_nieinterakt
+     20 nklas
+   1986 publ
+      8 qmow
+    387 urzed
+
+frekwencje kanałów
+    388 internet
+    817 ksiazka
+    363 mowiony
+    146 prasa
+   1744 prasa_dziennik
+    398 prasa_inne
+      5 prasa_miesiecznik
+     28 prasa_tygodnik
+
+frekwencje łączne typów-kanałów
+      127 fakt    ksiazka
+     56 inf-por ksiazka
+    283 konwers mowiony
+      2 listy   ksiazka
+    376 lit     ksiazka
+      1 lit_poezja      ksiazka
+     80 media   mowiony
+    175 nd      ksiazka
+    161 net_interakt    internet
+    227 net_nieinterakt internet
+     20 nklas   ksiazka
+     60 publ    ksiazka
+    146 publ    prasa
+   1744 publ    prasa_dziennik
+      3 publ    prasa_inne
+      5 publ    prasa_miesiecznik
+     28 publ    prasa_tygodnik
+      8 qmow    prasa_inne
+    387 urzed   prasa_inne
+
+    *)
+(*
 type id = {hash: bool; suffix: string; numbers: int list}