CONLL.ml
15.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
open Xstd
open ENIAMsubsyntaxTypes
open ENIAMtokenizerTypes
let alternative_string f mode alts = if List.exists (fun (m,_) -> mode = m) alts
then f (snd @@ List.find (fun (m,_) -> m = mode) alts)
else f (snd @@ List.find (fun (m,_) -> m = Struct) alts)
let string_of_token mode token conll_id super label =
let decompose_lemma = function
| Lemma(a,b,c) -> a,b,if c = [[]]
then "_"
else String.concat "][" @@ Xlist.map c (fun x ->
String.concat "|" @@ Xlist.map x ( fun y ->
String.concat "." y))
| t -> failwith ("string_of_token: not Lemma") in
match mode with
| Raw -> token.orth
| Struct -> failwith ("function string_of_token for mode Struct is not defined")
| CONLL -> let lemma,cat,interp = decompose_lemma token.token in
String.concat "\t" [string_of_int conll_id;
token.orth; lemma; cat; cat; interp; "_"; "_";
string_of_int token.beg; string_of_int token.len]
| Mate -> let lemma,cat,interp = decompose_lemma token.token in
String.concat "\t" [string_of_int conll_id;
token.orth; lemma; lemma; cat; cat; interp; interp; "_"; "_"; "_"; "_"; "_"; "_"]
| _ -> failwith "string_of_token: ni"
let string_of_paths mode tokens paths =
let l = Int.fold 1 (Array.length paths - 1) [] (fun l conll_id ->
let id,super,label = paths.(conll_id) in
(string_of_token mode (ExtArray.get tokens id) conll_id super label) :: l) in
String.concat "\n" (List.rev l) ^ "\n\n"
let rec string_of_sentence mode tokens = function
RawSentence s -> if mode = Raw then s else ""
| StructSentence (tokens, _) -> failwith ("string_of_sentence: StructSentence") (*String.concat "\n" @@ Xlist.map tokens (fun x -> string_of_token mode x)*)
| DepSentence (paths) -> string_of_paths mode tokens paths
| QuotedSentences _ -> failwith ("string_of_sentence: QuotedSentences")
| AltSentence alts -> alternative_string (string_of_sentence mode tokens) mode alts
let string_of_p_record mode tokens p_record =
(if p_record.id = "" then "" else p_record.id ^ "\n") ^
string_of_sentence mode tokens p_record.sentence
(*let rec string_of_paragraph mode tokens = function
RawParagraph s -> if mode = Raw then s else ""
| StructParagraph p_records -> String.concat "\n\n" @@ Xlist.map p_records (string_of_p_record mode tokens)
| AltParagraph alts -> alternative_string (string_of_paragraph mode) mode alts
let rec string_of_text mode tokens = function
RawText s -> if mode = Raw then s else ""
| StructText (paragraphs,_) -> String.concat "\n\n" @@ Xlist.map paragraphs (string_of_paragraph mode tokens)
| AltText alts -> alternative_string (string_of_text mode) mode alts*)
(******************)
(***
let establish_next tokens paths =
let n = ExtArray.size tokens in
Int.iter 1 (n - 2) (fun i ->
let f = ExtArray.get tokens i in
let s = ExtArray.get tokens (i+1) in
ExtArray.set tokens i {f with next = s.beg});
let last = ExtArray.get tokens (n-1) in
ExtArray.set tokens (n-1) {last with next = last.beg + last.len}
(*let rec pom res = function
h :: t -> let next = if res = []
then h.beg+h.len
else (List.hd res).beg in
pom ({h with next = next} :: res) t
| [] -> res in
pom [] rev_tokens*)
let rec establish_for_token i text tokens = function
(id,_,_) :: t as l->
let h = ExtArray.get tokens id in
if Xstring.check_prefix " " text
then establish_for_token (i+100) (Xstring.cut_prefix " " text) tokens l
else if Xstring.check_prefix h.orth text
then
let n = (List.length @@ Xunicode.utf8_chars_of_utf8_string h.orth) * 100 in
let n_h = {h with beg = i ; len = n} in
ExtArray.set tokens id n_h;
establish_for_token (i+n) (Xstring.cut_prefix h.orth text) tokens t
else failwith ("establish_for_token :" ^ h.orth ^ " " ^ text)
| [] -> 100, i
let rec establish_lengths text paths tokens =
let pbeg, plen = establish_for_token 100 text tokens (List.tl (Array.to_list paths)) in
establish_next tokens paths;
pbeg, plen-100
(******************)
exception ErrorInfoFile of string
let info_file = "../corpora/info_sentences.txt"
let info = Xstring.split "\n\n" @@ File.load_file_gen info_file
let add_to_map map info_str =
match Xstring.split "\n" info_str with
[id; text; info_token] -> StringMap.add map info_token (id, text)
| _ -> raise (ErrorInfoFile info_str)
let info_map =
Xlist.fold info StringMap.empty add_to_map
let match_sentence (p_record,tokens) =
let rec info_token s = match s with
RawSentence text -> failwith ("match_sentence: " ^ text)
| StructSentence (tokens, n) -> failwith ("match_sentence: StructSentence") (*String.concat " " @@ List.map (fun x -> x.orth) tokens*)
| DepSentence (paths) -> String.concat " " @@ List.map (fun (id,_,_) -> (ExtArray.get tokens id).orth) (List.tl (Array.to_list paths)), paths
| QuotedSentences _ -> failwith ("match_sentence: QuotedSentences")
| AltSentence alts -> failwith ("match_sentence: AltSentence")
(*if List.exists (fun (mode, s) -> mode = CONLL) alts
then info_token (snd (List.find (fun (mode, s) -> mode = CONLL) alts))
else failwith ("match_sentence: no CONLL mode in AltSentence")*) in
let info_token, paths = info_token p_record.psentence in
try
let id, text = StringMap.find info_map info_token in
let beg, len = establish_lengths text paths tokens (* -1, -1, p_record.psentence *) in
AltText[Raw,RawText text;CONLL,StructText([StructParagraph[{pid = id; pbeg = beg; plen = len; pnext = beg+len; pfile_prefix="";
psentence = AltSentence[Raw, RawSentence text; CONLL, DepSentence paths]}]],tokens)]
(* {s_id = id; s_text = text; s_tokens = sentence.s_tokens} *)
with _ -> AltText[CONLL,StructText([StructParagraph[p_record]],tokens)]
let match_corpus corpus =
Xlist.map corpus match_sentence***)
(******************)
type to_text = { t_orth: string; t_cat: string; t_interp: string list list list }
let empty_to_text = { t_orth = ""; t_cat = ""; t_interp = [[[]]] }
let get_text tokens =
let get i =
let cat = match (ExtArray.get tokens i).token with
Lemma(_,cat,_) -> cat
| _ -> "" in
let interp = match (ExtArray.get tokens i).token with
Lemma(_,_,i) -> i
| _ -> [[[]]] in
{ t_orth = (ExtArray.get tokens i).orth; t_cat = cat; t_interp = interp } in
let n_tokens = Int.fold_down (ExtArray.size tokens - 1) 0 []
(fun acc i -> (get i)::acc)in
let quote_open = ref false in
let hyphenated = ref false in
let maybe_add_space pre_previous previous token next =
if previous.t_orth = "" && token.t_orth = "\""
then quote_open := true;
if token.t_cat = "aglt" ||
(token.t_orth = "by" && previous.t_cat = "praet") ||
(previous.t_orth = "\"" && !quote_open) ||
previous.t_orth = "(" ||
previous.t_orth = "„" ||
previous.t_orth = "" ||
token.t_orth = "ń" || (* wyrażenie nań *)
(token.t_orth = "że" && (previous.t_orth = "czym" || previous.t_orth = "Czym")) || (*wyrażenie czymże*)
(* (token.orth = "r" && token.cat = "brev") || (*skrót r. - np. 1991r. *) *)
(pre_previous.t_cat = "adj" && previous.t_orth = "." &&
token.t_cat = "num" && token.t_interp = [[["pl"];["nom"];["f"];["rec"]]]) (* godzina - np 13.15*)
then token.t_orth
else if !hyphenated
then (hyphenated := false; token.t_orth)
else match token.t_orth with
"." -> "."
| "…" -> "…"
| "?" -> "?"
| "!" -> "!"
| "," -> ","
| ":" -> ":"
| ";" -> ";"
| ")" -> ")"
| "”" -> "”"
| "-" -> if previous.t_cat = "adja" ||
(previous.t_cat = "subst" && next.t_cat = "subst" && previous.t_interp = next.t_interp)
then (hyphenated := true; "-")
else " -"
| "\"" -> if !quote_open
then (quote_open := false; "\"")
else (quote_open := true; " \"")
| s -> " "^s in
let rec fold4 acc = function
a::b::c::d::t -> fold4 (acc^maybe_add_space a b c d) (b::c::d::t)
| a::b::c::[] -> fold4 (acc^maybe_add_space a b c empty_to_text) (b::c::[])
| a::b::[] -> acc
| _ -> failwith ("get_sentence") in
fold4 "" (empty_to_text::empty_to_text::n_tokens)
(******************)
let establish_next tokens paths =
let n = ExtArray.size tokens in
Int.iter 1 (n - 2) (fun i ->
let f = ExtArray.get tokens i in
let s = ExtArray.get tokens (i+1) in
ExtArray.set tokens i {f with next = s.beg});
let last = ExtArray.get tokens (n-1) in
ExtArray.set tokens (n-1) {last with next = last.beg + last.len}
(*let rec pom res = function
h :: t -> let next = if res = []
then h.beg+h.len
else (List.hd res).beg in
pom ({h with next = next} :: res) t
| [] -> res in
pom [] rev_tokens*)
let rec establish_for_token i text tokens = function
(id,_,_) :: t as l->
let h = ExtArray.get tokens id in
if Xstring.check_prefix " " text
then establish_for_token (i+100) (Xstring.cut_prefix " " text) tokens l
else if Xstring.check_prefix h.orth text
(* || (h.orth = "m.in." && Xstring.check_prefix "m.in" text) *)
then
let n = (List.length @@ Xunicode.utf8_chars_of_utf8_string h.orth) * 100 in
let n_h = {h with beg = i ; len = n} in
ExtArray.set tokens id n_h;
(* if Xstring.check_prefix h.orth text then *)
establish_for_token (i+n) (Xstring.cut_prefix h.orth text) tokens t
(* else establish_for_token (i+n) (Xstring.cut_prefix "m.in" text) tokens t *)
else (prerr_endline ("establish_for_token :" ^ h.orth ^ " " ^ text);
failwith ("establish_for_token :" ^ h.orth ^ " " ^ text))
| [] -> 100, i
let rec establish_lengths text paths tokens =
let pbeg, plen = establish_for_token 100 text tokens (List.tl (Array.to_list paths)) in
establish_next tokens paths;
pbeg, plen-100
(******************)
exception ErrorInfoFile of string
let info_file = "../corpora/info_sentences2.txt"
let info = Xstring.split "\n\n" @@ File.load_file_gen info_file
let add_to_map map info_str =
match Xstring.split "\n" info_str with
[id; text; info_token] -> StringMap.add map info_token (id, text)
| _ -> raise (ErrorInfoFile info_str)
let info_map =
Xlist.fold (List.tl info) StringMap.empty add_to_map
let match_sentence (p_record,tokens) =
let rec info_token s = match s with
RawSentence text -> failwith ("match_sentence: " ^ text)
| StructSentence (tokens, n) -> failwith ("match_sentence: StructSentence") (*String.concat " " @@ List.map (fun x -> x.orth) tokens*)
| DepSentence (paths) -> String.concat " " @@ List.map (fun (id,_,_) -> (ExtArray.get tokens id).orth) (List.tl (Array.to_list paths)), paths
| QuotedSentences _ -> failwith ("match_sentence: QuotedSentences")
| AltSentence alts -> failwith ("match_sentence: AltSentence")
(*if List.exists (fun (mode, s) -> mode = CONLL) alts
then info_token (snd (List.find (fun (mode, s) -> mode = CONLL) alts))
else failwith ("match_sentence: no CONLL mode in AltSentence")*) in
let info_token, paths = info_token p_record.sentence in
(* try *)
let id, text = try
StringMap.find info_map info_token
with
| _ -> p_record.id, get_text tokens in
let beg, len = establish_lengths text paths tokens (* -1, -1, p_record.psentence *) in
AltText[Raw,RawText text;CONLL,StructText[StructParagraph[{id = id; beg = beg; len = len; next = beg+len; file_prefix="";
sentence = AltSentence[Raw, RawSentence text; CONLL, DepSentence paths]}]]],tokens
(* {s_id = id; s_text = text; s_tokens = sentence.s_tokens} *)
(* with _ -> AltText[CONLL,StructText([StructParagraph[p_record]],tokens)] *)
let match_corpus corpus =
let rec pom f = function
[] -> []
| a::l -> try
let r = f a in r :: pom f l
with e -> (*print_endline (Printexc.to_string e);*) pom f l in
pom match_sentence corpus
(******************)
exception Empty_line
exception Empty_sentence
exception Id_line of string
let load_token in_channel =
let fail line =
(* failwith ("load_token: " ^ line) *)
() in
let int_of_super = function
"_" -> -1
| s -> int_of_string s in
let n_token id orth lemma cat interp super label =
let interp = if interp = "_"
then [[]]
else [Xlist.map (Xstring.split_delim "|" interp) (fun tag -> [tag])] in
{empty_token_env with orth = orth; token = Lemma(lemma,cat,interp);}, int_of_string id, int_of_super super, label in
let line = input_line in_channel in
(* print_endline ("load_token: " ^ line); *)
if line = ""
then raise Empty_line
else if line.[0] = '#'
then
if Xstring.check_prefix "# trees/" line && Xstring.check_sufix ".xml.trees" line
then let id = Xstring.cut_prefix "# trees/" @@ Xstring.cut_sufix ".xml.trees" line in
raise (Id_line id)
else if Xstring.check_prefix "# trees/" line && Xstring.check_sufix ".xml.tree" line
then let id = Xstring.cut_prefix "# trees/" @@ Xstring.cut_sufix ".xml.tree" line in
raise (Id_line id)
else failwith ("load_token: " ^ line)
else
match Xstring.split "\t" line with
[id; orth; lemma; cat; cat2; interp; super; label; "_"; "_"] ->
(if cat <> cat2 then fail line;
n_token id orth lemma cat interp super label)
| [id; orth; lemma; lemma2; cat; cat2; interp; interp2; "-1"; super; "_"; label; "_"; "_"] ->
(if (cat, lemma, interp) <> (cat2, lemma2, interp2) then fail line;
n_token id orth lemma cat interp super label)
| [id; orth; lemma; cat; cat2; interp; super; label_err; "_"] ->
(if cat <> cat2 && Xstring.check_sufix "_" label_err then fail line;
let label = Xstring.cut_sufix "_" label_err in
n_token id orth lemma cat interp super label)
| _ -> failwith ("load_token: " ^ line)
(* {c_id = List.nth pom 1;
c_lemma = List.nth pom 2;
c_cat = List.nth pom 3;
c_interp = (let interp = List.nth pom 5 in
if interp = "_"
then []
else Str.split (Str.regexp "|") interp);
c_super = -1; c_label = ""; c_beg = -1; c_len = -1} *)
let load_sentence in_channel =
let tokens = ExtArray.make 100 empty_token_env in
let _ = ExtArray.add tokens {empty_token_env with token = Interp "<conll_root>"} in
let rec pom rev_paths id =
(* print_endline "pom 1"; *)
try
(* print_endline "pom 2"; *)
let token, conll_id, super, label = load_token in_channel in
let id_a = ExtArray.add tokens token in
if id_a <> conll_id then failwith "load_sentence: different ids" else
(* print_endline "pom 3"; *)
pom ((id_a,super,label) :: rev_paths) id
with Id_line new_id -> (*print_endline "pom 4";*)pom rev_paths new_id
| Empty_line -> (*print_endline "pom 5";*)rev_paths, id
| End_of_file -> (*print_endline "pom 6";*)if rev_paths = []
then raise End_of_file
else rev_paths, id in
let rev_paths, id = pom [] "" in
{id = id; beg = -1; len = -1; next = -1; file_prefix = ""; sentence = DepSentence(Array.of_list ((0,-1,"") :: List.rev rev_paths))}, tokens
(* {s_id = id; s_text = ""; s_paths = (List.rev rev_paths)} *)
let load_corpus in_channel =
let rec pom res =
try
let conll_sentence, tokens = load_sentence in_channel in
pom ((conll_sentence, tokens) :: res)
with End_of_file -> res
| e -> prerr_endline (Printexc.to_string e); res in
(* match_corpus @@ *) List.rev @@ pom []