cleaner.py
15.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
"""Read Markdown file converted from ODT and fix as many errors as possible.
This uses heuristics to fix markdown, punctuation and even common misspellings.
"""
import re
LOWER = 'a-ząćęłńóśźż'
UPPER = 'A-ZĄĆĘŁŃÓŚŹŻ'
LETTER = 'A-Za-zĄĆĘŁŃÓŚŹŻąćęłńóśźż'
class Cleaner:
'The class to clean Markdown'
def __init__(self, verbose=''):
self.verbose = verbose
def clean(self, filename, output):
' Do all the work '
pars = self._read(filename)
filters = [self._merge_titles,
self._clear_odt,
self._merge_pars,
self._trim_interpelations,
self._fix_speakers,
self._fix_comments,
# self._simplify_formatting
]
for filter_function in filters:
pars = filter_function(pars)
self.save(pars, output)
# if self.args.mode != 'senate':
# pars = self._fix_senate(pars)
# if self.args.text:
# if self.args.speakers:
# self.extractSpeakers(filename, pars)
def _read(self, filename):
'Read data from file. Merge lines into paragraphs using empty line as separator'
pars = []
current = ''
for line in open(filename):
line = self._fix_punctuation(line).strip()
if line:
if current.endswith('- ') and not current.endswith('o- '):
current = current[:-2]
current += line + ' '
else:
pars.append(current.strip())
current = ''
if current.strip():
pars.append(current.strip())
return pars
def _merge_titles(self, pars):
'Merge split speakers with title in the first line and name in the second'
fixed = []
for par in pars:
if par.startswith('**') and fixed and ':' not in fixed[-1] and fixed[-1].endswith('**'):
fixed[-1] = (fixed[-1][:-2] + ' ' + par[2:]).replace(' ', ' ')
else:
fixed.append(par)
return fixed
def _fix_punctuation(self, line):
'Fix space punctuation and mdash'
line = re.sub(r'[\xc2\xad]', '', line) # ‍
line = re.sub(r'\s+', ' ', line) # spaces
line2 = line
replacements = {
r'(\s+|^),,': r'\1„',
r'"': r'”',
r'\s*---\s*': ' — ',
r'(\w)\s+\.': '\1.', # w ustawie . -> w ustawie.
r'\( ': '(', # ( razem) -> (raem)
r'(\w)\(': r'\1 (', # to(albo tamto) -> to (albo tamto)
r' \)': ')', # (razem ) -> (razem)
# biało- czerwony -> biało-czerwony
fr'([{LOWER}]{{3}}[wknłcz]o)-\s+([{LOWER}]{{4}})': r'\1-\2',
r'([{LOWER}])-\s+([{LOWER}]{{2}})': r'\1\2', # al- bo -> albo
r'(\d) — (\d)': r'\1-\2', # 12 - 20% -> 12-20%
}
for pattern, replacement in replacements.items():
line = re.sub(pattern, replacement, line)
self._debug('punctuation', ['punctuation changed',
line2, line], line != line2)
return line
def _clear_odt(self, pars):
'Remove leftovers from OpenOffice: bookmarks, footnotes, hyphenation, backslashes'
removed = ['[]', '\\*', '\\', '{.Apple-converted-spaces}', '(—)']
reremoved = [r'\s*\u00ad\s*', '{#anchor-?[0-9]*}', r'\{[0-9s.]*\}',
r'\[\W]*\]', r'\[\s*\]']
cleared = []
for par in pars:
for remove in removed:
par = par.replace(remove, ' ')
for remove in reremoved:
par = re.sub(remove, ' ', par)
if '*' in par:
par = self._fix_markdown(par)
if re.search(r'\S\s\S\s\S\s\S\s\S\s\S\s\S', par):
par = self._fix_spaceout(par)
cleared.append(par.strip())
return cleared
def _fix_spaceout(self, par):
'Try to merge spaced out words'
oldpar = par
match = re.search(r'\s\S\s\S\s\S\s\S\s\S\s\S\s', par)
if match:
start = match.start()
end = start
while end < len(par) - 1 and par[end] == ' ':
end += 2
spaceout = re.sub('(.)([A-ZĆŁÓŚŻŹ])', '\\1 \\2',
par[start:end].replace(' ', ''))
par = par[:start] + spaceout + par[end:]
self._debug('spaceout', ['spaceout removed', oldpar, par])
return par
def _fix_markdown(self, par):
'Simplify markdown asterisk. Fix spaces adjacent to asterisk'
oldpar = par
# Replace redundant asterisks
par = re.sub(r'\*\*([-=.]?)\*\*', r'\1', par)
for mark in ['**', '*']:
start = par.find(mark)
if start == -1 or start >= len(par) - len(mark) or par[start+len(mark)] == '*':
continue
while start > 0 and not par[start-1].isspace():
start -= 1
last = par.rfind(mark)
if last <= start:
continue
if par[last-1] == ' ':
mid = par[start:last-1].replace('*', '')
par = f'{par[:start]}{mark}{mid}{mark} {par[last+len(mark):]}'
else:
while (last < len(par) and par[last] != ' '):
last += 1
par = par[:start] + mark + \
par[start:last].replace('*', '') + mark + par[last:]
par = re.sub(r'\*\*(\s*)\*\*', '\\1', par, 0, re.MULTILINE)
# Add trailing * for starting * + bracket
par = re.sub(r'\s\*\(([^)]*)\)\.?\s+\*?', lambda m: ' *(' +
m.group(1).replace('*', '') + ')* ', par)
# Remove comment with no colon/bracket - some texts contain spurious italic
par = re.sub(
fr'\*+([{LETTER} ][{UPPER} !?-]*)\*+', '\\1', par)
self._debug('markdown', ['markdown fixed', oldpar, par], par != oldpar)
return par
def _fix_speakers(self, pars):
'Heuristics to remove spurious speaker formatting and to format some unformatted speakers'
fixed = []
for par in pars:
oldpar = par
if '**' in par:
# Remove ** around single words
par = re.sub(
r'\*\*([{LOWER}0-9%,.:/\s-]*)\*\*', '\\1', par)
# Remove ** not at the start of the line
if '**' in par and not par.startswith('**'):
par = par.replace('**', '')
# Move ** after the colon
par = re.sub(
r'^\*\*([^*:]*):\s+([^*]*)\s*\*\*', '**\\1:** \\2', par)
# Remove ** if the content does not look like person name
match = re.match(r'^\*\*([^\*]*):?\*\*', par)
if not match or not self._can_be_person(match.group(1)):
par = self._find_speakers(par.replace('**', ''))
self._debug('speakers', ['not a speaker', par])
else:
par = self._find_speakers(par)
fixed.append(par)
status = '' if par.startswith('**') else 'not a'
self._debug('speakers', [f'{status} speaker', par], oldpar != par)
return fixed
def _find_speakers(self, par):
'Add ** around unformatted speakers if applicable'
match = re.match(
fr'([{UPPER}][{LOWER}.]+\s+[{UPPER}][{LOWER}]+\s+[{UPPER}][{LETTER}-]+)[.:\s]*$', par)
if match:
if self._can_be_person(match.group(1)):
return f'**{match.group(1)}:**'
else:
return par
match = re.match(
fr'([{UPPER}][{LOWER}.]+\s+[{UPPER}][{LETTER}-]+):\s+(.*)$', par)
if match:
return f'**{match.group(1)}:**\n\n{match.group(2)}'
match = re.match(
fr'([{UPPER}][{LOWER}.]+\s+[{UPPER}][{LETTER}-]+\s+[{UPPER}][{LOWER}-]+):\s+(.*)$', par)
if match:
return f'**{match.group(1)}:**\n\n{match.group(2)}'
if par == u'Marszałek':
return u'**Marszałek:**'
return par
def _fix_comments(self, pars):
""" Add italic to some comments not formatted in the original file
They are assumed to start after a dot, be in brackets and start with uppercase.
Fragments with numbers are excluded to avoid formatting legal references.
Also fix some comment formatting.
"""
fixed = []
for par in pars:
oldpar = par
# Fix comments like '*(Part of* comment)'
par = par.replace('(*', '*(')
par = re.sub(r'\*\(([^\)]*)\*([^\)]*)\)', '*(\\1\\2)*', par)
# Add missing comments
par = re.sub(fr'\. \(([{UPPER}][^0-9)]*)\)\.?(\s|$|\*)',
lambda m: '. *(' + m.group(1).replace('*', '') + ')* ', par)
# Fix comments marked as new speaker
not_speaker = re.match(r'\*\*\((.*)\)\.?\s*\*\*$', par)
if not_speaker:
par = '*(' + not_speaker.group(1) + ')*'
self._debug('comments', [par], oldpar != par)
fixed.append(par)
return fixed
def _fix_senate(self, pars):
fixed = []
header = False
for par in pars:
if not header and not par.startswith('**'):
continue
header = True
if par.startswith('[]'):
continue
if re.match(r'\[\d+\]?$', par):
continue
if re.match(r'\*\(Początek posiedzenia o godzinie \d+ minut \d+\)\*', par):
continue
if re.match(r'\d+\. posiedzenie [\wąćęłńóśżźŹŻŚŁ\s,]+$', par):
continue
if re.match(r'\d+$', par):
continue
if re.match(r'w dniu \d+ \S+ [0-9]{4} r\.$', par):
continue
if re.match(r'\d+\. posiedzenie .*\*', par):
fixed.append(re.sub(r'^[^*]*(\*.*)$', '\\1', par))
else:
fixed.append(par)
if '*(Koniec posiedzenia' in par:
break
return fixed
def _can_be_person(self, person):
'Check if given fragment can be a person. Used to remove spurious bold around some titles'
if not 3 < len(person) < 151:
return False
for prefix in ['(', 'Obywatel', 'Wysoka', 'Proszę', 'Polski', 'Panie', 'Dziękuję',
'Sprawozdanie', 'Pan ', 'Pani ', 'Przystępuj']:
if person.startswith(prefix):
return False
for suffix in ['ego']:
if person.endswith(suffix):
return False
for infix in ['II', 'Sejm', 'rzystępujemy', 'nterpelacj', 'Warszawa']:
if infix in person:
return False
if re.search('[0-9]', person):
return False
return True
def _simplify_formatting(self, pars):
'Remove repeated or nested formatting'
text = []
for par in pars:
par = re.sub(r'\*\*[^\*]*\*\*', '', par)
par = re.sub(r'\*[^\*]*\*', '', par)
par = par.strip()
if par:
text.append(par)
return text
def _merge_pars(self, pars):
"""Heuristic merging of paragraphs. Adds weights suggesting it IS a new paragraph.
If the total weight is below the threshold, the paragraph is merged.
"""
merged = pars[:1]
for par in pars[1:]:
if not par:
continue
previous = merged[-1]
eol = 0
if previous.endswith(('!', '?', '.', ':', '"', ';')):
eol += 14
if '*' in previous:
eol += 5
if len(previous) < 60:
eol += 5
if '**' in par and len(par) < 80:
eol += 16
if previous.endswith('**'):
eol += 17
if par.startswith('--'):
eol += 12
if par[0].isupper():
eol += 5
if par.startswith('**') and len(par) > 2 and par[2].isupper():
eol += 17
if re.search(r' [a-z] [a-z] [a-z] [a-z] ', previous):
eol += 11
if eol >= 10:
self._debug('paragraphs', [
f'paragraphs not merged (weight {eol})', merged[-1], par],
par[0].islower())
merged.append(par)
else:
self._debug(
'paragraphs', [f'paragraphs merged (weight {eol})', merged[-1], par])
if merged[-1].endswith('-'): # Remove trailing hyphen when merging
merged[-1] = merged[-1][:-1] + par
else:
merged[-1] = merged[-1] + ' ' + par
return merged
def _trim_interpelations(self, pars):
'Remove interpelations'
trimmed = []
ignored = False
for par in pars:
if par.replace('*', '').startswith(u'Odpowiedź'): # and not self.interpellations
ignored = True
self._debug('interpellations', 'interpellation detected')
if not ignored:
trimmed.append(par)
elif self.verbose == 'interpellations':
self._debug('interpellations', ['interpellation ignored', par])
return trimmed
@ staticmethod
def save(pars, path):
'Write the content to given file'
with open(path, mode='w') as out:
for par in pars:
out.write(par)
out.write(u'\n\n')
# def extractSpeakers(self, filename, pars):
# 'Prepare a list of unique speaker names for checking'
# self.speakers.write(r'[{}]\n'.format(filename))
# speakers = set()
# for par in pars:
# match = re.match(r'^\*\*([^\*:]*)', par)
# if match:
# speakers.add(match.group(1))
# speakers = list(speakers)
# speakers.sort()
# for speaker in speakers:
# self.speakers.write(speaker + u'\n')
# self.speakers.write(u'\n')
def _debug(self, category, message, condition=True):
if condition and self.verbose == category:
if not isinstance(message, list):
message = [message]
print(f'{category.upper()}: {message[0]}')
for line in message[1:]:
print(f' • {line.rstrip()}')
# Read options
# parser = argparse.ArgumentParser()
# parser.add_argument('-o', '--output', help='output directory', default='out')
# parser.add_argument('-i', '--interpellations',
# help='include interpellations', action='store_true')
# parser.add_argument('-x', '--speakers',
# help='extract speakers', action='store_true')
# parser.add_argument('-t', '--text', help='remove metatext',
# action='store_true')
# parser.add_argument('-v', '--verbose', help='show messages', default='none',
# choices=['none', 'line', 'markdown', 'merge', 'spaceout',
# 'interpellations', 'speakers', 'comments'])
# parser.add_argument('-m', '--mode', help='special mode', default='default',
# choices=['default', 'senate'])
# parser.add_argument('filename', help='file to process', nargs='+')
# args = parser.parse_args()