cleaner.py
10.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
"""Read Markdown file converted from ODT and fix as many errors as possible.
This uses heuristics to fix markdown, punctuation and even common misspellings.
"""
import re
LOWER = 'a-ząćęłńóśźż'
UPPER = 'A-ZĄĆĘŁŃÓŚŹŻ'
NAME = f'[{UPPER}](?:[{LOWER}]+\\.?|\\.)'
SURNAME = f'[{UPPER}][{LOWER}]+(?:-[{UPPER}][{LOWER}]+)?'
LETTER = 'A-Za-zĄĆĘŁŃÓŚŹŻąćęłńóśźż'
PUNCT = ',;.?!:…'
UNIQUE = '~&^'
class Cleaner:
'The class to clean Markdown'
def __init__(self, verbose=''):
self.verbose = verbose
self.cache = []
def clean(self, filename, output):
'Apply all filters'
pars = self._read(filename)
filters = {
'lines': self._merge_lines,
'odt': self._clear_odt,
'asterisk': self._fix_asterisk,
'spaceout': self._fix_spaceout,
'whitespace': self._fix_whitespace,
'punctuation': self._fix_punctuation,
'titles': self._merge_titles,
'merge': self._merge_pars,
'nonspeakers': self._remove_nonspeakers,
'speakers': self._add_speakers,
'comments': self._fix_comments,
# self._simplify_formatting
}
for category, function in filters.items():
pars = self._filter(category, function, pars)
self.save(pars, output)
@staticmethod
def _read(filename):
'Read data from file. Merge lines into paragraphs using empty line as separator'
return [line.strip() for line in open(filename)]
def _filter(self, category, function, pars):
fixed = []
self.cache = []
for par in pars:
result = function(par)
if isinstance(result, list):
self._debug(category, [par] + result)
fixed.extend(result)
elif isinstance(result, str):
if result != par:
self._debug(category, [par, result])
fixed.append(result)
if self.cache:
fixed.append(self._cache())
return fixed
def _merge_lines(self, line):
'Merge lines into paragraphs'
if not line:
return self._cache()
elif self.cache and re.match(r'[a-np-ząćęłóńśźż]-', self.cache[-1][-2:]):
# and previous[-1].rindex(' ') > len(previous[-1]) - 5:
self.cache[-1] = f'{self.cache[-1][:-1]}{line}'
else:
self.cache.append(line)
def _clear_odt(self, par):
'Remove leftovers from OpenOffice: bookmarks, footnotes, hyphenation, backslashes'
removed = ['[]', '\\*', '\\', '{.Apple-converted-spaces}', '(—)']
reremoved = ['{#anchor-?[0-9]*}', r'\{[0-9s.]*\}',
r'\[\W]*\]', r'\[\s*\]']
for remove in removed:
par = par.replace(remove, '')
for remove in reremoved:
par = re.sub(remove, '', par)
return par.strip()
@staticmethod
def _fix_asterisk(par):
'Remove Markdown italics'
if '*' in par:
return par.replace('**', UNIQUE).replace('*', '').replace(UNIQUE, '**')
return par
@staticmethod
def _fix_spaceout(par):
'Try to merge spaced out words'
match = re.search(r'\s\S\s\S\s\S\s\S\s\S\s\S\s', par)
if match:
start = match.start()
end = start
while end < len(par) - 1 and par[end] == ' ':
end += 2
spaceout = re.sub('(.)([A-ZĆŁÓŚŻŹ])', r'\1 \2',
par[start:end].replace(' ', ''))
par = par[:start] + spaceout + par[end:]
return par
@staticmethod
def _fix_whitespace(par):
'Remove ‍ and double spaces'
return re.sub(r'\s+', ' ', (re.sub(r'[\xc2\xad]', '', par)))
@staticmethod
def _fix_punctuation(par):
'Fix space punctuation and mdash'
replacements = {
r'(\s+|^),,': r'\1„',
r'\\?\.\.\.': r'…', # więc\... -> więc…
r'"': r'”',
r'\s”': ' „',
r'\s*---\s*': ' — ',
r',\.': r'.', # dość,. -> dość.
fr'(\w)\s+([{PUNCT}])': r'\1\2', # w ustawie . -> w ustawie.
r' \)': ')', # (razem ) -> (razem)
r'\( ': '(', # ( razem) -> (raem)
r'(\w)\(': r'\1 (', # to(albo tamto) -> to (albo tamto)
# biało- czerwony -> biało-czerwony
fr'([{LOWER}]{{3}}[wknłcz]o)-\s+([{LOWER}]{{4}})': r'\1-\2',
r'(\d) — (\d)': r'\1-\2', # 12 - 20% -> 12-20%,
r'([IVX]+) - ([IVX]+)': r'\1-\2', # I - III -> I-III,
}
for pattern, replacement in replacements.items():
par = re.sub(pattern, replacement, par)
return par
def _merge_titles(self, par):
'Merge split speakers with title in the first line and name in the second'
if par.startswith('**') and self.cache and ':' not in self.cache[-1] and self.cache[-1].endswith('**'):
self.cache[-1] = (self.cache[-1][:-2] + ' ' +
par[2:]).replace(' ', ' ')
return None
return par
def _remove_nonspeakers(self, par):
'Heuristics to remove spurious speaker formatting and to format some unformatted speakers'
if '**' not in par:
return par
if not par.startswith('**'): # oraz **Poseł X**
return par.replace('**', '')
par = re.sub(r'\*\*([{LOWER}0-9%,.:/\s-]*)\*\*', r'\1', par) # **OK**
# Move ** after the colon
par = re.sub(r'^\*\*([^*:]*):\s+([^*]*)\s*\*\*', r'**\1:** \2', par)
# Remove ** if the content does not look like person name
match = re.match(r'^\*\*([^\*]*):?\*\*', par)
if not match or not self._can_be_person(match.group(1)):
return par.replace('**', '')
return par
def _add_speakers(self, par):
'Add ** around unformatted speakers if applicable'
if '**' in par:
return par
patterns = [
fr'({NAME}\s+{SURNAME})[.:\s]*$',
fr'({NAME}\s+{SURNAME}):\s+(.*)$',
fr'({NAME}\s+{NAME}\s+{SURNAME}):\s+(.*)$',
r'(Marszałek):\s+(.*)$'
]
for pattern in patterns:
match = re.match(pattern, par)
if match:
person = f'**{match.group(1)}:**'
return [person] + list(match.groups())[1:]
return par
def _fix_comments(self, par):
""" Add italic to some comments not formatted in the original file
They are assumed to start after a dot, be in brackets and start with uppercase.
Fragments with numbers are excluded to avoid formatting legal references.
Also fix some comment formatting.
"""
par = par.replace('(*', '*(') # (*Głos)*
par = re.sub(r'\*\((^\)*)\*(^\)*)\)', r'*(\1\2)*', par) # *(Głos*)
# Add missing comments
par = re.sub(fr'([.!?] |^)\(([{UPPER}][^0-9)]*)\)\.?(\s|$|\*)',
lambda m: f'{m.group(1)}*({m.group(2).replace("*", "")})* ', par)
# Fix comments marked as new speaker
not_speaker = re.match(r'\*\*\((.*)\)\.?\s*\*\*$', par)
if not_speaker:
par = '*(' + not_speaker.group(1) + ')*'
return par
def _can_be_person(self, person):
'Check if given fragment can be a person. Used to remove spurious bold around some titles'
if not 3 < len(person) < 151:
return False
for prefix in ['(', 'Obywatel', 'Wysoka', 'Proszę', 'Polski', 'Panie', 'Dziękuję',
'Sprawozdanie', 'Pan ', 'Pani ', 'Przystępuj']:
if person.startswith(prefix):
return False
for suffix in ['ego']:
if person.endswith(suffix):
return False
for infix in ['II', 'Sejm', 'rzystępujemy', 'nterpelacj', 'Warszawa']:
if infix in person:
return False
if re.search('[0-9]', person):
return False
return True
def _simplify_formatting(self, pars):
'Remove repeated or nested formatting'
text = []
for par in pars:
par = re.sub(r'\*\*[^\*]*\*\*', '', par)
par = re.sub(r'\*[^\*]*\*', '', par)
par = par.strip()
if par:
text.append(par)
return text
def _merge_pars(self, par):
"""Heuristic merging of paragraphs. Adds weights suggesting it IS a new paragraph.
If the total weight is below the threshold, the paragraph is merged.
"""
if not self.cache:
self.cache.append(par)
return None
eol = 0
previous = self.cache[-1]
if previous.endswith(('!', '?', '.', ':', '"', ';')):
eol += 14
if '*' in previous:
eol += 5
if len(previous) < 60:
eol += 5
if '**' in par and len(par) < 80:
eol += 16
if previous.endswith('**'):
eol += 17
if par.startswith('--'):
eol += 12
if par[0].isupper():
eol += 5
if par.startswith('**') and len(par) > 2 and par[2].isupper():
eol += 17
if re.search(r' [a-z] [a-z] [a-z] [a-z] ', previous):
eol += 11
if eol >= 10:
return self._cache([par])
if previous.endswith('-'): # Remove trailing hyphen
self.cache[-1] = self.cache[-1][: -1] + par
else:
self.cache.append(par)
return None
def _cache(self, content=None):
'Fetch and clear paragraph cache'
cache = ' '.join(self.cache)
self.cache = content or []
return cache
@ staticmethod
def save(pars, path):
'Write the content to given file'
with open(path, mode='w') as out:
for par in pars:
out.write(par)
out.write(u'\n\n')
def _debug(self, category, items):
'Print debug message if applicable'
if category == self.verbose:
print(f'{category.upper()}')
for item in items:
print(f' • {item.rstrip()}')