|
1
2
3
|
'''
Created on Oct 23, 2013
|
|
4
|
@author: mlenart
|
|
5
6
|
'''
|
|
7
8
9
10
|
from morfeuszbuilder.utils.serializationUtils import htons, serializeString
from morfeuszbuilder.utils import limits, exceptions
MAX_QUALIFIERS_COMBINATIONS = 1024
|
|
11
|
|
|
12
13
14
15
16
17
|
class Encoder(object):
'''
classdocs
'''
|
|
18
|
def __init__(self, lowercase, encoding='utf8'):
|
|
19
20
21
|
'''
Constructor
'''
|
|
22
|
self.lowercase = lowercase
|
|
23
|
self.encoding = encoding
|
|
24
|
self.qualifiersMap = { frozenset(): 0}
|
|
25
|
|
|
26
|
def encodeWord(self, word, lowercase=True):
|
|
27
|
assert type(word) == unicode
|
|
28
|
res = bytearray(word.lower() if self.lowercase and lowercase else word, self.encoding)
|
|
29
|
return res
|
|
30
31
|
def encodeData(self, data):
|
|
32
|
raise NotImplementedError()
|
|
33
34
|
def decodeData(self, rawData):
|
|
35
|
return NotImplementedError()
|
|
36
37
38
|
def decodeWord(self, rawWord):
return unicode(str(rawWord).strip('\x00'), self.encoding)
|
|
39
40
|
def word2SortKey(self, word):
|
|
41
42
|
normalizedWord = word.lower() if self.lowercase else word
return normalizedWord.encode(self.encoding)
|
|
43
|
|
|
44
|
def _encodeTypeNum(self, typenum):
|
|
45
46
47
|
exceptions.validate(
typenum <= limits.MAX_SEGMENT_TYPES,
u'Too many segment types. The limit is %d' % limits.MAX_SEGMENT_TYPES)
|
|
48
49
|
return bytearray([typenum])
|
|
50
51
52
53
54
55
56
57
|
def _encodeQualifiers(self, qualifiers):
res = bytearray()
key = frozenset(qualifiers)
if key in self.qualifiersMap:
n = self.qualifiersMap[key]
else:
n = len(self.qualifiersMap)
self.qualifiersMap[key] = n
|
|
58
59
60
|
exceptions.validate(
n <= limits.MAX_QUALIFIERS_COMBINATIONS,
u'Too many qualifiers combinations. The limit is %d' % limits.MAX_QUALIFIERS_COMBINATIONS)
|
|
61
|
res.extend(htons(n))
|
|
62
63
|
return res
|
|
64
|
def _hasUpperPrefix(self, casePattern):
|
|
65
|
for i in range(len(casePattern) + 1):
|
|
66
67
68
69
70
71
72
73
74
75
76
|
if all(casePattern[:i]) and not any(casePattern[i:]):
return True
return False
def _getUpperPrefixLength(self, casePattern):
assert self._hasUpperPrefix(casePattern)
for i in range(len(casePattern)):
if not casePattern[i]:
return i
return len(casePattern)
|
|
77
78
|
def _encodeTagNum(self, tagnum):
res = bytearray()
|
|
79
|
exceptions.validate(tagnum <= limits.MAX_TAGS, u'Too many tags. The limit is %d' % limits.MAX_TAGS)
|
|
80
81
82
83
84
|
res.append((tagnum & 0xFF00) >> 8)
res.append(tagnum & 0x00FF)
return res
def _encodeNameNum(self, namenum):
|
|
85
|
exceptions.validate(namenum <= limits.MAX_NAMES, u'Too many named entity types. The limit is %d' % limits.MAX_NAMES)
|
|
86
|
return bytearray([namenum])
|
|
87
88
89
90
91
92
93
94
|
def _groupInterpsByType(self, interpsList):
res = {}
for interp in interpsList:
res.setdefault(interp.typenum, [])
res[interp.typenum].append(interp)
return res
|
|
95
96
97
98
99
100
101
102
103
104
105
106
107
|
def _doEncodeData(self, interpsList):
assert type(interpsList) == frozenset
segnum2Interps = self._groupInterpsByType(interpsList)
res = bytearray()
for typenum, interpsList in segnum2Interps.iteritems():
res.extend(self._encodeInterps4Type(typenum, interpsList))
del interpsList
|
|
108
109
|
res = htons(len(res)) + res
|
|
110
111
112
113
114
115
116
117
118
119
120
|
return res
class MorphEncoder(Encoder):
def __init__(self, encoding='utf8'):
super(MorphEncoder, self).__init__(True, encoding)
def encodeData(self, interpsList):
return self._doEncodeData(interpsList)
def _getMinOrthCasePatterns(self, interpsList):
|
|
121
|
res = []
|
|
122
|
for interp in interpsList:
|
|
123
|
if not True in interp.orthCasePattern:
|
|
124
|
return []
|
|
125
|
else:
|
|
126
|
res.append(list(interp.orthCasePattern))
|
|
127
128
|
return res
|
|
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
def _encodeCasePattern(self, casePattern):
LEMMA_ONLY_LOWER = 0
LEMMA_UPPER_PREFIX = 1
LEMMA_MIXED_CASE = 2
res = bytearray()
if True not in casePattern:
res.append(LEMMA_ONLY_LOWER)
return res
elif self._hasUpperPrefix(casePattern):
res.append(LEMMA_UPPER_PREFIX)
res.append(self._getUpperPrefixLength(casePattern))
return res
else:
assert len(casePattern) < 256
res.append(LEMMA_MIXED_CASE)
res.append(len([c for c in casePattern if c]))
for idx in range(len(casePattern)):
if casePattern[idx]:
res.append(idx)
return res
def _casePatternsHaveOnlyLowercase(self, casePatterns):
return not any(map(lambda cp: cp and True in cp, casePatterns))
def _casePatternsAreOnlyTitles(self, casePatterns):
|
|
156
|
return all(map(lambda cp: cp and cp[0] == True and not True in cp[1:], casePatterns))
|
|
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
def _casePatternsAreEncodedInCompressByte(self, casePatterns):
return self._casePatternsHaveOnlyLowercase(casePatterns) or self._casePatternsAreOnlyTitles(casePatterns)
def _prefixCutsAreEncodedInCompressByte(self, prefixCuts):
return len(prefixCuts) == 1 and list(prefixCuts)[0] < 15
def _encodeCompressByte(self, orthCasePatterns, lemmaCasePatterns, prefixCuts):
ORTH_ONLY_LOWER = 128
ORTH_ONLY_TITLE = 64
LEMMA_ONLY_LOWER = 32
LEMMA_ONLY_TITLE = 16
PREFIX_CUT_MASK = 15
res = 0
if self._casePatternsHaveOnlyLowercase(orthCasePatterns):
res |= ORTH_ONLY_LOWER
elif self._casePatternsAreOnlyTitles(orthCasePatterns):
res |= ORTH_ONLY_TITLE
if self._casePatternsHaveOnlyLowercase(lemmaCasePatterns):
res |= LEMMA_ONLY_LOWER
elif self._casePatternsAreOnlyTitles(lemmaCasePatterns):
res |= LEMMA_ONLY_TITLE
if self._prefixCutsAreEncodedInCompressByte(prefixCuts):
res |= list(prefixCuts)[0]
else:
res |= PREFIX_CUT_MASK
return res
def _encodeInterps4Type(self, typenum, interpsList):
|
|
188
189
|
res = bytearray()
res.extend(self._encodeTypeNum(typenum))
|
|
190
|
|
|
191
|
encodedInterpsList = bytearray()
|
|
192
193
194
195
196
197
198
199
200
201
202
|
orthCasePatterns = set([tuple(interp.orthCasePattern) for interp in interpsList])
lemmaCasePatterns = set([tuple(interp.encodedForm.casePattern) for interp in interpsList])
prefixCuts = set([interp.encodedForm.prefixCutLength for interp in interpsList])
encodedInterpsList.append(self._encodeCompressByte(orthCasePatterns, lemmaCasePatterns, prefixCuts))
if not self._casePatternsAreEncodedInCompressByte(orthCasePatterns):
minOrthCasePatterns = self._getMinOrthCasePatterns(interpsList)
encodedInterpsList.append(len(minOrthCasePatterns))
for casePattern in minOrthCasePatterns:
|
|
203
|
encodedInterpsList.extend(self._encodeCasePattern(casePattern))
|
|
204
|
|
|
205
|
for interp in sorted(interpsList, key=lambda i: i.getSortKey()):
|
|
206
|
if not self._casePatternsAreEncodedInCompressByte(orthCasePatterns):
|
|
207
|
encodedInterpsList.extend(self._encodeCasePattern(interp.orthCasePattern))
|
|
208
209
|
if not self._prefixCutsAreEncodedInCompressByte(prefixCuts):
encodedInterpsList.append(interp.encodedForm.prefixCutLength)
|
|
210
|
encodedInterpsList.append(interp.encodedForm.cutLength)
|
|
211
|
encodedInterpsList.extend(serializeString(interp.encodedForm.suffixToAdd))
|
|
212
|
if not self._casePatternsAreEncodedInCompressByte(lemmaCasePatterns):
|
|
213
214
215
|
encodedInterpsList.extend(self._encodeCasePattern(interp.encodedForm.casePattern))
encodedInterpsList.extend(htons(interp.tagnum))
encodedInterpsList.append(interp.namenum)
|
|
216
|
encodedInterpsList.extend(self._encodeQualifiers(interp.qualifiers))
|
|
217
|
|
|
218
|
res.extend(htons(len(encodedInterpsList)))
|
|
219
220
|
res.extend(encodedInterpsList)
return res
|
|
221
222
223
224
|
class Encoder4Generator(Encoder):
def __init__(self, encoding='utf8'):
|
|
225
|
super(Encoder4Generator, self).__init__(False, encoding)
|
|
226
227
|
def encodeData(self, interpsList):
|
|
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
|
return self._doEncodeData(interpsList)
def _encodeInterps4Type(self, typenum, interpsList):
res = bytearray()
res.extend(self._encodeTypeNum(typenum))
encodedInterpsList = bytearray()
for interp in sorted(interpsList, key=lambda i: i.getSortKey()):
encodedInterpsList.extend(serializeString(interp.homonymId))
encodedInterpsList.extend(serializeString(interp.encodedForm.prefixToAdd))
encodedInterpsList.append(interp.encodedForm.cutLength)
encodedInterpsList.extend(serializeString(interp.encodedForm.suffixToAdd))
encodedInterpsList.extend(htons(interp.tagnum))
encodedInterpsList.append(interp.namenum)
encodedInterpsList.extend(self._encodeQualifiers(interp.qualifiers))
res.extend(htons(len(encodedInterpsList)))
res.extend(encodedInterpsList)
return res
|