|
1
2
3
4
5
6
7
8
9
|
'''
Created on Oct 8, 2013
@author: mlenart
'''
import state
import register
import logging
|
|
10
|
from morfeuszbuilder.utils import exceptions
|
|
11
12
13
14
15
16
17
|
class FSA(object):
'''
A finite state automaton
'''
|
|
18
|
def __init__(self, encoder, encodeData=True, encodeWords=True):
|
|
19
|
self.encodeWord = encoder.encodeWord if encodeWords else lambda x: x
|
|
20
21
|
self.encodeData = encoder.encodeData if encodeData else lambda x: x
self.decodeData = encoder.decodeData if encodeData else lambda x: x
|
|
22
|
self.encodedPrevWord = None
|
|
23
|
|
|
24
|
# self.tagset = tagset
|
|
25
26
|
self.initialState = state.State()
self.register = register.Register()
|
|
27
|
self.label2Freq = {}
|
|
28
29
|
self.n = 0
self.closed = False
|
|
30
|
|
|
31
32
|
def tryToRecognize(self, word, addFreq=False):
return self.decodeData(self.initialState.tryToRecognize(self.encodeWord(word), addFreq))
|
|
33
|
|
|
34
35
36
37
38
39
40
|
def addEntry(self, word, data):
assert not self.closed
assert data is not None
encodedWord = self.encodeWord(word)
assert encodedWord > self.encodedPrevWord
self._addSorted(encodedWord, self.encodeData(data))
self.encodedPrevWord = encodedWord
|
|
41
|
|
|
42
|
self.n += 1
|
|
43
|
|
|
44
|
# debug
|
|
45
|
if self.n < 10 or (self.n < 10000 and self.n % 1000 == 0) or self.n % 10000 == 0:
|
|
46
|
logging.info(u'%d %s' % (self.n, word))
|
|
47
48
49
50
|
for label in encodedWord:
self.label2Freq[label] = self.label2Freq.get(label, 0) + 1
def close(self):
|
|
51
52
|
if self.n == 0:
raise exceptions.FSABuilderException('empty input')
|
|
53
54
|
assert not self.closed
self.initialState = self._replaceOrRegister(self.initialState, self.encodedPrevWord)
|
|
55
|
self.encodedPrevWord = None
|
|
56
57
|
self.closed = True
|
|
58
|
def train(self, trainData):
|
|
59
|
self.label2Freq = {}
|
|
60
|
for idx, word in enumerate(trainData):
|
|
61
62
63
|
self.tryToRecognize(word, addFreq=True)
for label in self.encodeWord(word):
self.label2Freq[label] = self.label2Freq.get(label, 0) + 1
|
|
64
65
66
67
68
69
|
if idx % 100000 == 0:
logging.info(str(idx))
def dfs(self):
for state in self.initialState.dfs(set()):
yield state
|
|
70
71
72
|
def getStatesNum(self):
return self.register.getStatesNum()
|
|
73
74
75
76
77
78
|
def getTransitionsNum(self):
res = 0
for s in self.initialState.dfs(set()):
res += len(s.transitionsMap)
return res
|
|
79
80
81
|
def _addSorted(self, encodedWord, data):
assert self.encodedPrevWord < encodedWord
|
|
82
|
assert type(data) == bytearray
|
|
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
q = self.initialState
i = 0
while i <= len(encodedWord) and q.hasNext(encodedWord[i]):
q = q.getNext(encodedWord[i])
i += 1
if self.encodedPrevWord and i < len(self.encodedPrevWord):
nextState = q.getNext(self.encodedPrevWord[i])
q.setTransition(
self.encodedPrevWord[i],
self._replaceOrRegister(nextState, self.encodedPrevWord[i+1:]))
while i < len(encodedWord):
q.setTransition(encodedWord[i], state.State())
q = q.getNext(encodedWord[i])
i += 1
assert q.encodedData is None
q.encodedData = data
def _replaceOrRegister(self, q, encodedWord):
if encodedWord:
nextState = q.getNext(encodedWord[0])
q.setTransition(
encodedWord[0],
self._replaceOrRegister(nextState, encodedWord[1:]))
if self.register.containsEquivalentState(q):
return self.register.getEquivalentState(q)
else:
self.register.addState(q)
return q
def calculateOffsets(self, sizeCounter):
|
|
116
|
self.initialState.calculateOffsets(sizeCounter)
|
|
117
118
119
120
|
def debug(self):
for state in self.initialState.dfs(set()):
state.debug()
|