rulesFSA.py
2.25 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
'''
Created on 12 mar 2014
@author: mlenart
'''
import logging
from morfeuszbuilder.fsa import state
from morfeuszbuilder.utils.serializationUtils import htons
from morfeuszbuilder.utils import exceptions
MAX_FSA_SIZE = 65535
class RulesState(state.State):
def __init__(self):
super(RulesState, self).__init__()
self.weak = None
def setAsAccepting(self, weak):
self.weak = weak
self.encodedData = bytearray([1 if weak else 0])
def getEncodedSize(self):
stateInfoSize = 2 # accepting info + transitionsNum
transitionsSize = 4 * len(self.transitionsMap)
return stateInfoSize + transitionsSize
class RulesFSA(object):
def __init__(self):
self.initialState = state.State()
self.ACCEPTING_FLAG = 1
self.WEAK_FLAG = 2
def stateData2bytearray(self, state):
res = bytearray()
firstByte = 0
if state.isAccepting():
firstByte |= self.ACCEPTING_FLAG
if state.weak:
firstByte |= self.WEAK_FLAG
res.append(firstByte)
secondByte = len(state.transitionsMap)
res.append(secondByte)
return res
def transitionsData2bytearray(self, state):
res = bytearray()
# logging.debug('next')
for (segnum, shiftOrth), nextState in sorted(state.transitionsMap.items()):
res.append(segnum)
if shiftOrth:
res.append(1)
else:
res.append(0)
offset = nextState.offset
exceptions.validate(offset <= MAX_FSA_SIZE,
'Segmentation rules are too big and complicated' \
+ '- the resulting automaton would exceed its max size which is %d' \
% MAX_FSA_SIZE)
res.extend(htons(offset))
return res
def serialize(self):
self.initialState.calculateOffsets(sizeCounter=lambda s: s.getEncodedSize())
res = bytearray()
for state in sorted(self.initialState.dfs(set()), key=lambda s: s.offset):
res.extend(self.stateData2bytearray(state))
res.extend(self.transitionsData2bytearray(state))
return res