- ctest powinno znowu w miarę działać

git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@88 ff4e3ee1-f430-4e82-ade0-24591c43f1fd

- ctest powinno znowu w miarę działać
git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@88 ff4e3ee1-f430-4e82-ade0-24591c43f1fd
Michał Lenart
1 parent 4ea040d0
Showing 11 changed files with 59 additions and 29 deletions
fsabuilder/buildfsa.py
fsabuilder/morfeuszbuilder/fsa/serializer.pyc
fsabuilder/morfeuszbuilder/fsa/state.py
fsabuilder/morfeuszbuilder/fsa/state.pyc
fsabuilder/morfeuszbuilder/segrules/rules.py
fsabuilder/morfeuszbuilder/segrules/rulesManager.py
fsabuilder/morfeuszbuilder/segrules/rulesNFA.py
fsabuilder/morfeuszbuilder/segrules/rulesParser.py
fsabuilder/morfeuszbuilder/segrules/test/parserTest.py
fsabuilder/morfeuszbuilder/segrules/test/preprocessorTest.py
fsabuilder/morfeuszbuilder/segrules/test/segmenty.dat
@@ -164,9 +164,8 @@ def buildAnalyzerFromPoliMorf(inputFile, tagset):
     _printStats(fsa)
     return fsa
-def buildGeneratorFromPoliMorf(inputFile, tagsetFile):
+def buildGeneratorFromPoliMorf(inputFile, tagset):
     encoder = encode.Encoder4Generator()
-    tagset = Tagset(tagsetFile)
     fsa = FSA(encoder, tagset)
     inputData = _readPolimorfInput4Generator(inputFile, tagset, encoder)
     for word, data in inputData:
@@ -13,6 +13,7 @@ class State(object):
     def __init__(self, additionalData=None):
         self.transitionsMap = {}
+        self.transitionsDataMap = {}
         self.freq = 0
         self.encodedData = None
         self.reverseOffset = None
@@ -31,6 +32,9 @@ class State(object):
     def setTransition(self, byte, nextState):
         self.transitionsMap[byte] = nextState
+    def setTransitionData(self, byte, data):
+        self.transitionsDataMap[byte] = data
+    
     def hasNext(self, byte):
         return byte in self.transitionsMap
@@ -98,3 +98,15 @@ class ZeroOrMoreRule(UnaryRule):
         self.child._doAddToNFA(intermStartState, intermEndState)
         intermEndState.addTransition(None, endState)
         endState.addTransition(None, intermStartState)
+
+class ShiftOrthRule(UnaryRule):
+    
+    def __init__(self, child):
+        super(ShiftOrthRule, self).__init__(child)
+    
+    def addToNFA(self, fsa):
+        raise ValueError()
+    
+    def _doAddToNFA(self, startState, endState):
+        self.child._doAddToNFA(startState, endState)
+        startState.setTransitionData(self.child.segnum, 1)
@@ -12,7 +12,10 @@ class RulesManager(object):
     def _options2Key(self, optionsMap):
         return frozenset(optionsMap.items())
-    def addDFA4Options(self, optionsMap, dfa):
+    def getDFA(self, optionsMap):
+        return self.options2DFA[self._options2Key(optionsMap)]
+    
+    def addDFA(self, optionsMap, dfa):
         self.options2DFA[self._options2Key(optionsMap)] = dfa
     def serialize(self):
@@ -12,6 +12,7 @@ class RulesNFAState(object):
     def __init__(self, initial=False, final=False, weak=False):
         self.transitionsMap = {}
+        self.transitionsDataMap = {}
         self.initial = initial
         self.final = final
         self.weak = weak
@@ -21,6 +22,11 @@ class RulesNFAState(object):
     def addTransition(self, label, targetState):
         self.transitionsMap.setdefault(label, set())
         self.transitionsMap[label].add(targetState)
+        self.transitionsDataMap[label] = 0
+    
+    def setTransitionData(self, label, byte):
+        assert len(self.transitionsMap[label]) == 1
+        self.transitionsDataMap[label] = byte
     def getClosure(self, visited):
         if self in visited:
@@ -61,9 +67,10 @@ class RulesNFA(object):
         for nfaState in nfaStates:
             for label, nextStates in nfaState.transitionsMap.iteritems():
                 if label is not None:
-                    res.setdefault(label, set())
+                    transitionData = nfaState.transitionsDataMap[label]
+                    res.setdefault((label, transitionData), set())
                     for nextNFAState in nextStates:
-                        res[label] |= nextNFAState.getClosure(set())
+                        res[(label, transitionData)] |= nextNFAState.getClosure(set())
 #                         print 'closure of', nextNFAState.idx, 'is', [s.idx for s in sorted(nextNFAState.getClosure(), key=lambda s: s.idx)]
         return res
@@ -77,7 +84,7 @@ class RulesNFA(object):
             # dfaState should be final
             # and contain info about weakness
             dfaState.encodedData = bytearray([1 if weak else 0])
-        for label, nextNFAStates in self._groupOutputByLabels(nfaStates).iteritems():
+        for (label, transitionData), nextNFAStates in self._groupOutputByLabels(nfaStates).iteritems():
 #             print '============'
 #             print 'states:', [s.idx for s in sorted(nfaStates, key=lambda s: s.idx)]
 #             print 'label:', label
@@ -90,6 +97,7 @@ class RulesNFA(object):
                 nfaSubset2DFAState[key] = nextDFAState
                 self._doConvertState(nextDFAState, nextNFAStates, nfaSubset2DFAState)
             dfaState.setTransition(label, nextDFAState)
+            dfaState.setTransitionData(label, transitionData)
     def convertToDFA(self):
         dfa = fsa.FSA(encoder=None, encodeData=False, encodeWords=False)
@@ -50,7 +50,7 @@ class RulesParser(object):
             for rule in self._doParse(combinationEnumeratedLines, segtypesHelper):
                 rule.addToNFA(nfa)
             dfa = nfa.convertToDFA()
-            res.addDFA4Options(key2Def, dfa)
+            res.addDFA(key2Def, dfa)
         return res
     def _doParse(self, combinationEnumeratedLines, segtypesHelper):
@@ -67,9 +67,10 @@ class RulesParser(object):
     def _doParseOneLine(self, lineNum, line, segtypesHelper):
         rule = Forward()
-        tagRule = Word(alphanums+'_>')
+        tagRule = Word(alphanums+'_')
+        shiftOrthRule = tagRule + '>'
         parenRule = Suppress('(') + rule + Suppress(')')
-        atomicRule = tagRule ^ parenRule
+        atomicRule = tagRule ^ shiftOrthRule ^ parenRule
         zeroOrMoreRule = atomicRule + Suppress('*')
         oneOrMoreRule = atomicRule + Suppress('+')
         unaryRule = atomicRule ^ zeroOrMoreRule ^ oneOrMoreRule
@@ -79,6 +80,7 @@ class RulesParser(object):
         rule << concatRule
         tagRule.setParseAction(lambda string, loc, toks: self._createNewTagRule(toks[0], lineNum, line, segtypesHelper))
+        shiftOrthRule.setParseAction(lambda string, loc, toks: rules.ShiftOrthRule(toks[0]))
 #         parenRule.setParseAction(lambda string, loc, toks: toks[0])
         zeroOrMoreRule.setParseAction(lambda string, loc, toks: rules.ZeroOrMoreRule(toks[0]))
         oneOrMoreRule.setParseAction(lambda string, loc, toks: rules.ConcatRule([toks[0], rules.ZeroOrMoreRule(toks[0])]))
@@ -10,18 +10,20 @@ from morfeuszbuilder.tagset import tagset
 from morfeuszbuilder.fsa import visualizer, serializer
 class Test(unittest.TestCase):
-    print 'do test'
-    t = tagset.Tagset(os.path.join(os.path.dirname(__file__), 'polimorf.tagset'))
-    parser = rulesParser.RulesParser(t)
-    fsas = parser.parse(os.path.join(os.path.dirname(__file__), 'segmenty.dat'))
-    fsa = fsas[0]
-    for s in fsa.dfs():
-        s.debug()
-    print 'states:', len(list(fsa.dfs()))
-    print 'transitions:', fsa.getTransitionsNum()
-    visualizer.Visualizer().visualize(fsa, charLabels=False)
-    print 'size:', len(serializer.SimpleSerializer(fsa).fsa2bytearray(bytearray()))
-    print 'done'
+    
+    def testParser(self):
+        print 'do test'
+        t = tagset.Tagset(os.path.join(os.path.dirname(__file__), 'polimorf.tagset'))
+        parser = rulesParser.RulesParser(t)
+        rulesManager = parser.parse(os.path.join(os.path.dirname(__file__), 'segmenty.dat'))
+        fsa = rulesManager.getDFA({'aggl': 'permissive', 'praet': 'split'})
+        for s in fsa.dfs():
+            s.debug()
+        print 'states:', len(list(fsa.dfs()))
+        print 'transitions:', fsa.getTransitionsNum()
+        visualizer.Visualizer().visualize(fsa, charLabels=False)
+        print 'size:', len(serializer.SimpleSerializer(fsa).fsa2bytearray(bytearray()))
+        print 'done'
 if __name__ == "__main__":
     unittest.main()
@@ -16,7 +16,7 @@ class Test(unittest.TestCase):
     def testPreprocess(self):
         filename = os.path.join(os.path.dirname(__file__), 'segmenty.dat')
-        parsedFile = configFile.ConfigFile(filename, ['options', 'combinations', 'tags', 'lexemes'])
+        parsedFile = configFile.ConfigFile(filename, ['options', 'combinations', 'tags', 'lexemes', 'segment types'])
         linesEnum = parsedFile.enumerateLinesInSection('combinations')
         for lineNum, line in preprocessor.preprocess(linesEnum, ['extra', 'superextra']):
             print (lineNum, line)
@@ -111,7 +111,7 @@ moze_interp(z_on_agl)
 moze_interp(z_on_agl on_agl)
 # Liczba zapisana jako ciąg cyfr:
-#moze_interp( dig>* dig )
+moze_interp( dig>* dig )
 # Formacje prefiksalne
 #### trzeba wydzielić odpowiednie samodze!
@@ -133,11 +133,11 @@ adj dywiz samodz
 samodz dywiz adj
 [segment types]
-naj>
-nie>
+naj
+nie
 prefs
 prefv
-dig>
+dig
 adja
 adj
 adj_sup
@@ -156,11 +156,11 @@ praet_pl
 samodz
 [tags]
-naj>	naj
-nie>	nie
+naj	naj
+nie	nie
 prefs	prefs
 prefv	prefv
-dig>	dig
+dig	dig
 adja	adja
 adj	adj:%:pos
 adj_sup	adj:%:sup