- zrobiona konwersja NFA -> DFA dla automatów do zlepiania segmentów

- usunięcie "ignoreOrth" git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@87 ff4e3ee1-f430-4e82-ade0-24591c43f1fd

- zrobiona konwersja NFA -> DFA dla automatów do zlepiania segmentów
- usunięcie "ignoreOrth" git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@87 ff4e3ee1-f430-4e82-ade0-24591c43f1fd
Michał Lenart
1 parent 8d5a878e
Showing 18 changed files with 409 additions and 64 deletions
fsabuilder/PoliMorfTest.cek
fsabuilder/buildfsa.py
fsabuilder/morfeuszbuilder/fsa/fsa.py
fsabuilder/morfeuszbuilder/fsa/fsa.pyc
fsabuilder/morfeuszbuilder/fsa/serializer.py
fsabuilder/morfeuszbuilder/fsa/serializer.pyc
fsabuilder/morfeuszbuilder/fsa/state.py
fsabuilder/morfeuszbuilder/fsa/state.pyc
fsabuilder/morfeuszbuilder/fsa/visualizer.py
fsabuilder/morfeuszbuilder/fsa/visualizer.pyc
fsabuilder/morfeuszbuilder/segrules/preprocessor.py
fsabuilder/morfeuszbuilder/segrules/rules.py
fsabuilder/morfeuszbuilder/segrules/rulesManager.py
fsabuilder/morfeuszbuilder/segrules/rulesNFA.py
fsabuilder/morfeuszbuilder/segrules/rulesParser.py
fsabuilder/morfeuszbuilder/segrules/test/parserTest.py
fsabuilder/morfeuszbuilder/segrules/test/segmenty.dat
fsabuilder/morfeuszbuilder/segrules/test/segmenty1.dat
+bij	ABć+impt:sg:sec:imperf+pospolita
+bija	AAć+fin:sg:ter:imperf+pospolita
+bijacie	ADć+fin:pl:sec:imperf+pospolita
+bijaj	ABć+impt:sg:sec:imperf+pospolita
+bijajcie	AEć+impt:pl:sec:imperf+pospolita
+bijajmy	ADć+impt:pl:pri:imperf+pospolita
+bijają	ACć+fin:pl:ter:imperf+pospolita
+bijając	ADć+pcon:imperf+pospolita
+bijająca	AEć+pact:sg:nom.voc:f:imperf:aff+pospolita
+bijające	AEć+pact:pl:nom.acc.voc:m2.m3.f.n1.n2.p2.p3:imperf:aff+pospolita|AEć+pact:sg:nom.acc.voc:n1.n2:imperf:aff+pospolita
+bijającego	AGć+pact:sg:acc:m1.m2:imperf:aff+pospolita|AGć+pact:sg:gen:m1.m2.m3.n1.n2:imperf:aff+pospolita
+bijającej	AFć+pact:sg:gen.dat.loc:f:imperf:aff+pospolita
+bijającemu	AGć+pact:sg:dat:m1.m2.m3.n1.n2:imperf:aff+pospolita
+bijający	AEć+pact:pl:nom.voc:m1.p1:imperf:aff+pospolita|AEć+pact:sg:acc:m3:imperf:aff+pospolita|AEć+pact:sg:nom.voc:m1.m2.m3:imperf:aff+pospolita
+bijających	AGć+pact:pl:acc:m1.p1:imperf:aff+pospolita|AGć+pact:pl:gen.loc:m1.m2.m3.f.n1.n2.p1.p2.p3:imperf:aff+pospolita
+bijającym	AFć+pact:pl:dat:m1.m2.m3.f.n1.n2.p1.p2.p3:imperf:aff+pospolita|AFć+pact:sg:inst.loc:m1.m2.m3.n1.n2:imperf:aff+pospolita
+bijającymi	AGć+pact:pl:inst:m1.m2.m3.f.n1.n2.p1.p2.p3:imperf:aff+pospolita
+bijającą	AEć+pact:sg:acc.inst:f:imperf:aff+pospolita
+bijak	AA+subst:sg:acc:m3+pospolita|AA+subst:sg:nom:m3+pospolita
+bijaka	AB+subst:sg:gen:m3+pospolita
+bijakach	AD+subst:pl:loc:m3+pospolita
+bijakami	AD+subst:pl:inst:m3+pospolita
+bijaki	AB+subst:pl:acc:m3+pospolita|AB+subst:pl:nom:m3+pospolita|AB+subst:pl:voc:m3+pospolita
+bijakiem	AD+subst:sg:inst:m3+pospolita
+bijakom	AC+subst:pl:dat:m3+pospolita
+bijakowa	ABy+adj:sg:nom.voc:f:pos+pospolita
+bijakowe	ABy+adj:pl:acc:m2.m3.f.n1.n2.p2.p3:pos+pospolita|ABy+adj:pl:nom.voc:m2.m3.f.n1.n2.p2.p3:pos+pospolita|ABy+adj:sg:acc:n1.n2:pos+pospolita|ABy+adj:sg:nom.voc:n1.n2:pos+pospolita
+bijakowego	ADy+adj:sg:acc:m1.m2:pos+pospolita|ADy+adj:sg:gen:m1.m2.m3.n1.n2:pos+pospolita
+bijakowej	ACy+adj:sg:dat:f:pos+pospolita|ACy+adj:sg:gen:f:pos+pospolita|ACy+adj:sg:loc:f:pos+pospolita
+bijakowemu	ADy+adj:sg:dat:m1.m2.m3.n1.n2:pos+pospolita
+bijakowi	ABy+adj:pl:nom.voc:m1.p1:pos+pospolita|AD+subst:sg:dat:m3+pospolita
+bijakowo	ABy+adja+pospolita
+bijakowości	ACć+subst:pl:acc:f+pospolita|ACć+subst:pl:gen:f+pospolita|ACć+subst:pl:nom:f+pospolita|ACć+subst:pl:voc:f+pospolita|ACć+subst:sg:dat:f+pospolita|ACć+subst:sg:gen:f+pospolita|ACć+subst:sg:loc:f+pospolita|ACć+subst:sg:voc:f+pospolita
+bijakowościach	AFć+subst:pl:loc:f+pospolita
+bijakowościami	AFć+subst:pl:inst:f+pospolita
+bijakowościom	AEć+subst:pl:dat:f+pospolita
+bijakowością	ADć+subst:sg:inst:f+pospolita
+bijakowość	AA+subst:sg:acc:f+pospolita|AA+subst:sg:nom:f+pospolita
+bijakowy	AA+adj:sg:acc:m3:pos+pospolita|AA+adj:sg:nom.voc:m1.m2.m3:pos+pospolita
+bijakowych	AC+adj:pl:acc:m1.p1:pos+pospolita|AC+adj:pl:gen:m1.m2.m3.f.n1.n2.p1.p2.p3:pos+pospolita|AC+adj:pl:loc:m1.m2.m3.f.n1.n2.p1.p2.p3:pos+pospolita
+bijakowym	AB+adj:pl:dat:m1.m2.m3.f.n1.n2.p1.p2.p3:pos+pospolita|AB+adj:sg:inst:m1.m2.m3.n1.n2:pos+pospolita|AB+adj:sg:loc:m1.m2.m3.n1.n2:pos+pospolita
+bijakowymi	AC+adj:pl:inst:m1.m2.m3.f.n1.n2.p1.p2.p3:pos+pospolita
+bijakową	ABy+adj:sg:acc:f:pos+pospolita|ABy+adj:sg:inst:f:pos+pospolita
+bijaku	AB+subst:sg:loc:m3+pospolita|AB+subst:sg:voc:m3+pospolita
+bijaków	AC+subst:pl:gen:m3+pospolita
+bijali	ACć+praet:pl:m1.p1:imperf+pospolita
+bijam	ABć+fin:sg:pri:imperf+pospolita
+bijamy	ACć+fin:pl:pri:imperf+pospolita
+bijana	ACć+ppas:sg:nom.voc:f:imperf:aff+pospolita
+bijance	ACka+subst:sg:dat:f+pospolita|ACka+subst:sg:loc:f+pospolita
+bijane	ACć+ppas:pl:nom.acc.voc:m2.m3.f.n1.n2.p2.p3:imperf:aff+pospolita|ACć+ppas:sg:nom.acc.voc:n1.n2:imperf:aff+pospolita
+bijanego	AEć+ppas:sg:acc:m1.m2:imperf:aff+pospolita|AEć+ppas:sg:gen:m1.m2.m3.n1.n2:imperf:aff+pospolita
+bijanej	ADć+ppas:sg:gen.dat.loc:f:imperf:aff+pospolita
+bijanek	ACka+subst:pl:gen:f+pospolita
+bijanemu	AEć+ppas:sg:dat:m1.m2.m3.n1.n2:imperf:aff+pospolita
+bijani	ACć+ppas:pl:nom.voc:m1.p1:imperf:aff+pospolita
+bijania	ADć+ger:sg:gen:n2:imperf:aff+pospolita
+bijanie	ADć+ger:sg:nom.acc:n2:imperf:aff+pospolita
+bijaniem	AEć+ger:sg:inst:n2:imperf:aff+pospolita
+bijaniu	ADć+ger:sg:dat.loc:n2:imperf:aff+pospolita
+bijanka	AA+subst:sg:nom:f+pospolita
+bijankach	AC+subst:pl:loc:f+pospolita
+bijankami	AC+subst:pl:inst:f+pospolita
+bijanki	ABa+subst:pl:acc:f+pospolita|ABa+subst:pl:nom:f+pospolita|ABa+subst:pl:voc:f+pospolita|ABa+subst:sg:gen:f+pospolita
+bijanko	ABa+subst:sg:voc:f+pospolita
+bijankom	ACa+subst:pl:dat:f+pospolita
+bijanką	ABa+subst:sg:inst:f+pospolita
+bijankę	ABa+subst:sg:acc:f+pospolita
+bijano	ACć+imps:imperf+pospolita
+bijany	ACć+ppas:sg:acc:m3:imperf:aff+pospolita|ACć+ppas:sg:nom.voc:m1.m2.m3:imperf:aff+pospolita
+bijanych	AEć+ppas:pl:acc:m1.p1:imperf:aff+pospolita|AEć+ppas:pl:gen.loc:m1.m2.m3.f.n1.n2.p1.p2.p3:imperf:aff+pospolita
+bijanym	ADć+ppas:pl:dat:m1.m2.m3.f.n1.n2.p1.p2.p3:imperf:aff+pospolita|ADć+ppas:sg:inst.loc:m1.m2.m3.n1.n2:imperf:aff+pospolita
+bijanymi	AEć+ppas:pl:inst:m1.m2.m3.f.n1.n2.p1.p2.p3:imperf:aff+pospolita
+bijaną	ACć+ppas:sg:acc.inst:f:imperf:aff+pospolita
+bijasz	ACć+fin:sg:sec:imperf+pospolita
+bijatyce	ACka+subst:sg:dat:f+pospolita|ACka+subst:sg:loc:f+pospolita
+bijatyk	AAa+subst:pl:gen:f+pospolita
+bijatyka	AA+subst:sg:nom:f+pospolita
+bijatykach	AC+subst:pl:loc:f+pospolita
+bijatykami	AC+subst:pl:inst:f+pospolita
+bijatyki	ABa+subst:pl:acc:f+pospolita|ABa+subst:pl:nom:f+pospolita|ABa+subst:pl:voc:f+pospolita|ABa+subst:sg:gen:f+pospolita
+bijatyko	ABa+subst:sg:voc:f+pospolita
+bijatykom	ACa+subst:pl:dat:f+pospolita
+bijatyką	ABa+subst:sg:inst:f+pospolita
+bijatykę	ABa+subst:sg:acc:f+pospolita
+bijać	AA+inf:imperf+pospolita
+bijał	ABć+praet:sg:m1.m2.m3:imperf+pospolita
+bijała	ACć+praet:sg:f:imperf+pospolita
+bijało	ACć+praet:sg:n1.n2:imperf+pospolita
+bijały	ACć+praet:pl:m2.m3.f.n1.n2.p2.p3:imperf+pospolita
+bijcie	AEć+impt:pl:sec:imperf+pospolita
+bije	ACć+fin:sg:ter:imperf+pospolita
+bijecie	AFć+fin:pl:sec:imperf+pospolita
+bijekcja	AA+subst:sg:nom:f+pospolita
+bijekcjach	AC+subst:pl:loc:f+pospolita
+bijekcjami	AC+subst:pl:inst:f+pospolita
+bijekcje	ABa+subst:pl:acc:f+pospolita|ABa+subst:pl:nom:f+pospolita|ABa+subst:pl:voc:f+pospolita
+bijekcji	ABa+subst:pl:gen:f+pospolita|ABa+subst:sg:dat:f+pospolita|ABa+subst:sg:gen:f+pospolita|ABa+subst:sg:loc:f+pospolita
+bijekcjo	ABa+subst:sg:voc:f+pospolita
+bijekcjom	ACa+subst:pl:dat:f+pospolita
+bijekcją	ABa+subst:sg:inst:f+pospolita
+bijekcję	ABa+subst:sg:acc:f+pospolita
+bijekcyj	ACja+subst:pl:gen:f+pospolita
+bijemy	AEć+fin:pl:pri:imperf+pospolita
+bijesz	AEć+fin:sg:sec:imperf+pospolita
+bijmy	ADć+impt:pl:pri:imperf+pospolita
+bijnik	AA+subst:sg:acc:m3+pospolita|AA+subst:sg:nom:m3+pospolita
+bijnika	AB+subst:sg:gen:m3+pospolita
+bijnikach	AD+subst:pl:loc:m3+pospolita
+bijnikami	AD+subst:pl:inst:m3+pospolita
+bijniki	AB+subst:pl:acc:m3+pospolita|AB+subst:pl:nom:m3+pospolita|AB+subst:pl:voc:m3+pospolita
+bijnikiem	AD+subst:sg:inst:m3+pospolita
+bijnikom	AC+subst:pl:dat:m3+pospolita
+bijnikowi	AD+subst:sg:dat:m3+pospolita
+bijniku	AB+subst:sg:loc:m3+pospolita|AB+subst:sg:voc:m3+pospolita
+bijników	AC+subst:pl:gen:m3+pospolita
+biją	ACć+fin:pl:ter:imperf+pospolita
+bijąc	ADć+pcon:imperf+pospolita
+bijąca	AEć+pact:sg:nom.voc:f:imperf:aff+pospolita
+bijące	AEć+pact:pl:nom.acc.voc:m2.m3.f.n1.n2.p2.p3:imperf:aff+pospolita|AEć+pact:sg:nom.acc.voc:n1.n2:imperf:aff+pospolita
+bijącego	AGć+pact:sg:acc:m1.m2:imperf:aff+pospolita|AGć+pact:sg:gen:m1.m2.m3.n1.n2:imperf:aff+pospolita
+bijącej	AFć+pact:sg:gen.dat.loc:f:imperf:aff+pospolita
+bijącemu	AGć+pact:sg:dat:m1.m2.m3.n1.n2:imperf:aff+pospolita
+bijący	AEć+pact:pl:nom.voc:m1.p1:imperf:aff+pospolita|AEć+pact:sg:acc:m3:imperf:aff+pospolita|AEć+pact:sg:nom.voc:m1.m2.m3:imperf:aff+pospolita
+bijących	AGć+pact:pl:acc:m1.p1:imperf:aff+pospolita|AGć+pact:pl:gen.loc:m1.m2.m3.f.n1.n2.p1.p2.p3:imperf:aff+pospolita
+bijącym	AFć+pact:pl:dat:m1.m2.m3.f.n1.n2.p1.p2.p3:imperf:aff+pospolita|AFć+pact:sg:inst.loc:m1.m2.m3.n1.n2:imperf:aff+pospolita
+bijącymi	AGć+pact:pl:inst:m1.m2.m3.f.n1.n2.p1.p2.p3:imperf:aff+pospolita
+bijącą	AEć+pact:sg:acc.inst:f:imperf:aff+pospolita
@@ -10,9 +10,10 @@ import logging
 import codecs
 from morfeuszbuilder.fsa import encode
 from morfeuszbuilder.fsa import convertinput
-from morfeuszbuilder.fsa import common
 from morfeuszbuilder.fsa.fsa import FSA
 from morfeuszbuilder.fsa.serializer import VLengthSerializer1, VLengthSerializer2, SimpleSerializer
+from morfeuszbuilder.tagset.tagset import Tagset
+from morfeuszbuilder.segrules import rulesParser
 from optparse import OptionParser
 # class InputFormat():
@@ -50,6 +51,10 @@ def _parseOptions():
                         dest='tagsetFile',
                         metavar='FILE',
                         help='path to the file with tagset')
+    parser.add_option('--segments-file',
+                        dest='segmentsFile',
+                        metavar='FILE',
+                        help='path to the file with segment rules')
     parser.add_option('-o', '--output-file',
                         dest='outputFile',
                         metavar='FILE',
@@ -107,6 +112,8 @@ def _parseOptions():
     _checkOption(opts.serializationMethod, parser, "Serialization method file is missing")
     _checkExactlyOneOptionSet([opts.analyzer, opts.generator], 
                               parser, 'Must set exactly one FSA type: --analyzer or --generator')
+    if opts.analyzer:
+        _checkOption(opts.segmentsFile, parser, "Segment rules file is missing")
     if not opts.serializationMethod.upper() in [SerializationMethod.SIMPLE, SerializationMethod.V1, SerializationMethod.V2]:
         print >> sys.stderr, '--serialization-method must be one of ('+str([SerializationMethod.SIMPLE, SerializationMethod.V1, SerializationMethod.V2])+')'
@@ -147,9 +154,8 @@ def _printStats(fsa):
     logging.info('sink states num: '+str(sinkNum))
     logging.info('array states num: '+str(arrayNum))
-def buildAnalyzerFromPoliMorf(inputFile, tagsetFile):
+def buildAnalyzerFromPoliMorf(inputFile, tagset):
     encoder = encode.MorphEncoder()
-    tagset = common.Tagset(tagsetFile)
     fsa = FSA(encoder, tagset)
     inputData = _readPolimorfInput4Analyzer(inputFile, tagset, encoder)
     for word, data in inputData:
@@ -160,7 +166,7 @@ def buildAnalyzerFromPoliMorf(inputFile, tagsetFile):
 def buildGeneratorFromPoliMorf(inputFile, tagsetFile):
     encoder = encode.Encoder4Generator()
-    tagset = common.Tagset(tagsetFile)
+    tagset = Tagset(tagsetFile)
     fsa = FSA(encoder, tagset)
     inputData = _readPolimorfInput4Generator(inputFile, tagset, encoder)
     for word, data in inputData:
@@ -175,10 +181,15 @@ def main(opts):
     else:
         logging.basicConfig(level=logging.INFO)
+    tagset = Tagset(opts.tagsetFile)
+    
     if opts.analyzer:
-        fsa = buildAnalyzerFromPoliMorf(opts.inputFile, opts.tagsetFile)
+        fsa = buildAnalyzerFromPoliMorf(opts.inputFile, tagset)
+        segmentRulesManager = rulesParser.RulesParser(tagset).parse(opts.segmentsFile)
+        additionalData = segmentRulesManager.serialize()
     else:
-        fsa = buildGeneratorFromPoliMorf(opts.inputFile, opts.tagsetFile)
+        fsa = buildGeneratorFromPoliMorf(opts.inputFile, tagset)
+        additionalData = bytearray()
     if opts.trainFile:
         logging.info('training with '+opts.trainFile+' ...')
@@ -119,4 +119,3 @@ class FSA(object):
             state.reverseOffset = currReverseOffset
         for state in self.initialState.dfs(set()):
             state.offset = currReverseOffset - state.reverseOffset
-        
 \ No newline at end of file
@@ -45,16 +45,15 @@ class Serializer(object):
     def serialize2BinaryFile(self, fname):
         with open(fname, 'wb') as f:
-            f.write(self.fsa2bytearray())
+            f.write(self.fsa2bytearray(self.serializeTagset(self.fsa.tagset)))
     def getStateSize(self, state):
         raise NotImplementedError('Not implemented')
-    def fsa2bytearray(self):
+    def fsa2bytearray(self, additionalData=bytearray()):
         res = bytearray()
-        res.extend(self.serializePrologue(self.serializeTagset(self.fsa.tagset)))
+        res.extend(self.serializePrologue(additionalData))
         self.fsa.calculateOffsets(sizeCounter=lambda state: self.getStateSize(state))
-        logging.debug('SERIALIZE')
         for state in sorted(self.fsa.dfs(), key=lambda s: s.offset):
             res.extend(self.state2bytearray(state))
         return res
@@ -8,6 +8,8 @@ class State(object):
     '''
     A state in an automaton
     '''
+    
+    statesCounter = 0
     def __init__(self, additionalData=None):
         self.transitionsMap = {}
@@ -18,6 +20,9 @@ class State(object):
         self.label2Freq = {}
         self.serializeAsArray = False
         self.additionalData = additionalData
+        
+        self.idx = State.statesCounter
+        State.statesCounter += 1
     @property
     def transitionsNum(self):
@@ -51,10 +56,16 @@ class State(object):
         else:
             return self.encodedData
-    def dfs(self, alreadyVisited=set(), sortKey=lambda (_, state): -state.freq):
+    def dfs(self, alreadyVisited, sortKey=lambda (_, state): -state.freq):
         if not self in alreadyVisited:
+            alreadyVisited.add(self)
             for _, state in sorted(self.transitionsMap.iteritems(), key=sortKey):
                 for state1 in state.dfs(alreadyVisited):
                     yield state1
-            alreadyVisited.add(self)
             yield self
+    
+    def debug(self):
+        print '----------------'
+        print 'STATE:', self.idx
+        for label, s in self.transitionsMap.iteritems():
+            print label, '-->', s.idx
@@ -12,7 +12,7 @@ class Visualizer(object):
     def __init__(self):
         pass
-    def visualize(self, fsa):
+    def visualize(self, fsa, charLabels=True):
         G = nx.DiGraph()
         allStates = list(reversed(list(fsa.initialState.dfs(set()))))
         edgeLabelsMap = {}
@@ -21,10 +21,12 @@ class Visualizer(object):
             G.add_node(idx, offset=state.offset)
             for c, targetState in state.transitionsMap.iteritems():
                 G.add_edge(idx, allStates.index(targetState))
-                label = chr(c) if c <= 127 else '%'
+                label = (chr(c) if c <= 127 else '%') if charLabels \
+                    else c
                 edgeLabelsMap[(idx, allStates.index(targetState))] = label
             nodeLabelsMap[idx] = state.offset if not state.isAccepting() else state.encodedData + '(' + str(state.offset) + ')'
         pos=nx.shell_layout(G)
+#         pos=nx.random_layout(G)
         nx.draw_networkx_nodes(G,
                                pos, 
                                nodelist=list([allStates.index(s) for s in allStates if not s.isAccepting()]),
@@ -6,8 +6,7 @@ Created on 23 sty 2014
 import re
 from pyparsing import *
-identifier = Word(alphas, bodyChars=alphanums+'_')
-token = Word(alphas, bodyChars=alphanums+'_+>')
+identifier = Word(alphas, bodyChars=alphanums+'_>*+')
 define = Keyword('#define').suppress() + identifier + Optional(Suppress('(') + identifier + Suppress(')')) + restOfLine + LineEnd() + StringEnd()
 ifdef = Keyword('#ifdef').suppress() + identifier + LineEnd() + StringEnd()
 endif = Keyword('#endif').suppress() + LineEnd() + StringEnd()
@@ -34,6 +34,9 @@ class TagRule(SegmentRule):
     def _doAddToNFA(self, startState, endState):
         startState.addTransition(self.segnum, endState)
+    
+    def __str__(self):
+        return u''+self.segnum
 class UnaryRule(SegmentRule):
@@ -95,12 +98,3 @@ class ZeroOrMoreRule(UnaryRule):
         self.child._doAddToNFA(intermStartState, intermEndState)
         intermEndState.addTransition(None, endState)
         endState.addTransition(None, intermStartState)
-    
-class IgnoreOrthRule(UnaryRule):
-    
-    def __init__(self, child):
-        super(IgnoreOrthRule, self).__init__(child)
-    
-    def _doAddToNFA(self, startState, endState):
-        startState.addTransition(self.child.segnum, endState, ignoreOrth=True)
-
+'''
+Created on 20 lut 2014
+
+@author: mlenart
+'''
+
+class RulesManager(object):
+    
+    def __init__(self):
+        self.options2DFA = {}
+    
+    def _options2Key(self, optionsMap):
+        return frozenset(optionsMap.items())
+    
+    def addDFA4Options(self, optionsMap, dfa):
+        self.options2DFA[self._options2Key(optionsMap)] = dfa
+    
+    def serialize(self):
+        pass
 \ No newline at end of file
@@ -8,33 +8,98 @@ from morfeuszbuilder.fsa import fsa, state, encode
 class RulesNFAState(object):
-    def __init__(self, initial=False, final=False):
+    statesCounter = 0
+    
+    def __init__(self, initial=False, final=False, weak=False):
         self.transitionsMap = {}
         self.initial = initial
         self.final = final
+        self.weak = weak
+        self.idx = RulesNFAState.statesCounter
+        RulesNFAState.statesCounter += 1
+    
+    def addTransition(self, label, targetState):
+        self.transitionsMap.setdefault(label, set())
+        self.transitionsMap[label].add(targetState)
+    
+    def getClosure(self, visited):
+        if self in visited:
+            return set()
+        else:
+            visited.add(self)
+            res = set()
+            res.add(self)
+            for nextState in self.transitionsMap.get(None, []):
+                if self.idx in [6,8,4]:
+                    print nextState.idx
+                    print self.transitionsMap
+                res |= nextState.getClosure(visited)
+            return res
-    def addTransition(self, label, targetState, ignoreOrth=False):
-        assert not ignoreOrth or label is not None
-        self.transitionsMap.setdefault((label, ignoreOrth), set())
-        self.transitionsMap[(label, ignoreOrth)].add(targetState)
+    def dfs(self, visitedStates=set()):
+        if not self in visitedStates:
+            visitedStates.add(self)
+            yield self
+            for _, nextStates in self.transitionsMap.iteritems():
+                for state in nextStates:
+                    for state1 in state.dfs():
+                        yield state1
+    
+    def debug(self):
+        print '----------------'
+        print 'STATE:', self.idx
+        for label, nextStates in self.transitionsMap.iteritems():
+            print label, '-->', [s.idx for s in sorted(nextStates, key=lambda s: s.idx)]
 class RulesNFA(object):
-    def __init__(self, key2Def={}):
+    def __init__(self):
         self.initialState = RulesNFAState(initial=True)
-    def _doConvertState(self, dfaState, nfaStates):
-        for label, (nextIgnoreOrth, nextNFAStates) in self._groupOutputByLabels(nfaStates).iteritems():
-            nextDFAState = state.State(additionalData=nextIgnoreOrth)
+    def _groupOutputByLabels(self, nfaStates):
+        res = {}
+        for nfaState in nfaStates:
+            for label, nextStates in nfaState.transitionsMap.iteritems():
+                if label is not None:
+                    res.setdefault(label, set())
+                    for nextNFAState in nextStates:
+                        res[label] |= nextNFAState.getClosure(set())
+#                         print 'closure of', nextNFAState.idx, 'is', [s.idx for s in sorted(nextNFAState.getClosure(), key=lambda s: s.idx)]
+        return res
+    
+    def _doConvertState(self, dfaState, nfaStates, nfaSubset2DFAState):
+        assert all(map(lambda state: state.weak, nfaStates)) \
+            or not any(map(lambda state: state.weak, nfaStates))
+        weak = all(map(lambda state: state.weak, nfaStates))
+        final = any(map(lambda state: state.final, nfaStates))
+        assert not weak or not final
+        if final:
+            # dfaState should be final
+            # and contain info about weakness
+            dfaState.encodedData = bytearray([1 if weak else 0])
+        for label, nextNFAStates in self._groupOutputByLabels(nfaStates).iteritems():
+#             print '============'
+#             print 'states:', [s.idx for s in sorted(nfaStates, key=lambda s: s.idx)]
+#             print 'label:', label
+#             print 'nextStates:', [s.idx for s in sorted(nextNFAStates, key=lambda s: s.idx)]
+            key = frozenset(nextNFAStates)
+            if key in nfaSubset2DFAState:
+                nextDFAState = nfaSubset2DFAState[key]
+            else:
+                nextDFAState = state.State()
+                nfaSubset2DFAState[key] = nextDFAState
+                self._doConvertState(nextDFAState, nextNFAStates, nfaSubset2DFAState)
             dfaState.setTransition(label, nextDFAState)
-            dfaState.encodedData = bytearray()
-            self._doConvertState(nextDFAState, nextNFAStates)
     def convertToDFA(self):
-        dfa = fsa.FSA(encoder=None, encodeWords=False)
-        startStates = self.initialState.getClosure()
+        dfa = fsa.FSA(encoder=None, encodeData=False, encodeWords=False)
+        startStates = self.initialState.getClosure(set())
         assert not any(filter(lambda s: s.final, startStates))
         dfa.initialState = state.State(additionalData=False)
-        self._doConvertState(dfa.initialState, startStates)
-                
+        self._doConvertState(dfa.initialState, startStates, {frozenset(startStates): dfa.initialState})
+        return dfa
+    
+    def debug(self):
+        for state in self.initialState.dfs():
+            state.debug()
 \ No newline at end of file
@@ -3,7 +3,7 @@ from pyparsing import *
 ParserElement.enablePackrat()
 from morfeuszbuilder.tagset import segtypes
 from morfeuszbuilder.utils import configFile, exceptions
-from morfeuszbuilder.segrules import preprocessor, rules
+from morfeuszbuilder.segrules import preprocessor, rules, rulesManager
 import codecs
 import re
@@ -28,9 +28,9 @@ class RulesParser(object):
         return res
     def parse(self, filename):
-        res = []
+        res = rulesManager.RulesManager()
-        segtypesConfigFile = configFile.ConfigFile(filename, ['options', 'combinations', 'tags', 'lexemes'])
+        segtypesConfigFile = configFile.ConfigFile(filename, ['options', 'combinations', 'tags', 'lexemes', 'segment types'])
         key2Defs = self._getKey2Defs(segtypesConfigFile)
         segtypesHelper = segtypes.Segtypes(self.tagset, segtypesConfigFile)
@@ -39,14 +39,18 @@ class RulesParser(object):
             for define in defs:
                 def2Key[define] = key
+        firstNFA = None
         for defs in itertools.product(*key2Defs.values()):
             key2Def = dict([(def2Key[define], define) for define in defs])
-            nfa = rulesNFA.RulesNFA(key2Def)
+            nfa = rulesNFA.RulesNFA()
+            if not firstNFA:
+                firstNFA = nfa
             combinationEnumeratedLines = segtypesConfigFile.enumerateLinesInSection('combinations')
             combinationEnumeratedLines = list(preprocessor.preprocess(combinationEnumeratedLines, defs))
             for rule in self._doParse(combinationEnumeratedLines, segtypesHelper):
                 rule.addToNFA(nfa)
-            res.append(nfa)
+            dfa = nfa.convertToDFA()
+            res.addDFA4Options(key2Def, dfa)
         return res
     def _doParse(self, combinationEnumeratedLines, segtypesHelper):
@@ -58,14 +62,14 @@ class RulesParser(object):
         if not segtypesHelper.hasSegtype(segtype):
             raise exceptions.ConfigFileException(segtypesHelper.filename, lineNum, u'%s - invalid segment type: %s' % (line, segtype))
         else:
+#             return rules.TagRule(segtype)
             return rules.TagRule(segtypesHelper.getSegnum4Segtype(segtype))
     def _doParseOneLine(self, lineNum, line, segtypesHelper):
         rule = Forward()
-        tagRule = Word(alphanums+'_')
-        ignoreOrthRule = tagRule + Suppress('>')
+        tagRule = Word(alphanums+'_>')
         parenRule = Suppress('(') + rule + Suppress(')')
-        atomicRule = tagRule ^ ignoreOrthRule ^ parenRule
+        atomicRule = tagRule ^ parenRule
         zeroOrMoreRule = atomicRule + Suppress('*')
         oneOrMoreRule = atomicRule + Suppress('+')
         unaryRule = atomicRule ^ zeroOrMoreRule ^ oneOrMoreRule
@@ -75,19 +79,10 @@ class RulesParser(object):
         rule << concatRule
         tagRule.setParseAction(lambda string, loc, toks: self._createNewTagRule(toks[0], lineNum, line, segtypesHelper))
-        ignoreOrthRule.setParseAction(lambda string, loc, toks: rules.IgnoreOrthRule(toks[0]))
 #         parenRule.setParseAction(lambda string, loc, toks: toks[0])
         zeroOrMoreRule.setParseAction(lambda string, loc, toks: rules.ZeroOrMoreRule(toks[0]))
         oneOrMoreRule.setParseAction(lambda string, loc, toks: rules.ConcatRule([toks[0], rules.ZeroOrMoreRule(toks[0])]))
         oneOfRule.setParseAction(lambda string, loc, toks: rules.OrRule(toks))
         concatRule.setParseAction(lambda string, loc, toks: toks[0] if len(toks) == 1 else rules.ConcatRule(toks))
-        
-        
-#         rule << tagRule ^ ignoreOrthRule ^ zeroOrMoreRule ^ oneOrMoreRule ^ orRule ^ concatRule ^ parenRule
-        
-#         tagRule.setParseAction(lambda s,l,toks: doprint(toks))
-#         print lineNum, line
         parsedRule = rule.parseString(line, parseAll=True)[0]
-        print parsedRule
         return parsedRule
-#         print parsedLine
@@ -7,12 +7,20 @@ import unittest
 import os
 from morfeuszbuilder.segrules import rulesParser
 from morfeuszbuilder.tagset import tagset
+from morfeuszbuilder.fsa import visualizer, serializer
 class Test(unittest.TestCase):
     print 'do test'
     t = tagset.Tagset(os.path.join(os.path.dirname(__file__), 'polimorf.tagset'))
     parser = rulesParser.RulesParser(t)
-    parser.parse(os.path.join(os.path.dirname(__file__), 'segmenty.dat'))
+    fsas = parser.parse(os.path.join(os.path.dirname(__file__), 'segmenty.dat'))
+    fsa = fsas[0]
+    for s in fsa.dfs():
+        s.debug()
+    print 'states:', len(list(fsa.dfs()))
+    print 'transitions:', fsa.getTransitionsNum()
+    visualizer.Visualizer().visualize(fsa, charLabels=False)
+    print 'size:', len(serializer.SimpleSerializer(fsa).fsa2bytearray(bytearray()))
     print 'done'
 if __name__ == "__main__":
@@ -103,7 +103,7 @@ moze_interp( naj&gt; adj_sup )
 # Formy „zanegowane” gerundiów i imiesłowów:
 # np. „nie·czytanie”, „nie·przeczytany”, „nie·czytający”:
-moze_interp( nie > negat )
+moze_interp( nie> negat )
 # Przyimki akceptujące krótką formę „-ń”
 moze_interp(z_on_agl)
@@ -111,7 +111,7 @@ moze_interp(z_on_agl)
 moze_interp(z_on_agl on_agl)
 # Liczba zapisana jako ciąg cyfr:
-moze_interp( dig>* dig )
+#moze_interp( dig>* dig )
 # Formacje prefiksalne
 #### trzeba wydzielić odpowiednie samodze!
@@ -132,13 +132,35 @@ adj dywiz samodz
 # ?
 samodz dywiz adj
+[segment types]
+naj>
+nie>
+prefs
+prefv
+dig>
+adja
+adj
+adj_sup
+negat
+on_agl
+z_on_agl
+samotny
+interp
+aglsg
+aglpl
+praetcond
+praet_sg_agl
+praet_sg_na
+praet_sg
+praet_pl
+samodz
 [tags]
-naj	naj
-nie	nie
+naj>	naj
+nie>	nie
 prefs	prefs
 prefv	prefv
-dig	dig
+dig>	dig
 adja	adja
 adj	adj:%:pos
 adj_sup	adj:%:sup
+[options]
+aggl=permissive strict isolated
+praet=split composite
+
+[combinations]
+#define wsz_interp (interp|kropka|dywiz)*
+
+#define moze_interp(segmenty) wsz_interp segmenty wsz_interp
+
+moze_interp(samodz)
+samotny
+
+
+[segment types]
+naj>
+nie>
+prefs
+prefv
+dig
+adja
+adj
+adj_sup
+negat
+on_agl
+z_on_agl
+samotny
+interp
+aglsg
+aglpl
+praetcond
+praet_sg_agl
+praet_sg_na
+praet_sg
+praet_pl
+samodz
+
+[tags]
+naj	naj
+nie	nie
+prefs	prefs
+prefv	prefv
+dig	dig
+adja	adja
+adj	adj:%:pos
+adj_sup	adj:%:sup
+adj_sup	adv:sup
+negat	ger:%:neg
+negat	pact:%:neg
+negat	ppas:%:neg
+on_agl	ppron3:sg:gen.acc:m1.m2.m3:ter:nakc:praep
+z_on_agl	prep:%
+samotny	brev:pun
+samotny	brev:npun
+samotny	intrj
+interp	interp
+aglsg	aglt:sg:%
+aglpl	aglt:pl:%
+praetcond	cond:%
+praetcond	praet:%:pri:%
+praetcond	praet:%:sec:%
+praetcond	praet:%:ter:%
+praet_sg_agl	praet:sg:%:agl
+praet_sg_na	praet:sg:%:nagl
+praet_sg	praet:sg:%
+praet_pl	praet:pl:%
+praet_sg	winien:sg:%
+praet_pl	winien:pl:%
+samodz		%
+
+[lexemes]
+z_aglt	aby:comp
+z_aglt	bowiem:comp
+by	by:qub
+z_aglt	by:comp
+z_aglt	cóż:subst
+z_aglt	czemu:adv
+z_aglt	czyżby:qub
+z_aglt	choćby:comp
+z_aglt	chociażby:comp
+z_aglt	dlaczego:adv
+z_aglt	dopóki:comp
+z_aglt	dopóty:conj
+z_aglt	gdyby:comp
+z_aglt	gdzie:qub
+z_aglt	gdzie:adv
+z_aglt	jakby:comp
+z_aglt	jakoby:comp
+z_aglt	kiedy:adv
+z_aglt	kiedy:comp
+z_aglt	tylko:qub
+z_aglt	żeby:comp
+dywiz	-:interp
+kropka	.:interp