- działająca podstawowa wersja z tablicą w każdych dwóch pierwszych stanach

git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@11 ff4e3ee1-f430-4e82-ade0-24591c43f1fd

- działająca podstawowa wersja z tablicą w każdych dwóch pierwszych stanach
git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@11 ff4e3ee1-f430-4e82-ade0-24591c43f1fd
Michał Lenart
1 parent c2243119
Showing 12 changed files with 674 additions and 138 deletions
CMakeLists.txt
fsa/_fsa_impl.hpp
fsa/_fsaimpl.hpp
fsa/_vfsa_impl.hpp
fsa/fsa.hpp
fsa/test_speed.cpp
fsabuilder/fsa/buildfsa.py
fsabuilder/fsa/encode.py
fsabuilder/fsa/fsa.py
fsabuilder/fsa/serializer.py
fsabuilder/fsa/state.py
nbproject/configurations.xml
@@ -9,6 +9,6 @@ add_subdirectory (morfeusz)
  
 file(COPY fsabuilder testfiles DESTINATION .)
  
-add_test (TestBuildFSA python fsabuilder/fsa/buildfsa.py -i testfiles/dict.txt -o testfiles/test.fsa -t SPELL --input-format=PLAIN --output-format=BINARY)
+add_test (TestBuildFSA python fsabuilder/fsa/buildfsa.py -i testfiles/dict.txt -o testfiles/test.fsa -t SPELL --input-format=PLAIN --output-format=BINARY --use-arrays)
 add_test (TestRecognize fsa/test_recognize testfiles/test.fsa testfiles/dict.txt)
 add_test (TestNOTRecognize fsa/test_not_recognize testfiles/test.fsa testfiles/out_of_dict.txt)
@@ -115,7 +115,7 @@ bool FSA&lt;T&gt;::tryToRecognize(const char* input, T&amp; value) const {
         i++;
     }
     // input[i] == '\0'
-    currState.proceedToNext(0);
+//    currState.proceedToNext(0);
  
     if (currState.isAccepting()) {
         value = currState.getValue();
+/* 
+ * File:   _vfsa_impl.hpp
+ * Author: lennyn
+ *
+ * Created on October 29, 2013, 9:57 PM
+ */
+
+#ifndef _VFSA_IMPL_HPP
+#define	_VFSA_IMPL_HPP
+
+#include <algorithm>
+#include <utility>
+#include <iostream>
+#include <netinet/in.h>
+#include "fsa.hpp"
+
+using namespace std;
+
+#pragma pack(push)  /* push current alignment to stack */
+#pragma pack(1)     /* set alignment to 1 byte boundary */
+
+struct StateData2 {
+    unsigned transitionsNum: 6;
+    unsigned array : 1;
+    unsigned accepting : 1;
+};
+
+struct TransitionData2 {
+    unsigned offsetSize : 2;
+    unsigned shortLabel : 6;
+};
+
+
+#pragma pack(pop)   /* restore original alignment from stack */
+
+template <class T>
+int FSAImpl<T>::getMagicNumberOffset() {
+    return 0;
+}
+
+template <class T>
+int FSAImpl<T>::getVersionNumOffset() {
+    return getMagicNumberOffset() + sizeof (MAGIC_NUMBER);
+}
+
+template <class T>
+int FSAImpl<T>::getPopularCharsOffset() {
+    return getVersionNumOffset() + sizeof (VERSION_NUM);
+}
+
+template <class T>
+int FSAImpl<T>::getInitialStateOffset() {
+    return getPopularCharsOffset() + 256 + 1;
+}
+
+template <class T>
+vector<unsigned char> FSAImpl<T>::initializeChar2PopularCharIdx(const unsigned char* ptr) {
+    return vector<unsigned char>(ptr + getPopularCharsOffset(), ptr + getPopularCharsOffset() + 256);
+}
+
+template <class T>
+FSAImpl<T>::FSAImpl(const unsigned char* ptr, const Deserializer<T>& deserializer)
+: FSA<T>(ptr + getInitialStateOffset(), deserializer),
+label2ShortLabel(initializeChar2PopularCharIdx(ptr)) {
+    uint32_t magicNumber = ntohl(*((uint32_t*) ptr + getMagicNumberOffset()));
+    if (magicNumber != MAGIC_NUMBER) {
+        throw FSAException("Invalid magic number");
+    }
+    unsigned char versionNum = *(ptr + getVersionNumOffset());
+    if (versionNum != VERSION_NUM) {
+        throw FSAException("Invalid version number");
+    }
+    //    cerr << "initial state offset " << getInitialStateOffset() << endl;
+}
+
+template <class T>
+FSAImpl<T>::~FSAImpl() {
+
+}
+
+template <class T>
+void FSAImpl<T>::reallyDoProceed(
+        const unsigned char* statePtr,
+        State<T>& state) const {
+//    const unsigned char stateByte = *statePtr;
+    StateData2* sd = (StateData2*) statePtr;
+    if (sd->accepting) {
+//                                            cerr << "ACCEPTING" << endl;
+        T object;
+        int size = this->deserializer.deserialize(statePtr + 1, object);
+        state.setNext(statePtr - this->startPtr, object, size);
+    }
+    else {
+        state.setNext(statePtr - this->startPtr);
+    }
+}
+
+template <class T>
+void FSAImpl<T>::doProceedToNextByList(
+        const char c,
+        const unsigned char shortLabel,
+        const unsigned char* ptr,
+        const unsigned int transitionsNum,
+        State<T>& state) const {
+    register const unsigned char* currPtr = ptr;
+    //    TransitionData* foundTransition = (TransitionData*) (fromPointer + transitionsTableOffset);
+    bool found = false;
+    TransitionData2 td;
+    for (unsigned int i = 0; i < transitionsNum; i++) {
+        //        const_cast<Counter*>(&counter)->increment(1);
+        td = *((TransitionData2*) currPtr);
+        if (td.shortLabel == shortLabel) {
+            if (shortLabel == 0) {
+                currPtr++;
+                char label = (char) *currPtr;
+                if (label == c) {
+                    found = true;
+                    break;
+                }
+                else {
+                    currPtr += td.offsetSize + 1;
+                }
+            } else {
+                found = true;
+                break;
+            }
+        } 
+        else {
+            if (td.shortLabel == 0) {
+                currPtr++;
+            }
+            currPtr += td.offsetSize + 1;
+        }
+    }
+    //        const_cast<Counter*>(&counter)->increment(foundTransition - transitionsStart + 1);
+    if (!found) {
+//                                cerr << "SINK for " << c << endl;
+        state.setNextAsSink();
+    } else {
+        currPtr++;
+//                                        cerr << "offset size " << td.offsetSize << endl;
+//                            cerr << "offset " << offset << endl;
+        switch (td.offsetSize) {
+            case 0:
+                break;
+            case 1:
+                currPtr += *currPtr + 1;
+                break;
+            case 2:
+                currPtr += ntohs(*((uint16_t*) currPtr)) + 2;
+                break;
+            case 3:
+                currPtr += (((unsigned int) ntohs(*((uint16_t*) currPtr))) << 8) + currPtr[2] + 3;
+                break;
+        }
+//                                cerr << "FOUND " << c << " " << currPtr - this->startPtr << endl;
+        reallyDoProceed(currPtr, state);
+    }
+}
+
+template <class T>
+void FSAImpl<T>::doProceedToNextByArray(
+        const unsigned char shortLabel,
+        const uint32_t* ptr,
+        State<T>& state) const {
+    uint32_t offset = ntohl(ptr[shortLabel]);
+    if (offset != 0) {
+        const unsigned char* currPtr = this->startPtr + offset;
+        reallyDoProceed(currPtr, state);
+    }
+    else {
+        state.setNextAsSink();
+    }
+}
+
+template <class T>
+void FSAImpl<T>::proceedToNext(const char c, State<T>& state) const {
+//                    if (c <= 'z' && 'a' <= c)
+//                        cerr << "NEXT " << c << " from " << state.getOffset() << endl;
+//                    else
+//                        cerr << "NEXT " << (short) c << " from " << state.getOffset() << endl;
+    const unsigned char* fromPointer = this->startPtr + state.getOffset();
+    unsigned char shortLabel = this->label2ShortLabel[(const unsigned char) c];
+    unsigned int transitionsTableOffset = 1;
+    if (state.isAccepting()) {
+        transitionsTableOffset += state.getValueSize();
+//                                cerr << "transitionsTableOffset " << transitionsTableOffset + state.getOffset() << " because value is " << state.getValue() << endl;
+    }
+    StateData2* sd = (StateData2*) (fromPointer);
+//                cerr << "transitions num=" << sd->transitionsNum << endl;
+    if (sd->array) {
+        if (shortLabel > 0) {
+            this->doProceedToNextByArray(
+                    shortLabel,
+                    (uint32_t*) (fromPointer + transitionsTableOffset),
+                    state);
+        }
+        else {
+            reallyDoProceed((unsigned char*) fromPointer + transitionsTableOffset + 256, state);
+            proceedToNext(c, state);
+        }
+    } 
+    else {
+        this->doProceedToNextByList(
+                c,
+                shortLabel,
+                (unsigned char*) (fromPointer + transitionsTableOffset),
+                sd->transitionsNum,
+                state);
+    }
+}
+
+#endif	/* _VFSA_IMPL_HPP */
+
@@ -19,11 +19,11 @@ using namespace std;
 #pragma pack(push)  /* push current alignment to stack */
 #pragma pack(1)     /* set alignment to 1 byte boundary */
  
-//struct VTransitionData {
-//    unsigned label : 5;
-//    unsigned offsetSize : 2;
-//    unsigned last : 1;
-//};
+struct StateData2 {
+    unsigned transitionsNum : 6;
+    unsigned next : 1;
+    unsigned accepting : 1;
+};
  
 #pragma pack(pop)   /* restore original alignment from stack */
  
@@ -49,12 +49,13 @@ int FSAImpl&lt;T&gt;::getInitialStateOffset() {
  
 template <class T>
 vector<unsigned char> FSAImpl<T>::initializeChar2PopularCharIdx(const unsigned char* ptr) {
-    vector<unsigned char> res(256, FSAImpl<bool>::POPULAR_CHARS_NUM);
-    const unsigned char* popularChars = ptr + getPopularCharsOffset();
-    for (unsigned int i = 0; i < POPULAR_CHARS_NUM; i++) {
-        res[popularChars[i]] = i;
-    }
-    return res;
+    //    vector<unsigned char> res(256, FSAImpl<bool>::POPULAR_CHARS_NUM);
+    //    const unsigned char* popularChars = ptr + getPopularCharsOffset();
+    //    for (unsigned int i = 0; i < POPULAR_CHARS_NUM; i++) {
+    //        res[popularChars[i]] = i;
+    //    }
+    //    return res;
+    return vector<unsigned char>();
 }
  
 template <class T>
@@ -79,94 +80,165 @@ FSAImpl&lt;T&gt;::~FSAImpl() {
  
 template <class T>
 void FSAImpl<T>::proceedToNext(const char c, State<T>& state) const {
-    //    if (c <= 'z' && 'a' <= c)
-    //        cerr << "NEXT " << c << " from " << state.getOffset() << endl;
-    //    else
-    //        cerr << "NEXT " << (short) c << " from " << state.getOffset() << endl;
+//            if (c <= 'z' && 'a' <= c)
+//                cerr << "NEXT " << c << " from " << state.getOffset() << endl;
+//            else
+//                cerr << "NEXT " << (short) c << " from " << state.getOffset() << endl;
     const unsigned char* fromPointer = this->startPtr + state.getOffset();
-    unsigned int transitionsTableOffset = 0;
+    int transitionsTableOffset = sizeof (StateData2);
     if (state.isAccepting()) {
         transitionsTableOffset += state.getValueSize();
-        //        cerr << "transitionsTableOffset " << transitionsTableOffset + state.getOffset() << " because value is " << state.getValue() << endl;
+//                cerr << "transitionsTableOffset " << transitionsTableOffset + state.getOffset() << " because value is " << state.getValue() << endl;
     }
-    
+    StateData2 stateData = *(StateData2*) (fromPointer);
+//        cerr << "transitions num=" << stateData.transitionsNum << endl;
+    register unsigned char* currPtr = (unsigned char*) (fromPointer + transitionsTableOffset);
+    //    TransitionData* foundTransition = (TransitionData*) (fromPointer + transitionsTableOffset);
     bool found = false;
-//    bool failed = false;
-    unsigned int requiredShortLabel = char2PopularCharIdx[(unsigned char) c];
-    //    cerr << "NEXT " << c << " " << (int) shortLabel << endl;
-//    VTransitionData* td;
-//    unsigned char transitionByte = *currPtr;
-    unsigned int offsetSize;
-    register const unsigned char* currPtr = fromPointer + transitionsTableOffset;
-    
-    while (!found) {
-        
-        register unsigned char firstByte = *currPtr;
-        
-        unsigned int shortLabel = firstByte & 0b00011111;
-        bool last = (firstByte & 0b10000000);
-        offsetSize = (firstByte & 0b01100000) >> 5;
-
-        const_cast<FSAImpl<T>*>(this)->counter.increment(1);
-        
-        if (shortLabel != requiredShortLabel) {
-            if (last || shortLabel == POPULAR_CHARS_NUM) {
-                break;
-            }
-            currPtr += offsetSize + 1;
-            if (shortLabel == POPULAR_CHARS_NUM) {
-                currPtr++;
-            }
-        }
-        else if (shortLabel != POPULAR_CHARS_NUM) {
+    bool next = stateData.next;
+    for (unsigned int i = 0; i < stateData.transitionsNum; i++) {
+//                cerr << *currPtr << endl;      
+        if ((char) *currPtr == c) {
             found = true;
-            currPtr++;
+            next = next && i + 1 == stateData.transitionsNum;
+            break;
+        } else {
+            //            unsigned int offsetSize = currPtr[1] & 0b00000011;
+            currPtr += (currPtr[1] & 0b00000011) + 2;
         }
-        else {
+    }
+    //    const_cast<Counter*>(&counter)->increment(foundTransition - transitionsStart + 1);
+    if (!found) {
+//                cerr << "SINK for " << c << endl;
+        state.setNextAsSink();
+    }
+    else {
+        currPtr++;
+        if (!next) {
+            unsigned int offsetSize = *currPtr & 0b00000011;
+            unsigned int offset = *currPtr >> 2;
+//                    cerr << "offset size " << offsetSize << endl;
+//                    cerr << "offset " << offset << endl;
             currPtr++;
-            char realLabel = (char) *currPtr;
-            if (realLabel != c) {
-                if (last) {
+            //        currPtr += (*currPtr >> 2) + 1;
+            switch (offsetSize) {
+                case 0:
+                    currPtr += offset;
+                    break;
+                case 1:
+                    currPtr += (offset << 8) + *currPtr + 1;
+                    break;
+                case 2:
+                    currPtr += (offset << 16) + ntohs(*((uint16_t*) currPtr)) + 2;
+                    break;
+                case 3:
+                    currPtr += (offset << 24) + (((unsigned int) ntohs(*((uint16_t*) currPtr))) << 8) + currPtr[2] + 3;
                     break;
-                }
-                currPtr += offsetSize + 1;
-            }
-            else {
-                found = true;
-                currPtr++;
             }
         }
-    }
-
-    if (found) {
-        switch (offsetSize) {
-            case 0:
-                break;
-            case 1:
-                currPtr += *currPtr + 1;
-                break;
-            case 2:
-                currPtr += ntohs(*((uint16_t*) currPtr)) + 2;
-                break;
-            case 3:
-                currPtr += (((unsigned int) ntohs(*((uint16_t*) currPtr))) << 8) + currPtr[2] + 3;
-                break;
-        }
-        bool accepting = c == '\0';
-        if (accepting) {
-            T value;
-            int valueSize = this->deserializer.deserialize(currPtr, value);
-            currPtr += valueSize;
-            state.setNext(currPtr - this->startPtr, value, valueSize);
-        }
-        else {
+//                cerr << "FOUND " << c << " " << currPtr - this->startPtr << endl;
+        StateData* nextStateData = (StateData*) (currPtr);
+        if (nextStateData->accepting) {
+//                        cerr << "ACCEPTING" << endl;
+            T object;
+            int size = this->deserializer.deserialize(currPtr + sizeof (StateData), object);
+            state.setNext(currPtr - this->startPtr, object, size);
+        } else {
             state.setNext(currPtr - this->startPtr);
         }
     }
-    else {
-        state.setNextAsSink();
-    }
 }
  
+//template <class T>
+//void FSAImpl<T>::proceedToNext(const char c, State<T>& state) const {
+//    //    if (c <= 'z' && 'a' <= c)
+//    //        cerr << "NEXT " << c << " from " << state.getOffset() << endl;
+//    //    else
+//    //        cerr << "NEXT " << (short) c << " from " << state.getOffset() << endl;
+//    const unsigned char* fromPointer = this->startPtr + state.getOffset();
+//    unsigned int transitionsTableOffset = 0;
+//    if (state.isAccepting()) {
+//        transitionsTableOffset += state.getValueSize();
+//        //        cerr << "transitionsTableOffset " << transitionsTableOffset + state.getOffset() << " because value is " << state.getValue() << endl;
+//    }
+//    
+//    bool found = false;
+////    bool failed = false;
+//    unsigned int requiredShortLabel = char2PopularCharIdx[(unsigned char) c];
+//    //    cerr << "NEXT " << c << " " << (int) shortLabel << endl;
+////    VTransitionData* td;
+////    unsigned char transitionByte = *currPtr;
+//    unsigned int offsetSize;
+//    register const unsigned char* currPtr = fromPointer + transitionsTableOffset;
+//    
+//    while (!found) {
+//        
+//        register unsigned char firstByte = *currPtr;
+//        
+//        unsigned int shortLabel = firstByte & 0b00011111;
+//        bool last = (firstByte & 0b10000000);
+//        offsetSize = (firstByte & 0b01100000) >> 5;
+//
+//        const_cast<FSAImpl<T>*>(this)->counter.increment(1);
+//        
+//        if (shortLabel != requiredShortLabel) {
+//            if (last) {
+//                break;
+//            }
+//            currPtr += offsetSize + 1;
+//            if (shortLabel == POPULAR_CHARS_NUM) {
+//                currPtr++;
+//            }
+//        }
+//        else if (shortLabel != POPULAR_CHARS_NUM) {
+//            found = true;
+//            currPtr++;
+//        }
+//        else {
+//            currPtr++;
+//            char realLabel = (char) *currPtr;
+//            if (realLabel != c) {
+//                if (last) {
+//                    break;
+//                }
+//                currPtr += offsetSize + 1;
+//            }
+//            else {
+//                found = true;
+//                currPtr++;
+//            }
+//        }
+//    }
+//
+//    if (found) {
+//        switch (offsetSize) {
+//            case 0:
+//                break;
+//            case 1:
+//                currPtr += *currPtr + 1;
+//                break;
+//            case 2:
+//                currPtr += ntohs(*((uint16_t*) currPtr)) + 2;
+//                break;
+//            case 3:
+//                currPtr += (((unsigned int) ntohs(*((uint16_t*) currPtr))) << 8) + currPtr[2] + 3;
+//                break;
+//        }
+//        bool accepting = c == '\0';
+//        if (accepting) {
+//            T value;
+//            int valueSize = this->deserializer.deserialize(currPtr, value);
+//            currPtr += valueSize;
+//            state.setNext(currPtr - this->startPtr, value, valueSize);
+//        }
+//        else {
+//            state.setNext(currPtr - this->startPtr);
+//        }
+//    }
+//    else {
+//        state.setNextAsSink();
+//    }
+//}
+
 #endif	/* _VFSA_IMPL_HPP */
  
@@ -119,20 +119,36 @@ public:
     }
  
     static const uint32_t MAGIC_NUMBER = 0x8fc2bc1b;
-    static const unsigned char VERSION_NUM = 1;
-    static const unsigned int POPULAR_CHARS_NUM = 31;
+    static const unsigned char VERSION_NUM = 4;
+    
+    static const unsigned char ACCEPTING_FLAG =         0b10000000;
+    static const unsigned char ARRAY_FLAG =             0b01000000;
+    static const unsigned char TRANSITIONS_NUM_MASK =   0b00111111;
  
 protected:
     void proceedToNext(const char c, State<T>& state) const;
 private:
     Counter counter;
-    const std::vector<unsigned char> char2PopularCharIdx;
+    const std::vector<unsigned char> label2ShortLabel;
  
     static int getMagicNumberOffset();
     static int getVersionNumOffset();
     static int getPopularCharsOffset();
     static int getInitialStateOffset();
     static std::vector<unsigned char> initializeChar2PopularCharIdx(const unsigned char* ptr);
+    void doProceedToNextByList(
+        const char c,
+        const unsigned char shortLabel,
+        const unsigned char* ptr, 
+        const unsigned int transitionsNum, 
+        State<T>& state) const;
+    void doProceedToNextByArray(
+        const unsigned char shortLabel, 
+        const uint32_t* ptr, 
+        State<T>& state) const;
+    void reallyDoProceed(
+        const unsigned char* statePtr,
+        State<T>& state) const;
 };
  
 /**
@@ -201,7 +217,8 @@ private:
 };
  
 #include "_fsa_impl.hpp"
-#include "_vfsa_impl.hpp"
+#include "_fsaimpl.hpp"
+//#include "_vfsa_impl.hpp"
 #include "_state_impl.hpp"
  
 #endif	/* FSA_HPP */
@@ -30,13 +30,14 @@ int main(int argc, char** argv) {
     int unrecognized = 0;
     while (ifs.getline(line, 65536, '\n')) {
         char* val;
-//        cout << line << endl;
+//        cerr << line << endl;
         if (fsa.tryToRecognize(line, val)) {
 //            printf("%s: *OK*\n", line);
             recognized++;
         }
         else {
             unrecognized++;
+//            exit(1);
 //            printf("%s: NOT FOUND\n", line);
         }
     }
@@ -11,12 +11,10 @@ import codecs
 import encode
 import convertinput
 from fsa import FSA
-from serializer import VLengthSerializer
+from serializer import VLengthSerializer2, VLengthSerializer3
 from visualizer import Visualizer
 from optparse import OptionParser
  
-logging.basicConfig(level=logging.INFO)
-
 class OutputFormat():
     BINARY = 'BINARY'
     CPP = 'CPP'
@@ -52,6 +50,11 @@ def parseOptions():
     parser.add_option('--output-format',
                         dest='outputFormat',
                         help='output format - BINARY or CPP')
+    parser.add_option('--use-arrays',
+                        dest='useArrays',
+                        action='store_true',
+                        default=False,
+                        help='store states reachable by 2 transitions in arrays (should speed up recognition)')
     parser.add_option('--visualize',
                         dest='visualize',
                         action='store_true', 
@@ -60,6 +63,11 @@ def parseOptions():
     parser.add_option('--train-file',
                         dest='trainFile',
                         help='A text file used for training. Should contain words from some large corpus - one word in each line')
+    parser.add_option('--debug',
+                        dest='debug',
+                        action='store_true',
+                        default=False,
+                        help='output some debugging info')
  
     opts, args = parser.parse_args()
  
@@ -114,6 +122,10 @@ def readTrainData(trainFile):
  
 if __name__ == '__main__':
     opts = parseOptions()
+    if opts.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
     encoder = encode.Encoder()
     fsa = FSA(encoder)
  
@@ -129,16 +141,19 @@ if __name__ == &#39;__main__&#39;:
         logging.info('training with '+opts.trainFile+' ...')
         fsa.train(readTrainData(opts.trainFile))
         logging.info('done training')
-    serializer = VLengthSerializer(fsa)
+    serializer = VLengthSerializer3(fsa, useArrays=opts.useArrays)
     logging.info('states num: '+str(fsa.getStatesNum()))
     logging.info('transitions num: '+str(fsa.getTransitionsNum()))
     logging.info('accepting states num: '+str(len([s for s in fsa.initialState.dfs(set()) if s.isAccepting()])))
     logging.info('sink states num: '+str(len([s for s in fsa.initialState.dfs(set()) if len(s.transitionsMap.items()) == 0])))
+    logging.info('array states num: '+str(len([s for s in fsa.dfs() if s.serializeAsArray])))
     {
      OutputFormat.CPP: serializer.serialize2CppFile,
      OutputFormat.BINARY: serializer.serialize2BinaryFile
      }[opts.outputFormat](opts.outputFile)
     logging.info('size: '+str(fsa.initialState.reverseOffset))
+#     for s in fsa.dfs():
+#         logging.debug('%d %s' % (s.offset, str(s.transitionsMap)))
 #     for s in fsa.initialState.dfs(set()):
 #         logging.info(s.offset)
     if opts.visualize:
@@ -10,7 +10,7 @@ class Encoder(object):
     '''
  
  
-    def __init__(self, encoding='utf8', appendZero=True):
+    def __init__(self, encoding='utf8', appendZero=False):
         '''
         Constructor
         '''
@@ -21,7 +21,7 @@ class FSA(object):
         self.encodedPrevWord = None
         self.initialState = state.State()
         self.register = register.Register()
-        self.label2Freq = {0: float('inf')}
+        self.label2Freq = {}
  
     def tryToRecognize(self, word, addFreq=False):
         return self.decodeData(self.initialState.tryToRecognize(self.encodeWord(word), addFreq))
@@ -52,7 +52,7 @@ class FSA(object):
 #             self.tryToRecognize(w, True)
  
     def train(self, trainData):
-        self.label2Freq = {0: float('inf')}
+        self.label2Freq = {}
         for idx, word in enumerate(trainData):
             self.tryToRecognize(word, addFreq=True)
             for label in self.encodeWord(word):
@@ -115,6 +115,4 @@ class FSA(object):
             state.reverseOffset = currReverseOffset
         for state in self.initialState.dfs(set()):
             state.offset = currReverseOffset - state.reverseOffset
-        
-        
  
 \ No newline at end of file
@@ -5,6 +5,7 @@ Created on Oct 20, 2013
 '''
  
 import logging
+from state import State
  
 class Serializer(object):
  
@@ -162,10 +163,18 @@ class VLengthSerializer(Serializer):
         if state.isAccepting():
             res.extend(state.encodedData)
         return res
-     
+    
+    def getKey(self, state, label):
+        res = (-state.label2Freq.get(label, 0), -self.fsa.label2Freq.get(label, 0))
+#         logging.info(chr(label))
+#         logging.info(res)
+        return res
+
     def _transitionsData2bytearray(self, state):
         res = bytearray()
-        transitions = sorted(state.transitionsMap.iteritems(), key=lambda (label, nextState): (self.label2Index.get(label, float('inf')), -nextState.freq, -self.label2Count[label]))
+#         logging.info(self.fsa.label2Freq)
+        transitions = list(sorted(state.transitionsMap.iteritems(), key=lambda (label, nextState): self.getKey(state, label)))
+#         logging.info(str([chr(label) for label, _ in transitions]))
         thisIdx = self.state2Index[state]
         logging.debug('state '+str(state.offset))
         if len(transitions) == 0:
@@ -225,9 +234,9 @@ class VLengthSerializer(Serializer):
 class VLengthSerializer2(Serializer):
  
     MAGIC_NUMBER = 0x8fc2bc1b
-    VERSION = 2
-    FINAL_FLAG =    0b10000000
-    LAST_FLAG =     0b01000000
+    VERSION = 3
+    ACCEPTING_FLAG =    0b10000000
+    NEXT_FLAG =         0b01000000
  
     def __init__(self, fsa):
         super(VLengthSerializer2, self).__init__(fsa)
@@ -238,13 +247,13 @@ class VLengthSerializer2(Serializer):
         res = bytearray()
  
         # serialize magic number in big-endian order
-        res.append((VLengthSerializer.MAGIC_NUMBER & 0xFF000000) >> 24)
-        res.append((VLengthSerializer.MAGIC_NUMBER & 0x00FF0000) >> 16)
-        res.append((VLengthSerializer.MAGIC_NUMBER & 0x0000FF00) >> 8)
-        res.append(VLengthSerializer.MAGIC_NUMBER & 0x000000FF)
+        res.append((VLengthSerializer2.MAGIC_NUMBER & 0xFF000000) >> 24)
+        res.append((VLengthSerializer2.MAGIC_NUMBER & 0x00FF0000) >> 16)
+        res.append((VLengthSerializer2.MAGIC_NUMBER & 0x0000FF00) >> 8)
+        res.append(VLengthSerializer2.MAGIC_NUMBER & 0x000000FF)
  
         # serialize version number
-        res.append(VLengthSerializer.VERSION)
+        res.append(VLengthSerializer2.VERSION)
  
         return res
  
@@ -262,20 +271,37 @@ class VLengthSerializer2(Serializer):
         return res
  
     def _stateData2bytearray(self, state):
+        assert len(state.transitionsMap) < 64
         res = bytearray()
+        firstByte = 0
+        if state.isAccepting():
+            firstByte |= VLengthSerializer2.ACCEPTING_FLAG
+        transitions = list(sorted(state.transitionsMap.iteritems(), key=lambda (label, nextState): self.getKey(state, label)))
+        if transitions:
+            lastLabel, lastNextState = transitions[-1]
+            if self.state2Index[lastNextState] == self.state2Index[state] + 1:
+                firstByte |= VLengthSerializer2.NEXT_FLAG
+        firstByte |= len(state.transitionsMap)
+        assert firstByte < 256 and firstByte > 0
+        res.append(firstByte)
         if state.isAccepting():
             res.extend(state.encodedData)
         return res
+    
+    def getKey(self, state, label):
+        res = (-state.label2Freq.get(label, 0), -self.fsa.label2Freq.get(label, 0))
+#         logging.info(chr(label))
+#         logging.info(res)
+        return res
  
     def _transitionsData2bytearray(self, state):
         res = bytearray()
-        transitions = sorted(state.transitionsMap.iteritems(), key=lambda (label, nextState): (-nextState.freq, -self.label2Count[label]))
+        transitions = list(sorted(state.transitionsMap.iteritems(), key=lambda (label, nextState): self.getKey(state, label)))
         thisIdx = self.state2Index[state]
         logging.debug('state '+str(state.offset))
         if len(transitions) == 0:
             assert state.isAccepting()
-#             flags
-            return bytearray(0, )
+            return bytearray()
         else:
             stateAfterThis = self.statesTable[thisIdx + 1]
             for reversedN, (label, nextState) in enumerate(reversed(transitions)):
@@ -284,36 +310,178 @@ class VLengthSerializer2(Serializer):
                 assert stateAfterThis.reverseOffset is not None
                 logging.debug('next state reverse: '+str(nextState.reverseOffset))
                 logging.debug('after state reverse: '+str(stateAfterThis.reverseOffset))
-                n = len(transitions) - reversedN
-                
-                popularLabel = label in self.label2Index
-                firstByte = self.label2Index[label] if popularLabel else 31
  
-                last = len(transitions) == n
-                next = last and stateAfterThis == nextState
+                firstByte = label
  
-                if last:
-                    firstByte |= VLengthSerializer.LAST_FLAG
+                n = len(transitions) - reversedN
  
-                offsetSize = 0
-                offset = 0
-                if not next:
-                    offsetSize = 1
-#                     nextState.offset - stateAfterThis.offset
-                    offset = (stateAfterThis.reverseOffset - nextState.reverseOffset) + offsetSize + len(res) - 1
+                last = len(transitions) == n
+                isNext = last and stateAfterThis == nextState
+                if not isNext:
+                    offsetSize = 0
+    #                 offset = 0
+                    offset = (stateAfterThis.reverseOffset - nextState.reverseOffset) + len(res)
                     assert offset > 0
-                    if offset >= 256:
-#                         offset += 1
+                    if offset >= 64:
                         offsetSize += 1
-                    if offset >= 256 * 256:
-#                         offset += 1
+                    if offset >= 256 * 64:
                         offsetSize += 1
-                    assert offset < 256 * 256 * 256 #TODO - przerobic na jakis porzadny wyjatek
+                    if offset >= 256 * 256 * 64:
+                        offsetSize += 1
+                    assert offset < 256 * 256 * 256 * 64 #TODO - przerobic na jakis porzadny wyjatek
  
-                firstByte |= (32 * offsetSize)
+                    secondByte = offsetSize
+                    secondByte |= (offset >> (offsetSize * 8)) << 2
+                    
+                    transitionBytes.append(firstByte)
+                    transitionBytes.append(secondByte)
+                    # serialize offset in big-endian order
+                    if offsetSize == 3:
+                        transitionBytes.append((offset & 0x00FF0000) >> 16)
+                    if offsetSize >= 2:
+                        transitionBytes.append((offset & 0x0000FF00) >> 8)
+                    if offsetSize >= 1:
+                        transitionBytes.append(offset & 0x000000FF)
+                    for b in reversed(transitionBytes):
+                        res.insert(0, b)
+                    logging.debug('inserted transition at beginning '+chr(label)+' -> '+str(offset))
+                else:
+                    logging.debug('inserted transition at beginning '+chr(label)+' -> NEXT')
+                    res.insert(0, firstByte)
+        return res
+
+class VLengthSerializer3(Serializer):
+    
+    MAGIC_NUMBER = 0x8fc2bc1b
+    VERSION = 4
+    ACCEPTING_FLAG =    0b10000000
+    ARRAY_FLAG =         0b01000000
+    
+    def __init__(self, fsa, useArrays):
+        super(VLengthSerializer3, self).__init__(fsa)
+        self.statesTable = list(reversed(list(fsa.dfs())))
+        self.state2Index = dict([(state, idx) for (idx, state) in enumerate(self.statesTable)])
+        self._chooseArrayStates()
+        self.useArrays = useArrays
+    
+    def serializePrologue(self):
+        res = bytearray()
+        
+        # serialize magic number in big-endian order
+        res.append((VLengthSerializer3.MAGIC_NUMBER & 0xFF000000) >> 24)
+        res.append((VLengthSerializer3.MAGIC_NUMBER & 0x00FF0000) >> 16)
+        res.append((VLengthSerializer3.MAGIC_NUMBER & 0x0000FF00) >> 8)
+        res.append(VLengthSerializer3.MAGIC_NUMBER & 0x000000FF)
+        
+        # serialize version number
+        res.append(VLengthSerializer3.VERSION)
+        
+        # labels sorted by popularity
+        self.sortedLabels = [label for (label, freq) in sorted(self.fsa.label2Freq.iteritems(), key=lambda (label, freq): (-freq, label))]
+        remainingChars = [c for c in range(256) if not c in self.sortedLabels]
+#         while len(self.sortedLabels) < 256:
+#             self.sortedLabels.append(remainingChars.pop())
+        
+        # popular labels table
+        self.label2ShortLabel = dict([(label, self.sortedLabels.index(label) + 1) for label in self.sortedLabels[:63]])
+        
+        logging.debug(dict([(chr(label), shortLabel) for label, shortLabel in self.label2ShortLabel.items()]))
+        for label in range(256):
+            res.append(self.label2ShortLabel.get(label, 0))
+        
+        res.append(ord('^'))
+        
+        return res
+     
+    def getStateSize(self, state):
+        return len(self.state2bytearray(state))
+     
+    def getDataSize(self, state):
+        assert type(state.encodedData) == bytearray or not state.isAccepting()
+        return len(state.encodedData) if state.isAccepting() else 0
+    
+    def state2bytearray(self, state):
+        res = bytearray()
+        res.extend(self._stateData2bytearray(state))
+        res.extend(self._transitionsData2bytearray(state))
+        return res
+    
+    def stateShouldBeAnArray(self, state):
+#         return False
+#         return len(state.transitionsMap) >= 13
+        return self.useArrays and state.serializeAsArray
+     
+    def _stateData2bytearray(self, state):
+        assert len(state.transitionsMap) < 64
+        res = bytearray()
+        firstByte = 0
+        if state.isAccepting():
+            firstByte |= VLengthSerializer3.ACCEPTING_FLAG
+#         transitions = list(sorted(state.transitionsMap.iteritems(), key=lambda (label, nextState): self.getKey(state, label)))
+#         if transitions:
+#             lastLabel, lastNextState = transitions[-1]
+#         if self.state2Index[lastNextState] == self.state2Index[state] + 1:
+#             firstByte |= VLengthSerializer3.NEXT_FLAG
+        if self.stateShouldBeAnArray(state):
+            firstByte |= VLengthSerializer3.ARRAY_FLAG
+        firstByte |= len(state.transitionsMap)
+        assert firstByte < 256 and firstByte > 0
+        res.append(firstByte)
+        if state.isAccepting():
+            res.extend(state.encodedData)
+        return res
+    
+    def getKey(self, state, label):
+        res = (-state.label2Freq.get(label, 0))
+#         logging.info(chr(label))
+#         logging.info(res)
+        return res
+    
+    def _transitions2ListBytes(self, state, originalState=None):
+        res = bytearray()
+        transitions = list(sorted(state.transitionsMap.iteritems(), key=lambda (label, nextState): self.getKey(state, label)))
+        thisIdx = self.state2Index[originalState if originalState is not None else state]
+        logging.debug('state '+str(state.offset))
+        if len(transitions) == 0:
+            assert state.isAccepting()
+            return bytearray()
+        else:
+            stateAfterThis = self.statesTable[thisIdx + 1]
+            for reversedN, (label, nextState) in enumerate(reversed(transitions)):
+                transitionBytes = bytearray()
+                assert nextState.reverseOffset is not None
+                assert stateAfterThis.reverseOffset is not None
+                logging.debug('next state reverse: '+str(nextState.reverseOffset))
+                logging.debug('after state reverse: '+str(stateAfterThis.reverseOffset))
+                
+#                 firstByte = label
+                
+                n = len(transitions) - reversedN
+                hasShortLabel = label in self.label2ShortLabel
+                firstByte = self.label2ShortLabel[label] if hasShortLabel else 0
+                firstByte <<= 2
+                
+                last = len(transitions) == n
+                isNext = last and stateAfterThis == nextState
+                offsetSize = 0
+#                 offset = 0
+                offset = (stateAfterThis.reverseOffset - nextState.reverseOffset) + len(res)
+                assert offset > 0 or isNext
+                if offset > 0:
+                    offsetSize += 1
+                if offset >= 256:
+                    offsetSize += 1
+                if offset >= 256 * 256:
+                    offsetSize += 1
+                assert offset < 256 * 256 * 256 #TODO - przerobic na jakis porzadny wyjatek
+                assert offsetSize <= 3
+                assert offsetSize > 0 or isNext
+                firstByte |= offsetSize
+#                     secondByte = offsetSize
+#                     secondByte |= (offset >> (offsetSize * 8)) << 2
  
                 transitionBytes.append(firstByte)
-                if not popularLabel:
+                if not hasShortLabel:
                     transitionBytes.append(label)
                 # serialize offset in big-endian order
                 if offsetSize == 3:
@@ -325,4 +493,47 @@ class VLengthSerializer2(Serializer):
                 for b in reversed(transitionBytes):
                     res.insert(0, b)
                 logging.debug('inserted transition at beginning '+chr(label)+' -> '+str(offset))
+
         return res
+    
+    def _trimState(self, state):
+        newState = State()
+        newState.encodedData = state.encodedData
+        newState.reverseOffset = state.reverseOffset
+        newState.offset = state.offset
+        newState.transitionsMap = dict([(label, nextState) for (label, nextState) in state.transitionsMap.iteritems()])
+#         newState.transitionsMap = dict([(label, nextState) for (label, nextState) in state.transitionsMap.iteritems() if not label in self.label2ShortLabel or not self.label2ShortLabel[label] in range(1,64)])
+        newState.serializeAsArray = False
+        return newState
+    
+    def _transition2ArrayBytes(self, state):
+        res = bytearray()
+        array = [0] * 64
+        for label, nextState in state.transitionsMap.iteritems():
+            if label in self.label2ShortLabel:
+                shortLabel = self.label2ShortLabel[label]
+                array[shortLabel] = nextState.offset
+        logging.debug(array)
+        for offset in map(lambda x: x if x else 0, array):
+            res.append(0)
+            res.append((offset & 0xFF0000) >> 16)
+            res.append((offset & 0x00FF00) >> 8)
+            res.append(offset & 0x0000FF)
+        res.extend(self._stateData2bytearray(self._trimState(state)))
+        res.extend(self._transitions2ListBytes(self._trimState(state), originalState=state))
+        return res
+     
+    def _transitionsData2bytearray(self, state):
+        if self.stateShouldBeAnArray(state):
+            return self._transition2ArrayBytes(state)
+        else:
+            return self._transitions2ListBytes(state)
+    
+    def _chooseArrayStates(self):
+        for state1 in self.fsa.initialState.transitionsMap.values():
+            for state2 in state1.transitionsMap.values():
+#                 for state3 in state2.transitionsMap.values():
+#                     state3.serializeAsArray = True
+                state2.serializeAsArray = True
+            state1.serializeAsArray = True
+        self.fsa.initialState.serializeAsArray = True
@@ -15,6 +15,8 @@ class State(object):
         self.encodedData = None
         self.reverseOffset = None
         self.offset = None
+        self.label2Freq = {}
+        self.serializeAsArray = False
  
     def setTransition(self, byte, nextState):
         self.transitionsMap[byte] = nextState
@@ -25,6 +27,7 @@ class State(object):
     def getNext(self, byte, addFreq=False):
         if addFreq:
             self.freq += 1
+            self.label2Freq[byte] = self.label2Freq.get(byte, 0) + 1
         return self.transitionsMap.get(byte, None)
  
     def getRegisterKey(self):
@@ -2,6 +2,7 @@
 <configurationDescriptor version="90">
   <logicalFolder name="root" displayName="root" projectFiles="true" kind="ROOT">
     <df root="fsa" name="0">
+      <in>_fsaimpl.hpp</in>
       <in>test_not_recognize.cpp</in>
       <in>test_recognize.cpp</in>
       <in>test_speed.cpp</in>
@@ -38,7 +39,7 @@
           <buildCommandWorkingDir>build</buildCommandWorkingDir>
           <buildCommand>${MAKE} -f Makefile</buildCommand>
           <cleanCommand>${MAKE} -f Makefile clean</cleanCommand>
-          <executablePath>build/fsa/test_speed</executablePath>
+          <executablePath>build/fsa/test_dict</executablePath>
         </makeTool>
       </makefileType>
       <folder path="0">
@@ -56,13 +57,17 @@
           </incDir>
         </ccTool>
       </folder>
-      <item path="fsa/test_not_recognize.cpp" ex="false" tool="1" flavor2="4">
+      <item path="fsa/_fsaimpl.hpp" ex="false" tool="3" flavor2="0">
+      </item>
+      <item path="fsa/test_not_recognize.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
         </ccTool>
       </item>
-      <item path="fsa/test_recognize.cpp" ex="false" tool="1" flavor2="0">
+      <item path="fsa/test_recognize.cpp" ex="false" tool="1" flavor2="8">
+        <ccTool>
+        </ccTool>
       </item>
-      <item path="fsa/test_speed.cpp" ex="false" tool="1" flavor2="4">
+      <item path="fsa/test_speed.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
         </ccTool>
       </item>