- prawie działa rozpoznawanie informacji morfologicznej

git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@16 ff4e3ee1-f430-4e82-ade0-24591c43f1fd

- prawie działa rozpoznawanie informacji morfologicznej
git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@16 ff4e3ee1-f430-4e82-ade0-24591c43f1fd
Michał Lenart
1 parent d42f73bc
Showing 22 changed files with 498 additions and 140 deletions
fsa/CMakeLists.txt
fsa/cfsa1_impl.hpp
fsa/cfsa2_impl.hpp
fsa/const.cpp
fsa/const.hpp
fsa/fsa.hpp
fsa/fsa_impl.hpp
fsa/interpretation.hpp
fsa/simplefsa_impl.hpp
fsa/state_impl.hpp
fsa/utils.hpp
morfeusz/CMakeLists.txt
morfeusz/MorphDeserializer.cpp
morfeusz/MorphDeserializer.hpp
morfeusz/Tagset.cpp
morfeusz/Tagset.hpp
morfeusz/interpretations.cpp
morfeusz/interpretations.hpp
morfeusz/main.cpp
fsa/test_morph.cpp → morfeusz/test_morph.cpp
-add_executable (test_speed test_speed.cpp)
-add_executable (test_speed_profile test_speed.cpp)
-add_executable (test_recognize test_recognize.cpp)
-add_executable (test_not_recognize test_not_recognize.cpp)
-add_executable (test_morph test_morph.cpp)
+add_executable (test_speed test_speed.cpp const.cpp)
+add_executable (test_speed_profile test_speed.cpp const.cpp)
+add_executable (test_recognize test_recognize.cpp const.cpp)
+add_executable (test_not_recognize test_not_recognize.cpp const.cpp)
+
 set_target_properties ( test_speed PROPERTIES COMPILE_FLAGS "-std=gnu++0x -Wall -O2 -pedantic -Wcast-align -Wextra -Wmissing-noreturn -Wconversion -Wcast-qual -Wcast-align" )
 set_target_properties ( test_speed_profile PROPERTIES COMPILE_FLAGS "-std=gnu++0x -Wall -O2 -g" )
 set_target_properties ( test_recognize PROPERTIES COMPILE_FLAGS "-std=gnu++0x -Wall -O2" )
 set_target_properties ( test_not_recognize PROPERTIES COMPILE_FLAGS "-std=gnu++0x -Wall -O2" )
-set_target_properties ( test_morph PROPERTIES COMPILE_FLAGS "-std=gnu++0x -Wall -O2" )
@@ -79,7 +79,7 @@ void CompressedFSA1&lt;T&gt;::doProceedToNextByList(
     TransitionData2 td;
     for (unsigned int i = 0; i < transitionsNum; i++) {
         //        const_cast<Counter*>(&counter)->increment(1);
-        td = *((TransitionData2*) currPtr);
+        td = *(reinterpret_cast<const TransitionData2*>(currPtr));
         if (td.shortLabel == shortLabel) {
             if (shortLabel == 0) {
                 currPtr++;
@@ -107,7 +107,8 @@ void CompressedFSA1&lt;T&gt;::doProceedToNextByList(
     if (!found) {
 //                                cerr << "SINK for " << c << endl;
         state.setNextAsSink();
-    } else {
+    } 
+    else {
         currPtr++;
 //                                        cerr << "offset size " << td.offsetSize << endl;
 //                            cerr << "offset " << offset << endl;
@@ -152,12 +153,12 @@ void CompressedFSA1&lt;T&gt;::proceedToNext(const char c, State&lt;T&gt;&amp; state) const {
 //                        cerr << "NEXT " << (short) c << " from " << state.getOffset() << endl;
     const unsigned char* fromPointer = this->initialStatePtr + state.getOffset();
     unsigned char shortLabel = this->label2ShortLabel[(const unsigned char) c];
-    unsigned int transitionsTableOffset = 1;
+    unsigned long transitionsTableOffset = 1;
     if (state.isAccepting()) {
         transitionsTableOffset += state.getValueSize();
 //                                cerr << "transitionsTableOffset " << transitionsTableOffset + state.getOffset() << " because value is " << state.getValue() << endl;
     }
-    StateData2* sd = (StateData2*) (fromPointer);
+    const StateData2* sd = reinterpret_cast<const StateData2*>(fromPointer);
 //                cerr << "transitions num=" << sd->transitionsNum << endl;
     if (sd->array) {
         if (shortLabel > 0) {
@@ -135,7 +135,7 @@ void CompressedFSA2&lt;T&gt;::proceedToNext(const char c, State&lt;T&gt;&amp; state) const {
         cerr << "NEXT " << (short) c << " from " << state.getOffset() << endl;
 #endif
     const unsigned char* fromPointer = this->initialStatePtr + state.getOffset();
-    unsigned int transitionsTableOffset = 0;
+    unsigned long transitionsTableOffset = 0;
     if (state.isAccepting()) {
         transitionsTableOffset += state.getValueSize();
     }
+
+#include "const.hpp"
+
+extern const uint32_t MAGIC_NUMBER = 0x8fc2bc1b;
+extern const uint8_t VERSION_NUM = 9;
+
+extern const unsigned int VERSION_NUM_OFFSET = 4;
+extern const unsigned int IMPLEMENTATION_NUM_OFFSET = 5;
+extern const unsigned int ADDITIONAL_DATA_SIZE_OFFSET = 6;
+extern const unsigned int ADDITIONAL_DATA_OFFSET = 10;
+/* 
+ * File:   const.hpp
+ * Author: mlenart
+ *
+ * Created on 12 listopad 2013, 14:11
+ */
+
+#ifndef CONST_HPP
+#define	CONST_HPP
+
+#include <netinet/in.h>
+
+extern const uint32_t MAGIC_NUMBER;
+extern const uint8_t VERSION_NUM;
+
+extern const unsigned int VERSION_NUM_OFFSET;
+extern const unsigned int IMPLEMENTATION_NUM_OFFSET;
+extern const unsigned int ADDITIONAL_DATA_SIZE_OFFSET;
+extern const unsigned int ADDITIONAL_DATA_OFFSET;
+
+#endif	/* CONST_HPP */
+
@@ -9,13 +9,12 @@
 #define FSA_HPP
 //#include <iostream>
-//#include <cstring>
-#include <typeinfo>
+#include <cstring>
 #include <cassert>
+#include <typeinfo>
 #include <exception>
 #include <string>
 #include <vector>
-#include "interpretation.hpp"
 template <class T> class State;
 template <class T> class FSA;
@@ -44,16 +43,12 @@ public:
      * Returns number of bytes read or -1 on error.
      */
     long deserialize(const unsigned char* ptr, char*& text) const {
-        //        text = const_cast<char*> (reinterpret_cast<const char*> (ptr));
-        //        return strlen(text) + 1;
-        return 1;
+        text = const_cast<char*> (reinterpret_cast<const char*> (ptr));
+        return strlen(text) + 1;
+//        return 1;
     }
 };
-class MorphDeserializer: public Deserializer<std::vector<Interpretation>> {
-    long deserialize(const unsigned char* ptr, std::vector<Interpretation>& interp) const;
-};
-
 class Counter {
 public:
@@ -88,8 +83,6 @@ public:
      */
     static FSA<T>* getFSA(const unsigned char* ptr, const Deserializer<T>& deserializer);
-    static const uint32_t MAGIC_NUMBER = 0x8fc2bc1b;
-    static const uint8_t VERSION_NUM = 8;
 protected:
     /**
@@ -105,10 +98,6 @@ protected:
     const Deserializer<T>& deserializer;
     friend class State<T>;
 private:
-    static int getMagicNumberOffset();
-    static int getVersionNumOffset();
-    static int getPopularCharsOffset();
-    static int getInitialStateOffset();
     //    FSA();
 };
@@ -220,12 +209,12 @@ public:
      * Makes sense only for accepting states.
      * For non-accepting states is throws an exception.
      */
-    unsigned int getValueSize() const;
+    unsigned long getValueSize() const;
     unsigned long getOffset() const;
     void setNext(const unsigned long offset);
-    void setNext(const unsigned long offset, const T& value, const unsigned int valueSize);
+    void setNext(const unsigned long offset, const T& value, const unsigned long valueSize);
     void setNextAsSink();
     explicit State(const FSA<T>& fsa);
@@ -237,7 +226,7 @@ private:
     bool accepting;
     bool sink;
     T value;
-    int valueSize;
+    long valueSize;
 };
 class FSAException : public std::exception {
@@ -14,14 +14,11 @@
 #include <iostream>
 #include <vector>
 #include <netinet/in.h>
-#include "fsa.hpp"
 #include "utils.hpp"
+#include "const.hpp"
 using namespace std;
-
-static const unsigned int VERSION_NUM_OFFSET = 4;
-static const unsigned int IMPLEMENTATION_NUM_OFFSET = 5;
-static const unsigned int FSA_OFFSET = 6;
+//static const unsigned int FSA_OFFSET = 6;
 template <class T>
 bool FSA<T>::tryToRecognize(const char* input, T& value) const {
@@ -73,7 +70,9 @@ FSA&lt;T&gt;* FSA&lt;T&gt;::getFSA(const unsigned char* ptr, const Deserializer&lt;T&gt;&amp; deserial
     }
     uint8_t implementationNum = *(ptr + IMPLEMENTATION_NUM_OFFSET);
-    const unsigned char* startPtr = ptr + FSA_OFFSET;
+    
+    uint32_t additionalDataSize = ntohl(*(reinterpret_cast<const uint32_t*>(ptr + ADDITIONAL_DATA_SIZE_OFFSET)));
+    const unsigned char* startPtr = ptr + ADDITIONAL_DATA_OFFSET + additionalDataSize;
     switch (implementationNum) {
         case 0:
             return new SimpleFSA<T>(startPtr, deserializer);
@@ -86,34 +85,4 @@ FSA&lt;T&gt;* FSA&lt;T&gt;::getFSA(const unsigned char* ptr, const Deserializer&lt;T&gt;&amp; deserial
     }
 }
-static void deserializeLemma(const unsigned char*& ptr, Lemma& lemma) {
-    // XXX uważać na poprawność danych
-    lemma.suffixToCut = *ptr;
-    ptr++;
-    lemma.suffixToAdd = (const char*) ptr;
-    ptr += strlen((const char*) ptr) + 1;
-}
-
-static void deserializeInterp(const unsigned char*& ptr, Interpretation& interp) {
-    deserializeLemma(ptr, interp.lemma);
-    interp.tag = ntohs(*(reinterpret_cast<const uint16_t*>(ptr)));
-    ptr += 2;
-    interp.nameClassifier = *ptr;
-    ptr++;
-}
-
-long MorphDeserializer::deserialize(const unsigned char* ptr, vector<Interpretation>& interps) const {
-    const unsigned char* currPtr = ptr;
-    uint8_t interpsNum = *ptr;
-    interps.clear();
-    interps.reserve(interpsNum);
-    currPtr++;
-    for (unsigned int i = 0; i < interpsNum; i++) {
-        Interpretation interp;
-        deserializeInterp(currPtr, interp);
-        interps.push_back(interp);
-    }
-    return currPtr - ptr;
-}
-
 #endif	/* _SIMPLE_FSA_IMPL_HPP */
-/* 
- * File:   interpretation.hpp
- * Author: mlenart
- *
- * Created on November 4, 2013, 3:11 PM
- */
-
-#ifndef INTERPRETATION_HPP
-#define	INTERPRETATION_HPP
-
-#include <string>
-#include <list>
-
-using namespace std;
-
-struct Lemma {
-    unsigned short suffixToCut;
-    const char* suffixToAdd;
-};
-
-struct Interpretation {
-    Lemma lemma;
-    unsigned int tag;      // np. subst:sg:nom:m1
-    unsigned short nameClassifier; // np. "pospolita"
-    unsigned short qualifier;      // np. "dawne" lub "potoczne"
-};
-
-#endif	/* INTERPRETATION_HPP */
-
@@ -37,7 +37,7 @@ static unsigned int decodeOffset(const unsigned char* ptr) {
 template <class T>
 void SimpleFSA<T>::proceedToNext(const char c, State<T>& state) const {
     const unsigned char* fromPointer = this->initialStatePtr + state.getOffset();
-    int transitionsTableOffset = sizeof (StateData);
+    long transitionsTableOffset = sizeof (StateData);
     if (state.isAccepting()) {
         transitionsTableOffset += state.getValueSize();
     }
@@ -60,7 +60,7 @@ void SimpleFSA&lt;T&gt;::proceedToNext(const char c, State&lt;T&gt;&amp; state) const {
         const StateData* nextStateData = reinterpret_cast<const StateData*>(nextStatePointer);
         if (nextStateData->accepting) {
             T object;
-            int size = this->deserializer.deserialize(nextStatePointer + sizeof (StateData), object);
+            long size = this->deserializer.deserialize(nextStatePointer + sizeof (StateData), object);
             state.setNext(offset, object, size);
         } else {
             state.setNext(offset);
@@ -51,7 +51,7 @@ T State&lt;T&gt;::getValue() const {
 }
 template <class T>
-unsigned int State<T>::getValueSize() const {
+unsigned long State<T>::getValueSize() const {
     assert(this->isAccepting());
     return this->valueSize;
 }
@@ -69,7 +69,7 @@ void State&lt;T&gt;::setNext(const unsigned long offset) {
 }
 template <class T>
-void State<T>::setNext(const unsigned long offset, const T& value, const unsigned int valueSize) {
+void State<T>::setNext(const unsigned long offset, const T& value, const unsigned long valueSize) {
 //    assert(!this->isSink());
     this->offset = offset;
     this->accepting = true;
@@ -9,14 +9,15 @@
 #define	UTILS_HPP
 #include <iostream>
+#include <fstream>
 #include <sstream>
 #include <string>
 #include <fstream>
 #include <vector>
-using namespace std;
+//using namespace std;
-//#define DEBUG_BUILD
+#define DEBUG_BUILD
 #ifdef DEBUG_BUILD
 #  define DEBUG(x) do { std::cerr << x << std::endl; } while (0)
@@ -24,14 +25,14 @@ using namespace std;
 #  define DEBUG(x)
 #endif
-void validate(const bool cond, const std::string& msg) {
+inline void validate(const bool cond, const std::string& msg) {
     if (!cond) {
         std::cerr << msg << std::endl;
         exit(1);
     }
 }
-std::vector<std::string> &split(const std::string &s, char delim, std::vector<std::string> &elems) {
+inline std::vector<std::string> &split(const std::string &s, char delim, std::vector<std::string> &elems) {
     std::stringstream ss(s);
     std::string item;
     while (std::getline(ss, item, delim)) {
@@ -41,25 +42,25 @@ std::vector&lt;std::string&gt; &amp;split(const std::string &amp;s, char delim, std::vector&lt;st
 }
-std::vector<std::string> split(const std::string &s, char delim) {
+inline std::vector<std::string> split(const std::string &s, char delim) {
     std::vector<std::string> elems;
     split(s, delim, elems);
     return elems;
 }
-string &rtrim(string &s) {
-        s.erase(find_if(s.rbegin(), s.rend(), not1(ptr_fun<int, int>(isspace))).base(), s.end());
-        return s;
-}
+//string &rtrim(string &s) {
+//        s.erase(find_if(s.rbegin(), s.rend(), not1(ptr_fun<int, int>(isspace))).base(), s.end());
+//        return s;
+//}
-unsigned char* readFile(const char* fname) {
-    ifstream ifs;
+inline unsigned char* readFile(const char* fname) {
+    std::ifstream ifs;
     ifs.exceptions(std::ifstream::failbit | std::ifstream::badbit);
-    ifs.open(fname, ios::in | ios::binary | ios::ate);
+    ifs.open(fname, std::ios::in | std::ios::binary | std::ios::ate);
     //    if (ifs.is_open()) {
-    int size = ifs.tellg();
+    long size = ifs.tellg();
     unsigned char* memblock = new unsigned char [size];
-    ifs.seekg(0, ios::beg);
+    ifs.seekg(0, std::ios::beg);
     ifs.read(reinterpret_cast<char*> (memblock), size);
     ifs.close();
     return memblock;
@@ -6,7 +6,10 @@
 include_directories (${Morfeusz_SOURCE_DIR}/fsa) 
 add_library (morfeusz2 morfeusz.hpp morfeusz.cpp)
 add_executable (morfeusz2_analyze main.cpp)
+add_executable (test_morph test_morph.cpp interpretations.cpp MorphDeserializer.cpp Tagset.cpp ../fsa/const.cpp)
 # Link the executable to the Hello library. 
 target_link_libraries (morfeusz2_analyze morfeusz2)
 set_target_properties ( morfeusz2_analyze PROPERTIES COMPILE_FLAGS "-std=gnu++0x" )
+
+set_target_properties ( test_morph PROPERTIES COMPILE_FLAGS "-std=gnu++0x -Wall -O2" )
+/* 
+ * File:   MorphDeserializer.cpp
+ * Author: mlenart
+ * 
+ * Created on 12 listopad 2013, 15:31
+ */
+
+#include "MorphDeserializer.hpp"
+
+MorphDeserializer::MorphDeserializer() {
+}
+
+MorphDeserializer::MorphDeserializer(const MorphDeserializer& orig) {
+}
+
+MorphDeserializer::~MorphDeserializer() {
+}
+
+static void deserializeLemma(const unsigned char*& ptr, Lemma& lemma) {
+    // XXX uważać na poprawność danych
+    lemma.suffixToCut = *ptr;
+    ptr++;
+    lemma.suffixToAdd = (const char*) ptr;
+    ptr += strlen((const char*) ptr) + 1;
+}
+
+static void deserializeInterp(const unsigned char*& ptr, Interpretation& interp) {
+    deserializeLemma(ptr, interp.lemma);
+    interp.tag = ntohs(*(reinterpret_cast<const uint16_t*>(ptr)));
+    ptr += 2;
+    interp.nameClassifier = *ptr;
+    ptr++;
+}
+
+long MorphDeserializer::deserialize(const unsigned char* ptr, vector<Interpretation>& interps) const {
+    const unsigned char* currPtr = ptr;
+    uint8_t interpsNum = *ptr;
+    interps.clear();
+    interps.reserve(interpsNum);
+    currPtr++;
+    for (unsigned int i = 0; i < interpsNum; i++) {
+        Interpretation interp;
+        deserializeInterp(currPtr, interp);
+        interps.push_back(interp);
+    }
+    return currPtr - ptr;
+}
+
+/* 
+ * File:   MorphDeserializer.hpp
+ * Author: mlenart
+ *
+ * Created on 12 listopad 2013, 15:31
+ */
+
+#ifndef MORPHDESERIALIZER_HPP
+#define	MORPHDESERIALIZER_HPP
+
+#include <vector>
+#include "fsa.hpp"
+#include "interpretations.hpp"
+
+class MorphDeserializer: public Deserializer<std::vector<Interpretation>> {
+public:
+    MorphDeserializer();
+    MorphDeserializer(const MorphDeserializer& orig);
+    virtual ~MorphDeserializer();
+    long deserialize(
+        const unsigned char* ptr, 
+        std::vector<Interpretation>& interps) const;
+private:
+
+};
+
+#endif	/* MORPHDESERIALIZER_HPP */
+
+
+#include <string>
+#include <netinet/in.h>
+#include "Tagset.hpp"
+#include "const.hpp"
+#include "utils.hpp"
+
+using namespace std;
+
+static uint16_t readInt16(const unsigned char*& currPtr) {
+    DEBUG("readInt16");
+    uint16_t res = htons(*reinterpret_cast<const uint16_t*>(currPtr));
+    DEBUG("still alive " + to_string(res));
+    currPtr += 2;
+    DEBUG("still alive after ptr add");
+    return res;
+}
+
+static string readString(const unsigned char*& currPtr) {
+    DEBUG("readString");
+    string res(reinterpret_cast<const char*>(currPtr));
+    currPtr += res.length();
+    currPtr++;
+    return res;
+}
+
+static void readTags(const unsigned char*& currPtr, vector<string>& tags) {
+    tags.clear();
+    tags.resize(65536);
+    uint16_t tagsNum = readInt16(currPtr);
+    DEBUG("hi there");
+    DEBUG("tagsNum="+to_string((int) tagsNum));
+    for (unsigned int i = 0; i < tagsNum; i++) {
+        unsigned int tagNum = readInt16(currPtr);
+        tags[tagNum] = readString(currPtr);
+    }
+}
+
+Tagset::Tagset(const unsigned char* fsaData) {
+    const unsigned char* currPtr = fsaData + ADDITIONAL_DATA_OFFSET;
+//    uint32_t tagsNum = ntohl(*reinterpret_cast<const uint32_t*>(currPtr));
+//    uint32_t namesNum = ntohl(*reinterpret_cast<const uint32_t*>(fsaData + ADDITIONAL_DATA_OFFSET + 4));
+//    const unsigned char* currPtr = fsaData + 8;
+    DEBUG("will read tags");
+    readTags(currPtr, this->tags);
+    DEBUG("will read names");
+    readTags(currPtr, this->names);
+}
+
+const string& Tagset::getTag(const int tagNum) const {
+    return this->tags.at(tagNum);
+}
+
+const string& Tagset::getName(const int nameNum) const {
+    return this->names.at(nameNum);
+}
+/* 
+ * File:   tagset.hpp
+ * Author: mlenart
+ *
+ * Created on 12 listopad 2013, 14:09
+ */
+
+#ifndef TAGSET_HPP
+#define	TAGSET_HPP
+
+#include <string>
+#include <vector>
+
+class Tagset {
+public:
+    explicit Tagset(const unsigned char* fsaData);
+    const std::string& getTag(const int tagNum) const;
+    const std::string& getName(const int nameNum) const;
+private:
+    std::vector<std::string> tags;
+    std::vector<std::string> names;
+};
+
+#endif	/* TAGSET_HPP */
+
+
+#include "interpretations.hpp"
+#include "Tagset.hpp"
+
+using namespace std;
+
+Interpretation::Interpretation()
+: lemma(), tag(), nameClassifier() {
+
+}
+
+Interpretation::Interpretation(const Lemma& lemma, const int tag, const int name)
+: lemma(lemma), tag(tag), nameClassifier(name) {
+
+}
+
+StringInterpretation::StringInterpretation(
+        const string& lemma,
+        const string& tag,
+        const string& name)
+: lemma(lemma), tag(tag), name(name) {
+
+}
+
+string StringInterpretation::toString() const {
+    std::stringstream ss;
+    ss << lemma << ":" << tag << ":" << name;
+    return ss.str();
+}
+
+string LemmaConverter::convertLemma(
+        const string& orth,
+        const Lemma& lemma) const {
+    string res(orth);
+    res.erase(
+            res.end() - lemma.suffixToCut,
+            res.end());
+    res.append(lemma.suffixToAdd);
+    return res;
+}
+
+InterpretationsConverter::InterpretationsConverter(const unsigned char* data)
+: tagset(Tagset(data)) {
+
+}
+
+StringInterpretation InterpretationsConverter::convertInterpretation(
+        const string& orth,
+        const Interpretation& interp) const {
+    string lemma = this->lemmaConverter.convertLemma(orth, interp.lemma);
+    const string& tag = this->tagset.getTag(interp.tag);
+    const string& name = this->tagset.getName(interp.nameClassifier);
+    return StringInterpretation(lemma, tag, name);
+}
+
+
+/* 
+ * File:   interpretation.hpp
+ * Author: mlenart
+ *
+ * Created on November 4, 2013, 3:11 PM
+ */
+
+#ifndef INTERPRETATION_HPP
+#define	INTERPRETATION_HPP
+
+#include <string>
+#include <sstream>
+#include "Tagset.hpp"
+
+using namespace std;
+
+struct Lemma {
+    int suffixToCut;
+    string suffixToAdd;
+};
+
+struct Interpretation {
+    Interpretation();
+    Interpretation(const Lemma& lemma, const int tag, const int name);
+    Lemma lemma;
+    int tag;      // np. subst:sg:nom:m1
+    int nameClassifier; // np. "pospolita"
+//    int qualifier;      // np. "dawne" lub "potoczne"
+};
+
+struct StringInterpretation {
+    StringInterpretation(const std::string& lemma, const std::string& tag, const std::string& name);
+    const std::string lemma;
+    const std::string& tag;      // np. subst:sg:nom:m1
+    const std::string& name; // np. "pospolita"
+//    std::string qualifier;      // np. "dawne" lub "potoczne"
+    std::string toString() const;
+};
+
+class LemmaConverter {
+public:
+    std::string convertLemma(const std::string& orth, const Lemma& interp) const;
+};
+
+
+class InterpretationsConverter {
+public:
+    explicit InterpretationsConverter(const unsigned char* data);
+    StringInterpretation convertInterpretation(
+            const std::string& orth, 
+            const Interpretation& interp) const;
+private:
+    LemmaConverter lemmaConverter;
+    Tagset tagset;
+};
+
+#endif	/* INTERPRETATION_HPP */
+
@@ -9,6 +9,7 @@
 #include <iostream>
 #include "fsa.hpp"
 #include "default_fsa.hpp"
+#include "Tagset.hpp"
 using namespace std;
@@ -5,11 +5,13 @@
  * Created on November 8, 2013, 4:12 PM
  */
-#include <cstdlib>
+//#include <cstdlib>
 #include <sstream>
 #include <iostream>
 #include "fsa.hpp"
+#include "interpretations.hpp"
 #include "utils.hpp"
+#include "MorphDeserializer.hpp"
 using namespace std;
@@ -24,7 +26,14 @@ void debug(const string&amp; key, const vector&lt;Interpretation&gt; value) {
     cerr << "==================" << endl;
 }
-void doTest(const FSA<vector<Interpretation>>& fsa, const char* fname) {
+void debug(const string& key, const StringInterpretation& value) {
+    cerr << key << '\t' << value.toString() << endl;
+}
+
+void doTest(
+        const FSA<vector<Interpretation>>& fsa, 
+        const InterpretationsConverter& interpsConverter, 
+        const char* fname) {
     ifstream ifs;
     //    ifs.exceptions(std::ifstream::failbit | std::ifstream::badbit);
     ifs.open(fname, ios::binary);
@@ -32,22 +41,42 @@ void doTest(const FSA&lt;vector&lt;Interpretation&gt;&gt;&amp; fsa, const char* fname) {
     while (getline(ifs, line)) {
         vector<string> splitVector(split(line, '\t'));
         string key = splitVector[0];
+        string lemma = splitVector[1];
+        string tag = splitVector[2];
+        string name = splitVector[3];
         vector<Interpretation> value2;
         fsa.tryToRecognize(key.c_str(), value2);
-        debug(key, value2);
+        vector<StringInterpretation> parsedValues;
+        bool found = false;
+        for (Interpretation interp: value2) {
+            StringInterpretation parsedValue = interpsConverter.convertInterpretation(key, interp);
+//            parsedValues.push_back(parsedValue);
+            debug(key, parsedValue);
+            if (lemma == parsedValue.lemma && tag == parsedValue.tag && name == parsedValue.name) {
+                found = true;
+            }
+        }
+        validate(found, "Failed to recognize " + key + " " + lemma + ":" + tag + ":" + name);
+//        debug(key, value2);
 //        validate(fsa.tryToRecognize(key.c_str(), value2), "Failed to recognize " + key);
     }
     validate(ifs.eof(), "Failed to read the input file to the end");
 }
 int main(int argc, char** argv) {
-    validate(argc == 3, "Must provide exactly two arguments - FSA filename and dictionary filename.");
+    DEBUG("start test");
+    validate(argc == 3, "Must provide exactly two arguments - FSA filename, and dictionary filename.");
     const unsigned char* fsaData = readFile(argv[1]);
     MorphDeserializer deserializer;
+    DEBUG("will read FSA");
     FSA<vector<Interpretation>>* fsa = FSA<vector<Interpretation>>::getFSA(fsaData, deserializer);
-    doTest(*fsa, argv[2]);
+    DEBUG("DONE read FSA");
+    DEBUG("will read tagset");
+    InterpretationsConverter converter(fsaData);
+    DEBUG("DONE read tagset");
+    DEBUG("still alive");
+    doTest(*fsa, converter, argv[2]);
     //    cout << argc << endl;
     delete fsa;
     return 0;
 }
-
 <?xml version="1.0" encoding="UTF-8"?>
 <configurationDescriptor version="90">
   <logicalFolder name="root" displayName="root" projectFiles="true" kind="ROOT">
+    <logicalFolder name="2.8.11.2"
+                   displayName="2.8.11.2"
+                   projectFiles="true"
+                   root="build/CMakeFiles/2.8.11.2">
+      <logicalFolder name="CompilerIdC" displayName="CompilerIdC" projectFiles="true">
+        <itemPath>build/CMakeFiles/2.8.11.2/CompilerIdC/CMakeCCompilerId.c</itemPath>
+      </logicalFolder>
+      <logicalFolder name="CompilerIdCXX"
+                     displayName="CompilerIdCXX"
+                     projectFiles="true">
+        <itemPath>build/CMakeFiles/2.8.11.2/CompilerIdCXX/CMakeCXXCompilerId.cpp</itemPath>
+      </logicalFolder>
+    </logicalFolder>
     <df root="fsa" name="0">
-      <in>cfsa1_impl.hpp</in>
-      <in>cfsa2_impl.hpp</in>
-      <in>interpretation.hpp</in>
-      <in>simplefsa_impl.hpp</in>
-      <in>test_morph.cpp</in>
       <in>test_not_recognize.cpp</in>
       <in>test_recognize.cpp</in>
       <in>test_speed.cpp</in>
     </df>
+    <logicalFolder name="Modules"
+                   displayName="Modules"
+                   projectFiles="true"
+                   root="/usr/share/cmake-2.8/Modules">
+      <itemPath>/usr/share/cmake-2.8/Modules/CMakeCCompilerABI.c</itemPath>
+      <itemPath>/usr/share/cmake-2.8/Modules/CMakeCXXCompilerABI.cpp</itemPath>
+      <itemPath>/usr/share/cmake-2.8/Modules/CMakeCompilerABI.h</itemPath>
+    </logicalFolder>
     <df root="morfeusz" name="1">
+      <in>interpretations.cpp</in>
       <in>main.cpp</in>
       <in>morfeusz.cpp</in>
     </df>
@@ -22,7 +39,6 @@
       <itemPath>CMakeLists.txt</itemPath>
       <itemPath>build/Makefile</itemPath>
     </logicalFolder>
-    <itemPath>cfsa1_impl.hpp</itemPath>
   </logicalFolder>
   <sourceFolderFilter>^(nbproject)$</sourceFolderFilter>
   <sourceRootList>
@@ -44,10 +60,41 @@
           <buildCommandWorkingDir>build</buildCommandWorkingDir>
           <buildCommand>${MAKE} -f Makefile</buildCommand>
           <cleanCommand>${MAKE} -f Makefile clean</cleanCommand>
-          <executablePath>build/fsa/test_dict</executablePath>
+          <executablePath>build/fsa/test_speed</executablePath>
+          <cTool>
+            <incDir>
+              <pElem>build/CMakeFiles/CMakeTmp</pElem>
+            </incDir>
+          </cTool>
         </makeTool>
       </makefileType>
-      <item path="cfsa1_impl.hpp" ex="false" tool="3" flavor2="0">
+      <item path="/usr/share/cmake-2.8/Modules/CMakeCCompilerABI.c"
+            ex="false"
+            tool="0"
+            flavor2="2">
+        <cTool>
+        </cTool>
+      </item>
+      <item path="/usr/share/cmake-2.8/Modules/CMakeCXXCompilerABI.cpp"
+            ex="false"
+            tool="1"
+            flavor2="4">
+        <ccTool>
+        </ccTool>
+      </item>
+      <item path="build/CMakeFiles/2.8.11.2/CompilerIdC/CMakeCCompilerId.c"
+            ex="false"
+            tool="0"
+            flavor2="2">
+        <cTool>
+        </cTool>
+      </item>
+      <item path="build/CMakeFiles/2.8.11.2/CompilerIdCXX/CMakeCXXCompilerId.cpp"
+            ex="false"
+            tool="1"
+            flavor2="4">
+        <ccTool>
+        </ccTool>
       </item>
       <folder path="0">
         <ccTool>
@@ -56,23 +103,27 @@
           </incDir>
         </ccTool>
       </folder>
-      <folder path="1">
+      <folder path="2.8.11.2">
+        <ccTool>
+          <incDir>
+            <pElem>build/CMakeFiles/CMakeTmp</pElem>
+          </incDir>
+        </ccTool>
+      </folder>
+      <folder path="Modules">
+        <ccTool>
+          <incDir>
+            <pElem>build/CMakeFiles/CMakeTmp</pElem>
+          </incDir>
+        </ccTool>
+      </folder>
+      <item path="fsa/const.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
           <incDir>
             <pElem>fsa</pElem>
             <pElem>build/morfeusz</pElem>
           </incDir>
         </ccTool>
-      </folder>
-      <item path="fsa/cfsa1_impl.hpp" ex="false" tool="3" flavor2="0">
-      </item>
-      <item path="fsa/cfsa2_impl.hpp" ex="false" tool="3" flavor2="0">
-      </item>
-      <item path="fsa/interpretation.hpp" ex="false" tool="3" flavor2="0">
-      </item>
-      <item path="fsa/simplefsa_impl.hpp" ex="false" tool="3" flavor2="0">
-      </item>
-      <item path="fsa/test_morph.cpp" ex="false" tool="1" flavor2="0">
       </item>
       <item path="fsa/test_not_recognize.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
@@ -86,12 +137,53 @@
         <ccTool>
         </ccTool>
       </item>
+      <item path="morfeusz/MorphDeserializer.cpp" ex="false" tool="1" flavor2="8">
+        <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
+        </ccTool>
+      </item>
+      <item path="morfeusz/Tagset.cpp" ex="false" tool="1" flavor2="8">
+        <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
+        </ccTool>
+      </item>
+      <item path="morfeusz/interpretations.cpp" ex="false" tool="1" flavor2="8">
+        <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
+        </ccTool>
+      </item>
       <item path="morfeusz/main.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
         </ccTool>
       </item>
       <item path="morfeusz/morfeusz.cpp" ex="false" tool="1" flavor2="4">
         <ccTool>
+          <incDir>
+            <pElem>build/CMakeFiles/CMakeTmp</pElem>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
+        </ccTool>
+      </item>
+      <item path="morfeusz/test_morph.cpp" ex="false" tool="1" flavor2="8">
+        <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
         </ccTool>
       </item>
     </conf>
@@ -4,9 +4,9 @@
     <configuration>
         <data xmlns="http://www.netbeans.org/ns/make-project/1">
             <name>morfeusz</name>
-            <c-extensions/>
+            <c-extensions>c</c-extensions>
             <cpp-extensions>cpp</cpp-extensions>
-            <header-extensions>hpp</header-extensions>
+            <header-extensions>h,hpp</header-extensions>
             <sourceEncoding>UTF-8</sourceEncoding>
             <make-dep-projects/>
             <sourceRootList>