- podstawa analizy tekstu już działa

- obsług ign-ów w zasadzie też git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@23 ff4e3ee1-f430-4e82-ade0-24591c43f1fd

- podstawa analizy tekstu już działa
- obsług ign-ów w zasadzie też git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@23 ff4e3ee1-f430-4e82-ade0-24591c43f1fd
Michał Lenart
1 parent e5220b90
Showing 17 changed files with 223 additions and 65 deletions
CMakeLists.txt
morfeusz/CMakeLists.txt
morfeusz/EncodedInterpretation.hpp
morfeusz/FlexionGraph.cpp
morfeusz/FlexionGraph.hpp
morfeusz/Morfeusz.cpp
morfeusz/Morfeusz.hpp
morfeusz/MorphInterpretation.cpp
morfeusz/MorphInterpretation.hpp
morfeusz/charset/CharsetConverter.cpp
morfeusz/charset/CharsetConverter.hpp
morfeusz/morfeusz.cpp
morfeusz/morfeusz.hpp
morfeusz/test_simple.cpp
nbproject/configurations.xml
testfiles/PoliMorfSmall.tab
testfiles/polimorf.tagset
  
 cmake_minimum_required (VERSION 2.8) 
 project (Morfeusz) 
-
+set(CMAKE_BUILD_TYPE "Debug")
 enable_testing()
  
 add_subdirectory (fsa)
@@ -4,15 +4,14 @@
 # Make sure the linker can find the Hello library once it is built. 
 #link_directories (${Morfeusz_BINARY_DIR}/Hello) 
 include_directories (${Morfeusz_SOURCE_DIR}/fsa)
-add_library (morfeusz2 morfeusz.hpp morfeusz.cpp)
-add_executable (morfeusz2_analyze main.cpp)
+# add_executable (morfeusz2_analyze main.cpp)
 add_executable (test_morph test_morph.cpp MorphDeserializer.cpp Tagset.cpp ../fsa/const.cpp MorphInterpretation.cpp)
 add_executable (test_morfeusz test_morph.cpp MorphDeserializer.cpp Tagset.cpp ../fsa/const.cpp MorphInterpretation.cpp Morfeusz.cpp charset/CharsetConverter.cpp FlexionGraph.cpp)
 add_executable (test_simple test_simple.cpp MorphDeserializer.cpp Tagset.cpp ../fsa/const.cpp MorphInterpretation.cpp Morfeusz.cpp charset/CharsetConverter.cpp FlexionGraph.cpp)
  
 # Link the executable to the Hello library. 
-target_link_libraries (morfeusz2_analyze morfeusz2)
-set_target_properties ( morfeusz2_analyze PROPERTIES COMPILE_FLAGS "-std=gnu++0x" )
+#target_link_libraries (morfeusz2_analyze morfeusz2)
+#set_target_properties ( morfeusz2_analyze PROPERTIES COMPILE_FLAGS "-std=gnu++0x" )
  
 set_target_properties ( test_morph PROPERTIES COMPILE_FLAGS "-std=gnu++0x -Wall -O2" )
 set_target_properties ( test_morfeusz PROPERTIES COMPILE_FLAGS "-std=gnu++0x -Wall -O2" )
@@ -31,8 +31,6 @@ struct EncodedInterpretation {
     int type;
     int tag;
     int nameClassifier;
-    int startNode;
-    int endNode;
 };
  
 #endif	/* INTERPRETATION_HPP */
  
+#include <string>
+#include "utils.hpp"
 #include "FlexionGraph.hpp"
  
 FlexionGraph::FlexionGraph(int startNode)
@@ -6,29 +8,56 @@ FlexionGraph::FlexionGraph(int startNode)
  
 }
  
+static inline void debugPath(const std::vector<InterpretedChunk>& path) {
+    for (const InterpretedChunk& chunk: path) {
+        std::string text(chunk.chunk, chunk.chunkLength);
+        DEBUG(text);
+        DEBUG(chunk.chunkLength);
+    }
+}
+
+void FlexionGraph::addStartEdge(const Edge& e) {
+    if (this->graph.empty()) {
+        this->graph.push_back(vector<Edge>());
+    }
+    this->graph[0].push_back(e);
+}
+    
+void FlexionGraph::addMiddleEdge(const Edge& e) {
+    this->graph.push_back(vector<Edge>(1, e));
+}
+
 void FlexionGraph::addPath(const std::vector<InterpretedChunk>& path) {
+//    debugPath(path);
     for (const InterpretedChunk& chunk: path) {
-        if (&chunk == &(path.back())) {
+        if (&chunk == &(path.front()) 
+                && &chunk == &(path.back())) {
             Edge e = { chunk, -1 };
-            vector<Edge> v;
-            v.push_back(e);
-            this->graph.push_back(v);
-//            this->graph[node].push_back(e);
+            this->addStartEdge(e);
         }
         else if (&chunk == &(path.front())) {
-            Edge e = { chunk, (int) this->graph.size() };
-            this->graph[0].push_back(e);
+            Edge e = { chunk, (int) this->graph.size() + 1 };
+            this->addStartEdge(e);
+        }
+        else if (&chunk == &(path.back())) {
+            Edge e = { chunk, -1 };
+            this->addMiddleEdge(e);
         }
         else {
-            Edge e = { chunk, (int) this->graph.size() };
-            vector<Edge> v;
-            v.push_back(e);
-            this->graph.push_back(v);
+            Edge e = { chunk, (int) this->graph.size() + 1 };
+            this->addMiddleEdge(e);
         }
     }
 }
  
+void FlexionGraph::minimizeGraph() {
+    if (this->graph.size() > 2) {
+        
+    }
+}
+
 void FlexionGraph::appendToResults(const Tagset& tagset, std::vector<MorphInterpretation>& results) {
+    this->minimizeGraph();
     int endNode = graph.size();
     for (unsigned int i = 0; i < graph.size(); i++) {
         vector<Edge>& edges = graph[i];
@@ -41,3 +70,7 @@ void FlexionGraph::appendToResults(const Tagset&amp; tagset, std::vector&lt;MorphInterp
         }
     }
 }
+
+bool FlexionGraph::empty() const {
+    return this->graph.empty();
+}
@@ -16,6 +16,22 @@ struct Edge {
     int nextNode;
 };
  
+//struct EdgeLabel {
+//    int type;
+//    const char* textStart;
+//    int textLength;
+//    
+//    bool operator==(const EdgeLabel &el) const {
+//        return this->type == el.type 
+//                && this->textStart == el.textStart
+//                && this->textLength == el.textLength;
+//    }
+//
+//    bool operator<(const coord &o) {
+//        return x < o.x || (x == o.x && y < o.y);
+//    }
+//};
+
 class FlexionGraph {
 public:
  
@@ -25,8 +41,17 @@ public:
  
     void appendToResults(const Tagset& tagset, std::vector<MorphInterpretation>& results);
  
+    bool empty() const;
+    
 //    virtual ~FlexionGraph();
 private:
+    
+    void addStartEdge(const Edge& e);
+    
+    void addMiddleEdge(const Edge& e);
+    
+    void minimizeGraph();
+    
     int startNode;
     std::vector< std::vector<Edge> > graph;
 };
@@ -38,27 +38,37 @@ static Tagset* initializeTagset(const string&amp; filename) {
 }
  
 Morfeusz::Morfeusz(const string& filename)
-: fsa(initializeFSA(filename)), 
-        charsetConverter(initializeCharsetConverter()),
-        tagset(initializeTagset(filename)) {
+: fsa(initializeFSA(filename)),
+charsetConverter(initializeCharsetConverter()),
+tagset(initializeTagset(filename)) {
  
 }
  
 Morfeusz::~Morfeusz() {
-    delete &this->fsa;
-    delete &this->charsetConverter;
+    //    delete &this->fsa;
+    //    delete &this->charsetConverter;
 }
  
 void Morfeusz::processOneWord(
         const char*& inputData,
         const char* inputEnd,
-        const int startNodeNum,
+        int startNodeNum,
         std::vector<MorphInterpretation>& results) const {
+    while (inputData != inputEnd 
+            && isEndOfWord(this->charsetConverter->peek(inputData, inputEnd))) {
+        this->charsetConverter->next(inputData, inputEnd);
+    }
+    const char* wordStart = inputData;
     vector<InterpretedChunk> accum;
     FlexionGraph graph(startNodeNum);
     const char* currInput = inputData;
     doProcessOneWord(currInput, inputEnd, accum, graph);
-    graph.appendToResults(*this->tagset, results);
+    if (!graph.empty()) {
+        graph.appendToResults(*this->tagset, results);
+    }
+    else if (wordStart != currInput) {
+        this->appendIgnotiumToResults(string(wordStart, currInput), startNodeNum, results);
+    }
     inputData = currInput;
 }
  
@@ -67,38 +77,56 @@ void Morfeusz::doProcessOneWord(
         const char* inputEnd,
         vector<InterpretedChunk>& accum,
         FlexionGraph& graph) const {
+    bool endOfWord = inputData == inputEnd;
     const char* currInput = inputData;
-    StateType state = this->fsa->getInitialState();
-    int codepoint = this->charsetConverter->next(currInput, inputEnd);
+    const char* prevInput = inputData;
+    int codepoint = endOfWord ? 0 : this->charsetConverter->next(currInput, inputEnd);
  
-    if (!accum.empty() && isEndOfWord(codepoint)) {
-        graph.addPath(accum);
-    }
-    else
-        while (!isEndOfWord(codepoint)) {
-            this->feedState(state, codepoint);
-            codepoint = this->charsetConverter->next(currInput, inputEnd);
-            if (state.isAccepting()) {
-                for (InterpsGroup& ig : state.getValue()) {
-                    InterpretedChunk ic = {inputData, currInput - inputData, ig};
-                    accum.push_back(ic);
-                    doProcessOneWord(currInput, inputEnd, accum, graph);
-                    accum.pop_back();
-                }
+    StateType state = this->fsa->getInitialState();
+
+    while (!isEndOfWord(codepoint)) {
+        this->feedState(state, codepoint);
+        if (state.isAccepting()) {
+            for (InterpsGroup& ig : state.getValue()) {
+                InterpretedChunk ic = {inputData, currInput - inputData, ig};
+                accum.push_back(ic);
+                const char* newCurrInput = currInput;
+                doProcessOneWord(newCurrInput, inputEnd, accum, graph);
+                accum.pop_back();
             }
         }
+        prevInput = currInput;
+        codepoint = currInput == inputEnd ? 0 : this->charsetConverter->next(currInput, inputEnd);
+    }
+    if (state.isAccepting()) {
+        for (InterpsGroup& ig : state.getValue()) {
+            InterpretedChunk ic = {inputData, prevInput - inputData, ig};
+            accum.push_back(ic);
+            graph.addPath(accum);
+            accum.pop_back();
+        }
+    }
+    inputData = currInput;
 }
  
 void Morfeusz::feedState(
         StateType& state,
-        const int codepoint) const {
+        int codepoint) const {
     vector<char> chars;
     this->charsetConverter->append(codepoint, chars);
-    for (char c: chars) {
+    for (char c : chars) {
         state.proceedToNext(c);
     }
 }
  
+void Morfeusz::appendIgnotiumToResults(
+        const string& word,
+        int startNodeNum,
+        std::vector<MorphInterpretation>& results) const {
+    MorphInterpretation interp = MorphInterpretation::createIgn(startNodeNum, word, *this->tagset);
+    results.push_back(interp);
+}
+
 ResultsIterator Morfeusz::analyze(const string& text) {
     //    const char* textStart = text.c_str();
     //    const char* textEnd = text.c_str() + text.length();
@@ -106,7 +134,12 @@ ResultsIterator Morfeusz::analyze(const string&amp; text) {
 }
  
 void Morfeusz::analyze(const string& text, vector<MorphInterpretation>& results) {
-    
+    const char* input = text.c_str();
+    const char* inputEnd = input + text.length();
+    while (input != inputEnd) {
+        int startNode = results.empty() ? 0 : results.back().getEndNode();
+        this->processOneWord(input, inputEnd, startNode, results);
+    }
 }
  
 ResultsIterator::ResultsIterator(const string& text, const Morfeusz& morfeusz)
 /* 
  * File:   Morfeusz.hpp
- * Author: lennyn
+ * Author: mlenart
  *
  * Created on November 13, 2013, 5:21 PM
  */
@@ -37,7 +37,7 @@ public:
     void processOneWord(
         const char*& inputData,
         const char* inputEnd,
-        const int startNodeNum, 
+        int startNodeNum, 
         std::vector<MorphInterpretation>& result) const;
  
 //    Morfeusz();
@@ -52,7 +52,12 @@ private:
  
     void feedState(
         StateType& state,
-        const int codepoint) const;
+        int codepoint) const;
+    
+    void appendIgnotiumToResults(
+        const std::string& word,
+        int startNodeNum,
+        std::vector<MorphInterpretation>& results) const;
  
     FSAType* fsa;
     CharsetConverter* charsetConverter;
@@ -39,6 +39,25 @@ MorphInterpretation::MorphInterpretation(
  
 }
  
+MorphInterpretation::MorphInterpretation(
+        int startNode,
+        const std::string& orth,
+        const Tagset& tagset)
+: startNode(startNode), 
+        endNode(startNode + 1), 
+        orth(orth),
+        lemma(orth),
+        tagnum(0),
+        namenum(0),
+        tag(tagset.getTag(0)),
+        name(tagset.getName(0)) {
+
+}
+
+MorphInterpretation MorphInterpretation::createIgn(int startNode, const std::string& orth, const Tagset& tagset) {
+    return MorphInterpretation(startNode, orth, tagset);
+}
+
 MorphInterpretation::~MorphInterpretation() {
 }
  
@@ -20,6 +20,7 @@ public:
             const std::string& orth,
             const EncodedInterpretation& encodedInterp,
             const Tagset& tagset);
+    static MorphInterpretation createIgn(int startNode, const std::string& orth, const Tagset& tagset);
     virtual ~MorphInterpretation();
     int getStartNode() const;
     int getEndNode() const;
@@ -30,6 +31,10 @@ public:
     const std::string& getTag() const;
     const std::string& getName() const;
 private:
+    MorphInterpretation(
+            int startNode,
+            const std::string& orth,
+            const Tagset& tagset);
     int startNode;
     int endNode;
     std::string orth;
@@ -6,6 +6,10 @@
  
 using namespace std;
  
+uint32_t UTF8CharsetConverter::peek(const char*& it, const char* end) const {
+    return utf8::peek_next(it, end);
+}
+
 uint32_t UTF8CharsetConverter::next(const char*& it, const char* end) const {
     return utf8::next(it, end);
 }
@@ -10,6 +10,7 @@
  
 class CharsetConverter {
 public:
+    virtual uint32_t peek(const char*& it, const char* end) const = 0;
     virtual uint32_t next(const char*& it, const char* end) const = 0;
     virtual void append(uint32_t cp, std::vector<char>& result) const = 0;
 private:
@@ -17,6 +18,7 @@ private:
  
 class UTF8CharsetConverter: public CharsetConverter {
 public:
+    uint32_t peek(const char*& it, const char* end) const;
     uint32_t next(const char*& it, const char* end) const;
     void append(uint32_t cp, std::vector<char>& result) const;
 private:
@@ -24,6 +26,7 @@ private:
  
 class UTF16CharsetConverter: public CharsetConverter {
 public:
+    uint32_t peek(const char*& it, const char* end) const;
     uint32_t next(const char*& it, const char* end) const;
     void append(uint32_t cp, std::vector<char>& result) const;
 private:
@@ -31,6 +34,7 @@ private:
  
 class UTF32CharsetConverter: public CharsetConverter {
 public:
+    uint32_t peek(const char*& it, const char* end) const;
     uint32_t next(const char*& it, const char* end) const;
     void append(uint32_t cp, std::vector<char>& result) const;
 private:
@@ -38,6 +42,7 @@ private:
  
 class ISO8859_2_CharsetConverter: public CharsetConverter {
 public:
+    uint32_t peek(const char*& it, const char* end) const;
     uint32_t next(const char*& it, const char* end) const;
     void append(uint32_t cp, std::vector<char>& result) const;
 private:
-
-
@@ -7,6 +7,7 @@
  
 #include <cstdlib>
  
+#include "utils.hpp"
 #include "Morfeusz.hpp"
 #include "MorphInterpretation.hpp"
  
@@ -16,11 +17,11 @@ using namespace std;
  * 
  */
 int main(int argc, char** argv) {
-    Morfeusz morfeusz(argv[1]);
+    Morfeusz morfeusz("/tmp/test-SIMPLE-PoliMorfSmall.tab.fsa");
     vector<MorphInterpretation> res;
-    string word = "mijałem";
-    const char* ptr = word.c_str();
-    morfeusz.processOneWord(ptr, word.c_str() + word.size(), 0, res);
+    string word = "  mijałem   fasdfasd abdominalności   ";
+    morfeusz.analyze(word, res);
+    DEBUG("znaleziono "+to_string(res.size()));
     for (MorphInterpretation& mi: res) {
         cerr << mi.getStartNode() << " " << mi.getEndNode() << " " << mi.getLemma() << " " << mi.getTag() << " " << mi.getName() << endl;
     }
@@ -17,9 +17,9 @@
       <in>MorphInterpretation.cpp</in>
       <in>Tagset.cpp</in>
       <in>main.cpp</in>
-      <in>morfeusz.cpp</in>
       <in>test_morfeusz.cpp</in>
       <in>test_morph.cpp</in>
+      <in>test_simple.cpp</in>
     </df>
     <logicalFolder name="ExternalFiles"
                    displayName="Important Files"
@@ -49,7 +49,7 @@
           <buildCommandWorkingDir>build</buildCommandWorkingDir>
           <buildCommand>${MAKE} -f Makefile</buildCommand>
           <cleanCommand>${MAKE} -f Makefile clean</cleanCommand>
-          <executablePath>build/fsa/test_dict</executablePath>
+          <executablePath>build/morfeusz/test_simple</executablePath>
         </makeTool>
       </makefileType>
       <folder path="1">
@@ -120,10 +120,6 @@
         <ccTool>
         </ccTool>
       </item>
-      <item path="morfeusz/morfeusz.cpp" ex="false" tool="1" flavor2="4">
-        <ccTool>
-        </ccTool>
-      </item>
       <item path="morfeusz/test_morfeusz.cpp" ex="false" tool="1" flavor2="4">
         <ccTool>
         </ccTool>
@@ -132,12 +128,8 @@
         <ccTool>
         </ccTool>
       </item>
-      <item path="morfeusz/test_simple.cpp" ex="false" tool="1" flavor2="0">
+      <item path="morfeusz/test_simple.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
-          <incDir>
-            <pElem>fsa</pElem>
-            <pElem>build/morfeusz</pElem>
-          </incDir>
         </ccTool>
       </item>
     </conf>
@@ -579,3 +579,43 @@ abdominoplastyki	abdominoplastyka	subst:pl:voc:f	pospolita
 abdominoplastyki	abdominoplastyka	subst:sg:gen:f	pospolita
 abdominoplastyko	abdominoplastyka	subst:sg:voc:f	pospolita
 abdominoplastykom	abdominoplastyka	subst:pl:dat:f	pospolita
+mijał	mijać	praet:sg:m1.m2.m3:imperf	pospolita
+mijała	mijać	praet:sg:f:imperf	pospolita
+mijało	mijać	praet:sg:n1.n2:imperf	pospolita
+mijały	mijać	praet:pl:m2.m3.f.n1.n2.p2.p3:imperf	pospolita
+omijał	omijać	praet:sg:m1.m2.m3:imperf	pospolita
+omijała	omijać	praet:sg:f:imperf	pospolita
+omijało	omijać	praet:sg:n1.n2:imperf	pospolita
+omijały	omijać	praet:pl:m2.m3.f.n1.n2.p2.p3:imperf	pospolita
+pomijał	pomijać	praet:sg:m1.m2.m3:imperf	pospolita
+pomijała	pomijać	praet:sg:f:imperf	pospolita
+pomijało	pomijać	praet:sg:n1.n2:imperf	pospolita
+pomijały	pomijać	praet:pl:m2.m3.f.n1.n2.p2.p3:imperf	pospolita
+powymijał	powymijać	praet:sg:m1.m2.m3:perf	pospolita
+powymijała	powymijać	praet:sg:f:perf	pospolita
+powymijało	powymijać	praet:sg:n1.n2:perf	pospolita
+powymijały	powymijać	praet:pl:m2.m3.f.n1.n2.p2.p3:perf	pospolita
+przemijał	przemijać	praet:sg:m1.m2.m3:imperf	pospolita
+przemijała	przemijać	praet:sg:f:imperf	pospolita
+przemijało	przemijać	praet:sg:n1.n2:imperf	pospolita
+przemijały	przemijać	praet:pl:m2.m3.f.n1.n2.p2.p3:imperf	pospolita
+rozmijał	rozmijać	praet:sg:m1.m2.m3:imperf	pospolita
+rozmijała	rozmijać	praet:sg:f:imperf	pospolita
+rozmijało	rozmijać	praet:sg:n1.n2:imperf	pospolita
+rozmijały	rozmijać	praet:pl:m2.m3.f.n1.n2.p2.p3:imperf	pospolita
+wymijał	wymijać	praet:sg:m1.m2.m3:imperf	pospolita
+wymijała	wymijać	praet:sg:f:imperf	pospolita
+wymijało	wymijać	praet:sg:n1.n2:imperf	pospolita
+wymijały	wymijać	praet:pl:m2.m3.f.n1.n2.p2.p3:imperf	pospolita
+zmijał	zmijać	praet:sg:m1.m2.m3:imperf	pospolita
+zmijała	zmijać	praet:sg:f:imperf	pospolita
+zmijało	zmijać	praet:sg:n1.n2:imperf	pospolita
+zmijały	zmijać	praet:pl:m2.m3.f.n1.n2.p2.p3:imperf	pospolita
+em	być	aglt:sg:pri:imperf:wok	pospolita
+eś	być	aglt:sg:sec:imperf:wok	pospolita
+eście	być	aglt:pl:sec:imperf:wok	pospolita
+eśmy	być	aglt:pl:pri:imperf:wok	pospolita
+m	być	aglt:sg:pri:imperf:nwok	pospolita
+ś	być	aglt:sg:sec:imperf:nwok	pospolita
+ście	być	aglt:pl:sec:imperf:nwok	pospolita
+śmy	być	aglt:pl:pri:imperf:nwok	pospolita
@@ -2,7 +2,7 @@
  
 [TAGS]
  
-0	adj:pl:acc:m1.p1:com
+0	ign
 1	adj:pl:acc:m1.p1:pos
 2	adj:pl:acc:m1.p1:sup
 3	adj:pl:acc:m2.m3.f.n1.n2.p2.p3:com
@@ -576,6 +576,7 @@
 571	winien:sg:f:imperf
 572	winien:sg:m1.m2.m3:imperf
 573	winien:sg:n1.n2:imperf
+574	adj:pl:acc:m1.p1:com
  
 [NAMES]