- praca nad grafem fleksyjnym

git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@21 ff4e3ee1-f430-4e82-ade0-24591c43f1fd

- praca nad grafem fleksyjnym
git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@21 ff4e3ee1-f430-4e82-ade0-24591c43f1fd
Michał Lenart
1 parent a9d3e65c
Showing 13 changed files with 258 additions and 89 deletions
morfeusz/CMakeLists.txt
morfeusz/FlexionGraph.cpp
morfeusz/FlexionGraph.hpp
morfeusz/InterpretedChunk.hpp
morfeusz/InterpsGroup.hpp
morfeusz/Morfeusz.cpp
morfeusz/Morfeusz.hpp
morfeusz/Morfeusz_impl.hpp
morfeusz/charset/CharsetConverter.cpp
morfeusz/charset/CharsetConverter.hpp
morfeusz/charset/charset_utils.hpp
morfeusz/test_morph.cpp
nbproject/configurations.xml
@@ -7,7 +7,7 @@ include_directories (${Morfeusz_SOURCE_DIR}/fsa)
 add_library (morfeusz2 morfeusz.hpp morfeusz.cpp)
 add_executable (morfeusz2_analyze main.cpp)
 add_executable (test_morph test_morph.cpp MorphDeserializer.cpp Tagset.cpp ../fsa/const.cpp MorphInterpretation.cpp)
-add_executable (test_morfeusz test_morph.cpp MorphDeserializer.cpp Tagset.cpp ../fsa/const.cpp MorphInterpretation.cpp Morfeusz.cpp charset/CharsetConverter.cpp)
+add_executable (test_morfeusz test_morph.cpp MorphDeserializer.cpp Tagset.cpp ../fsa/const.cpp MorphInterpretation.cpp Morfeusz.cpp charset/CharsetConverter.cpp FlexionGraph.cpp)
 # Link the executable to the Hello library. 
 target_link_libraries (morfeusz2_analyze morfeusz2)
+
+#include "FlexionGraph.hpp"
+
+FlexionGraph::FlexionGraph(int startNode)
+: startNode(startNode) {
+    
+}
+
+void FlexionGraph::addPath(const std::vector<InterpretedChunk>& path) {
+    for (const InterpretedChunk& chunk: path) {
+        if (&chunk == &(path.back())) {
+            Edge e = { chunk, -1 };
+            vector<Edge> v;
+            v.push_back(e);
+            this->graph.push_back(v);
+//            this->graph[node].push_back(e);
+        }
+        else if (&chunk == &(path.front())) {
+            Edge e = { chunk, (int) this->graph.size() };
+            this->graph[0].push_back(e);
+        }
+        else {
+            Edge e = { chunk, (int) this->graph.size() };
+            vector<Edge> v;
+            v.push_back(e);
+            this->graph.push_back(v);
+        }
+    }
+}
+
+void FlexionGraph::appendToResults(const Tagset& tagset, std::vector<MorphInterpretation>& results) {
+    int endNode = graph.size();
+    for (int i = 0; i < graph.size(); i++) {
+        vector<Edge>& edges = graph[i];
+        for (Edge& e: edges) {
+            int realStartNode = i + this->startNode;
+            int realEndNode = e.nextNode == -1 ? (endNode + this->startNode) : (i + e.nextNode);
+            string orth(e.chunk.chunk, e.chunk.chunkLength);
+            vector<MorphInterpretation> interps = e.chunk.interpsGroup.getRealInterps(orth, realStartNode, realEndNode, tagset);
+            results.insert(results.end(), interps.begin(), interps.end());
+        }
+    }
+}
+/* 
+ * File:   FlexionGraph.hpp
+ * Author: mlenart
+ *
+ * Created on 18 listopad 2013, 15:03
+ */
+
+#ifndef FLEXIONGRAPH_HPP
+#define	FLEXIONGRAPH_HPP
+
+#include <vector>
+#include "InterpretedChunk.hpp"
+
+struct Edge {
+    InterpretedChunk chunk;
+    int nextNode;
+};
+
+class FlexionGraph {
+public:
+    
+    explicit FlexionGraph(int startNode);
+    
+    void addPath(const std::vector<InterpretedChunk>& path);
+    
+    void appendToResults(const Tagset& tagset, std::vector<MorphInterpretation>& results);
+    
+//    virtual ~FlexionGraph();
+private:
+    int startNode;
+    std::vector< std::vector<Edge> > graph;
+};
+
+#endif	/* FLEXIONGRAPH_HPP */
+
+/* 
+ * File:   InterpretedChunk.hpp
+ * Author: mlenart
+ *
+ * Created on 18 listopad 2013, 15:00
+ */
+
+#ifndef INTERPRETEDCHUNK_HPP
+#define	INTERPRETEDCHUNK_HPP
+
+#include "InterpsGroup.hpp"
+
+struct InterpretedChunk {
+    const char* chunk;
+    long chunkLength;
+    InterpsGroup& interpsGroup;
+};
+
+#endif	/* INTERPRETEDCHUNK_HPP */
+
 /* 
  * File:   GroupedInterpretations.hpp
- * Author: lennyn
+ * Author: mlenart
  *
  * Created on November 16, 2013, 7:58 PM
  */
-#ifndef GROUPEDINTERPRETATIONS_HPP
-#define	GROUPEDINTERPRETATIONS_HPP
+#ifndef INTERPSGROUP_HPP
+#define	INTERPSGROUP_HPP
 #include <vector>
 #include <string>
@@ -26,7 +26,11 @@ public:
     }
-    std::vector<MorphInterpretation> getRealInterps(const std::string& orth, const Tagset& tagset) {
+    std::vector<MorphInterpretation> getRealInterps(
+                const std::string& orth, 
+                const int startNode, 
+                const int endNode,
+                const Tagset& tagset) {
         std::vector<MorphInterpretation> res;
         for (EncodedInterpretation& ei: interps) {
             res.push_back(MorphInterpretation(startNode, endNode, orth, ei, tagset));
@@ -39,8 +43,7 @@ public:
     }
     int type;
-    int startNode;
-    int endNode;
+    
 private:
     std::vector<EncodedInterpretation> interps;
 };
@@ -11,6 +11,9 @@
 #include "Morfeusz.hpp"
 #include "MorphDeserializer.hpp"
 #include "charset/CharsetConverter.hpp"
+#include "charset/charset_utils.hpp"
+
+// TODO - konstruktor kopiujący działający Tak-Jak-Trzeba
 using namespace std;
@@ -30,16 +33,64 @@ Morfeusz::Morfeusz(const string&amp; filename)
 }
-//Morfeusz::Morfeusz(const Morfeusz& orig) {
-//}
-
 Morfeusz::~Morfeusz() {
     delete &this->fsa;
+    delete &this->charsetConverter;
+}
+
+void Morfeusz::processOneWord(
+        const char*& inputData,
+        const char* inputEnd,
+        const int startNodeNum,
+        std::vector<MorphInterpretation>& results) const {
+    vector<InterpretedChunk> accum;
+    FlexionGraph graph(startNodeNum);
+    const char* currInput = inputData;
+    doProcessOneWord(currInput, inputEnd, accum, graph);
+    graph.appendToResults(this->tagset, results);
+    inputData = currInput;
+}
+
+void Morfeusz::doProcessOneWord(
+        const char*& inputData,
+        const char* inputEnd,
+        vector<InterpretedChunk>& accum,
+        FlexionGraph& graph) const {
+    const char* currInput = inputData;
+    StateType state = this->fsa->getInitialState();
+    int codepoint = this->charsetConverter->next(currInput, inputEnd);
+    
+    if (!accum.empty() && isEndOfWord(codepoint)) {
+        graph.addPath(accum);
+    }
+    else
+        while (!isEndOfWord(codepoint)) {
+            this->feedState(state, codepoint);
+            codepoint = this->charsetConverter->next(currInput, inputEnd);
+            if (state.isAccepting()) {
+                for (InterpsGroup& ig : state.getValue()) {
+                    InterpretedChunk ic = {inputData, currInput - inputData, ig};
+                    accum.push_back(ic);
+                    doProcessOneWord(currInput, inputEnd, accum, graph);
+                    accum.pop_back();
+                }
+            }
+        }
+}
+
+void Morfeusz::feedState(
+        StateType& state,
+        const int codepoint) const {
+    vector<char> chars;
+    this->charsetConverter->append(codepoint, chars);
+    for (char c: chars) {
+        state.proceedToNext(c);
+    }
 }
 ResultsIterator Morfeusz::analyze(const std::string& text) {
-//    const char* textStart = text.c_str();
-//    const char* textEnd = text.c_str() + text.length();
+    //    const char* textStart = text.c_str();
+    //    const char* textEnd = text.c_str() + text.length();
     return ResultsIterator(text, *this);
 }
@@ -49,13 +100,13 @@ morfeusz(morfeusz) {
 }
 MorphInterpretation ResultsIterator::getNext() {
-//    if (resultsBuffer.empty()) {
-//        morfeusz.processOneWord(rawInput, startNode, back_inserter(resultsBuffer));
-//    }
-//    startNode = resultsBuffer.back().getEndNode();
-//    MorphInterpretation res = resultsBuffer.front();
-//    resultsBuffer.pop_front();
-//    return res;
+    //    if (resultsBuffer.empty()) {
+    //        morfeusz.processOneWord(rawInput, startNode, back_inserter(resultsBuffer));
+    //    }
+    //    startNode = resultsBuffer.back().getEndNode();
+    //    MorphInterpretation res = resultsBuffer.front();
+    //    resultsBuffer.pop_front();
+    //    return res;
 }
 bool ResultsIterator::hasNext() {
@@ -16,6 +16,8 @@
 #include "MorphInterpretation.hpp"
 #include "InterpsGroup.hpp"
 #include "charset/CharsetConverter.hpp"
+#include "InterpretedChunk.hpp"
+#include "FlexionGraph.hpp"
 class Morfeusz;
 //class AnalyzeResult;
@@ -34,17 +36,26 @@ public:
 //    Morfeusz();
     friend class ResultsIterator;
 private:
-    template <class OutputIterator>
-//    void processOneWord(const char*& inputData, int startNodeNum, OutputIterator resInterps) const;
+    void processOneWord(
+        const char*& inputData,
+        const char* inputEnd,
+        const int startNodeNum, 
+        std::vector<MorphInterpretation>& result) const;
-    int doProcessOneWord(const char*& inputData, int startNodeNum, std::vector<InterpsGroup>& interps) const;
+    void doProcessOneWord(
+        const char*& inputData,
+        const char* inputEnd,
+        std::vector<InterpretedChunk>& accum,
+        FlexionGraph& graph) const;
-    const FSAType* fsa;
+    void feedState(
+        StateType& state,
+        const int codepoint) const;
+    
+    FSAType* fsa;
     CharsetConverter* charsetConverter;
 };
-#include "Morfeusz_impl.hpp"
-
 class ResultsIterator {
 public:
     ResultsIterator(const std::string& text, const Morfeusz& morfeusz);
-/* 
- * File:   Morfeusz_impl.hpp
- * Author: lennyn
- *
- * Created on November 15, 2013, 1:43 PM
- */
-
-#ifndef MORFEUSZ_IMPL_HPP
-#define	MORFEUSZ_IMPL_HPP
-
-#include <cassert>
-#include "Morfeusz.hpp"
-
-//template <class OutputIterator>
-//void Morfeusz::processOneWord(const char*& inputData, const char* inputEnd, int startNodeNum, OutputIterator output, bool insertIgn = true) const {
-//    if (inputData == inputEnd) {
-//        return;
-//    }
-//    const char* start = inputData;
-//    StateType state = fsa->getInitialState();
-//    int currNodeNum = startNodeNum;
-//    do {
-//        int codepoint = this->charsetConverter->next(inputData, inputEnd);
-//        if (!isSpace(codepoint) && codepoint != 0) {
-//            feedAutomaton(state, codepoint);
-//            if (state.isAccepting()) {
-//                int currInput = inputData;
-//                vector<MorphInterpretation> additionalInterps;
-//                processOneWord(
-//                        currInput, inputEnd, 
-//                        currNodeNum + 1, 
-//                        back_inserter(additionalInterps), false);
-//                if (!additionalInterps.empty()) {
-//                    currNodeNum = additionalInterps.back().getEndNode();
-//                }
-//            }
-//        }
-//    }
-//}
-
-#endif	/* MORFEUSZ_IMPL_HPP */
-
-/* 
- * File:   EncodingConverter.cpp
- * Author: mlenart
- * 
- * Created on 14 listopad 2013, 17:28
- */
+#include <vector>
+#include <iterator>
 #include "utf8.h"
 #include "CharsetConverter.hpp"
+using namespace std;
+
 uint32_t UTF8CharsetConverter::next(const char*& it, const char* end) const {
     return utf8::next(it, end);
 }
-char* UTF8CharsetConverter::append(uint32_t cp, char* result) const {
-    return utf8::append(cp, result);
+
+void UTF8CharsetConverter::append(uint32_t cp, vector<char>& result) const {
+    utf8::append(cp, back_inserter(result));
 }
@@ -11,35 +11,35 @@
 class CharsetConverter {
 public:
     virtual uint32_t next(const char*& it, const char* end) const = 0;
-    virtual char* append(uint32_t cp, char* result) const = 0;
+    virtual void append(uint32_t cp, std::vector<char>& result) const = 0;
 private:
 };
 class UTF8CharsetConverter: public CharsetConverter {
 public:
     uint32_t next(const char*& it, const char* end) const;
-    char* append(uint32_t cp, char* result) const;
+    void append(uint32_t cp, std::vector<char>& result) const;
 private:
 };
 class UTF16CharsetConverter: public CharsetConverter {
 public:
     uint32_t next(const char*& it, const char* end) const;
-    char* append(uint32_t cp, char* result) const;
+    void append(uint32_t cp, std::vector<char>& result) const;
 private:
 };
 class UTF32CharsetConverter: public CharsetConverter {
 public:
     uint32_t next(const char*& it, const char* end) const;
-    char* append(uint32_t cp, char* result) const;
+    void append(uint32_t cp, std::vector<char>& result) const;
 private:
 };
 class ISO8859_2_CharsetConverter: public CharsetConverter {
 public:
     uint32_t next(const char*& it, const char* end) const;
-    char* append(uint32_t cp, char* result) const;
+    void append(uint32_t cp, std::vector<char>& result) const;
 private:
 };
@@ -8,7 +8,12 @@
 #ifndef CHARSET_UTILS_HPP
 #define	CHARSET_UTILS_HPP
+#include <set>
+bool isEndOfWord(int codepoint) {
+    static std::set<int> whitespaces = { 0x00, 0x0A, 0x20 };
+    return whitespaces.count(codepoint) != 0;
+}
 #endif	/* CHARSET_UTILS_HPP */
@@ -38,7 +38,7 @@ void doTest(
 //        vector<TaggedInterpretation> parsedValues;
         bool found = false;
         for (InterpsGroup gi: value2)
-        for (MorphInterpretation interp: gi.getRealInterps(orth, tagset)) {
+        for (MorphInterpretation interp: gi.getRealInterps(orth, 0, 0, tagset)) {
 //            TaggedInterpretation parsedValue = interpsConverter.getInterpretation(key, interp);
 //            (0, 0, orth, encodedInterp, tagset);
 //            parsedValues.push_back(parsedValue);
@@ -10,11 +10,15 @@
     <df root="morfeusz" name="1">
       <df name="charset">
         <in>CharsetConverter.cpp</in>
-        <in>charset_utils.hpp</in>
       </df>
-      <in>InterpsGroup.hpp</in>
+      <df name="encoding">
+        <in>CharsetConverter.cpp</in>
+      </df>
+      <df name="flexion">
+        <in>FlexionGraph.cpp</in>
+      </df>
+      <in>FlexionGraph.cpp</in>
       <in>Morfeusz.cpp</in>
-      <in>Morfeusz_impl.hpp</in>
       <in>MorphDeserializer.cpp</in>
       <in>MorphInterpretation.cpp</in>
       <in>Tagset.cpp</in>
@@ -53,7 +57,7 @@
           <executablePath>build/fsa/test_dict</executablePath>
         </makeTool>
       </makefileType>
-      <folder path="1">
+      <folder path="1/charset">
         <ccTool>
           <incDir>
             <pElem>fsa</pElem>
@@ -90,24 +94,44 @@
           </incDir>
         </ccTool>
       </item>
-      <item path="morfeusz/InterpsGroup.hpp" ex="false" tool="3" flavor2="0">
+      <item path="morfeusz/FlexionGraph.cpp" ex="false" tool="1" flavor2="8">
+        <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
+        </ccTool>
       </item>
       <item path="morfeusz/Morfeusz.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
         </ccTool>
       </item>
-      <item path="morfeusz/Morfeusz_impl.hpp" ex="false" tool="3" flavor2="0">
-      </item>
       <item path="morfeusz/MorphDeserializer.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
         </ccTool>
       </item>
       <item path="morfeusz/MorphInterpretation.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
         </ccTool>
       </item>
       <item path="morfeusz/Tagset.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
         </ccTool>
       </item>
       <item path="morfeusz/charset/CharsetConverter.cpp"
@@ -117,18 +141,38 @@
         <ccTool>
         </ccTool>
       </item>
-      <item path="morfeusz/charset/charset_utils.hpp" ex="false" tool="3" flavor2="0">
+      <item path="morfeusz/encoding/CharsetConverter.cpp"
+            ex="false"
+            tool="1"
+            flavor2="4">
+        <ccTool>
+        </ccTool>
+      </item>
+      <item path="morfeusz/flexion/FlexionGraph.cpp" ex="false" tool="1" flavor2="4">
+        <ccTool>
+        </ccTool>
       </item>
       <item path="morfeusz/main.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
         </ccTool>
       </item>
       <item path="morfeusz/morfeusz.cpp" ex="false" tool="1" flavor2="4">
         <ccTool>
+          <incDir>
+            <pElem>morfeusz</pElem>
+          </incDir>
         </ccTool>
       </item>
       <item path="morfeusz/test_morph.cpp" ex="false" tool="1" flavor2="8">
         <ccTool>
+          <incDir>
+            <pElem>fsa</pElem>
+            <pElem>build/morfeusz</pElem>
+          </incDir>
         </ccTool>
       </item>
     </conf>