diff --git a/CMakeLists.txt b/CMakeLists.txt
index e9fe8dc..81c621a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,7 +1,7 @@
cmake_minimum_required (VERSION 2.8)
project (Morfeusz)
-
+set(CMAKE_BUILD_TYPE "Debug")
enable_testing()
add_subdirectory (fsa)
diff --git a/morfeusz/CMakeLists.txt b/morfeusz/CMakeLists.txt
index 52f710c..714ef5f 100644
--- a/morfeusz/CMakeLists.txt
+++ b/morfeusz/CMakeLists.txt
@@ -4,15 +4,14 @@
# Make sure the linker can find the Hello library once it is built.
#link_directories (${Morfeusz_BINARY_DIR}/Hello)
include_directories (${Morfeusz_SOURCE_DIR}/fsa)
-add_library (morfeusz2 morfeusz.hpp morfeusz.cpp)
-add_executable (morfeusz2_analyze main.cpp)
+# add_executable (morfeusz2_analyze main.cpp)
add_executable (test_morph test_morph.cpp MorphDeserializer.cpp Tagset.cpp ../fsa/const.cpp MorphInterpretation.cpp)
add_executable (test_morfeusz test_morph.cpp MorphDeserializer.cpp Tagset.cpp ../fsa/const.cpp MorphInterpretation.cpp Morfeusz.cpp charset/CharsetConverter.cpp FlexionGraph.cpp)
add_executable (test_simple test_simple.cpp MorphDeserializer.cpp Tagset.cpp ../fsa/const.cpp MorphInterpretation.cpp Morfeusz.cpp charset/CharsetConverter.cpp FlexionGraph.cpp)
# Link the executable to the Hello library.
-target_link_libraries (morfeusz2_analyze morfeusz2)
-set_target_properties ( morfeusz2_analyze PROPERTIES COMPILE_FLAGS "-std=gnu++0x" )
+#target_link_libraries (morfeusz2_analyze morfeusz2)
+#set_target_properties ( morfeusz2_analyze PROPERTIES COMPILE_FLAGS "-std=gnu++0x" )
set_target_properties ( test_morph PROPERTIES COMPILE_FLAGS "-std=gnu++0x -Wall -O2" )
set_target_properties ( test_morfeusz PROPERTIES COMPILE_FLAGS "-std=gnu++0x -Wall -O2" )
diff --git a/morfeusz/EncodedInterpretation.hpp b/morfeusz/EncodedInterpretation.hpp
index f969cf6..6eb7ca5 100644
--- a/morfeusz/EncodedInterpretation.hpp
+++ b/morfeusz/EncodedInterpretation.hpp
@@ -31,8 +31,6 @@ struct EncodedInterpretation {
int type;
int tag;
int nameClassifier;
- int startNode;
- int endNode;
};
#endif /* INTERPRETATION_HPP */
diff --git a/morfeusz/FlexionGraph.cpp b/morfeusz/FlexionGraph.cpp
index 1aa59be..3ff28c8 100644
--- a/morfeusz/FlexionGraph.cpp
+++ b/morfeusz/FlexionGraph.cpp
@@ -1,4 +1,6 @@
+#include <string>
+#include "utils.hpp"
#include "FlexionGraph.hpp"
FlexionGraph::FlexionGraph(int startNode)
@@ -6,29 +8,56 @@ FlexionGraph::FlexionGraph(int startNode)
}
+static inline void debugPath(const std::vector<InterpretedChunk>& path) {
+ for (const InterpretedChunk& chunk: path) {
+ std::string text(chunk.chunk, chunk.chunkLength);
+ DEBUG(text);
+ DEBUG(chunk.chunkLength);
+ }
+}
+
+void FlexionGraph::addStartEdge(const Edge& e) {
+ if (this->graph.empty()) {
+ this->graph.push_back(vector<Edge>());
+ }
+ this->graph[0].push_back(e);
+}
+
+void FlexionGraph::addMiddleEdge(const Edge& e) {
+ this->graph.push_back(vector<Edge>(1, e));
+}
+
void FlexionGraph::addPath(const std::vector<InterpretedChunk>& path) {
+// debugPath(path);
for (const InterpretedChunk& chunk: path) {
- if (&chunk == &(path.back())) {
+ if (&chunk == &(path.front())
+ && &chunk == &(path.back())) {
Edge e = { chunk, -1 };
- vector<Edge> v;
- v.push_back(e);
- this->graph.push_back(v);
-// this->graph[node].push_back(e);
+ this->addStartEdge(e);
}
else if (&chunk == &(path.front())) {
- Edge e = { chunk, (int) this->graph.size() };
- this->graph[0].push_back(e);
+ Edge e = { chunk, (int) this->graph.size() + 1 };
+ this->addStartEdge(e);
+ }
+ else if (&chunk == &(path.back())) {
+ Edge e = { chunk, -1 };
+ this->addMiddleEdge(e);
}
else {
- Edge e = { chunk, (int) this->graph.size() };
- vector<Edge> v;
- v.push_back(e);
- this->graph.push_back(v);
+ Edge e = { chunk, (int) this->graph.size() + 1 };
+ this->addMiddleEdge(e);
}
}
}
+void FlexionGraph::minimizeGraph() {
+ if (this->graph.size() > 2) {
+
+ }
+}
+
void FlexionGraph::appendToResults(const Tagset& tagset, std::vector<MorphInterpretation>& results) {
+ this->minimizeGraph();
int endNode = graph.size();
for (unsigned int i = 0; i < graph.size(); i++) {
vector<Edge>& edges = graph[i];
@@ -41,3 +70,7 @@ void FlexionGraph::appendToResults(const Tagset& tagset, std::vector<MorphInterp
}
}
}
+
+bool FlexionGraph::empty() const {
+ return this->graph.empty();
+}
diff --git a/morfeusz/FlexionGraph.hpp b/morfeusz/FlexionGraph.hpp
index dbb9478..e84770d 100644
--- a/morfeusz/FlexionGraph.hpp
+++ b/morfeusz/FlexionGraph.hpp
@@ -16,6 +16,22 @@ struct Edge {
int nextNode;
};
+//struct EdgeLabel {
+// int type;
+// const char* textStart;
+// int textLength;
+//
+// bool operator==(const EdgeLabel &el) const {
+// return this->type == el.type
+// && this->textStart == el.textStart
+// && this->textLength == el.textLength;
+// }
+//
+// bool operator<(const coord &o) {
+// return x < o.x || (x == o.x && y < o.y);
+// }
+//};
+
class FlexionGraph {
public:
@@ -25,8 +41,17 @@ public:
void appendToResults(const Tagset& tagset, std::vector<MorphInterpretation>& results);
+ bool empty() const;
+
// virtual ~FlexionGraph();
private:
+
+ void addStartEdge(const Edge& e);
+
+ void addMiddleEdge(const Edge& e);
+
+ void minimizeGraph();
+
int startNode;
std::vector< std::vector<Edge> > graph;
};
diff --git a/morfeusz/Morfeusz.cpp b/morfeusz/Morfeusz.cpp
index 99ca9ac..575cd2e 100644
--- a/morfeusz/Morfeusz.cpp
+++ b/morfeusz/Morfeusz.cpp
@@ -38,27 +38,37 @@ static Tagset* initializeTagset(const string& filename) {
}
Morfeusz::Morfeusz(const string& filename)
-: fsa(initializeFSA(filename)),
- charsetConverter(initializeCharsetConverter()),
- tagset(initializeTagset(filename)) {
+: fsa(initializeFSA(filename)),
+charsetConverter(initializeCharsetConverter()),
+tagset(initializeTagset(filename)) {
}
Morfeusz::~Morfeusz() {
- delete &this->fsa;
- delete &this->charsetConverter;
+ // delete &this->fsa;
+ // delete &this->charsetConverter;
}
void Morfeusz::processOneWord(
const char*& inputData,
const char* inputEnd,
- const int startNodeNum,
+ int startNodeNum,
std::vector<MorphInterpretation>& results) const {
+ while (inputData != inputEnd
+ && isEndOfWord(this->charsetConverter->peek(inputData, inputEnd))) {
+ this->charsetConverter->next(inputData, inputEnd);
+ }
+ const char* wordStart = inputData;
vector<InterpretedChunk> accum;
FlexionGraph graph(startNodeNum);
const char* currInput = inputData;
doProcessOneWord(currInput, inputEnd, accum, graph);
- graph.appendToResults(*this->tagset, results);
+ if (!graph.empty()) {
+ graph.appendToResults(*this->tagset, results);
+ }
+ else if (wordStart != currInput) {
+ this->appendIgnotiumToResults(string(wordStart, currInput), startNodeNum, results);
+ }
inputData = currInput;
}
@@ -67,38 +77,56 @@ void Morfeusz::doProcessOneWord(
const char* inputEnd,
vector<InterpretedChunk>& accum,
FlexionGraph& graph) const {
+ bool endOfWord = inputData == inputEnd;
const char* currInput = inputData;
- StateType state = this->fsa->getInitialState();
- int codepoint = this->charsetConverter->next(currInput, inputEnd);
+ const char* prevInput = inputData;
+ int codepoint = endOfWord ? 0 : this->charsetConverter->next(currInput, inputEnd);
- if (!accum.empty() && isEndOfWord(codepoint)) {
- graph.addPath(accum);
- }
- else
- while (!isEndOfWord(codepoint)) {
- this->feedState(state, codepoint);
- codepoint = this->charsetConverter->next(currInput, inputEnd);
- if (state.isAccepting()) {
- for (InterpsGroup& ig : state.getValue()) {
- InterpretedChunk ic = {inputData, currInput - inputData, ig};
- accum.push_back(ic);
- doProcessOneWord(currInput, inputEnd, accum, graph);
- accum.pop_back();
- }
+ StateType state = this->fsa->getInitialState();
+
+ while (!isEndOfWord(codepoint)) {
+ this->feedState(state, codepoint);
+ if (state.isAccepting()) {
+ for (InterpsGroup& ig : state.getValue()) {
+ InterpretedChunk ic = {inputData, currInput - inputData, ig};
+ accum.push_back(ic);
+ const char* newCurrInput = currInput;
+ doProcessOneWord(newCurrInput, inputEnd, accum, graph);
+ accum.pop_back();
}
}
+ prevInput = currInput;
+ codepoint = currInput == inputEnd ? 0 : this->charsetConverter->next(currInput, inputEnd);
+ }
+ if (state.isAccepting()) {
+ for (InterpsGroup& ig : state.getValue()) {
+ InterpretedChunk ic = {inputData, prevInput - inputData, ig};
+ accum.push_back(ic);
+ graph.addPath(accum);
+ accum.pop_back();
+ }
+ }
+ inputData = currInput;
}
void Morfeusz::feedState(
StateType& state,
- const int codepoint) const {
+ int codepoint) const {
vector<char> chars;
this->charsetConverter->append(codepoint, chars);
- for (char c: chars) {
+ for (char c : chars) {
state.proceedToNext(c);
}
}
+void Morfeusz::appendIgnotiumToResults(
+ const string& word,
+ int startNodeNum,
+ std::vector<MorphInterpretation>& results) const {
+ MorphInterpretation interp = MorphInterpretation::createIgn(startNodeNum, word, *this->tagset);
+ results.push_back(interp);
+}
+
ResultsIterator Morfeusz::analyze(const string& text) {
// const char* textStart = text.c_str();
// const char* textEnd = text.c_str() + text.length();
@@ -106,7 +134,12 @@ ResultsIterator Morfeusz::analyze(const string& text) {
}
void Morfeusz::analyze(const string& text, vector<MorphInterpretation>& results) {
-
+ const char* input = text.c_str();
+ const char* inputEnd = input + text.length();
+ while (input != inputEnd) {
+ int startNode = results.empty() ? 0 : results.back().getEndNode();
+ this->processOneWord(input, inputEnd, startNode, results);
+ }
}
ResultsIterator::ResultsIterator(const string& text, const Morfeusz& morfeusz)
diff --git a/morfeusz/Morfeusz.hpp b/morfeusz/Morfeusz.hpp
index b7a82a8..1006409 100644
--- a/morfeusz/Morfeusz.hpp
+++ b/morfeusz/Morfeusz.hpp
@@ -1,6 +1,6 @@
/*
* File: Morfeusz.hpp
- * Author: lennyn
+ * Author: mlenart
*
* Created on November 13, 2013, 5:21 PM
*/
@@ -37,7 +37,7 @@ public:
void processOneWord(
const char*& inputData,
const char* inputEnd,
- const int startNodeNum,
+ int startNodeNum,
std::vector<MorphInterpretation>& result) const;
// Morfeusz();
@@ -52,7 +52,12 @@ private:
void feedState(
StateType& state,
- const int codepoint) const;
+ int codepoint) const;
+
+ void appendIgnotiumToResults(
+ const std::string& word,
+ int startNodeNum,
+ std::vector<MorphInterpretation>& results) const;
FSAType* fsa;
CharsetConverter* charsetConverter;
diff --git a/morfeusz/MorphInterpretation.cpp b/morfeusz/MorphInterpretation.cpp
index c6894c7..7872b0d 100644
--- a/morfeusz/MorphInterpretation.cpp
+++ b/morfeusz/MorphInterpretation.cpp
@@ -39,6 +39,25 @@ MorphInterpretation::MorphInterpretation(
}
+MorphInterpretation::MorphInterpretation(
+ int startNode,
+ const std::string& orth,
+ const Tagset& tagset)
+: startNode(startNode),
+ endNode(startNode + 1),
+ orth(orth),
+ lemma(orth),
+ tagnum(0),
+ namenum(0),
+ tag(tagset.getTag(0)),
+ name(tagset.getName(0)) {
+
+}
+
+MorphInterpretation MorphInterpretation::createIgn(int startNode, const std::string& orth, const Tagset& tagset) {
+ return MorphInterpretation(startNode, orth, tagset);
+}
+
MorphInterpretation::~MorphInterpretation() {
}
diff --git a/morfeusz/MorphInterpretation.hpp b/morfeusz/MorphInterpretation.hpp
index eab9c1a..c3bf61a 100644
--- a/morfeusz/MorphInterpretation.hpp
+++ b/morfeusz/MorphInterpretation.hpp
@@ -20,6 +20,7 @@ public:
const std::string& orth,
const EncodedInterpretation& encodedInterp,
const Tagset& tagset);
+ static MorphInterpretation createIgn(int startNode, const std::string& orth, const Tagset& tagset);
virtual ~MorphInterpretation();
int getStartNode() const;
int getEndNode() const;
@@ -30,6 +31,10 @@ public:
const std::string& getTag() const;
const std::string& getName() const;
private:
+ MorphInterpretation(
+ int startNode,
+ const std::string& orth,
+ const Tagset& tagset);
int startNode;
int endNode;
std::string orth;
diff --git a/morfeusz/charset/CharsetConverter.cpp b/morfeusz/charset/CharsetConverter.cpp
index bd42d65..667627d 100644
--- a/morfeusz/charset/CharsetConverter.cpp
+++ b/morfeusz/charset/CharsetConverter.cpp
@@ -6,6 +6,10 @@
using namespace std;
+uint32_t UTF8CharsetConverter::peek(const char*& it, const char* end) const {
+ return utf8::peek_next(it, end);
+}
+
uint32_t UTF8CharsetConverter::next(const char*& it, const char* end) const {
return utf8::next(it, end);
}
diff --git a/morfeusz/charset/CharsetConverter.hpp b/morfeusz/charset/CharsetConverter.hpp
index 13331da..c812ea7 100644
--- a/morfeusz/charset/CharsetConverter.hpp
+++ b/morfeusz/charset/CharsetConverter.hpp
@@ -10,6 +10,7 @@
class CharsetConverter {
public:
+ virtual uint32_t peek(const char*& it, const char* end) const = 0;
virtual uint32_t next(const char*& it, const char* end) const = 0;
virtual void append(uint32_t cp, std::vector<char>& result) const = 0;
private:
@@ -17,6 +18,7 @@ private:
class UTF8CharsetConverter: public CharsetConverter {
public:
+ uint32_t peek(const char*& it, const char* end) const;
uint32_t next(const char*& it, const char* end) const;
void append(uint32_t cp, std::vector<char>& result) const;
private:
@@ -24,6 +26,7 @@ private:
class UTF16CharsetConverter: public CharsetConverter {
public:
+ uint32_t peek(const char*& it, const char* end) const;
uint32_t next(const char*& it, const char* end) const;
void append(uint32_t cp, std::vector<char>& result) const;
private:
@@ -31,6 +34,7 @@ private:
class UTF32CharsetConverter: public CharsetConverter {
public:
+ uint32_t peek(const char*& it, const char* end) const;
uint32_t next(const char*& it, const char* end) const;
void append(uint32_t cp, std::vector<char>& result) const;
private:
@@ -38,6 +42,7 @@ private:
class ISO8859_2_CharsetConverter: public CharsetConverter {
public:
+ uint32_t peek(const char*& it, const char* end) const;
uint32_t next(const char*& it, const char* end) const;
void append(uint32_t cp, std::vector<char>& result) const;
private:
diff --git a/morfeusz/morfeusz.cpp b/morfeusz/morfeusz.cpp
deleted file mode 100644
index e69de29..0000000
--- a/morfeusz/morfeusz.cpp
+++ /dev/null
diff --git a/morfeusz/morfeusz.hpp b/morfeusz/morfeusz.hpp
deleted file mode 100644
index 139597f..0000000
--- a/morfeusz/morfeusz.hpp
+++ /dev/null
@@ -1,2 +0,0 @@
-
-
diff --git a/morfeusz/test_simple.cpp b/morfeusz/test_simple.cpp
index b8afe79..1a1b3ed 100644
--- a/morfeusz/test_simple.cpp
+++ b/morfeusz/test_simple.cpp
@@ -7,6 +7,7 @@
#include <cstdlib>
+#include "utils.hpp"
#include "Morfeusz.hpp"
#include "MorphInterpretation.hpp"
@@ -16,11 +17,11 @@ using namespace std;
*
*/
int main(int argc, char** argv) {
- Morfeusz morfeusz(argv[1]);
+ Morfeusz morfeusz("/tmp/test-SIMPLE-PoliMorfSmall.tab.fsa");
vector<MorphInterpretation> res;
- string word = "mijałem";
- const char* ptr = word.c_str();
- morfeusz.processOneWord(ptr, word.c_str() + word.size(), 0, res);
+ string word = " mijałem fasdfasd abdominalności ";
+ morfeusz.analyze(word, res);
+ DEBUG("znaleziono "+to_string(res.size()));
for (MorphInterpretation& mi: res) {
cerr << mi.getStartNode() << " " << mi.getEndNode() << " " << mi.getLemma() << " " << mi.getTag() << " " << mi.getName() << endl;
}
diff --git a/nbproject/configurations.xml b/nbproject/configurations.xml
index 60de1ce..02dc701 100644
--- a/nbproject/configurations.xml
+++ b/nbproject/configurations.xml
@@ -17,9 +17,9 @@
<in>MorphInterpretation.cpp</in>
<in>Tagset.cpp</in>
<in>main.cpp</in>
- <in>morfeusz.cpp</in>
<in>test_morfeusz.cpp</in>
<in>test_morph.cpp</in>
+ <in>test_simple.cpp</in>
</df>
<logicalFolder name="ExternalFiles"
displayName="Important Files"
@@ -49,7 +49,7 @@
<buildCommandWorkingDir>build</buildCommandWorkingDir>
<buildCommand>${MAKE} -f Makefile</buildCommand>
<cleanCommand>${MAKE} -f Makefile clean</cleanCommand>
- <executablePath>build/fsa/test_dict</executablePath>
+ <executablePath>build/morfeusz/test_simple</executablePath>
</makeTool>
</makefileType>
<folder path="1">
@@ -120,10 +120,6 @@
<ccTool>
</ccTool>
</item>
- <item path="morfeusz/morfeusz.cpp" ex="false" tool="1" flavor2="4">
- <ccTool>
- </ccTool>
- </item>
<item path="morfeusz/test_morfeusz.cpp" ex="false" tool="1" flavor2="4">
<ccTool>
</ccTool>
@@ -132,12 +128,8 @@
<ccTool>
</ccTool>
</item>
- <item path="morfeusz/test_simple.cpp" ex="false" tool="1" flavor2="0">
+ <item path="morfeusz/test_simple.cpp" ex="false" tool="1" flavor2="8">
<ccTool>
- <incDir>
- <pElem>fsa</pElem>
- <pElem>build/morfeusz</pElem>
- </incDir>
</ccTool>
</item>
</conf>
diff --git a/testfiles/PoliMorfSmall.tab b/testfiles/PoliMorfSmall.tab
index c9bc25b..801a359 100644
--- a/testfiles/PoliMorfSmall.tab
+++ b/testfiles/PoliMorfSmall.tab
@@ -579,3 +579,43 @@ abdominoplastyki abdominoplastyka subst:pl:voc:f pospolita
abdominoplastyki abdominoplastyka subst:sg:gen:f pospolita
abdominoplastyko abdominoplastyka subst:sg:voc:f pospolita
abdominoplastykom abdominoplastyka subst:pl:dat:f pospolita
+mijał mijać praet:sg:m1.m2.m3:imperf pospolita
+mijała mijać praet:sg:f:imperf pospolita
+mijało mijać praet:sg:n1.n2:imperf pospolita
+mijały mijać praet:pl:m2.m3.f.n1.n2.p2.p3:imperf pospolita
+omijał omijać praet:sg:m1.m2.m3:imperf pospolita
+omijała omijać praet:sg:f:imperf pospolita
+omijało omijać praet:sg:n1.n2:imperf pospolita
+omijały omijać praet:pl:m2.m3.f.n1.n2.p2.p3:imperf pospolita
+pomijał pomijać praet:sg:m1.m2.m3:imperf pospolita
+pomijała pomijać praet:sg:f:imperf pospolita
+pomijało pomijać praet:sg:n1.n2:imperf pospolita
+pomijały pomijać praet:pl:m2.m3.f.n1.n2.p2.p3:imperf pospolita
+powymijał powymijać praet:sg:m1.m2.m3:perf pospolita
+powymijała powymijać praet:sg:f:perf pospolita
+powymijało powymijać praet:sg:n1.n2:perf pospolita
+powymijały powymijać praet:pl:m2.m3.f.n1.n2.p2.p3:perf pospolita
+przemijał przemijać praet:sg:m1.m2.m3:imperf pospolita
+przemijała przemijać praet:sg:f:imperf pospolita
+przemijało przemijać praet:sg:n1.n2:imperf pospolita
+przemijały przemijać praet:pl:m2.m3.f.n1.n2.p2.p3:imperf pospolita
+rozmijał rozmijać praet:sg:m1.m2.m3:imperf pospolita
+rozmijała rozmijać praet:sg:f:imperf pospolita
+rozmijało rozmijać praet:sg:n1.n2:imperf pospolita
+rozmijały rozmijać praet:pl:m2.m3.f.n1.n2.p2.p3:imperf pospolita
+wymijał wymijać praet:sg:m1.m2.m3:imperf pospolita
+wymijała wymijać praet:sg:f:imperf pospolita
+wymijało wymijać praet:sg:n1.n2:imperf pospolita
+wymijały wymijać praet:pl:m2.m3.f.n1.n2.p2.p3:imperf pospolita
+zmijał zmijać praet:sg:m1.m2.m3:imperf pospolita
+zmijała zmijać praet:sg:f:imperf pospolita
+zmijało zmijać praet:sg:n1.n2:imperf pospolita
+zmijały zmijać praet:pl:m2.m3.f.n1.n2.p2.p3:imperf pospolita
+em być aglt:sg:pri:imperf:wok pospolita
+eś być aglt:sg:sec:imperf:wok pospolita
+eście być aglt:pl:sec:imperf:wok pospolita
+eśmy być aglt:pl:pri:imperf:wok pospolita
+m być aglt:sg:pri:imperf:nwok pospolita
+ś być aglt:sg:sec:imperf:nwok pospolita
+ście być aglt:pl:sec:imperf:nwok pospolita
+śmy być aglt:pl:pri:imperf:nwok pospolita
diff --git a/testfiles/polimorf.tagset b/testfiles/polimorf.tagset
index 6a51843..27b3093 100644
--- a/testfiles/polimorf.tagset
+++ b/testfiles/polimorf.tagset
@@ -2,7 +2,7 @@
[TAGS]
-0 adj:pl:acc:m1.p1:com
+0 ign
1 adj:pl:acc:m1.p1:pos
2 adj:pl:acc:m1.p1:sup
3 adj:pl:acc:m2.m3.f.n1.n2.p2.p3:com
@@ -576,6 +576,7 @@
571 winien:sg:f:imperf
572 winien:sg:m1.m2.m3:imperf
573 winien:sg:n1.n2:imperf
+574 adj:pl:acc:m1.p1:com
[NAMES]