poprawienie buga w dodawaniu replacement char, ogólne porządki w kodzie

git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@168 ff4e3ee1-f430-4e82-ade0-24591c43f1fd

poprawienie buga w dodawaniu replacement char, ogólne porządki w kodzie
git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@168 ff4e3ee1-f430-4e82-ade0-24591c43f1fd
Michał Lenart
1 parent 39810b81
Showing 15 changed files with 58 additions and 86 deletions
CMakeLists.txt
morfeusz/CMakeLists.txt
morfeusz/Morfeusz.cpp
morfeusz/Morfeusz.hpp
morfeusz/MorphDeserializer.cpp
morfeusz/Qualifiers.cpp
morfeusz/charset/CharsetConverter.cpp
morfeusz/charset/CharsetConverter.hpp
morfeusz/cli/cli.cpp
morfeusz/cli/cli.hpp
morfeusz/deserializationUtils.hpp
morfeusz/endianness.hpp
morfeusz/fsa/fsa_impl.hpp
morfeusz/fsa/simplefsa_impl.hpp
morfeusz/segrules/segrules.cpp
@@ -5,7 +5,7 @@ project (Morfeusz)
 set (Morfeusz_VERSION_MAJOR 2)
 set (Morfeusz_VERSION_MINOR 0)
 set (Morfeusz_VERSION_PATCH 0)
-#~ set (CMAKE_BUILD_TYPE "Release")
+#~ set (CMAKE_BUILD_TYPE "Debug")
 enable_testing()
@@ -3,13 +3,13 @@
 ########## generate default dictionary data #################
 add_custom_command (
         OUTPUT "${INPUT_DICTIONARY_CPP}"
-        COMMAND python ${PROJECT_SOURCE_DIR}/fsabuilder/morfeusz_builder --analyzer --input-files="${INPUT_DICTIONARIES}" -o "${INPUT_DICTIONARY_CPP}" --tagset-file="${INPUT_TAGSET}" --segments-file="${SEGMENT_RULES_FILE}" --cpp --serialization-method=V2 --trim-supneg
+        COMMAND python ${PROJECT_SOURCE_DIR}/fsabuilder/morfeusz_builder --analyzer --input-files="${INPUT_DICTIONARIES}" -o "${INPUT_DICTIONARY_CPP}" --tagset-file="${INPUT_TAGSET}" --segments-file="${SEGMENT_RULES_FILE}" --cpp --serialization-method=V1 --trim-supneg
         DEPENDS "${INPUT_DICTIONARY}"
         COMMENT "Building default dictionary C++ file"
 )
 add_custom_command (
         OUTPUT "${INPUT_SYNTH_DICTIONARY_CPP}"
-        COMMAND python ${PROJECT_SOURCE_DIR}/fsabuilder/morfeusz_builder --generator --input-files="${INPUT_DICTIONARIES}" -o "${INPUT_SYNTH_DICTIONARY_CPP}" --tagset-file="${INPUT_TAGSET}" --segments-file="${SEGMENT_RULES_FILE}" --cpp --serialization-method=V2
+        COMMAND python ${PROJECT_SOURCE_DIR}/fsabuilder/morfeusz_builder --generator --input-files="${INPUT_DICTIONARIES}" -o "${INPUT_SYNTH_DICTIONARY_CPP}" --tagset-file="${INPUT_TAGSET}" --segments-file="${SEGMENT_RULES_FILE}" --cpp --serialization-method=SIMPLE
         DEPENDS "${INPUT_DICTIONARY}"
         COMMENT "Building default dictionary C++ file"
 )
@@ -66,13 +66,6 @@ void Morfeusz::setGeneratorFile(const string&amp; filename) {
 Morfeusz::~Morfeusz() {
 }
-string Morfeusz::prepareStringToProcess(const std::string& str) const {
-    string res;
-    res.reserve(str.size());
-    utf8::replace_invalid(str.begin(), str.end(), back_inserter(res));
-    return res;
-}
-
 void Morfeusz::processOneWord(
         const Environment& env,
         const char*& inputStart,
@@ -88,9 +81,9 @@ void Morfeusz::processOneWord(
     InflexionGraph graph;
     const char* currInput = inputStart;
     const SegrulesFSA& segrulesFSA = env.getCurrentSegrulesFSA();
-    
+
     doProcessOneWord(env, currInput, inputEnd, segrulesFSA.initialState, accum, graph);
-    
+
     if (!graph.empty()) {
         const InterpretedChunksDecoder& interpretedChunksDecoder = env.getInterpretedChunksDecoder();
         int srcNode = startNodeNum;
@@ -148,10 +141,10 @@ void Morfeusz::doProcessOneWord(
         SegrulesState segrulesState,
         vector<InterpretedChunk>& accum,
         InflexionGraph& graph) const {
-    //    if (this->options.debug) {
-    //        cerr << "----------" << endl;
-    //        cerr << "PROCESS: '" << inputData << "', already recognized: " << debugAccum(accum) << endl;
-    //    }
+    if (this->options.debug) {
+        cerr << "----------" << endl;
+        cerr << "PROCESS: '" << inputData << "', already recognized: " << debugAccum(accum) << endl;
+    }
     //    cerr << "doAnalyzeOneWord " << inputData << endl;
     const char* inputStart = inputData;
     const char* currInput = inputData;
@@ -303,9 +296,8 @@ ResultsIterator Morfeusz::analyze(const string&amp; text) const {
 }
 void Morfeusz::analyze(const string& text, vector<MorphInterpretation>& results) const {
-    string preparedText = this->prepareStringToProcess(text);
-    const char* input = preparedText.c_str();
-    const char* inputEnd = input + preparedText.length();
+    const char* input = text.c_str();
+    const char* inputEnd = input + text.length();
     while (input != inputEnd) {
         int startNode = results.empty() ? 0 : results.back().getEndNode();
         this->processOneWord(this->analyzerEnv, input, inputEnd, startNode, results);
@@ -324,8 +316,7 @@ ResultsIterator Morfeusz::generate(const string&amp; text, int tagnum) const {
     return ResultsIterator(res);
 }
-void Morfeusz::generate(const string& text, vector<MorphInterpretation>& results) const {
-    string lemma = this->prepareStringToProcess(text);
+void Morfeusz::generate(const string& lemma, vector<MorphInterpretation>& results) const {
     const char* input = lemma.c_str();
     const char* inputEnd = input + lemma.length();
     int startNode = 0;
@@ -158,8 +158,6 @@ public:
     friend class ResultsIterator;
 private:
-    std::string prepareStringToProcess(const std::string& input) const;
-
     void processOneWord(
             const Environment& env,
             const char*& inputData,
@@ -11,6 +11,7 @@
 #include "MorphDeserializer.hpp"
 #include "EncodedInterpretation.hpp"
 #include "InterpsGroup.hpp"
+#include "deserializationUtils.hpp"
 //const uint8_t LEMMA_ONLY_LOWER = 0;
 //const uint8_t LEMMA_UPPER_PREFIX = 1;
@@ -25,16 +26,13 @@ MorphDeserializer::~MorphDeserializer() {
 long MorphDeserializer::deserialize(const unsigned char* ptr, vector<InterpsGroup>& interps) const {
     const unsigned char* currPtr = ptr;
-    uint8_t interpTypesNum = *currPtr;
-    currPtr++;
+    uint8_t interpTypesNum = readInt8(currPtr);
     interps.clear();
     interps.reserve(interpTypesNum);
     for (unsigned int i = 0; i < interpTypesNum; i++) {
         InterpsGroup ig;
-        ig.type = *currPtr;
-        currPtr++;
-        ig.size = ntohs(*(reinterpret_cast<const uint16_t*>(currPtr)));
-        currPtr += 2;
+        ig.type = readInt8(currPtr);
+        ig.size = readInt16(currPtr);
         ig.ptr = currPtr;
         currPtr += ig.size;
         interps.push_back(ig);
@@ -14,7 +14,7 @@ using namespace std;
 Qualifiers::Qualifiers(const unsigned char* ptr):
 qualifiers() {
-    uint32_t fsaSize = htonl(*reinterpret_cast<const uint32_t*>(ptr + FSA_DATA_SIZE_OFFSET));
+    uint32_t fsaSize = readInt32Const(ptr + FSA_DATA_SIZE_OFFSET);
     const unsigned char* currPtr = ptr + FSA_DATA_OFFSET + fsaSize + 4;
     vector<string> _dupa;
     readTags(currPtr, _dupa);
@@ -23,7 +23,7 @@ qualifiers() {
     uint16_t allCombinationsSize = readInt16(currPtr);
     this->qualifiers.reserve(allCombinationsSize);
     for (unsigned int i = 0; i < allCombinationsSize; i++) {
-        unsigned char qualsNum = *currPtr++;
+        unsigned char qualsNum = readInt8(currPtr);
         vector<string> quals;
         for (unsigned int j = 0; j < qualsNum; j++) {
             quals.push_back(readString(currPtr));
@@ -11,7 +11,7 @@
 using namespace std;
-const char DEFAULT_UNDEFINED_CHAR = static_cast<char>(0xF7);
+const char DEFAULT_UNDEFINED_CHAR = static_cast<char> (0xF7);
 string CharsetConverter::toString(const vector<uint32_t>& codepoints) const {
     string res;
@@ -22,22 +22,31 @@ string CharsetConverter::toString(const vector&lt;uint32_t&gt;&amp; codepoints) const {
 }
 CharsetConverter::~CharsetConverter() {
-    
+
 }
-uint32_t UTF8CharsetConverter::peek(const char*& it, const char* end) const {
-    return utf8::unchecked::peek_next(it);
+uint32_t CharsetConverter::peek(const char* it, const char* end) const {
+    return this->next(it, end);
+}
+
+static inline void iterateThroughInvalidUtf8Sequence(const char*& it, const char* end) {
+    uint32_t _dupa;
+    while (it != end && utf8::internal::validate_next(it, end, _dupa) != utf8::internal::UTF8_OK) {
+        it++;
+    }
 }
 uint32_t UTF8CharsetConverter::next(const char*& it, const char* end) const {
-    return utf8::unchecked::next(it);
-//    catch (utf8::exception ex) {
-//        cerr << "WARNING: Replacing invalid character: " << hex << (uint16_t) *it << dec << " with replacement char: 0xFFFD" << endl;
-//        while (it != end && peek(it, end) == 0xFFFD) {
-//            utf8::unchecked::next(it);
-//        }
-//        return 0xFFFD;
-//    }
+    uint32_t cp = 0;
+    utf8::internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
+    if (err_code == utf8::internal::UTF8_OK) {
+        return cp;
+    }
+    else {
+        cerr << "WARNING: Replacing invalid sequence with replacement char: 0xFFFD" << endl;
+        iterateThroughInvalidUtf8Sequence(it, end);
+        return 0xFFFD;
+    }
 }
 void UTF8CharsetConverter::append(uint32_t cp, string& result) const {
@@ -52,7 +61,8 @@ static vector&lt;char&gt; reverseArray(const uint32_t* array) {
         res.resize(max(static_cast<uint32_t> (res.size()), codepoint + 1), DEFAULT_UNDEFINED_CHAR);
         res[codepoint] = static_cast<char> (c);
         c++;
-    }    while (c != 255);
+    }
+    while (c != 255);
     return res;
 }
@@ -63,18 +73,15 @@ codepoint2Char(reverseArray(array)) {
 // TODO - sprawdzanie zakresu
-uint32_t OneByteCharsetConverter::peek(const char*& it, const char* end) const {
-    return this->array[static_cast<unsigned char> (*it)];
-}
-
 uint32_t OneByteCharsetConverter::next(const char*& it, const char* end) const {
-    return this->array[static_cast<unsigned char> (*(it++))];
+    return this->array[static_cast<unsigned char> (*it++)];
 }
 void OneByteCharsetConverter::append(uint32_t cp, std::string& result) const {
     if (cp < this->codepoint2Char.size()) {
         result.push_back(this->codepoint2Char[cp]);
-    } else {
+    }
+    else {
         result.push_back(DEFAULT_UNDEFINED_CHAR);
     }
 }
@@ -15,7 +15,7 @@
 class CharsetConverter {
 public:
-    virtual uint32_t peek(const char*& it, const char* end) const = 0;
+    uint32_t peek(const char* it, const char* end) const;
     virtual uint32_t next(const char*& it, const char* end) const = 0;
     virtual void append(uint32_t cp, std::string& result) const = 0;
     virtual std::string fromUTF8(const std::string& input) const;
@@ -28,7 +28,6 @@ private:
 class UTF8CharsetConverter : public CharsetConverter {
 public:
-    uint32_t peek(const char*& it, const char* end) const;
     uint32_t next(const char*& it, const char* end) const;
     void append(uint32_t cp, std::string& result) const;
     //    std::string fromUTF8(const std::string& input) const;
@@ -41,7 +40,6 @@ private:
 class OneByteCharsetConverter : public CharsetConverter {
 public:
     explicit OneByteCharsetConverter(const uint32_t* array);
-    uint32_t peek(const char*& it, const char* end) const;
     uint32_t next(const char*& it, const char* end) const;
     void append(uint32_t cp, std::string& result) const;
 private:
@@ -139,10 +139,7 @@ void initializeMorfeusz(ezOptionParser&amp; opt, Morfeusz&amp; morfeusz) {
         cerr << "setting case sensitive to FALSE" << endl;
         morfeusz.setCaseSensitive(false);
     }
-#ifdef _WIN32
-    morfeusz.setCharset(CP852);
-#endif
-#ifdef _WIN64
+#if defined(_WIN64) || defined(_WIN32)
     morfeusz.setCharset(CP852);
 #endif
 }
@@ -8,14 +8,9 @@
 #ifndef CLI_HPP
 #define	CLI_HPP
-#ifdef _WIN64
+#if defined(_WIN64) || defined(_WIN32)
 #define TMPDUPA_IN IN
 #define IN  IN
-#else
-#ifdef _WIN32
-#define TMPDUPA_IN IN
-#define IN  IN
-#endif
 #endif
 #include <iostream>
@@ -40,12 +35,8 @@ void initializeMorfeusz(ez::ezOptionParser&amp; opt, Morfeusz&amp; morfeusz);
 #pragma GCC diagnostic pop
-#ifdef _WIN64
+#if defined(_WIN64) || defined(_WIN32)
 #define IN  TMPDUPA_IN
-#else
-#ifdef _WIN32
-#define IN  TMPDUPA_IN
-#endif
 #endif
 #endif	/* CLI_HPP */
@@ -27,6 +27,11 @@ inline uint32_t readInt32(const unsigned char*&amp; currPtr) {
     return res;
 }
+inline uint32_t readInt32Const(const unsigned char* const currPtr) {
+    uint32_t res = htonl(*reinterpret_cast<const uint32_t*> (currPtr));
+    return res;
+}
+
 inline std::string readString(const unsigned char*& currPtr) {
     std::string res((const char*) currPtr);
     currPtr += res.length();
@@ -8,15 +8,11 @@
 #ifndef ENDIANNESS_HPP
 #define	ENDIANNESS_HPP
-#ifdef _WIN64
-#include <winsock2.h>
-#else
-#ifdef _WIN32
+#if defined(_WIN64) || defined(_WIN32)
 #include <winsock2.h>
 #else
 #include <netinet/in.h>
 #endif
-#endif
 #endif	/* ENDIANNESS_HPP */
@@ -78,8 +78,6 @@ FSA&lt;T&gt;* FSA&lt;T&gt;::getFSA(const unsigned char* ptr, const Deserializer&lt;T&gt;&amp; deserial
     uint8_t implementationNum = *(ptr + IMPLEMENTATION_NUM_OFFSET);
-//    uint32_t additionalDataSize = ntohl(*(reinterpret_cast<const uint32_t*>(ptr + ADDITIONAL_DATA_SIZE_OFFSET)));
-    
     const unsigned char* startPtr = ptr + FSA_DATA_OFFSET;
     switch (implementationNum) {
         case 0:
@@ -63,7 +63,6 @@ void SimpleFSA&lt;T&gt;::proceedToNext(const char c, State&lt;T&gt;&amp; state) const {
             break;
         }
     }
-    //    const_cast<Counter*>(&counter)->increment(foundTransition - transitionsStart + 1);
     if (!found) {
         state.setNextAsSink();
     }
@@ -14,16 +14,12 @@ static inline void skipSeparatorsList(const unsigned char*&amp; ptr) {
 static inline const unsigned char* getSeparatorsListPtr(const unsigned char* ptr) {
     const unsigned char* additionalDataPtr = ptr 
         + FSA_DATA_OFFSET 
-        + ntohl(*reinterpret_cast<const uint32_t*>(ptr + FSA_DATA_SIZE_OFFSET));
-    const unsigned char* res = additionalDataPtr + readInt32(additionalDataPtr) + 4;
+        + readInt32Const(ptr + FSA_DATA_SIZE_OFFSET);
+    const unsigned char* res = additionalDataPtr + readInt32Const(additionalDataPtr) + 4;
     return res;
 }
 static inline const unsigned char* getFSAsMapPtr(const unsigned char* ptr) {
-//    const unsigned char* additionalDataPtr = ptr 
-//        + FSA_DATA_OFFSET 
-//        + ntohl(*reinterpret_cast<const uint32_t*>(ptr + FSA_DATA_SIZE_OFFSET));
-//    const unsigned char* res = additionalDataPtr + deserializeUint32(additionalDataPtr) + 4;
     const unsigned char* res = getSeparatorsListPtr(ptr);
     skipSeparatorsList(res);
     return res;
@@ -84,11 +80,9 @@ SegrulesFSA* getDefaultSegrulesFSA(
 vector<uint32_t> getSeparatorsList(const unsigned char* ptr) {
     ptr = getSeparatorsListPtr(ptr);
     vector<uint32_t> res;
-    uint16_t listSize = ntohs(*reinterpret_cast<const uint16_t*>(ptr));
-    ptr += 2;
+    uint16_t listSize = readInt16(ptr);
     for (unsigned int i = 0; i < listSize; i++) {
-        res.push_back(ntohl(*reinterpret_cast<const uint32_t*>(ptr)));
-        ptr += 4;
+        res.push_back(readInt32(ptr));
     }
     return res;
 }