dodanie opcji "debug" dla analizatora i generatora

git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@140 ff4e3ee1-f430-4e82-ade0-24591c43f1fd

dodanie opcji "debug" dla analizatora i generatora
git-svn-id: svn://svn.nlp.ipipan.waw.pl/morfeusz/morfeusz@140 ff4e3ee1-f430-4e82-ade0-24591c43f1fd
Michał Lenart
1 parent 53f1a33d
Showing 9 changed files with 72 additions and 26 deletions
fsabuilder/morfeuszbuilder/fsa/fsa.py
fsabuilder/morfeuszbuilder/segrules/rulesFSA.py
fsabuilder/morfeuszbuilder/tagset/segtypes.py
morfeusz/InterpretedChunk.hpp
morfeusz/InterpsGroup.hpp
morfeusz/Morfeusz.cpp
morfeusz/Morfeusz.hpp
morfeusz/MorfeuszOptions.hpp
morfeusz/cli/cli.cpp
@@ -41,9 +41,9 @@ class FSA(object):
         self.n += 1
         # debug
-        if self.n % 10000 == 0:
-            logging.info(word)
-            logging.info(str(self.register.getStatesNum()))
+        if self.n % 100000 == 0:
+            logging.info(u'%d %s' % (self.n, word))
+#             logging.info(str(self.register.getStatesNum()))
     #             allWords.append(word)
         for label in encodedWord:
             self.label2Freq[label] = self.label2Freq.get(label, 0) + 1
@@ -68,6 +68,6 @@ class RulesFSA(object):
             res.extend(self.stateData2bytearray(state))
             res.extend(self.transitionsData2bytearray(state))
-        logging.info('Segmentation automaton size: %d bytes', len(res))
-        print list(res)
+#         logging.info('Segmentation automaton size: %d bytes', len(res))
+#         print list(res)
         return res
@@ -5,6 +5,7 @@ Created on 17 lut 2014
 '''
 import re
 import logging
+import sys
 from morfeuszbuilder.utils import exceptions
 def _cutHomonymFromLemma(lemma):
@@ -33,10 +34,12 @@ class Segtypes(object):
         self._readTags(segrulesConfigFile)
         self._indexSegnums()
-        print self._lemmaTagnum2Segnum
-        print self._tagnum2Segnum
-        
-        print self.segnum2Segtype
+#         print self._lemmaTagnum2Segnum
+#         print self._tagnum2Segnum
+        logging.info('segment number -> segment type')
+        logging.info('------------------------------')
+        logging.info(str(self.segnum2Segtype))
+        logging.info('------------------------------')
 #         self._debugSegnums()
@@ -13,6 +13,7 @@
 struct InterpretedChunk {
     const char* chunkStartPtr;
+    const char* chunkEndPtr;
     std::vector<uint32_t> originalCodepoints;
     std::vector<uint32_t> lowercaseCodepoints;
     InterpsGroup interpsGroup;
@@ -15,25 +15,9 @@
 #include "Tagset.hpp"
 struct InterpsGroup {
-//public:
-//    
-//    InterpsGroup() {
-//        
-//    }
-//    
-//    explicit InterpsGroup(const unsigned char type)
-//    : type(type) {
-//        
-//    }
-//    
-//    void addInterpretation(const EncodedInterpretation& interp) {
-//        interps.push_back(interp);
-//    }
-    
     unsigned char type;
     uint16_t size;
     const unsigned char* ptr;
-//    std::vector<EncodedInterpretation> interps;
 };
 #endif	/* GROUPEDINTERPRETATIONS_HPP */
@@ -28,6 +28,7 @@ static MorfeuszOptions createDefaultOptions() {
     MorfeuszOptions res;
     res.caseSensitive = true;
     res.encoding = UTF8;
+    res.debug = false;
     return res;
 }
@@ -102,6 +103,21 @@ static inline void doShiftOrth(InterpretedChunk&amp; from, InterpretedChunk&amp; to) {
     to.chunkStartPtr = from.chunkStartPtr;
 }
+static inline string debugInterpsGroup(unsigned char type, const char* startPtr, const char* endPtr) {
+    stringstream res;
+    res << "(" << (int) type << ", " << string(startPtr, endPtr) << "), ";
+    return res.str();
+}
+
+static inline string debugAccum(vector<InterpretedChunk>& accum) {
+    stringstream res;
+    for (unsigned int i = 0; i < accum.size(); i++) {
+        res << debugInterpsGroup(accum[i].interpsGroup.type, accum[i].chunkStartPtr, accum[i].chunkEndPtr);
+//        res << "(" << (int) accum[i].interpsGroup.type << ", " << string(accum[i].chunkStartPtr, accum[i].chunkStartPtr) << "), ";
+    }
+    return res.str();
+}
+
 void Morfeusz::doProcessOneWord(
         const Environment& env,
         const char*& inputData,
@@ -109,7 +125,12 @@ void Morfeusz::doProcessOneWord(
         SegrulesState segrulesState,
         vector<InterpretedChunk>& accum,
         InflexionGraph& graph) const {
+//    if (this->options.debug) {
+//        cerr << "----------" << endl;
+//        cerr << "PROCESS: '" << inputData << "', already recognized: " << debugAccum(accum) << endl;
+//    }
 //    cerr << "doAnalyzeOneWord " << inputData << endl;
+    const char* inputStart = inputData;
     const char* currInput = inputData;
     uint32_t codepoint = inputData == inputEnd ? 0 : env.getCharsetConverter().next(currInput, inputEnd);
     vector<uint32_t> originalCodepoints;
@@ -139,9 +160,15 @@ void Morfeusz::doProcessOneWord(
             vector<InterpsGroup> val(state.getValue());
             for (unsigned int i = 0; i < val.size(); i++) {
                 InterpsGroup& ig = val[i];
+                if (this->options.debug) {
+                    cerr << "recognized: " << debugInterpsGroup(ig.type, inputStart, currInput) << " at: '" << inputStart << "'" << endl;
+                }
 //                cerr << "accept at '" << currInput << "' type=" << (int) ig.type << endl;
                 set<SegrulesState> newSegrulesStates;
                 env.getCurrentSegrulesFSA().proceedToNext(ig.type, segrulesState, newSegrulesStates);
+                if (this->options.debug && newSegrulesStates.empty()) {
+                    cerr << "NOT ACCEPTING " << debugAccum(accum) << debugInterpsGroup(ig.type, inputStart, currInput) << endl;
+                }
 //                cerr << "newSegrulesStates.size() " << newSegrulesStates.size() << endl;
                 for (
                         set<SegrulesState>::iterator it = newSegrulesStates.begin();
@@ -149,7 +176,8 @@ void Morfeusz::doProcessOneWord(
                         ++it) {
                     SegrulesState newSegrulesState = *it;
                     InterpretedChunk ic = {
-                        inputData,
+                        inputStart,
+                        currInput,
                         originalCodepoints,
                         normalizedCodepoints,
                         ig,
@@ -164,6 +192,9 @@ void Morfeusz::doProcessOneWord(
                     accum.push_back(ic);
                     if (isEndOfWord(codepoint) 
                             && newSegrulesState.accepting) {
+                        if (this->options.debug) {
+                            cerr << "ACCEPTING " << debugAccum(accum) << endl;
+                        }
                         graph.addPath(accum, newSegrulesState.weak);
                     }
                     else if (!isEndOfWord(codepoint)) {
@@ -255,6 +286,10 @@ void Morfeusz::setPraet(const std::string&amp; praet) {
     this->generatorEnv.setSegrulesOption("praet", praet);
 }
+void Morfeusz::setDebug(bool debug) {
+    this->options.debug = debug;
+}
+
 ResultsIterator::ResultsIterator(const vector<MorphInterpretation>& res) {
     resultsBuffer.insert(resultsBuffer.begin(), res.begin(), res.end());
 }
@@ -139,6 +139,13 @@ public:
      * @param praet
      */
     void setPraet(const std::string& praet);
+    
+    /**
+     * Set debug option value.
+     * 
+     * @param praet
+     */
+    void setDebug(bool debug);
     friend class ResultsIterator;
 private:
@@ -13,6 +13,7 @@
 struct MorfeuszOptions {
     bool caseSensitive;
     MorfeuszCharset encoding;
+    bool debug;
 };
 #endif	/* MORFEUSZOPTIONS_HPP */
@@ -65,6 +65,17 @@ ezOptionParser* getOptions(int argc, const char** argv, const string&amp; titleText)
             "-praet", // Flag token.
             "--praet" // Flag token.
             );
+    
+    opt.add(
+            "", // Default.
+            0, // Required?
+            0, // Number of args expected.
+            0, // Delimiter if expecting multiple args.
+            "praet option.", // Help description.
+            "-d", // Flag token. 
+            "-debug", // Flag token.
+            "--debug" // Flag token.
+            );
     opt.parse(argc, argv);
@@ -105,6 +116,10 @@ void initializeMorfeusz(ezOptionParser&amp; opt, Morfeusz&amp; morfeusz) {
         cerr << "setting praet option to " << praet << endl;
         morfeusz.setPraet(praet);
     }
+    if (opt.isSet("-d")) {
+        cerr << "setting debug to TRUE" << endl;
+        morfeusz.setDebug(true);
+    }
 #ifdef _WIN32
     morfeusz.setCharset(CP852);
 #endif