Merge branch 'master' of https://github.com/meertensinstituut/mtas

# Conflicts: # conf/parser/mtas/crm_test.xml # conf/parser/mtas/folia_ddd.xml # pom.xml # src/mtas/analysis/MtasTokenizer.java # src/mtas/analysis/parser/MtasCRMParser.java # src/mtas/analysis/token/MtasTokenCollection.java # src/mtas/codec/MtasCodecPostingsFormat.java # src/mtas/codec/MtasFieldsConsumer.java # src/mtas/codec/util/CodecComponent.java # src/mtas/codec/util/CodecInfo.java # src/mtas/codec/util/DataCollector.java # src/mtas/codec/util/collector/MtasDataAdvanced.java # src/mtas/codec/util/collector/MtasDataBasic.java # src/mtas/codec/util/collector/MtasDataDoubleAdvanced.java # src/mtas/codec/util/collector/MtasDataDoubleBasic.java # src/mtas/codec/util/collector/MtasDataDoubleFull.java # src/mtas/codec/util/collector/MtasDataFull.java # src/mtas/codec/util/collector/MtasDataItem.java # src/mtas/codec/util/collector/MtasDataItemAdvanced.java # src/mtas/codec/util/collector/MtasDataItemBasic.java # src/mtas/codec/util/collector/MtasDataItemDoubleAdvanced.java # src/mtas/codec/util/collector/MtasDataItemDoubleBasic.java # src/mtas/codec/util/collector/MtasDataItemDoubleFull.java # src/mtas/codec/util/collector/MtasDataItemFull.java # src/mtas/codec/util/collector/MtasDataItemLongAdvanced.java # src/mtas/codec/util/collector/MtasDataItemLongBasic.java # src/mtas/codec/util/collector/MtasDataLongAdvanced.java # src/mtas/codec/util/collector/MtasDataLongBasic.java # src/mtas/codec/util/collector/MtasDataLongFull.java # src/mtas/parser/cql/util/MtasCQLParserDefaultPrefixCondition.java # src/mtas/parser/cql/util/MtasCQLParserSentenceCondition.java # src/mtas/parser/cql/util/MtasCQLParserSentencePartCondition.java # src/mtas/solr/handler/component/MtasSolrSearchComponent.java # src/mtas/solr/search/MtasCQLQParser.java # src/mtas/solr/update/processor/MtasUpdateRequestProcessorFactory.java # src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultWriter.java # src/site/markdown/download.md.vm # src/site/markdown/index.md # src/site/markdown/installation.md # src/site/markdown/installation_lucene.md # src/site/markdown/installation_solr.md # src/site/site.xml

Merge branch 'master' of https://github.com/meertensinstituut/mtas
# Conflicts: # conf/parser/mtas/crm_test.xml # conf/parser/mtas/folia_ddd.xml # pom.xml # src/mtas/analysis/MtasTokenizer.java # src/mtas/analysis/parser/MtasCRMParser.java # src/mtas/analysis/token/MtasTokenCollection.java # src/mtas/codec/MtasCodecPostingsFormat.java # src/mtas/codec/MtasFieldsConsumer.java # src/mtas/codec/util/CodecComponent.java # src/mtas/codec/util/CodecInfo.java # src/mtas/codec/util/DataCollector.java # src/mtas/codec/util/collector/MtasDataAdvanced.java # src/mtas/codec/util/collector/MtasDataBasic.java # src/mtas/codec/util/collector/MtasDataDoubleAdvanced.java # src/mtas/codec/util/collector/MtasDataDoubleBasic.java # src/mtas/codec/util/collector/MtasDataDoubleFull.java # src/mtas/codec/util/collector/MtasDataFull.java # src/mtas/codec/util/collector/MtasDataItem.java # src/mtas/codec/util/collector/MtasDataItemAdvanced.java # src/mtas/codec/util/collector/MtasDataItemBasic.java # src/mtas/codec/util/collector/MtasDataItemDoubleAdvanced.java # src/mtas/codec/util/collector/MtasDataItemDoubleBasic.java # src/mtas/codec/util/collector/MtasDataItemDoubleFull.java # src/mtas/codec/util/collector/MtasDataItemFull.java # src/mtas/codec/util/collector/MtasDataItemLongAdvanced.java # src/mtas/codec/util/collector/MtasDataItemLongBasic.java # src/mtas/codec/util/collector/MtasDataLongAdvanced.java # src/mtas/codec/util/collector/MtasDataLongBasic.java # src/mtas/codec/util/collector/MtasDataLongFull.java # src/mtas/parser/cql/util/MtasCQLParserDefaultPrefixCondition.java # src/mtas/parser/cql/util/MtasCQLParserSentenceCondition.java # src/mtas/parser/cql/util/MtasCQLParserSentencePartCondition.java # src/mtas/solr/handler/component/MtasSolrSearchComponent.java # src/mtas/solr/search/MtasCQLQParser.java # src/mtas/solr/update/processor/MtasUpdateRequestProcessorFactory.java # src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultWriter.java # src/site/markdown/download.md.vm # src/site/markdown/index.md # src/site/markdown/installation.md # src/site/markdown/installation_lucene.md # src/site/markdown/installation_solr.md # src/site/site.xml
Matthijs Brouwer
2 parents 8148835a cac6601c
Showing 22 changed files with 466 additions and 425 deletions
conf/parser/mtas/crm_test.xml
conf/parser/mtas/folia_ddd.xml
pom.xml
src/mtas/analysis/MtasTokenizer.java
src/mtas/analysis/parser/MtasSketchParser.java
src/mtas/analysis/token/MtasTokenCollection.java
src/mtas/codec/MtasCodecPostingsFormat.java
src/mtas/codec/MtasFieldsConsumer.java
src/mtas/codec/util/CodecCollector.java
src/mtas/codec/util/CodecComponent.java
src/mtas/codec/util/CodecInfo.java
src/mtas/codec/util/DataCollector.java
src/mtas/codec/util/collector/MtasDataAdvanced.java
src/mtas/codec/util/collector/MtasDataBasic.java
src/mtas/codec/util/collector/MtasDataCollector.java
src/mtas/codec/util/collector/MtasDataFull.java
src/mtas/codec/util/collector/MtasDataItem.java
src/mtas/parser/cql/util/MtasCQLParserSentenceCondition.java
src/mtas/solr/handler/component/MtasSolrSearchComponent.java
src/site/markdown/download.md.vm
@@ -17,16 +17,20 @@
 	<!-- START CONFIGURATION MTAS FOLIA PARSER -->
 	<parser name="mtas.analysis.parser.MtasCRMParser">
+<<<<<<< HEAD
 		<!-- START GENERAL SETTINGS MTAS PARSER -->
 		<autorepair value="true" />
 		<makeunique value="true" />
 		<!-- END GENERAL SETTINGS MTAS PARSER -->
+=======
+>>>>>>> cac6601c611fd300b075447ee1eb8299e5bffc90
 		<mappings>
 			<mapping type="word">
 			</mapping>
+<<<<<<< HEAD
 			<mapping type="wordAnnotation" name="0">
 				<token type="string" offset="false" parent="false">
 					<pre>
@@ -71,12 +75,19 @@
 				<token type="string" offset="false" parent="false">
 					<pre>
 						<item type="string" value="t2" />
+=======
+			<mapping type="wordAnnotation" name="2">
+				<token type="string" offset="false" parent="false">
+					<pre>
+						<item type="string" value="t" />
+>>>>>>> cac6601c611fd300b075447ee1eb8299e5bffc90
 					</pre>
 					<post>
 						<item type="text" />
 					</post>
 				</token>
 			</mapping>
+<<<<<<< HEAD
 			<mapping type="wordAnnotation" name="2">
 				<token type="string" offset="false" parent="false">
 					<pre>
@@ -87,6 +98,8 @@
 					</post>
 				</token>
 			</mapping>
+=======
+>>>>>>> cac6601c611fd300b075447ee1eb8299e5bffc90
 			<mapping type="wordAnnotation" name="3">
 				<token type="string" offset="false" parent="false">
 					<pre>
@@ -98,6 +111,7 @@
 				</token>
 			</mapping>
 			<mapping type="wordAnnotation" name="4">
+<<<<<<< HEAD
 				<token type="string" offset="false" parent="false">
 					<pre>
 						<item type="string" value="crm" />
@@ -114,12 +128,34 @@
 				</condition>
 			</mapping>
 			<mapping type="crmPair" name="part">
+=======
+			</mapping>
+			<mapping type="wordAnnotation" name="5">			  
+      </mapping>
+			<mapping type="wordAnnotation" name="6">        
+      </mapping>
+      <mapping type="wordAnnotation" name="7">
+        <token type="string" offset="false" parent="false">
+          <pre>
+            <item type="string" value="sentence" />
+          </pre>
+          <post>
+            <item type="text" />
+          </post>         
+        </token>
+        <condition>
+          <item type="text" not="true" condition="-" />
+        </condition>
+      </mapping>
+      <mapping type="wordAnnotation" name="pos">
+>>>>>>> cac6601c611fd300b075447ee1eb8299e5bffc90
 				<token type="string" offset="false" parent="false">
 					<pre>
 						<item type="name" />
 					</pre>
 					<post>
 						<item type="text" />
+<<<<<<< HEAD
 					</post>
 				</token>
 			</mapping>
@@ -282,11 +318,111 @@
 				</condition>
 			</function>
 			<function type="wordAnnotation" name="4" split="+">
+=======
+					</post>					
+				</token>
+			</mapping>
+      <mapping type="wordAnnotation" name="feat.getal">
+        <token type="string" offset="false" parent="false">
+          <pre>
+            <item type="name" />
+          </pre>
+          <post>
+            <item type="text" />
+          </post>         
+        </token>
+      </mapping>
+      <mapping type="wordAnnotation" name="feat.persoon">
+        <token type="string" offset="false" parent="false">
+          <pre>
+            <item type="name" />
+          </pre>
+          <post>
+            <item type="text" />
+          </post>         
+        </token>
+      </mapping>
+      <mapping type="wordAnnotation" name="feat.ntype">
+        <token type="string" offset="false" parent="false">
+          <pre>
+            <item type="name" />
+          </pre>
+          <post>
+            <item type="text" />
+          </post>         
+        </token>
+      </mapping>
+      <mapping type="wordAnnotation" name="feat.pvtijd">
+        <token type="string" offset="false" parent="false">
+          <pre>
+            <item type="name" />
+          </pre>
+          <post>
+            <item type="text" />
+          </post>         
+        </token>
+      </mapping>
+      <mapping type="wordAnnotation" name="feat.wvorm">
+        <token type="string" offset="false" parent="false">
+          <pre>
+            <item type="name" />
+          </pre>
+          <post>
+            <item type="text" />
+          </post>         
+        </token>
+      </mapping>
+      <mapping type="wordAnnotation" name="feat.numtype">
+        <token type="string" offset="false" parent="false">
+          <pre>
+            <item type="name" />
+          </pre>
+          <post>
+            <item type="text" />
+          </post>         
+        </token>
+      </mapping>
+      <mapping type="wordAnnotation" name="feat.vwtype">
+        <token type="string" offset="false" parent="false">
+          <pre>
+            <item type="name" />
+          </pre>
+          <post>
+            <item type="text" />
+          </post>         
+        </token>
+      </mapping>
+      <mapping type="wordAnnotation" name="feat.lwtype">
+        <token type="string" offset="false" parent="false">
+          <pre>
+            <item type="name" />
+          </pre>
+          <post>
+            <item type="text" />
+          </post>         
+        </token>
+      </mapping>
+      <mapping type="wordAnnotation" name="feat.probleemgeval">
+        <token type="string" offset="false" parent="false">
+          <pre>
+            <item type="name" />
+          </pre>
+          <post>
+            <item type="text" />
+          </post>         
+        </token>
+      </mapping>
+		</mappings>
+
+		<functions>
+			<function name="4" split="+">
+>>>>>>> cac6601c611fd300b075447ee1eb8299e5bffc90
 				<condition value="000,001,002,003,004,005,006,009">
 					<output name="pos" value="N" />
 					<output name="feat.getal" value="ev" />
 				</condition>
 				<condition value="010,011,012,013,014,015,016,019">
+<<<<<<< HEAD
 					<output name="pos" value="N" />
 					<output name="feat.getal" value="mv" />
 				</condition>
@@ -602,6 +738,167 @@
 					<output name="feat.form" value="unclear" />
 				</condition>
+=======
+          <output name="pos" value="N" />
+          <output name="feat.getal" value="mv" />
+        </condition>
+        <condition value="020,021,022,023,024,025,026,029">
+          <output name="pos" value="N" />
+          <output name="feat.ntype" value="eigen" />
+        </condition>
+        <condition value="090,091,092,093,094,095,096,099">
+          <output name="pos" value="N" />
+          <output name="feat.probleemgeval" />
+        </condition>
+        <condition value="100,101,102,103,104,105,106,109">
+          <output name="pos" value="ADJ" />
+          <output name="feat.getal" value="ev" />
+        </condition>
+        <condition value="110,111,112,113,114,115,116,119">
+          <output name="pos" value="ADJ" />
+          <output name="feat.getal" value="mv" />
+        </condition>
+        <condition value="190,191,192,193,194,195,196,199">
+          <output name="pos" value="ADJ" />
+          <output name="feat.probleemgeval" />
+        </condition>
+        
+        
+        <condition value="200,201,202,203,204,205,206,209">
+          <output name="pos" value="WW" />
+          <output name="feat.pvtijd" value="tgw" />
+        </condition>
+        <condition value="210,211,212,213,214,215,216,219">
+          <output name="pos" value="WW" />
+          <output name="feat.pvtijd" value="tgw" />
+        </condition>
+        <condition value="220,221,222,223,224,225,226,229">
+          <output name="pos" value="WW" />
+          <output name="feat.pvtijd" value="verl" />
+        </condition>
+        <condition value="230,231,232,233,234,235,236,239">
+          <output name="pos" value="WW" />
+          <output name="feat.pvtijd" value="verl" />
+        </condition> 
+        <condition value="240,241,242,243,244,245,246,249">
+          <output name="pos" value="WW" />          
+        </condition> 
+        <condition value="250,251,252,253,254,255,256,259">
+          <output name="pos" value="WW" /> 
+          <output name="feat.wvorm" value="inf" />        
+        </condition> <condition value="260,261,262,263,264,265,266,269">
+          <output name="pos" value="WW" />        
+          <output name="feat.wvorm" value="inf" />  
+        </condition> <condition value="270,271,272,273,274,275,276,279">
+          <output name="pos" value="WW" />          
+        </condition> <condition value="280,281,282,283,284,285,286,289">
+          <output name="pos" value="WW" />          
+        </condition>        
+        <condition value="290,291,292,293,294,295,296,299">
+          <output name="pos" value="WW" />
+          <output name="feat.probleemgeval" />
+        </condition>
+        
+        
+        <condition value="300,301,302,303,304,305,306,309">
+          <output name="pos" value="TW" />
+          <output name="feat.numtype" value="hoofd" />
+        </condition>
+        <condition value="310,311,312,313,314,315,316,319">
+          <output name="pos" value="TW" />
+          <output name="feat.numtype" value="rang" />
+        </condition>
+        <condition value="320,321,322,323,324,325,326,329">
+          <output name="pos" value="TW" />
+        </condition>
+        <condition value="390,391,392,393,394,395,396,399">
+          <output name="pos" value="TW" />
+          <output name="feat.probleemgeval" />
+        </condition>
+        
+        <condition value="401">
+          <output name="pos" value="VNW" />
+          <output name="feat.getal" value="ev" />
+          <output name="feat.persoon" value="1" />
+        </condition>
+        <condition value="402">
+          <output name="pos" value="VNW" />
+          <output name="feat.getal" value="ev" />
+          <output name="feat.persoon" value="2" />
+        </condition>
+        <condition value="403">
+          <output name="pos" value="VNW" />
+          <output name="feat.getal" value="ev" />
+          <output name="feat.persoon" value="3" />
+        </condition>
+        <condition value="404">
+          <output name="pos" value="VNW" />
+          <output name="feat.getal" value="mv" />
+          <output name="feat.persoon" value="1" />
+        </condition>
+        <condition value="405">
+          <output name="pos" value="VNW" />
+          <output name="feat.getal" value="mv" />
+          <output name="feat.persoon" value="2" />
+        </condition>
+        <condition value="406">
+          <output name="pos" value="VNW" />
+          <output name="feat.getal" value="mv" />
+          <output name="feat.persoon" value="3" />
+        </condition>
+        <condition value="409">
+          <output name="pos" value="VNW" />
+          <output name="feat.probleemgeval" />
+        </condition>
+        <condition value="410,411,412,413,414,415,416,419">
+          <output name="pos" value="VNW" />
+          <output name="feat.vwtype" value="aanw" />
+        </condition>
+        <condition value="420,421,422,423,424,425,426,429">
+          <output name="pos" value="VNW" />
+          <output name="feat.vwtype" value="betr" />
+        </condition>
+        <condition value="430,431,432,433,434,435,436,439">
+          <output name="pos" value="VNW" />
+          <output name="feat.vwtype" value="vb" />
+        </condition>
+        <condition value="434,441,442,443,444,445,446,449">
+          <output name="pos" value="VNW" />
+          <output name="feat.vwtype" value="vb" />
+        </condition>
+        <condition value="440,441,442,443,444,445,446,449">
+          <output name="pos" value="VNW" />
+          <output name="feat.lwtype" value="onbep" />
+        </condition>
+        <condition value="450,451,452,453,454,455,456,459">
+          <output name="pos" value="VNW" />
+          <output name="feat.vwtype" value="bez" />
+        </condition>
+        
+        
+        <condition value="001,011,021,091">
+          <output name="feat.form" value="-e" />
+        </condition>
+        <condition value="002,012,022,092">
+          <output name="feat.form" value="-s/-th" />
+        </condition>
+        <condition value="003,013,023,092">
+          <output name="feat.form" value="-t" />
+        </condition>
+        <condition value="004,014,024,092">
+          <output name="feat.form" value="-n" />
+        </condition>
+        <condition value="005,015,025,095">
+          <output name="feat.form" value="-r/-re" />
+        </condition>
+        <condition value="006,016,026,096">
+          <output name="feat.form" value="-a" />
+        </condition>
+        <condition value="009,019,029,099">
+          <output name="feat.form" value="unclear" />
+        </condition>
+        
+>>>>>>> cac6601c611fd300b075447ee1eb8299e5bffc90
 			</function>
 		</functions>
@@ -19,8 +19,12 @@
 		<!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
 		<autorepair value="true" />
+<<<<<<< HEAD
 		<makeunique value="true" />
     <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
+=======
+		<!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
+>>>>>>> cac6601c611fd300b075447ee1eb8299e5bffc90
 		<!-- START REFERENCES -->
 		<references>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<properties>
-		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+	<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<currentDevelopmentVersion>6.2.0</currentDevelopmentVersion>
 		<currentDevelopmentRelease>20160802</currentDevelopmentRelease>
 	</properties>
@@ -84,7 +84,7 @@
 						<addMavenDescriptor>false</addMavenDescriptor>
 					</archive>
 				</configuration>
-			</plugin>
+			</plugin>						
 		</plugins>
 	</build>
 	<reporting>
@@ -194,5 +194,4 @@
 			</dependency>
 		</dependencies>
 	</dependencyManagement>
-
 </project>
 \ No newline at end of file
@@ -92,7 +92,7 @@ public final class MtasTokenizer&lt;T&gt; extends Tokenizer {
    *           Signals that an I/O exception has occurred.
    */
   public MtasTokenizer(MtasConfiguration config) throws IOException {
-    processConfiguration(config);
+    processConfiguration(config);    
   }
   /**
@@ -41,7 +41,8 @@ final public class MtasSketchParser extends MtasBasicParser {
   /**
    * Instantiates a new mtas sketch parser.
    *
-   * @param config the config
+   * @param config
+   *          the config
    */
   public MtasSketchParser(MtasConfiguration config) {
     super(config);
@@ -363,7 +364,8 @@ final public class MtasSketchParser extends MtasBasicParser {
   /**
    * Prints the config types.
    *
-   * @param types the types
+   * @param types
+   *          the types
    * @return the string
    */
   private String printConfigTypes(HashMap<?, MtasParserType> types) {
@@ -378,15 +380,8 @@ final public class MtasSketchParser extends MtasBasicParser {
     return text;
   }
-  /**
-   * The Class MtasSketchParserMappingWord.
-   */
   private class MtasSketchParserMappingWord
       extends MtasParserMapping<MtasSketchParserMappingWord> {
-
-    /**
-     * Instantiates a new mtas sketch parser mapping word.
-     */
     public MtasSketchParserMappingWord() {
       super();
       this.position = SOURCE_OWN;
@@ -394,18 +389,14 @@ final public class MtasSketchParser extends MtasBasicParser {
       this.offset = SOURCE_OWN;
       this.type = MAPPING_TYPE_WORD;
     }
-
-    /*
-     * (non-Javadoc)
-     * 
-     * @see mtas.analysis.parser.MtasBasicParser.MtasParserMapping#self()
-     */
+    
     @Override
     protected MtasSketchParserMappingWord self() {
       return this;
     }
   }
+
   /**
    * The Class MtasSketchParserMappingWordAnnotation.
    */
 package mtas.analysis.token;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -95,7 +96,7 @@ public class MtasTokenCollection {
     Iterator<MtasToken<?>> it = this.iterator();
     while (it.hasNext()) {
       MtasToken<?> token = it.next();
-      System.out.println(token);
+      System.out.println(token);      
     }
   }
@@ -237,7 +238,7 @@ public class MtasTokenCollection {
         trash.add(i);
       } else if ((token.getPositionStart() == null)
           || (token.getPositionEnd() == null)) {
-        trash.add(i);
+        trash.add(i);        
       } else if (token.getValue() == null || (token.getValue().equals(""))) {
         trash.add(i);
       } else if (token.getPrefix() == null || (token.getPrefix().equals(""))) {
@@ -285,6 +285,10 @@ public class MtasCodecPostingsFormat extends PostingsFormat {
     } catch (Exception e) {
       throw new IOException(e.getMessage());
     }
+    Long termRef = inObject.readVLong();
+    inTerm.seek(termRef);
+    token.setTermRef(termRef);
+    token.setValue(inTerm.readString());
     return token;
   }
@@ -29,289 +29,21 @@ import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.MergeState;
 import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.Lock;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 /**
  * The Class MtasFieldsConsumer.
- *
- * 
- * The Class MtasFieldsConsumer constructs several temporal and permanent files
- * to provide a forward index
- *
- * <ul>
- * <li><b>Temporary files</b><br>
- * <ul>
- * <li><b>Temporary file {@link #mtasTmpFieldFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TMP_FIELD_EXTENSION} </b><br>
- * Contains for each field a reference to the list of documents. Structure of
- * content:
- * <ul>
- * <li><b>String</b>: field</li>
- * <li><b>VLong</b>: reference to {@link #mtasDocFileName}</li>
- * <li><b>VInt</b>: number of documents</li>
- * <li><b>VLong</b>: reference to {@link #mtasTermFileName}</li>
- * <li><b>VInt</b>: number of terms</li>
- * <li><b>VLong</b>: reference to {@link #mtasPrefixFileName}</li>
- * <li><b>VInt</b>: number of prefixes</li>
- * </ul>
- * </li>
- * <li><b>Temporary file {@link #mtasTmpObjectFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TMP_OBJECT_EXTENSION}</b><br>
- * Contains for a specific field all objects constructed by
- * {@link createObjectAndRegisterPrefix}. For all fields, the objects are later
- * on copied to {@link #mtasObjectFileName} while statistics are collected.
- * Structure of content identical to {@link #mtasObjectFileName}.</li>
- * <li><b>Temporary file {@link #mtasTmpDocsFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TMP_DOCS_EXTENSION}</b> <br>
- * Contains for a specific field for each doc multiple fragments. Each occurring
- * term results in a fragment. Structure of content:
- * <ul>
- * <li><b>VInt</b>: docId</li>
- * <li><b>VInt</b>: number of objects in this fragment</li>
- * <li><b>VLong</b>: offset references to {@link #mtasTmpObjectFileName}</li>
- * <li><b>VInt</b>,<b>VLong</b>: mtasId object, reference temporary object in
- * {@link #mtasTmpObjectFileName} minus offset</li>
- * <li><b>VInt</b>,<b>VLong</b>: ...</li>
- * </ul>
- * </li>
- * <li><b>Temporary file {@link #mtasTmpDocsChainedFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TMP_DOCS_CHAINED_EXTENSION}
- * </b><br>
- * Contains for a specific field for each doc multiple chained fragments.
- * Structure of content:
- * <ul>
- * <li><b>VInt</b>: docId</li>
- * <li><b>VInt</b>: number of objects in this fragment</li>
- * <li><b>VLong</b>: offset references to {@link #mtasTmpObjectFileName}</li>
- * <li><b>VInt</b>,<b>VLong</b>: mtasId object, reference temporary object in
- * {@link #mtasTmpObjectFileName} minus offset</li>
- * <li><b>VInt</b>,<b>VLong</b>: ...</li>
- * <li><b>VLong</b>: reference to next fragment in
- * {@link #mtasTmpDocsChainedFileName}, self reference indicates end of chain
- * </ul>
- * </li>
- * <li><b>Temporary file {@link #mtasTmpDocFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TMP_DOC_EXTENSION}</b><br>
- * For each document
- * <ul>
- * <li><b>VInt</b>: docId</li>
- * <li><b>VLong</b>: reference to {@link #mtasIndexObjectIdFileName}</li>
- * <li><b>VLong</b>: reference first object, used as offset for tree index
- * <li><b>VInt</b>: slope used in approximation reference objects index on id
- * </li>
- * <li><b>ZLong</b>: offset used in approximation reference objects index on id
- * </li>
- * <li><b>Byte</b>: flag indicating how corrections on the approximation
- * references objects for the index on id are stored:
- * {@link MtasCodecPostingsFormat#MTAS_STORAGE_BYTE},
- * {@link MtasCodecPostingsFormat#MTAS_STORAGE_SHORT},
- * {@link MtasCodecPostingsFormat#MTAS_STORAGE_INTEGER} or
- * {@link MtasCodecPostingsFormat#MTAS_STORAGE_LONG}</li>
- * <li><b>VInt</b>: number of objects in this document</li>
- * <li><b>VInt</b>: first position</li>
- * <li><b>VInt</b>: last position</li>
- * </ul>
- * </li>
- * </ul>
- * </li>
- * <li><b>Final files</b><br>
- * <ul>
- * <li><b>File {@link #mtasIndexFieldFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_FIELD_EXTENSION}</b><br>
- * Contains for each field a reference to the list of documents and the
- * prefixes. Structure of content:
- * <ul>
- * <li><b>String</b>: field</li>
- * <li><b>VLong</b>: reference to {@link #mtasDocFileName}</li>
- * <li><b>VLong</b>: reference to {@link #mtasIndexDocIdFileName}</li>
- * <li><b>VInt</b>: number of documents</li>
- * <li><b>VLong</b>: reference to {@link #mtasTermFileName}</li>
- * <li><b>VInt</b>: number of terms</li>
- * <li><b>VLong</b>: reference to {@link #mtasPrefixFileName}</li>
- * <li><b>VInt</b>: number of prefixes</li>
- * </ul>
- * </li>
- * <li><b>File {@link #mtasTermFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_TERM_EXTENSION}</b><br>
- * For each field, all unique terms are stored here. Structure of content:
- * <ul>
- * <li><b>String</b>: term</li>
- * </ul>
- * </li>
- * <li><b>File {@link #mtasPrefixFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_PREFIX_EXTENSION}</b><br>
- * For each field, all unique prefixes are stored here. Structure of content:
- * <ul>
- * <li><b>String</b>: prefix</li>
- * </ul>
- * </li>
- * <li><b>File {@link #mtasObjectFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_OBJECT_EXTENSION}</b><br>
- * Contains all objects for all fields. Structure of content:
- * <ul>
- * <li><b>VInt</b>: mtasId</li>
- * <li><b>VInt</b>: objectFlags
- * <ul>
- * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_PARENT}</li>
- * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_RANGE}</li>
- * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_SET}</li>
- * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_OFFSET}</li>
- * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_REALOFFSET}</li>
- * <li>{@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_PAYLOAD}</li>
- * </ul>
- * </li>
- * <li>Only if {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_PARENT}<br>
- * <b>VInt</b>: parentId
- * <li>Only if
- * {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_RANGE}<br>
- * <b>VInt</b>,<b>VInt</b>: startPosition and (endPosition-startPosition)
- * <li>Only if {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_SET}<br>
- * <b>VInt</b>,<b>VInt</b>,<b>VInt</b>,...: number of positions, firstPosition,
- * (position-previousPosition),...
- * <li>Only if no {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_RANGE}
- * or {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_POSITION_SET}<br>
- * <b>VInt</b>: position
- * <li>Only if {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_OFFSET}<br>
- * <b>VInt</b>,<b>VInt</b>: startOffset, (endOffset-startOffset)
- * <li>Only if {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_REALOFFSET}<br>
- * <b>VInt</b>,<b>VInt</b>: startRealOffset, (endRealOffset-startRealOffset)
- * <li>Only if {@link MtasCodecPostingsFormat#MTAS_OBJECT_HAS_PAYLOAD}<br>
- * <b>VInt</b>,<b>Bytes</b>: number of bytes, payload
- * <li><b>VLong</b>: reference to Term in {@link #mtasTermFileName}</li>
- * </ul>
- * </li>
- * <li><b>File {@link #mtasIndexDocIdFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_INDEX_DOC_ID_EXTENSION}
- * </b><br>
- * Contains for each field a tree structure {@link MtasTree} to search reference
- * to {@link #mtasDocFileName} by id. Structure of content for each node:
- * <ul>
- * <li><b>VLong</b>: offset references to {@link #mtasIndexDocIdFileName}, only
- * available in root node</li>
- * <li><b>Byte</b>: flag, should be zero for this tree, only available in root
- * node</li>
- * <li><b>VInt</b>: left</li>
- * <li><b>VInt</b>: right</li>
- * <li><b>VInt</b>: max</li>
- * <li><b>VLong</b>: left reference to {@link #mtasIndexDocIdFileName} minus the
- * offset stored in the root node</li>
- * <li><b>VLong</b>: right reference to {@link #mtasIndexDocIdFileName} minus
- * the offset stored in the root node</li>
- * <li><b>VInt</b>: number of objects on this node (always 1 for this tree)</li>
- * <li><b>VLong</b>: reference to {@link #mtasDocFileName} minus offset</li>
- * </ul>
- * </li>
- * <li><b>File {@link #mtasDocFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_DOC_EXTENSION}</b><br>
- * For each document
- * <ul>
- * <li><b>VInt</b>: docId</li>
- * <li><b>VLong</b>: reference to {@link #mtasIndexObjectIdFileName}</li>
- * <li><b>VLong</b>: reference to {@link #mtasIndexObjectPositionFileName}</li>
- * <li><b>VLong</b>: reference to {@link #mtasIndexObjectParentFileName}</li>
- * <li><b>VLong</b>: reference first object, used as offset for tree index
- * <li><b>VInt</b>: slope used in approximation reference objects index on id
- * </li>
- * <li><b>ZLong</b>: offset used in approximation reference objects index on id
- * </li>
- * <li><b>Byte</b>: flag indicating how corrections on the approximation
- * references objects for the index on id are stored:
- * {@link MtasCodecPostingsFormat#MTAS_STORAGE_BYTE},
- * {@link MtasCodecPostingsFormat#MTAS_STORAGE_SHORT},
- * {@link MtasCodecPostingsFormat#MTAS_STORAGE_INTEGER} or
- * {@link MtasCodecPostingsFormat#MTAS_STORAGE_LONG}</li>
- * <li><b>VInt</b>: number of objects</li>
- * <li><b>VInt</b>: first position</li>
- * <li><b>VInt</b>: last position</li>
- * </ul>
- * </li>
- * <li><b>File {@link #mtasIndexObjectIdFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_INDEX_OBJECT_ID_EXTENSION}
- * </b><br>
- * Provides for each mtasId the reference to {@link #mtasObjectFileName}. These
- * references are grouped by document, sorted by mtasId, and because the
- * mtasId's for each document will always start with 0 and are sequential
- * without gaps, a reference can be computed if the position of the first
- * reference for a document is known from {@link #mtasDocFileName}. The
- * reference is approximated by the reference to the first object plus the
- * mtasId times a slope. Only a correction to this approximation is stored.
- * Structure of content:
- * <ul>
- * <li><b>Byte</b>/<b>Short</b>/<b>Int</b>/<b>Long</b>: correction reference to
- * {@link #mtasObjectFileName}</li>
- * </ul>
- * </li>
- * <li><b>File {@link #mtasIndexObjectPositionFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_INDEX_OBJECT_POSITION_EXTENSION}
- * </b><br>
- * Contains for each document a tree structure {@link MtasTree} to search
- * objects by position. Structure of content for each node:
- * <ul>
- * <li><b>VLong</b>: offset references to
- * {@link #mtasIndexObjectPositionFileName}, only available in root node</li>
- * <li><b>Byte</b>: flag, should be zero for this tree, only available in root
- * node</li>
- * <li><b>VInt</b>: left</li>
- * <li><b>VInt</b>: right</li>
- * <li><b>VInt</b>: max</li>
- * <li><b>VLong</b>: left reference to {@link #mtasIndexObjectPositionFileName}
- * minus the offset stored in the root node</li>
- * <li><b>VLong</b>: right reference to {@link #mtasIndexObjectPositionFileName}
- * minus the offset stored in the root node</li>
- * <li><b>VInt</b>: number of objects on this node</li>
- * <li><b>VLong</b>,<b>VInt</b>,<b>VLong</b>: set of the first reference to
- * {@link #mtasObjectFileName} minus offset, the prefixId referring to the
- * position the prefix in {@link #mtasPrefixFileName} and the reference to
- * {@link #mtasTermFileName} minus offset</li>
- * <li><b>VLong</b>,<b>VInt</b>,<b>VLong</b>,...: for optional other sets of
- * reference to {@link #mtasObjectFileName}, position of the prefix in
- * {@link #mtasPrefixFileName} and the reference to {@link #mtasTermFileName};
- * for the first item the difference between this reference minus the previous
- * reference is stored</li>
- * </ul>
- * </li>
- * <li><b>File {@link #mtasIndexObjectParentFileName} with extension
- * {@value mtas.codec.MtasCodecPostingsFormat#MTAS_INDEX_OBJECT_PARENT_EXTENSION}
- * </b><br>
- * Contains for each document a tree structure {@link MtasTree} to search
- * objects by parent. Structure of content for each node:
- * <ul>
- * <li><b>VLong</b>: offset references to {@link #mtasIndexObjectParentFileName}
- * , only available in root node</li>
- * <li><b>Byte</b>: flag, for this tree equal to
- * {@link mtas.codec.tree.MtasTree#SINGLE_POSITION_TREE} indicating a tree with
- * exactly one point at each node, only available in root node</li>
- * <li><b>VInt</b>: left</li>
- * <li><b>VInt</b>: right</li>
- * <li><b>VInt</b>: max</li>
- * <li><b>VLong</b>: left reference to {@link #mtasIndexObjectParentFileName}
- * minus the offset stored in the root node</li>
- * <li><b>VLong</b>: right reference to {@link #mtasIndexObjectParentFileName}
- * minus the offset stored in the root node</li>
- * <li><b>VInt</b>: number of objects on this node</li>
- * <li><b>VLong</b>,<b>VInt</b>,<b>VLong</b>: set of the first reference to
- * {@link #mtasObjectFileName} minus offset, the prefixId referring to the
- * position the prefix in {@link #mtasPrefixFileName} and the reference to
- * {@link #mtasTermFileName} minus offset</li>
- * <li><b>VLong</b>,<b>VInt</b>,<b>VLong</b>,...: for optional other sets of
- * reference to {@link #mtasObjectFileName}, position of the prefix in
- * {@link #mtasPrefixFileName} and the reference to {@link #mtasTermFileName};
- * for the first item the difference between this reference minus the previous
- * reference is stored</li>
- * </ul>
- * </li>
- * </ul>
- * </li>
- * </ul>
- * 
  */
 public class MtasFieldsConsumer extends FieldsConsumer {
@@ -362,14 +94,10 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Instantiates a new mtas fields consumer.
    *
-   * @param fieldsConsumer
-   *          the fields consumer
-   * @param state
-   *          the state
-   * @param name
-   *          the name
-   * @param delegatePostingsFormatName
-   *          the delegate postings format name
+   * @param fieldsConsumer the fields consumer
+   * @param state the state
+   * @param name the name
+   * @param delegatePostingsFormatName the delegate postings format name
    */
   public MtasFieldsConsumer(FieldsConsumer fieldsConsumer,
       SegmentWriteState state, String name, String delegatePostingsFormatName) {
@@ -426,14 +154,10 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Register prefix.
    *
-   * @param field
-   *          the field
-   * @param prefix
-   *          the prefix
-   * @param outPrefix
-   *          the out prefix
-   * @throws IOException
-   *           Signals that an I/O exception has occurred.
+   * @param field the field
+   * @param prefix the prefix
+   * @param outPrefix the out prefix
+   * @throws IOException Signals that an I/O exception has occurred.
    */
   private void registerPrefix(String field, String prefix,
       IndexOutput outPrefix) throws IOException {
@@ -452,14 +176,10 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Register prefix stats single position value.
    *
-   * @param field
-   *          the field
-   * @param value
-   *          the value
-   * @param outPrefix
-   *          the out prefix
-   * @throws IOException
-   *           Signals that an I/O exception has occurred.
+   * @param field the field
+   * @param value the value
+   * @param outPrefix the out prefix
+   * @throws IOException Signals that an I/O exception has occurred.
    */
   public void registerPrefixStatsSinglePositionValue(String field, String value,
       IndexOutput outPrefix) throws IOException {
@@ -474,14 +194,10 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Register prefix stats range position value.
    *
-   * @param field
-   *          the field
-   * @param value
-   *          the value
-   * @param outPrefix
-   *          the out prefix
-   * @throws IOException
-   *           Signals that an I/O exception has occurred.
+   * @param field the field
+   * @param value the value
+   * @param outPrefix the out prefix
+   * @throws IOException Signals that an I/O exception has occurred.
    */
   public void registerPrefixStatsRangePositionValue(String field, String value,
       IndexOutput outPrefix) throws IOException {
@@ -495,14 +211,10 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Register prefix stats set position value.
    *
-   * @param field
-   *          the field
-   * @param value
-   *          the value
-   * @param outPrefix
-   *          the out prefix
-   * @throws IOException
-   *           Signals that an I/O exception has occurred.
+   * @param field the field
+   * @param value the value
+   * @param outPrefix the out prefix
+   * @throws IOException Signals that an I/O exception has occurred.
    */
   public void registerPrefixStatsSetPositionValue(String field, String value,
       IndexOutput outPrefix) throws IOException {
@@ -517,8 +229,7 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Inits the prefix stats field.
    *
-   * @param field
-   *          the field
+   * @param field the field
    */
   private void initPrefixStatsField(String field) {
     if (!singlePositionPrefix.containsKey(field)) {
@@ -535,8 +246,7 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Gets the prefix stats single position prefix attribute.
    *
-   * @param field
-   *          the field
+   * @param field the field
    * @return the prefix stats single position prefix attribute
    */
   public String getPrefixStatsSinglePositionPrefixAttribute(String field) {
@@ -547,8 +257,7 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Gets the prefix stats multiple position prefix attribute.
    *
-   * @param field
-   *          the field
+   * @param field the field
    * @return the prefix stats multiple position prefix attribute
    */
   public String getPrefixStatsMultiplePositionPrefixAttribute(String field) {
@@ -559,8 +268,7 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Gets the prefix stats set position prefix attribute.
    *
-   * @param field
-   *          the field
+   * @param field the field
    * @return the prefix stats set position prefix attribute
    */
   public String getPrefixStatsSetPositionPrefixAttribute(String field) {
@@ -585,6 +293,14 @@ public class MtasFieldsConsumer extends FieldsConsumer {
     return text;
   }
+  /* (non-Javadoc)
+   * @see org.apache.lucene.codecs.FieldsConsumer#merge(org.apache.lucene.index.MergeState)
+   */
+  @Override
+  public void merge(MergeState mergeState) throws IOException {
+    delegateFieldsConsumer.merge(mergeState);
+  }
+
   /*
    * (non-Javadoc)
    * 
@@ -600,12 +316,9 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Write.
    *
-   * @param fieldInfos
-   *          the field infos
-   * @param fields
-   *          the fields
-   * @throws IOException
-   *           Signals that an I/O exception has occurred.
+   * @param fieldInfos the field infos
+   * @param fields the fields
+   * @throws IOException Signals that an I/O exception has occurred.
    */
   private void write(FieldInfos fieldInfos, Fields fields) {
     IndexOutput outField, outDoc, outIndexDocId, outIndexObjectId,
@@ -1258,27 +971,17 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Creates the object and register prefix.
    *
-   * @param field
-   *          the field
-   * @param out
-   *          the out
-   * @param term
-   *          the term
-   * @param termRef
-   *          the term ref
-   * @param startPosition
-   *          the start position
-   * @param payload
-   *          the payload
-   * @param startOffset
-   *          the start offset
-   * @param endOffset
-   *          the end offset
-   * @param outPrefix
-   *          the out prefix
+   * @param field the field
+   * @param out the out
+   * @param term the term
+   * @param termRef the term ref
+   * @param startPosition the start position
+   * @param payload the payload
+   * @param startOffset the start offset
+   * @param endOffset the end offset
+   * @param outPrefix the out prefix
    * @return the integer
-   * @throws IOException
-   *           Signals that an I/O exception has occurred.
+   * @throws IOException Signals that an I/O exception has occurred.
    */
   private Integer createObjectAndRegisterPrefix(String field, IndexOutput out,
       BytesRef term, Long termRef, int startPosition, BytesRef payload,
@@ -1392,15 +1095,11 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Store tree.
    *
-   * @param tree
-   *          the tree
-   * @param out
-   *          the out
-   * @param refApproxOffset
-   *          the ref approx offset
+   * @param tree the tree
+   * @param out the out
+   * @param refApproxOffset the ref approx offset
    * @return the long
-   * @throws IOException
-   *           Signals that an I/O exception has occurred.
+   * @throws IOException Signals that an I/O exception has occurred.
    */
   private Long storeTree(MtasTree<?> tree, IndexOutput out,
       long refApproxOffset) throws IOException {
@@ -1411,21 +1110,14 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Store tree.
    *
-   * @param node
-   *          the node
-   * @param isSinglePoint
-   *          the is single point
-   * @param storeAdditionalInformation
-   *          the store additional information
-   * @param out
-   *          the out
-   * @param nodeRefApproxOffset
-   *          the node ref approx offset
-   * @param refApproxOffset
-   *          the ref approx offset
+   * @param node the node
+   * @param isSinglePoint the is single point
+   * @param storeAdditionalInformation the store additional information
+   * @param out the out
+   * @param nodeRefApproxOffset the node ref approx offset
+   * @param refApproxOffset the ref approx offset
    * @return the long
-   * @throws IOException
-   *           Signals that an I/O exception has occurred.
+   * @throws IOException Signals that an I/O exception has occurred.
    */
   private Long storeTree(MtasTreeNode<?> node, boolean isSinglePoint,
       boolean storeAdditionalInformation, IndexOutput out,
@@ -1510,10 +1202,8 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Token stats add.
    *
-   * @param min
-   *          the min
-   * @param max
-   *          the max
+   * @param min the min
+   * @param max the max
    */
   private void tokenStatsAdd(int min, int max) {
     tokenStatsNumber++;
@@ -1532,16 +1222,11 @@ public class MtasFieldsConsumer extends FieldsConsumer {
   /**
    * Copy object and update stats.
    *
-   * @param id
-   *          the id
-   * @param in
-   *          the in
-   * @param inRef
-   *          the in ref
-   * @param out
-   *          the out
-   * @throws IOException
-   *           Signals that an I/O exception has occurred.
+   * @param id the id
+   * @param in the in
+   * @param inRef the in ref
+   * @param out the out
+   * @throws IOException Signals that an I/O exception has occurred.
    */
   private void copyObjectAndUpdateStats(int id, IndexInput in, Long inRef,
       IndexOutput out) throws IOException {
@@ -2138,8 +2138,9 @@ public class CodecCollector {
             .checkExistenceNecessaryKeys()) {
           needSecondRound = true;
         }
-      }
-    }
+        termVector.subComponentFunction.dataCollector.reduceToSegmentKeys();
+      }      
+    }    
     return needSecondRound;
   }
@@ -23,7 +23,10 @@ import mtas.parser.function.util.MtasFunctionParserFunction;
 import mtas.parser.function.util.MtasFunctionParserFunctionDefault;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.util.automaton.Automata;
+import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 /**
@@ -274,7 +274,7 @@ public class CodecInfo {
     IndexInput inTerm = indexInputList.get("term");
     for (MtasTreeHit<?> hit : hits) {
       MtasToken<String> token = MtasCodecPostingsFormat.getToken(inObject,
-          inTerm, hit.ref);
+          inTerm, hit.ref); 
       if (token != null) {
         if (prefixes.size() > 0) {
           if (prefixes.contains(token.getPrefix())) {
@@ -151,4 +151,6 @@ public class DataCollector {
       throw new IOException("unknown dataType " + dataType);
     }
   }
+
+
 }
@@ -201,6 +201,32 @@ abstract class MtasDataAdvanced&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 extends N
         tmpOldSize);
   }
+  public void reduceToSegmentKeys() {
+    if(segmentRegistration != null) {
+      int sizeCopy = size;
+      String[] keyListCopy = keyList.clone();
+      T1[] advancedValueSumListCopy = advancedValueSumList.clone(); 
+      T1[] advancedValueMaxListCopy = advancedValueMaxList.clone(); 
+      T1[] advancedValueMinListCopy = advancedValueMinList.clone(); 
+      T1[] advancedValueSumOfSquaresListCopy = advancedValueSumOfSquaresList.clone(); 
+      T2[] advancedValueSumOfLogsListCopy = advancedValueSumOfLogsList.clone(); 
+      long[] advancedValueNListCopy = advancedValueNList.clone(); 
+      size = 0;
+      for(int i=0; i< sizeCopy; i++) {
+        if(segmentKeys.contains(keyListCopy[i])) {
+          keyList[size] = keyListCopy[i];
+          advancedValueSumList[size] = advancedValueSumListCopy[i];
+          advancedValueMaxList[size] = advancedValueMaxListCopy[i];
+          advancedValueMinList[size] = advancedValueMinListCopy[i];
+          advancedValueSumOfSquaresList[size] = advancedValueSumOfSquaresListCopy[i];
+          advancedValueSumOfLogsList[size] = advancedValueSumOfLogsListCopy[i];
+          advancedValueNList[size] = advancedValueNListCopy[i];
+          size++;
+        }
+      } 
+    }  
+  }
+  
   /*
    * (non-Javadoc)
    * 
@@ -228,6 +228,24 @@ abstract class MtasDataBasic&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 extends Numb
         tmpOldSize);
   }
+  public void reduceToSegmentKeys() {
+    if (segmentRegistration != null) {
+      int sizeCopy = size;
+      String[] keyListCopy = keyList.clone();
+      T1[] basicValueSumListCopy = basicValueSumList.clone();
+      long[] basicValueNListCopy = basicValueNList.clone();
+      size = 0;
+      for (int i = 0; i < sizeCopy; i++) {
+        if (segmentKeys.contains(keyListCopy[i])) {
+          keyList[size] = keyListCopy[i];
+          basicValueSumList[size] = basicValueSumListCopy[i];
+          basicValueNList[size] = basicValueNListCopy[i];
+          size++;
+        }
+      }
+    }
+  }
+
   /*
    * (non-Javadoc)
    * 
@@ -891,7 +891,7 @@ public abstract class MtasDataCollector&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 e
   /**
    * Reduce to segment keys.
    */
-  public final void reduceToSegmentKeys() {
+  public void reduceToSegmentKeys() {
     if (segmentRegistration != null) {
       reduceToKeys(segmentKeys);
     }
@@ -158,6 +158,22 @@ abstract class MtasDataFull&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 extends Numbe
     System.arraycopy(tmpNewFullValueList, 0, newFullValueList, 0, tmpOldSize);
   }
+  public void reduceToSegmentKeys() {
+    if(segmentRegistration != null) {
+      int sizeCopy = size;
+      String[] keyListCopy = keyList.clone();
+      T1[][] fullValueListCopy = fullValueList.clone(); 
+      size = 0;
+      for(int i=0; i< sizeCopy; i++) {
+        if(segmentKeys.contains(keyListCopy[i])) {
+          keyList[size] = keyListCopy[i];
+          fullValueList[size] = fullValueListCopy[i];
+          size++;
+        }
+      } 
+    }  
+  }
+
   /*
    * (non-Javadoc)
    * 
@@ -38,7 +38,7 @@ public abstract class MtasDataItem&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 extend
   /** The error list. */
   protected HashMap<String, Integer> errorList;
-
+  
   /** The comparable sort value. */
   protected NumberComparator<?> comparableSortValue;
@@ -101,7 +101,7 @@ public abstract class MtasDataItem&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 extend
    */
   public abstract Map<String, Object> rewrite(boolean showDebugInfo)
       throws IOException;
-
+  
   /**
    * Gets the sub.
    *
@@ -47,7 +47,7 @@ public class MtasCQLParserSentenceCondition {
   public MtasCQLParserSentenceCondition(MtasCQLParserBasicSentenceCondition s)
       throws ParseException {
     sequenceList = new ArrayList<List<MtasCQLParserSentenceCondition>>();
-    basicSentence = s;
+    basicSentence = s;    
     minimumOccurence = 1;
     maximumOccurence = 1;
     simplified = false;
@@ -436,8 +436,8 @@ public class MtasSolrSearchComponent extends SearchComponent {
    * (non-Javadoc)
    * 
    * @see
-   * org.apache.solr.handler.component.SearchComponent#finishStage(org.apache.
-   * solr.handler.component.ResponseBuilder)
+   * org.apache.solr.handler.component.SearchComponent#distributedProcess(org.
+   * apache.solr.handler.component.ResponseBuilder)
    */
   @Override
   public void finishStage(ResponseBuilder rb) {
@@ -549,7 +549,6 @@ public class MtasSolrSearchComponent extends SearchComponent {
           return STAGE_GROUP;
         }
       }
-
     }
     return ResponseBuilder.STAGE_DONE;
   }
@@ -562,15 +561,9 @@ public class MtasSolrSearchComponent extends SearchComponent {
    * @return the mtas fields
    */
-  /**
-   * Gets the mtas fields.
-   *
-   * @param rb
-   *          the rb
-   * @return the mtas fields
-   */
   private ComponentFields getMtasFields(ResponseBuilder rb) {
     return (ComponentFields) rb.req.getContext().get(ComponentFields.class);
   }
+
 }
@@ -13,10 +13,10 @@
     <tr>
       <td>$context.get("currentDevelopmentVersion")</td>
       <td>$context.get("currentDevelopmentRelease")</td>
-      <td><a href='https://github.com/meertensinstituut/mtas/releases/download/${currentDevelopmentRelease}/mtas-${currentDevelopmentVersion}.jar'>Binary (jar)</a></td>
-      <td><a href='https://github.com/meertensinstituut/mtas/archive/${currentDevelopmentRelease}.tar.gz'>Source (tgz)</a></td>
-      <td><a href='https://github.com/meertensinstituut/mtas/archive/${currentDevelopmentRelease}.zip'>Source (zip)</a></td>
+      <td><a href='https://github.com/matthijsbrouwer/mtas/releases/download/${currentDevelopmentRelease}/mtas-${currentDevelopmentVersion}.jar'>Binary (jar)</a></td>
+      <td><a href='https://github.com/matthijsbrouwer/mtas/archive/${currentDevelopmentRelease}.tar.gz'>Source (tgz)</a></td>
+      <td><a href='https://github.com/matthijsbrouwer/mtas/archive/${currentDevelopmentRelease}.zip'>Source (zip)</a></td>
       <td>Development version</td>
     </tr>
   </tbody>
 -</table>
+</table>
 \ No newline at end of file
+#Getting started
+
+
 #Mtas and Solr
-
-