diff --git a/conf/parser/mtas.xml b/conf/parser/mtas.xml
index e4d9f13..ee796b4 100644
--- a/conf/parser/mtas.xml
+++ b/conf/parser/mtas.xml
@@ -1,12 +1,18 @@
<?xml version="1.0" encoding="UTF-8" ?>
<mtas>
<configurations type="mtas.analysis.util.MtasTokenizerFactory">
+ <configuration name="test" file="mtas/folia_test.xml" />
+ <configuration name="CRM" file="mtas/crm_test.xml" />
<configuration name="DBNL" file="mtas/folia_dbnl.xml" />
+ <configuration name="DDD" file="mtas/folia_ddd.xml" />
<configuration name="EDBO" file="mtas/folia_edbo.xml" />
<configuration name="SONAR" file="mtas/folia_sonar.xml" />
</configurations>
<configurations type="mtas.analysis.util.MtasCharFilterFactory">
+ <configuration name="test" type="file" />
+ <configuration name="CRM" type="file" prefix="/Users/matthijs/Software/Mtas/data/CRM/data/files/" postfix=".txt" />
<configuration name="DBNL" type="url" prefix="https://openskos.meertens.knaw.nl/nederlab/archief/get/" />
+ <configuration name="DDD" type="url" prefix="https://openskos.meertens.knaw.nl/nederlab/archief/get/" />
<configuration name="EDBO" type="url" prefix="https://openskos.meertens.knaw.nl/nederlab/archief/get/" />
<configuration name="SONAR" type="url" prefix="https://openskos.meertens.knaw.nl/nederlab/archief/get/" />
</configurations>
diff --git a/conf/parser/mtas/crm_test.xml b/conf/parser/mtas/crm_test.xml
new file mode 100644
index 0000000..a204b2c
--- /dev/null
+++ b/conf/parser/mtas/crm_test.xml
@@ -0,0 +1,612 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<mtas>
+
+ <!-- START MTAS INDEX CONFIGURATION -->
+ <index>
+ <!-- START GENERAL SETTINGS MTAS INDEX PROCESS -->
+ <payload index="false" />
+ <offset index="false" />
+ <realoffset index="false" />
+ <parent index="true" />
+ <!-- END GENERAL SETTINGS MTAS INDEX PROCESS -->
+ </index>
+ <!-- END MTAS INDEX CONFIGURATION -->
+
+
+
+ <!-- START CONFIGURATION MTAS FOLIA PARSER -->
+ <parser name="mtas.analysis.parser.MtasCRMParser">
+
+ <!-- START GENERAL SETTINGS MTAS PARSER -->
+ <autorepair value="true" />
+ <makeunique value="true" />
+ <!-- END GENERAL SETTINGS MTAS PARSER -->
+
+ <mappings>
+
+ <mapping type="word">
+ </mapping>
+
+ <mapping type="wordAnnotation" name="0">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="string" value="t" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="0">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="string" value="t_lc" />
+ </pre>
+ <post>
+ <item type="text" filter="ascii,lowercase" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="1">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="string" value="t1" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="1">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="string" value="t1_lc" />
+ </pre>
+ <post>
+ <item type="text" filter="ascii,lowercase" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="2">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="string" value="t2" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="2">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="string" value="t2_lc" />
+ </pre>
+ <post>
+ <item type="text" filter="ascii,lowercase" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="3">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="string" value="lemma" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="4">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="string" value="crm" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+
+ <mapping type="crmPair" name="6">
+ <condition>
+ <item type="text" not="true" condition="-" />
+ </condition>
+ </mapping>
+ <mapping type="crmPair" name="part">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+
+ <mapping type="crmSentence" name="7">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="string" value="s"/>
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ <condition>
+ <item type="text" not="true" condition="-" />
+ <item type="text" not="true" condition="2" />
+ <item type="text" not="true" condition="4" />
+ <item type="text" not="true" condition="5" />
+ <item type="text" not="true" condition="6" />
+ <item type="text" not="true" condition="8" />
+ </condition>
+ </mapping>
+ <mapping type="crmClause" name="7">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="string" value="sc"/>
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ <condition>
+ <item type="text" not="true" condition="-" />
+ <item type="text" not="true" condition="0" />
+ <item type="text" not="true" condition="1" />
+ </condition>
+ </mapping>
+ <mapping type="crmClause" name="7">
+ <condition>
+ <item type="text" not="true" condition="-" />
+ </condition>
+ </mapping>
+
+ <mapping type="wordAnnotation" name="pos">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="feat.getal">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="feat.persoon">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="feat.ntype">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="feat.pvtijd">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="feat.wvorm">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="feat.numtype">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="feat.vwtype">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="feat.lwtype">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="feat.form">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="wordAnnotation" name="feat.probleemgeval">
+ <token type="string" offset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ </mapping>
+ </mappings>
+
+ <functions>
+ <function type="crmPair" name="6" split="+">
+ <condition value="">
+ <output name="part" />
+ </condition>
+ </function>
+ <function type="wordAnnotation" name="4" split="+">
+ <condition value="000,001,002,003,004,005,006,009">
+ <output name="pos" value="N" />
+ <output name="feat.getal" value="ev" />
+ </condition>
+ <condition value="010,011,012,013,014,015,016,019">
+ <output name="pos" value="N" />
+ <output name="feat.getal" value="mv" />
+ </condition>
+ <condition value="020,021,022,023,024,025,026,029">
+ <output name="pos" value="N" />
+ <output name="feat.ntype" value="eigen" />
+ </condition>
+ <condition value="090,091,092,093,094,095,096,099">
+ <output name="pos" value="N" />
+ <output name="feat.probleemgeval" />
+ </condition>
+ <condition value="100,101,102,103,104,105,106,109">
+ <output name="pos" value="ADJ" />
+ <output name="feat.getal" value="ev" />
+ </condition>
+ <condition value="110,111,112,113,114,115,116,119">
+ <output name="pos" value="ADJ" />
+ <output name="feat.getal" value="mv" />
+ </condition>
+ <condition value="190,191,192,193,194,195,196,199">
+ <output name="pos" value="ADJ" />
+ <output name="feat.probleemgeval" />
+ </condition>
+
+ <condition value="200,201,202,203,204,205,206,209">
+ <output name="pos" value="WW" />
+ <output name="feat.pvtijd" value="tgw" />
+ </condition>
+ <condition value="210,211,212,213,214,215,216,219">
+ <output name="pos" value="WW" />
+ <output name="feat.pvtijd" value="tgw" />
+ </condition>
+ <condition value="220,221,222,223,224,225,226,229">
+ <output name="pos" value="WW" />
+ <output name="feat.pvtijd" value="verl" />
+ </condition>
+ <condition value="230,231,232,233,234,235,236,239">
+ <output name="pos" value="WW" />
+ <output name="feat.pvtijd" value="verl" />
+ </condition>
+ <condition value="240,241,242,243,244,245,246,249">
+ <output name="pos" value="WW" />
+ </condition>
+ <condition value="250,251,252,253,254,255,256,259">
+ <output name="pos" value="WW" />
+ <output name="feat.wvorm" value="inf" />
+ </condition>
+ <condition value="260,261,262,263,264,265,266,269">
+ <output name="pos" value="WW" />
+ <output name="feat.wvorm" value="inf" />
+ </condition>
+ <condition value="270,271,272,273,274,275,276,279">
+ <output name="pos" value="WW" />
+ </condition>
+ <condition value="280,281,282,283,284,285,286,289">
+ <output name="pos" value="WW" />
+ </condition>
+ <condition value="290,291,292,293,294,295,296,299">
+ <output name="pos" value="WW" />
+ <output name="feat.probleemgeval" />
+ </condition>
+
+
+ <condition value="300,301,302,303,304,305,306,309">
+ <output name="pos" value="TW" />
+ <output name="feat.numtype" value="hoofd" />
+ </condition>
+ <condition value="310,311,312,313,314,315,316,319">
+ <output name="pos" value="TW" />
+ <output name="feat.numtype" value="rang" />
+ </condition>
+ <condition value="320,321,322,323,324,325,326,329">
+ <output name="pos" value="TW" />
+ </condition>
+ <condition value="390,391,392,393,394,395,396,399">
+ <output name="pos" value="TW" />
+ <output name="feat.probleemgeval" />
+ </condition>
+
+ <condition value="401">
+ <output name="pos" value="VNW" />
+ <output name="feat.getal" value="ev" />
+ <output name="feat.persoon" value="1" />
+ </condition>
+ <condition value="402">
+ <output name="pos" value="VNW" />
+ <output name="feat.getal" value="ev" />
+ <output name="feat.persoon" value="2" />
+ </condition>
+ <condition value="403">
+ <output name="pos" value="VNW" />
+ <output name="feat.getal" value="ev" />
+ <output name="feat.persoon" value="3" />
+ </condition>
+ <condition value="404">
+ <output name="pos" value="VNW" />
+ <output name="feat.getal" value="mv" />
+ <output name="feat.persoon" value="1" />
+ </condition>
+ <condition value="405">
+ <output name="pos" value="VNW" />
+ <output name="feat.getal" value="mv" />
+ <output name="feat.persoon" value="2" />
+ </condition>
+ <condition value="406">
+ <output name="pos" value="VNW" />
+ <output name="feat.getal" value="mv" />
+ <output name="feat.persoon" value="3" />
+ </condition>
+ <condition value="409">
+ <output name="pos" value="VNW" />
+ <output name="feat.probleemgeval" />
+ </condition>
+ <condition value="410,411,412,413,414,415,416,419">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="aanw" />
+ </condition>
+ <condition value="420,421,422,423,424,425,426,429">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="betr" />
+ </condition>
+ <condition value="430,431,432,433,434,435,436,439">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="vb" />
+ </condition>
+ <condition value="434,441,442,443,444,445,446,449">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="vb" />
+ </condition>
+ <condition value="440,441,442,443,444,445,446,449">
+ <output name="pos" value="VNW" />
+ <output name="feat.lwtype" value="onbep" />
+ </condition>
+ <condition value="450,451,452,453,454,455,456,459">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="bez" />
+ </condition>
+ <condition value="461">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="refl" />
+ <output name="feat.getal" value="ev" />
+ <output name="feat.persoon" value="1" />
+ </condition>
+ <condition value="462">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="refl" />
+ <output name="feat.getal" value="ev" />
+ <output name="feat.persoon" value="2" />
+ </condition>
+ <condition value="463">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="refl" />
+ <output name="feat.getal" value="ev" />
+ <output name="feat.persoon" value="3" />
+ </condition>
+ <condition value="464">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="refl" />
+ <output name="feat.getal" value="mv" />
+ <output name="feat.persoon" value="1" />
+ </condition>
+ <condition value="465">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="refl" />
+ <output name="feat.getal" value="mv" />
+ <output name="feat.persoon" value="2" />
+ </condition>
+ <condition value="466">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="refl" />
+ <output name="feat.getal" value="mv" />
+ <output name="feat.persoon" value="3" />
+ </condition>
+ <condition value="469">
+ <output name="pos" value="VNW" />
+ <output name="feat.vwtype" value="refl" />
+ <output name="feat.probleemgeval" />
+ </condition>
+ <condition value="470,471,472,473,474,475,476,479">
+ <output name="pos" value="LID" />
+ </condition>
+ <condition value="480,481,482,483,484,485,486,489">
+ <output name="pos" value="LID" />
+ </condition>
+ <condition value="490,491,492,493,494,495,496,499">
+ <output name="pos" value="VNW" />
+ <output name="feat.probleemgeval" />
+ </condition>
+
+ <condition value="500,501,502,503,504,505,506,509">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="510,511,512,513,514,515,516,519">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="520,521,522,523,524,525,526,529">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="530,531,532,533,534,535,536,539">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="540,541,542,543,544,545,546,549">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="550,551,552,553,554,555,556,559">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="560,561,562,563,564,565,566,569">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="590,591,592,593,594,595,596,599">
+ <output name="pos" value="BW" />
+ <output name="feat.probleemgeval" />
+ </condition>
+
+ <condition value="600,601,602,603,604,605,606,609">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="610,611,612,613,614,615,616,619">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="620,621,622,623,624,625,626,629">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="630,631,632,633,634,635,636,639">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="640,641,642,643,644,645,646,649">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="650,651,652,653,654,655,656,659">
+ <output name="pos" value="BW" />
+ </condition>
+ <condition value="690,691,692,693,694,695,696,699">
+ <output name="pos" value="BW" />
+ <output name="feat.probleemgeval" />
+ </condition>
+
+ <condition value="700,701,702,703,704,705,706,709">
+ <output name="pos" value="VZ" />
+ </condition>
+ <condition value="790,791,792,793,794,795,796,799">
+ <output name="pos" value="VZ" />
+ </condition>
+
+ <condition value="800,801,802,803,804,805,806,809">
+ <output name="pos" value="VG" />
+ </condition>
+ <condition value="810,811,812,813,814,815,816,819">
+ <output name="pos" value="VG" />
+ </condition>
+ <condition value="820,821,822,823,824,825,826,829">
+ <output name="pos" value="VG" />
+ </condition>
+ <condition value="830,831,832,833,834,835,836,839">
+ <output name="pos" value="VG" />
+ </condition>
+ <condition value="840,841,842,843,844,845,846,849">
+ <output name="pos" value="VG" />
+ </condition>
+ <condition value="850,851,852,853,854,855,856,859">
+ <output name="pos" value="VG" />
+ </condition>
+ <condition value="860,861,862,863,864,865,866,869">
+ <output name="pos" value="VG" />
+ </condition>
+ <condition value="870,871,872,873,874,875,876,879">
+ <output name="pos" value="VG" />
+ </condition>
+ <condition value="880,881,882,883,884,885,886,889">
+ <output name="pos" value="VG" />
+ </condition>
+ <condition value="890,891,892,893,894,895,896,899">
+ <output name="pos" value="VG" />
+ <output name="feat.probleemgeval" />
+ </condition>
+
+ <condition value="900,901,902,903,904,905,906,909">
+ <output name="feat.probleemgeval" />
+ </condition>
+ <condition value="900,901,902,903,904,905,906,909">
+ <output name="feat.probleemgeval" />
+ </condition>
+ <condition value="990,991,992,993,994,995,996,999">
+ <output name="feat.probleemgeval" />
+ </condition>
+
+ <condition
+ value="001,011,021,091,101,111,191,201,211,221,231,241,251,261,271,281,291,301,311,321,391,411,421,431,441,451,471,481,491,501,511,521,531,541,551,561,591,601,611,621,631,641,651,691,701,791,801,811,821,831,841,851,861,871,881,891,901,911,991">
+ <output name="feat.form" value="-e" />
+ </condition>
+ <condition
+ value="002,012,022,092,102,112,192,202,212,222,232,242,252,262,272,282,292,302,312,322,392,412,422,432,442,452,472,482,492,502,512,522,532,542,552,562,592,602,612,622,632,642,652,692,702,792,802,812,822,832,842,852,862,872,882,892,902,912,992">
+ <output name="feat.form" value="-s/-th" />
+ </condition>
+ <condition
+ value="003,013,023,093,103,113,193,203,213,223,233,243,253,263,273,283,293,303,313,323,393,413,423,433,443,453,473,483,493,503,513,523,533,543,553,563,593,603,613,623,633,643,653,693,703,793,803,813,823,833,843,853,863,873,883,893,903,913,993">
+ <output name="feat.form" value="-t" />
+ </condition>
+ <condition
+ value="004,014,024,094,104,114,194,204,214,224,234,244,254,264,274,284,294,304,314,324,394,414,424,434,444,454,474,484,494,504,514,524,534,544,554,564,594,604,614,624,634,644,654,694,704,794,804,814,824,834,844,854,864,874,884,894,904,914,994">
+ <output name="feat.form" value="-n" />
+ </condition>
+ <condition
+ value="005,015,025,095,105,115,195,205,215,225,235,245,255,265,275,285,295,305,315,325,395,415,425,435,445,455,475,485,495,505,515,525,535,545,555,565,595,605,615,625,635,645,655,695,705,795,805,815,825,835,845,855,865,875,885,895,905,915,995">
+ <output name="feat.form" value="-r/-re" />
+ </condition>
+ <condition
+ value="006,016,026,096,106,116,196,206,216,226,236,246,256,266,276,286,296,306,316,326,396,416,426,436,446,456,476,486,496,506,516,526,536,546,556,566,596,606,616,626,636,646,656,696,706,796,806,816,826,836,846,856,866,876,886,896,906,916,996">
+ <output name="feat.form" value="-a" />
+ </condition>
+ <condition value="009,019,029,099">
+ <output name="feat.form" value="unclear" />
+ </condition>
+
+ </function>
+ </functions>
+
+ </parser>
+ <!-- END CONFIGURATION MTAS FOLIA PARSER -->
+
+
+</mtas>
\ No newline at end of file
diff --git a/conf/parser/mtas/elan_mks.xml b/conf/parser/mtas/elan_mks.xml
index 294f721..01f7696 100644
--- a/conf/parser/mtas/elan_mks.xml
+++ b/conf/parser/mtas/elan_mks.xml
@@ -17,9 +17,10 @@
<!-- START CONFIGURATION MTAS FOLIA PARSER -->
<parser name="mtas.analysis.parser.MtasElanParser">
- <!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
+ <!-- START GENERAL SETTINGS MTAS PARSER -->
<autorepair value="true" />
- <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
+ <makeunique value="true" />
+ <!-- END GENERAL SETTINGS MTAS PARSER -->
<!-- START REFERENCES -->
<references>
diff --git a/conf/parser/mtas/folia_dbnl.xml b/conf/parser/mtas/folia_dbnl.xml
index cd1e2d8..3fb5b0f 100644
--- a/conf/parser/mtas/folia_dbnl.xml
+++ b/conf/parser/mtas/folia_dbnl.xml
@@ -19,7 +19,8 @@
<!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
<autorepair value="true" />
- <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
+ <makeunique value="true" />
+ <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
<!-- START REFERENCES -->
<references>
diff --git a/conf/parser/mtas/folia_ddd.xml b/conf/parser/mtas/folia_ddd.xml
new file mode 100644
index 0000000..d0e5362
--- /dev/null
+++ b/conf/parser/mtas/folia_ddd.xml
@@ -0,0 +1,146 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<mtas>
+
+ <!-- START MTAS INDEX CONFIGURATION -->
+ <index>
+ <!-- START GENERAL SETTINGS MTAS INDEX PROCESS -->
+ <payload index="false" />
+ <offset index="false" />
+ <realoffset index="false" />
+ <parent index="true" />
+ <!-- END GENERAL SETTINGS MTAS INDEX PROCESS -->
+ </index>
+ <!-- END MTAS INDEX CONFIGURATION -->
+
+
+
+ <!-- START CONFIGURATION MTAS FOLIA PARSER -->
+ <parser name="mtas.analysis.parser.MtasFoliaParser">
+
+ <!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
+ <autorepair value="true" />
+ <makeunique value="true" />
+ <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
+
+ <!-- START REFERENCES -->
+ <references>
+ <reference name="wref" ref="id" />
+ </references>
+ <!-- END REFERENCES -->
+
+ <!-- START MAPPINGS -->
+ <mappings>
+
+ <!-- START WORDS -->
+ <mapping type="word" name="w">
+ </mapping>
+ <mapping type="word" name="w">
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ <condition>
+ <item type="attribute" name="class" />
+ <item type="attribute" name="class" not="true" condition="WORD" />
+ </condition>
+ </mapping>
+ <!-- END WORDS -->
+
+ <!-- START WORD ANNOTATIONS -->
+ <mapping type="wordAnnotation" name="t">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ <item type="string" value="_lc" />
+ </pre>
+ <post>
+ <item type="text" filter="ascii,lowercase" />
+ </post>
+ </token>
+ <condition>
+ <item type="ancestor" number="0" />
+ <item type="ancestorWord" number="1" />
+ <item type="unknownAncestor" number="0" />
+ </condition>
+ </mapping>
+ <!-- END WORD ANNOTATIONS -->
+
+ <!-- START RELATIONS -->
+ <!-- END RELATIONS -->
+
+ <!-- START GROUPS -->
+ <mapping type="group" name="s">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="group" name="p">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="group" name="div">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="group" name="head">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <!-- END GROUPS -->
+
+ <!-- START GROUP ANNOTATIONS -->
+ <mapping type="groupAnnotation" name="lang">
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <!-- END GROUP ANNOTATIONS -->
+
+ </mappings>
+ <!-- END MAPPINGS -->
+
+ </parser>
+ <!-- END CONFIGURATION MTAS FOLIA PARSER -->
+
+
+</mtas>
\ No newline at end of file
diff --git a/conf/parser/mtas/folia_edbo.xml b/conf/parser/mtas/folia_edbo.xml
index cc27d34..34554ee 100644
--- a/conf/parser/mtas/folia_edbo.xml
+++ b/conf/parser/mtas/folia_edbo.xml
@@ -17,6 +17,7 @@
<!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
<autorepair value="true" />
+ <makeunique value="true" />
<!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
<!-- START REFERENCES -->
@@ -72,30 +73,6 @@
<item type="unknownAncestor" number="0" />
</condition>
</mapping>
- <mapping type="wordAnnotation" name="aref">
- <token type="string" offset="false">
- <pre>
- <item type="string" value="translated.t" />
- </pre>
- <post>
- <item type="attribute" name="t" />
- </post>
- </token>
- <token type="string" offset="false" realoffset="false" parent="false">
- <pre>
- <item type="string" value="translated.t" />
- <item type="string" value="_lc" />
- </pre>
- <post>
- <item type="attribute" name="t" filter="ascii,lowercase" />
- </post>
- </token>
- <condition>
- <item type="ancestor" number="0" />
- <item type="ancestorWord" number="1" />
- <item type="unknownAncestor" number="1" />
- </condition>
- </mapping>
<mapping type="wordAnnotation" name="lemma">
<token type="string" offset="false" realoffset="false" parent="false">
<pre>
@@ -109,24 +86,6 @@
<item type="attribute" name="class" />
<item type="ancestor" number="0" />
<item type="unknownAncestor" number="0" />
- <item type="attribute" name="set" condition="original.http://ilk.uvt.nl/folia/sets/frog-mblem-nl" />
- </condition>
- </mapping>
- <mapping type="wordAnnotation" name="lemma">
- <token type="string" offset="false" realoffset="false" parent="false">
- <pre>
- <item type="string" value="translated." />
- <item type="name" />
- </pre>
- <post>
- <item type="attribute" name="class" />
- </post>
- </token>
- <condition>
- <item type="attribute" name="class" />
- <item type="ancestor" number="0" />
- <item type="unknownAncestor" number="1" />
- <item type="attribute" name="set" condition="translated.http://ilk.uvt.nl/folia/sets/frog-mblem-nl" />
</condition>
</mapping>
<mapping type="wordAnnotation" name="morphology">
@@ -166,54 +125,11 @@
<item type="ancestor" number="0" />
<item type="unknownAncestor" number="0" />
<item type="attribute" name="class" />
- <item type="attribute" name="set" condition="original.http://ilk.uvt.nl/folia/sets/frog-mbpos-cgn" />
- </condition>
- </mapping>
- <mapping type="wordAnnotation" name="pos">
- <token type="string" offset="false" realoffset="false" parent="false">
- <pre>
- <item type="string" value="translated." />
- <item type="name" />
- </pre>
- <post>
- <item type="attribute" name="head" />
- </post>
- <payload>
- <item type="attribute" name="confidence" />
- </payload>
- </token>
- <condition>
- <item type="ancestor" number="0" />
- <item type="unknownAncestor" number="1" />
- <item type="attribute" name="class" />
- <item type="attribute" name="set" condition="translated.http://ilk.uvt.nl/folia/sets/frog-mbpos-cgn" />
- </condition>
- </mapping>
- <mapping type="wordAnnotation" name="feat">
- <token type="string" offset="false" realoffset="false" parent="false">
- <pre>
- <item type="name" />
- <item type="attribute" name="subset" prefix="." />
- </pre>
- <post>
- <item type="attribute" name="class" />
- </post>
- <payload>
- <item type="ancestorAttribute" distance="0" name="confidence" />
- </payload>
- </token>
- <condition>
- <item type="ancestor" number="1" />
- <item type="unknownAncestor" number="0" />
- <item type="attribute" name="class" />
- <item type="attribute" name="subset" />
- <item type="ancestorAttribute" name="set" condition="original.http://ilk.uvt.nl/folia/sets/frog-mbpos-cgn" />
</condition>
</mapping>
<mapping type="wordAnnotation" name="feat">
<token type="string" offset="false" realoffset="false" parent="false">
<pre>
- <item type="string" value="translated." />
<item type="name" />
<item type="attribute" name="subset" prefix="." />
</pre>
@@ -229,7 +145,6 @@
<item type="unknownAncestor" number="0" />
<item type="attribute" name="class" />
<item type="attribute" name="subset" />
- <item type="ancestorAttribute" name="set" condition="translated.http://ilk.uvt.nl/folia/sets/frog-mbpos-cgn" />
</condition>
</mapping>
<!-- END WORD ANNOTATIONS -->
diff --git a/conf/parser/mtas/folia_mimore.xml b/conf/parser/mtas/folia_mimore.xml
index d6bd815..bfb65d2 100644
--- a/conf/parser/mtas/folia_mimore.xml
+++ b/conf/parser/mtas/folia_mimore.xml
@@ -18,7 +18,8 @@
<!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
<autorepair value="false" />
- <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
+ <makeunique value="true" />
+ <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
<!-- START REFERENCES -->
<references>
diff --git a/conf/parser/mtas/folia_mtas.xml b/conf/parser/mtas/folia_mtas.xml
index cd1e2d8..3fb5b0f 100644
--- a/conf/parser/mtas/folia_mtas.xml
+++ b/conf/parser/mtas/folia_mtas.xml
@@ -19,7 +19,8 @@
<!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
<autorepair value="true" />
- <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
+ <makeunique value="true" />
+ <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
<!-- START REFERENCES -->
<references>
diff --git a/conf/parser/mtas/folia_oeaw.xml b/conf/parser/mtas/folia_oeaw.xml
new file mode 100644
index 0000000..f85087f
--- /dev/null
+++ b/conf/parser/mtas/folia_oeaw.xml
@@ -0,0 +1,205 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<mtas>
+
+ <!-- START MTAS INDEX CONFIGURATION -->
+ <index>
+ <!-- START GENERAL SETTINGS MTAS INDEX PROCESS -->
+ <payload index="false" />
+ <offset index="false" />
+ <realoffset index="false" />
+ <parent index="true" />
+ <!-- END GENERAL SETTINGS MTAS INDEX PROCESS -->
+ </index>
+ <!-- END MTAS INDEX CONFIGURATION -->
+
+
+
+ <!-- START CONFIGURATION MTAS FOLIA PARSER -->
+ <parser name="mtas.analysis.parser.MtasFoliaParser">
+
+ <!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
+ <autorepair value="true" />
+ <makeunique value="true" />
+ <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
+
+ <!-- START REFERENCES -->
+ <references>
+ <reference name="wref" ref="id" />
+ </references>
+ <!-- END REFERENCES -->
+
+ <!-- START MAPPINGS -->
+ <mappings>
+
+ <!-- START WORDS -->
+ <mapping type="word" name="w">
+ </mapping>
+ <mapping type="word" name="w">
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ <condition>
+ <item type="attribute" name="class" />
+ <item type="attribute" name="class" not="true" condition="WORD" />
+ </condition>
+ </mapping>
+ <!-- END WORDS -->
+
+ <!-- START WORD ANNOTATIONS -->
+ <mapping type="wordAnnotation" name="t">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ <item type="string" value="_lc" />
+ </pre>
+ <post>
+ <item type="text" filter="ascii,lowercase" />
+ </post>
+ </token>
+ <condition>
+ <item type="ancestor" number="0" />
+ <item type="ancestorWord" number="1" />
+ <item type="unknownAncestor" number="0" />
+ </condition>
+ </mapping>
+ <mapping type="wordAnnotation" name="lemma">
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ <condition>
+ <item type="attribute" name="class" />
+ <item type="ancestor" number="0" />
+ <item type="unknownAncestor" number="0" />
+ </condition>
+ </mapping>
+ <mapping type="wordAnnotation" name="pos">
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="attribute" name="set" />
+ </pre>
+ <post>
+ <item type="attribute" name="head" />
+ </post>
+ </token>
+ <condition>
+ <item type="ancestor" number="0" />
+ <item type="unknownAncestor" number="0" />
+ <item type="attribute" name="class" />
+ <item type="attribute" name="set" />
+ </condition>
+ </mapping>
+ <mapping type="wordAnnotation" name="feat">
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ <item type="attribute" name="subset" prefix="." />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ <condition>
+ <item type="ancestor" number="1" />
+ <item type="unknownAncestor" number="0" />
+ <item type="attribute" name="class" />
+ <item type="attribute" name="subset" />
+ </condition>
+ </mapping>
+ <!-- END WORD ANNOTATIONS -->
+
+ <!-- START RELATIONS -->
+ <mapping type="relation" name="entities">
+ </mapping>
+ <mapping type="relation" name="entity">
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ <condition>
+ <item type="ancestor" number="1" />
+ <item type="ancestorName" condition="entities" />
+ </condition>
+ </mapping>
+ <!-- END RELATIONS -->
+
+ <!-- START RELATION ANNOTATIONS -->
+ <mapping type="relationAnnotation" name="feat">
+ <token type="string" offset="false" realoffset="false">
+ <pre>
+ <item type="ancestorRelationName" />
+ <item type="name" prefix="." />
+ <item type="attribute" name="subset" prefix="." />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <!-- END RELATION ANNOTATIONS -->
+
+ <!-- START GROUPS -->
+ <mapping type="group" name="s">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="group" name="p">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="group" name="div">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <!-- END GROUPS -->
+
+ <!-- START GROUP ANNOTATIONS -->
+ <!-- END GROUP ANNOTATIONS -->
+
+ </mappings>
+ <!-- END MAPPINGS -->
+
+ </parser>
+ <!-- END CONFIGURATION MTAS FOLIA PARSER -->
+
+
+</mtas>
\ No newline at end of file
diff --git a/conf/parser/mtas/folia_sonar.xml b/conf/parser/mtas/folia_sonar.xml
index 3e9bbe9..6a9da6e 100644
--- a/conf/parser/mtas/folia_sonar.xml
+++ b/conf/parser/mtas/folia_sonar.xml
@@ -18,6 +18,7 @@
<!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
<autorepair value="true" />
+ <makeunique value="true" />
<!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
<!-- START REFERENCES -->
diff --git a/conf/parser/mtas/folia_test.xml b/conf/parser/mtas/folia_test.xml
index 7610b1c..df44ae9 100644
--- a/conf/parser/mtas/folia_test.xml
+++ b/conf/parser/mtas/folia_test.xml
@@ -19,7 +19,8 @@
<!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
<autorepair value="true" />
- <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
+ <makeunique value="true" />
+ <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
<!-- START REFERENCES -->
<references>
diff --git a/conf/parser/mtas/sketch_acdh.xml b/conf/parser/mtas/sketch_acdh.xml
index 5b907a9..fd81486 100644
--- a/conf/parser/mtas/sketch_acdh.xml
+++ b/conf/parser/mtas/sketch_acdh.xml
@@ -18,7 +18,8 @@
<parser name="mtas.analysis.parser.MtasSketchParser">
<!-- START GENERAL SETTINGS MTAS SKETCH PARSER -->
<autorepair value="true" />
- <!-- END GENERAL SETTINGS MTAS SKETCH PARSER -->
+ <makeunique value="true" />
+ <!-- END GENERAL SETTINGS MTAS SKETCH PARSER -->
<mappings>
diff --git a/conf/parser/mtas/tei_test.xml b/conf/parser/mtas/tei_test.xml
index 34b127e..b7e905e 100644
--- a/conf/parser/mtas/tei_test.xml
+++ b/conf/parser/mtas/tei_test.xml
@@ -19,7 +19,8 @@
<!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
<autorepair value="true" />
- <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
+ <makeunique value="true" />
+ <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
<!-- START REFERENCES -->
<references>
diff --git a/conf/parser/mtasSource.xml b/conf/parser/mtasSource.xml
new file mode 100644
index 0000000..7098996
--- /dev/null
+++ b/conf/parser/mtasSource.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<mtas>
+ <configurations type="mtas.analysis.util.MtasTokenizerFactory">
+ <configuration name="EDBO" file="mtasSource/folia_edbo.xml" />
+ </configurations>
+ <configurations type="mtas.analysis.util.MtasCharFilterFactory">
+ <configuration name="EDBO" type="url" prefix="https://openskos.meertens.knaw.nl/nederlab/archief/get/" />
+ </configurations>
+</mtas>
diff --git a/conf/parser/mtasSource/folia_edbo.xml b/conf/parser/mtasSource/folia_edbo.xml
new file mode 100644
index 0000000..62502bd
--- /dev/null
+++ b/conf/parser/mtasSource/folia_edbo.xml
@@ -0,0 +1,204 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<mtas>
+
+ <!-- START MTAS INDEX CONFIGURATION -->
+ <index>
+ <!-- START GENERAL SETTINGS MTAS INDEX PROCESS -->
+ <payload index="false" />
+ <offset index="false" />
+ <realoffset index="false" />
+ <parent index="true" />
+ <!-- END GENERAL SETTINGS MTAS INDEX PROCESS -->
+ </index>
+ <!-- END MTAS INDEX CONFIGURATION -->
+
+ <!-- START CONFIGURATION MTAS FOLIA PARSER -->
+ <parser name="mtas.analysis.parser.MtasFoliaParser">
+
+ <!-- START GENERAL SETTINGS MTAS FOLIA PARSER -->
+ <autorepair value="true" />
+ <makeunique value="true" />
+ <!-- END GENERAL SETTINGS MTAS FOLIA PARSER -->
+
+ <!-- START REFERENCES -->
+ <references>
+ </references>
+ <!-- END REFERENCES -->
+
+ <!-- START MAPPINGS -->
+ <mappings>
+
+ <!-- START WORDS -->
+ <mapping type="word" name="str">
+ </mapping>
+ <!-- END WORDS -->
+
+ <!-- START WORD ANNOTATIONS -->
+ <mapping type="wordAnnotation" name="t">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ <item type="string" value="_lc" />
+ </pre>
+ <post>
+ <item type="text" filter="ascii,lowercase" />
+ </post>
+ </token>
+ <condition>
+ <item type="ancestor" number="0" />
+ <item type="ancestorWord" number="1" />
+ <item type="unknownAncestor" number="0" />
+ <item type="attribute" name="class" condition="Ticcl"/>
+ </condition>
+ </mapping>
+ <mapping type="wordAnnotation" name="correction">
+ </mapping>
+ <mapping type="wordAnnotation" name="new">
+ </mapping>
+ <mapping type="wordAnnotation" name="original">
+ </mapping>
+ <mapping type="wordAnnotation" name="suggestion">
+ </mapping>
+ <mapping type="wordAnnotation" name="t">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ <item type="string" value="_lc" />
+ </pre>
+ <post>
+ <item type="text" filter="ascii,lowercase" />
+ </post>
+ </token>
+ <condition>
+ <item type="ancestor" number="2" />
+ <item type="ancestorName" condition="new" />
+ <item type="unknownAncestor" number="0" />
+ <item type="attribute" name="class" condition="Ticcl"/>
+ </condition>
+ </mapping>
+ <mapping type="wordAnnotation" name="t">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ <item type="ancestorName" prefix="."/>
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ <item type="string" value="_lc" />
+ <item type="ancestorName" prefix="."/>
+ </pre>
+ <post>
+ <item type="text" filter="ascii,lowercase" />
+ </post>
+ </token>
+ <condition>
+ <item type="ancestor" number="2" />
+ <item type="ancestorName" condition="original" />
+ <item type="unknownAncestor" number="0" />
+ </condition>
+ </mapping>
+ <mapping type="wordAnnotation" name="t">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ <item type="ancestorName" prefix="."/>
+ </pre>
+ <post>
+ <item type="text" />
+ </post>
+ </token>
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ <item type="string" value="_lc" />
+ <item type="ancestorName" prefix="."/>
+ </pre>
+ <post>
+ <item type="text" filter="ascii,lowercase" />
+ </post>
+ </token>
+ <condition>
+ <item type="ancestor" number="2" />
+ <item type="ancestorName" condition="suggestion" />
+ <item type="unknownAncestor" number="0" />
+ </condition>
+ </mapping>
+ <!-- END WORD ANNOTATIONS -->
+
+ <!-- START RELATIONS -->
+ <!-- END RELATIONS -->
+
+ <!-- START GROUPS -->
+ <mapping type="group" name="p">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="group" name="div">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <mapping type="group" name="head">
+ <token type="string" offset="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <!-- END GROUPS -->
+
+ <!-- START GROUP ANNOTATIONS -->
+ <mapping type="groupAnnotation" name="lang">
+ <token type="string" offset="false" realoffset="false" parent="false">
+ <pre>
+ <item type="name" />
+ </pre>
+ <post>
+ <item type="attribute" name="class" />
+ </post>
+ </token>
+ </mapping>
+ <!-- END GROUP ANNOTATIONS -->
+
+ </mappings>
+ <!-- END MAPPINGS -->
+
+ </parser>
+ <!-- END CONFIGURATION MTAS FOLIA PARSER -->
+
+</mtas>
\ No newline at end of file
diff --git a/conf/solr/schemaNederlab.xml b/conf/solr/schemaNederlab.xml
index ff5b98d..96c24c3 100644
--- a/conf/solr/schemaNederlab.xml
+++ b/conf/solr/schemaNederlab.xml
@@ -255,8 +255,8 @@
<field name="NLContent_folia_available" type="nederlab_boolean"
required="false" multiValued="false" indexed="true" stored="true" />
<field name="NLContent_mtas" type="mtas_text" indexed="true"
- stored="true" />
- <field name="NLContent_mtas_error" type="nederlab_string"
+ stored="true" />
+ <field name="NLContent_mtas_error" type="nederlab_string"
indexed="true" stored="true" />
<field name="NLContent_mtas_numberOfTokens" type="nederlab_int"
indexed="true" stored="true" />
@@ -264,7 +264,17 @@
indexed="true" stored="true" />
<field name="NLContent_mtas_size" type="nederlab_int" indexed="true"
stored="true" />
- <!-- Combined Field Metadata -->
+ <field name="NLContent_mtasSource" type="mtasSource_text" indexed="true"
+ stored="true" />
+ <field name="NLContent_mtasSource_error" type="nederlab_string"
+ indexed="true" stored="true" />
+ <field name="NLContent_mtasSource_numberOfTokens" type="nederlab_int"
+ indexed="true" stored="true" />
+ <field name="NLContent_mtasSource_numberOfPositions" type="nederlab_int"
+ indexed="true" stored="true" />
+ <field name="NLContent_mtasSource_size" type="nederlab_int" indexed="true"
+ stored="true" />
+ <!-- Combined Field Metadata -->
<field name="NLMetadata" type="nederlab_text" required="false"
multiValued="true" indexed="true" stored="false" />
<copyField source="NLCore_NLIdentification_nederlabID" dest="NLMetadata" />
@@ -420,5 +430,27 @@
prefix="t" />
</analyzer>
</fieldType>
+
+ <fieldType name="mtasSource_text_example_config" class="solr.TextField"
+ postingsFormat="MtasCodec">
+ <analyzer type="index">
+ <charFilter class="mtas.analysis.util.MtasCharFilterFactory"
+ config="mtasSource.xml" />
+ <tokenizer class="mtas.analysis.util.MtasTokenizerFactory"
+ config="mtasSource.xml" />
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="mtasSource_text" class="mtas.solr.schema.MtasPreAnalyzedField"
+ followIndexAnalyzer="mtasSource_text_example_config"
+ configurationFromField="NLCore_NLAdministrative_sourceCollection" setNumberOfTokens="NLContent_mtasSource_numberOfTokens"
+ setNumberOfPositions="NLContent_mtasSource_numberOfPositions" setSize="NLContent_mtasSource_size"
+ setError="NLContent_mtasSource_error" postingsFormat="MtasCodec">
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory" />
+ <filter class="mtas.analysis.util.MtasPrefixTokenFilterFactory"
+ prefix="t" />
+ </analyzer>
+ </fieldType>
</schema>
diff --git a/conf/solr/schemaOeaw.xml b/conf/solr/schemaOeaw.xml
new file mode 100644
index 0000000..ccb993b
--- /dev/null
+++ b/conf/solr/schemaOeaw.xml
@@ -0,0 +1,392 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<schema name="nederlab" version="1.5">
+
+ <field name="_version_" type="nederlab_long" indexed="true"
+ stored="true" />
+
+ <!-- component Profile -->
+ <field name="NLProfile_name" type="nederlab_string" required="true"
+ multiValued="false" indexed="true" stored="true" />
+
+ <!-- component ResourceProxy -->
+ <field name="ResourceProxy_resourceRef" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <dynamicField name="ResourceProxy_resourceRef_mimeType_*"
+ type="nederlab_string" required="false" multiValued="true" indexed="true"
+ stored="true" />
+
+ <!-- component NLCore -->
+ <field name="NLCore_NLIdentification_nederlabID" type="nederlab_uuid"
+ required="true" multiValued="false" indexed="true" stored="true" />
+ <field name="NLCore_NLIdentification_editorialCode" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLCore_NLIdentification_versionID" type="nederlab_string"
+ required="true" multiValued="false" indexed="true" stored="true" />
+ <field name="NLCore_NLIdentification_sourceRef" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLCore_NLIdentification_sourceUrl" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLCore_NLIdentification_sourceRefUrl_serialized"
+ type="nederlab_string" required="false" multiValued="true" indexed="false"
+ stored="true" />
+ <field name="NLCore_NLAdministrative_ingestTime" type="nederlab_date"
+ required="true" multiValued="false" indexed="true" stored="true" />
+ <field name="NLCore_NLAdministrative_expirationTime" type="nederlab_date"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLCore_NLAdministrative_lastEditedBy" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLCore_NLAdministrative_modificationTime" type="nederlab_date"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLCore_NLAdministrative_editorialNote" type="nederlab_text"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLCore_NLAdministrative_sourceCollection" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLCore_NLAdministrative_isThesaurusElement" type="nederlab_boolean"
+ required="true" multiValued="false" indexed="true" stored="true" />
+ <field name="NLCore_NLExternalReference_organizationName" type="nederlab_text"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLCore_NLExternalReference_collectionName" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLCore_NLExternalReference_resourceRef" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLCore_NLExternalReference_serialized" type="nederlab_string"
+ required="false" multiValued="true" indexed="false" stored="true" />
+
+ <!-- component NLTitle -->
+ <field name="NLTitle_title" type="nederlab_text" required="false"
+ multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_subtitle" type="nederlab_text" required="false"
+ multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_genre" type="nederlab_string" required="false"
+ multiValued="true" indexed="true" stored="true" />
+ <field name="NLTitle_category" type="nederlab_string" required="false"
+ multiValued="true" indexed="true" stored="true" />
+ <field name="NLTitle_yearOfPublicationMin" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_yearOfPublicationMax" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_yearOfPublicationApprox" type="nederlab_boolean"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_yearOfPublicationLabel" type="nederlab_text"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_edition" type="nederlab_string" required="false"
+ multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_inNederlabAs" type="nederlab_uuid" required="false"
+ multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_NLPublicationPlace_placeOfPublication" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLTitle_NLPublicationPlace_placeID" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLTitle_NLPublicationPlace_placeOfPublicationOriginal"
+ type="nederlab_text" required="false" multiValued="true" indexed="true"
+ stored="true" />
+ <field name="NLTitle_numberOfPages" type="nederlab_int" required="false"
+ multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_numberOfWords" type="nederlab_int" required="false"
+ multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_primaryLanguage" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_isTranslation" type="nederlab_boolean"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_characterEncoding" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_codingStandard" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLTitle_textQuality" type="nederlab_text" required="false"
+ multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_processingMethod" type="nederlab_text"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_autopsyPerformed" type="nederlab_boolean"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_NLPersonRef_personID" type="nederlab_uuid"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLTitle_NLPersonRef_role" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <dynamicField name="NLTitle_NLPersonRef_personID_role_*"
+ type="nederlab_uuid" required="false" multiValued="true" indexed="true"
+ stored="true" />
+ <field name="NLTitle_contains" type="nederlab_uuid" required="false"
+ multiValued="true" indexed="true" stored="true" />
+ <field name="NLTitle_seriesTitleID" type="nederlab_uuid"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLTitle_seriesTitleID_parent" type="nederlab_uuid"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLTitle_seriesTitleID_root" type="nederlab_uuid"
+ required="false" multiValued="false" indexed="true" stored="true" />
+
+ <!-- component NLDependentTitle -->
+ <field name="NLDependentTitle_title" type="nederlab_text"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLDependentTitle_subtitle" type="nederlab_text"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLDependentTitle_primaryLanguage" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLDependentTitle_parentTitleID" type="nederlab_uuid"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLDependentTitle_inNederlabAs" type="nederlab_uuid"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLDependentTitle_NLPersonRef_personID" type="nederlab_uuid"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLDependentTitle_NLPersonRef_role" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <dynamicField name="NLDependentTitle_NLPersonRef_personID_role_*"
+ type="nederlab_uuid" required="false" multiValued="true" indexed="true"
+ stored="true" />
+ <field name="NLDependentTitle_startPage" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLDependentTitle_endPage" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+
+ <!-- component NLPerson -->
+ <field name="NLPerson_NLPersonName_nameId" type="nederlab_uuid"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_lastName" type="nederlab_text"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_firstName" type="nederlab_text"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_infixes" type="nederlab_text"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_firstNameFull" type="nederlab_text"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_fullName" type="nederlab_text"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_fullName_serialized" type="nederlab_string"
+ required="false" multiValued="true" indexed="false" stored="true" />
+ <field name="NLPerson_NLPersonName_preferredNameID" type="nederlab_uuid"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_preferredLastName" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_preferredFirstName" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_preferredFirstNameFull" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_preferredInfixes" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_preferredFullName" type="nederlab_text"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_NLPersonName_preferredFullName_serialized"
+ type="nederlab_string" required="false" multiValued="false" indexed="false"
+ stored="true" />
+ <field name="NLPerson_dateOfBirthDayMonth" type="nederlab_text"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_dateOfBirthMonth" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_dateOfBirthDay" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_yearOfBirthMin" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_yearOfBirthMax" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_yearOfBirthApprox" type="nederlab_boolean"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_yearOfBirthLabel" type="nederlab_text"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_placeOfBirth" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_placeOfBirthID" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_dateOfDeathDayMonth" type="nederlab_text"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_dateOfDeathMonth" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_dateOfDeathDay" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_yearOfDeathMin" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_yearOfDeathMax" type="nederlab_int"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_yearOfDeathApprox" type="nederlab_boolean"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_yearOfDeathLabel" type="nederlab_text"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_placeOfDeath" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_placeOfDeathID" type="nederlab_string"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_gender" type="nederlab_string" required="false"
+ multiValued="false" indexed="true" stored="true" />
+ <field name="NLPerson_profession" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLPerson_education" type="nederlab_string" required="false"
+ multiValued="true" indexed="true" stored="true" />
+ <field name="NLPerson_inThesaurusAs" type="nederlab_uuid"
+ required="false" multiValued="false" indexed="true" stored="true" />
+
+ <!-- component NLSeriesTitle -->
+ <field name="NLSeriesTitle_title" type="nederlab_text" required="false"
+ multiValued="false" indexed="true" stored="true" />
+ <field name="NLSeriesTitle_years" type="nederlab_text" required="false"
+ multiValued="false" indexed="true" stored="true" />
+ <field name="NLSeriesTitle_description" type="nederlab_text"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLSeriesTitle_inNederlabAs" type="nederlab_uuid"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLSeriesTitle_seriesTitleID" type="nederlab_uuid"
+ required="false" multiValued="true" indexed="true" stored="true" />
+ <field name="NLSeriesTitle_seriesTitleID_parent" type="nederlab_uuid"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLSeriesTitle_seriesTitleID_root" type="nederlab_uuid"
+ required="false" multiValued="false" indexed="true" stored="true" />
+
+ <!-- component NLCollectionSpecific -->
+ <dynamicField name="NLCollectionSpecific_*" type="nederlab_string"
+ required="false" multiValued="true" indexed="true" stored="true" />
+
+ <!-- component NLContent old -->
+
+ <field name="NLContent_text_available" type="nederlab_boolean"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLContent_text" type="nederlab_content" required="false"
+ multiValued="false" indexed="true" stored="true" termVectors="true"
+ termPositions="true" termOffsets="true" />
+ <field name="NLContent_text_lowercase" type="nederlab_content_lowercase"
+ required="false" multiValued="false" indexed="true" stored="true"
+ termVectors="true" termPositions="true" termOffsets="true" />
+ <copyField source="NLContent_text" dest="NLContent_text_lowercase" />
+ <field name="NLContent_ticcl_available" type="nederlab_boolean"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLContent_ticcl_lowercase" type="nederlab_content_lowercase"
+ required="false" multiValued="false" indexed="true" stored="true"
+ termVectors="true" termPositions="true" termOffsets="true" />
+
+ <!-- component NLContent -->
+
+ <field name="NLContent_folia_available" type="nederlab_boolean"
+ required="false" multiValued="false" indexed="true" stored="true" />
+ <field name="NLContent_mtas" type="mtas_text" indexed="true"
+ stored="true" />
+ <field name="NLContent_mtas_error" type="nederlab_string"
+ indexed="true" stored="true" />
+ <field name="NLContent_mtas_numberOfTokens" type="nederlab_int"
+ indexed="true" stored="true" />
+ <field name="NLContent_mtas_numberOfPositions" type="nederlab_int"
+ indexed="true" stored="true" />
+ <field name="NLContent_mtas_size" type="nederlab_int" indexed="true"
+ stored="true" />
+ <!-- Combined Field Metadata -->
+ <field name="NLMetadata" type="nederlab_text" required="false"
+ multiValued="true" indexed="true" stored="false" />
+ <copyField source="NLCore_NLIdentification_nederlabID" dest="NLMetadata" />
+ <copyField source="NLCore_NLIdentification_editorialCode"
+ dest="NLMetadata" />
+ <copyField source="NLCore_NLIdentification_sourceRef" dest="NLMetadata" />
+ <copyField source="NLCore_NLAdministrative_editorialNote"
+ dest="NLMetadata" />
+ <copyField source="NLCore_NLAdministrative_sourceCollection"
+ dest="NLMetadata" />
+ <copyField source="NLCore_NLExternalReference_organizationName"
+ dest="NLMetadata" />
+ <copyField source="NLCore_NLExternalReference_collectionName"
+ dest="NLMetadata" />
+ <copyField source="NLCore_NLExternalReference_resourceRef"
+ dest="NLMetadata" />
+ <copyField source="NLTitle_title" dest="NLMetadata" />
+ <copyField source="NLTitle_subtitle" dest="NLMetadata" />
+ <copyField source="NLTitle_genre" dest="NLMetadata" />
+ <copyField source="NLTitle_category" dest="NLMetadata" />
+ <copyField source="NLTitle_yearOfPublicationMin" dest="NLMetadata" />
+ <copyField source="NLTitle_yearOfPublicationMax" dest="NLMetadata" />
+ <copyField source="NLTitle_yearOfPublicationLabel" dest="NLMetadata" />
+ <copyField source="NLTitle_edition" dest="NLMetadata" />
+ <copyField source="NLTitle_NLPublicationPlace_placeOfPublication"
+ dest="NLMetadata" />
+ <copyField source="NLTitle_NLPublicationPlace_placeID" dest="NLMetadata" />
+ <copyField source="NLTitle_NLPublicationPlace_placeOfPublicationOriginal"
+ dest="NLMetadata" />
+ <copyField source="NLTitle_primaryLanguage" dest="NLMetadata" />
+ <copyField source="NLTitle_characterEncoding" dest="NLMetadata" />
+ <copyField source="NLTitle_codingStandard" dest="NLMetadata" />
+ <copyField source="NLTitle_textQuality" dest="NLMetadata" />
+ <copyField source="NLTitle_processingMethod" dest="NLMetadata" />
+ <copyField source="NLTitle_NLPersonRef_role" dest="NLMetadata" />
+ <copyField source="NLDependentTitle_title" dest="NLMetadata" />
+ <copyField source="NLDependentTitle_subtitle" dest="NLMetadata" />
+ <copyField source="NLDependentTitle_primaryLanguage" dest="NLMetadata" />
+ <copyField source="NLDependentTitle_NLPersonRef_role" dest="NLMetadata" />
+ <copyField source="NLPerson_NLPersonName_lastName" dest="NLMetadata" />
+ <copyField source="NLPerson_NLPersonName_firstName" dest="NLMetadata" />
+ <copyField source="NLPerson_NLPersonName_infixes" dest="NLMetadata" />
+ <copyField source="NLPerson_NLPersonName_firstNameFull" dest="NLMetadata" />
+ <copyField source="NLPerson_NLPersonName_fullName" dest="NLMetadata" />
+ <copyField source="NLPerson_dateOfBirthDayMonth" dest="NLMetadata" />
+ <copyField source="NLPerson_yearOfBirthMin" dest="NLMetadata" />
+ <copyField source="NLPerson_yearOfBirthMax" dest="NLMetadata" />
+ <copyField source="NLPerson_yearOfBirthLabel" dest="NLMetadata" />
+ <copyField source="NLPerson_placeOfBirth" dest="NLMetadata" />
+ <copyField source="NLPerson_placeOfBirthID" dest="NLMetadata" />
+ <copyField source="NLPerson_dateOfDeathDayMonth" dest="NLMetadata" />
+ <copyField source="NLPerson_yearOfDeathMin" dest="NLMetadata" />
+ <copyField source="NLPerson_yearOfDeathMax" dest="NLMetadata" />
+ <copyField source="NLPerson_yearOfDeathLabel" dest="NLMetadata" />
+ <copyField source="NLPerson_placeOfDeath" dest="NLMetadata" />
+ <copyField source="NLPerson_placeOfDeathID" dest="NLMetadata" />
+ <copyField source="NLPerson_gender" dest="NLMetadata" />
+ <copyField source="NLPerson_profession" dest="NLMetadata" />
+ <copyField source="NLPerson_education" dest="NLMetadata" />
+ <copyField source="NLSeriesTitle_title" dest="NLMetadata" />
+ <copyField source="NLSeriesTitle_years" dest="NLMetadata" />
+ <copyField source="NLSeriesTitle_description" dest="NLMetadata" />
+ <copyField source="NLCollectionSpecific_*" dest="NLMetadata" />
+
+ <uniqueKey>NLCore_NLIdentification_versionID</uniqueKey>
+
+ <fieldType name="nederlab_string" class="solr.StrField"
+ sortMissingLast="true" />
+ <fieldType name="nederlab_uuid" class="solr.StrField"
+ sortMissingLast="true" />
+ <fieldType name="nederlab_boolean" class="solr.BoolField"
+ sortMissingLast="true" />
+ <fieldType name="nederlab_int" class="solr.TrieIntField"
+ precisionStep="8" positionIncrementGap="0" />
+ <fieldType name="nederlab_long" class="solr.TrieLongField"
+ precisionStep="0" positionIncrementGap="0" />
+ <fieldType name="nederlab_date" class="solr.TrieDateField"
+ precisionStep="6" positionIncrementGap="0" />
+ <fieldtype name="nederlab_binary" class="solr.BinaryField" />
+
+ <fieldType name="nederlab_text" class="solr.TextField"
+ positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.LowerCaseFilterFactory" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.LowerCaseFilterFactory" />
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="nederlab_content" class="solr.TextField"
+ positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="nederlab_content_lowercase" class="solr.TextField"
+ positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.LowerCaseFilterFactory" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.LowerCaseFilterFactory" />
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="mtas_text" class="solr.TextField"
+ postingsFormat="MtasCodec">
+ <analyzer type="index">
+ <charFilter class="mtas.analysis.util.MtasCharFilterFactory"
+ type="file" prefix="/local/data/" />
+ <tokenizer class="mtas.analysis.util.MtasTokenizerFactory"
+ configFile="mtas/folia_oeaw.xml" />
+ </analyzer>
+ </fieldType>
+
+</schema>
diff --git a/conf/solr/schemaTest.xml b/conf/solr/schemaTest.xml
index 1d89138..b443b15 100644
--- a/conf/solr/schemaTest.xml
+++ b/conf/solr/schemaTest.xml
@@ -115,11 +115,7 @@
WARNING: The _text_ catch-all field will significantly increase your index size.
If you don't need it, consider removing it and the corresponding copyField directive.
- -->
- <!
- <fieldType name="string_simpletext" class="solr.StrField" postingsFormat="SimpleText" />
- <field name="simple_string" type="string_simpletext" indexed="true" stored="true" required="false" multiValued="false" />
- -->
+ -->
<fieldType name="mtas_text" class="solr.TextField" postingsFormat="MtasCodec">
<analyzer type="index">
diff --git a/junit/mtas/parser/MtasCQLParserTestSentence.java b/junit/mtas/parser/MtasCQLParserTestSentence.java
index 69e11bb..81227f3 100644
--- a/junit/mtas/parser/MtasCQLParserTestSentence.java
+++ b/junit/mtas/parser/MtasCQLParserTestSentence.java
@@ -29,11 +29,11 @@ public class MtasCQLParserTestSentence {
basicTests();
}
- private void testCQLParse(String field, String cql, SpanQuery q) {
+ private void testCQLParse(String field, String defaultPrefix, String cql, SpanQuery q) {
MtasCQLParser p = new MtasCQLParser(new BufferedReader(new StringReader(cql)));
try {
System.out.print("CQL parsing:\t"+cql);
- assertEquals(p.parse(field) ,q);
+ assertEquals(p.parse(field, defaultPrefix) ,q);
System.out.print("\n");
} catch (ParseException e) {
System.out.println("Error CQL parsing:\t"+cql);
@@ -41,12 +41,12 @@ public class MtasCQLParserTestSentence {
}
}
- private void testCQLEquivalent(String field, String cql1, String cql2) {
+ private void testCQLEquivalent(String field, String defaultPrefix, String cql1, String cql2) {
MtasCQLParser p1 = new MtasCQLParser(new BufferedReader(new StringReader(cql1)));
MtasCQLParser p2 = new MtasCQLParser(new BufferedReader(new StringReader(cql2)));
try {
System.out.print("CQL equivalent:\t"+cql1+" and "+cql2);
- assertEquals(p1.parse(field) ,p2.parse(field));
+ assertEquals(p1.parse(field, defaultPrefix) ,p2.parse(field, defaultPrefix));
System.out.print("\n");
} catch (ParseException e) {
System.out.println("Error CQL equivalent:\t"+cql1+" and "+cql2);
@@ -73,6 +73,7 @@ public class MtasCQLParserTestSentence {
basicTest16();
basicTest17();
basicTest18();
+ basicTest19();
}
private void basicTest1() {
@@ -84,14 +85,14 @@ public class MtasCQLParserTestSentence {
items.add(new MtasSpanSequenceItem(q1, false));
items.add(new MtasSpanSequenceItem(q2, false));
SpanQuery q = new MtasSpanSequenceQuery(items);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest2() {
String field = "testveld";
String cql1 = "[pos=\"LID\"] [] []? [] [lemma=\"koe\"]";
String cql2 = "[pos=\"LID\"] []{2,3} [lemma=\"koe\"]";
- testCQLEquivalent(field, cql1, cql2);
+ testCQLEquivalent(field, null, cql1, cql2);
}
private void basicTest3() {
@@ -100,7 +101,7 @@ public class MtasCQLParserTestSentence {
SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID");
SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe");
SpanQuery q = new MtasSpanOrQuery(q1,q2);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest4() {
@@ -114,28 +115,28 @@ public class MtasCQLParserTestSentence {
items.add(new MtasSpanSequenceItem(q3, false));
SpanQuery q4 = new MtasSpanSequenceQuery(items);
SpanQuery q = new MtasSpanOrQuery(q1,q4);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest5() {
String field = "testveld";
String cql1 = "([pos=\"LID\"]([pos=\"ADJ\"][lemma=\"koe\"]))";
String cql2 = "[pos=\"LID\"][pos=\"ADJ\"][lemma=\"koe\"]";
- testCQLEquivalent(field, cql1, cql2);
+ testCQLEquivalent(field, null, cql1, cql2);
}
private void basicTest6() {
String field = "testveld";
String cql1 = "([pos=\"LID\"]|[lemma=\"de\"][lemma=\"koe\"])|([pos=\"ADJ\"]|([lemma=\"het\"]([lemma=\"paard\"])))";
String cql2 = "[pos=\"LID\"]|[lemma=\"de\"][lemma=\"koe\"]|[pos=\"ADJ\"]|[lemma=\"het\"][lemma=\"paard\"]";
- testCQLEquivalent(field, cql1, cql2);
+ testCQLEquivalent(field, null, cql1, cql2);
}
private void basicTest7() {
String field = "testveld";
String cql1 = "[pos=\"LID\"] []{0,1} []{3,5} []{2,4}";
String cql2 = "[pos=\"LID\"] []{5,10}";
- testCQLEquivalent(field, cql1, cql2);
+ testCQLEquivalent(field, null, cql1, cql2);
}
private void basicTest8() {
@@ -149,7 +150,7 @@ public class MtasCQLParserTestSentence {
items.add(new MtasSpanSequenceItem(q1, false));
items.add(new MtasSpanSequenceItem(q4, false));
SpanQuery q = new MtasSpanSequenceQuery(items);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest9() {
@@ -165,7 +166,7 @@ public class MtasCQLParserTestSentence {
items.add(new MtasSpanSequenceItem(q5, false));
items.add(new MtasSpanSequenceItem(q4, false));
SpanQuery q = new MtasSpanSequenceQuery(items);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest10() {
@@ -179,7 +180,7 @@ public class MtasCQLParserTestSentence {
items.add(new MtasSpanSequenceItem(new MtasSpanRecurrenceQuery(q2,1,3), false));
items.add(new MtasSpanSequenceItem(q3, false));
SpanQuery q = new MtasSpanSequenceQuery(items);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest11() {
@@ -188,7 +189,7 @@ public class MtasCQLParserTestSentence {
SpanQuery q1 = new MtasCQLParserGroupQuery(field,"sentence");
SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe");
SpanQuery q = new SpanContainingQuery(q1, q2);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest12() {
@@ -197,7 +198,7 @@ public class MtasCQLParserTestSentence {
SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe");
SpanQuery q2 = new MtasCQLParserGroupQuery(field,"sentence");
SpanQuery q = new SpanWithinQuery(q2, q1);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest13() {
@@ -211,7 +212,7 @@ public class MtasCQLParserTestSentence {
items.add(new MtasSpanSequenceItem(q1, false));
items.add(new MtasSpanSequenceItem(q4, false));
SpanQuery q = new MtasSpanSequenceQuery(items);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest14() {
@@ -225,7 +226,7 @@ public class MtasCQLParserTestSentence {
items.add(new MtasSpanSequenceItem(q3, false));
items.add(new MtasSpanSequenceItem(q4, false));
SpanQuery q = new MtasSpanSequenceQuery(items);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest15() {
@@ -246,7 +247,7 @@ public class MtasCQLParserTestSentence {
items2.add(new MtasSpanSequenceItem(q1, false));
items2.add(new MtasSpanSequenceItem(q8, false));
SpanQuery q = new MtasSpanSequenceQuery(items2);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest16() {
@@ -258,7 +259,7 @@ public class MtasCQLParserTestSentence {
SpanQuery q4 = new SpanContainingQuery(q2, q3);
SpanQuery q5 = new SpanWithinQuery(q4, q1);
SpanQuery q = new SpanNotQuery(q5,new SpanContainingQuery(q5, q3));
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest17() {
@@ -271,11 +272,23 @@ public class MtasCQLParserTestSentence {
items.add(new MtasSpanSequenceItem(q2, false));
items.add(new MtasSpanSequenceItem(new MtasSpanMatchAllQuery(field), false));
SpanQuery q = new MtasSpanSequenceQuery(items);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest18() {
String field = "testveld";
+ String cql = "\"de\" [pos=\"N\"]";
+ SpanQuery q1 = new MtasCQLParserWordQuery(field,"t_lc","de");
+ SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","N");
+ List<MtasSpanSequenceItem> items = new ArrayList<MtasSpanSequenceItem>();
+ items.add(new MtasSpanSequenceItem(q1, false));
+ items.add(new MtasSpanSequenceItem(q2, false));
+ SpanQuery q = new MtasSpanSequenceQuery(items);
+ testCQLParse(field, "t_lc", cql, q);
+ }
+
+ private void basicTest19() {
+ String field = "testveld";
String cql = "([]<entity=\"loc\"/>{1,2}[]){3,4}";
SpanQuery q1 = new MtasCQLParserGroupQuery(field,"entity","loc");
SpanQuery q2 = new MtasSpanRecurrenceQuery(q1,1,2);
@@ -285,7 +298,7 @@ public class MtasCQLParserTestSentence {
items.add(new MtasSpanSequenceItem(new MtasSpanMatchAllQuery(field), false));
SpanQuery q3 = new MtasSpanSequenceQuery(items);
SpanQuery q = new MtasSpanRecurrenceQuery(q3,3,4);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
}
diff --git a/junit/mtas/parser/MtasCQLParserTestWord.java b/junit/mtas/parser/MtasCQLParserTestWord.java
index c702d18..5299431 100644
--- a/junit/mtas/parser/MtasCQLParserTestWord.java
+++ b/junit/mtas/parser/MtasCQLParserTestWord.java
@@ -23,10 +23,10 @@ public class MtasCQLParserTestWord {
basicNotTests();
}
- private void testCQLParse(String field, String cql, SpanQuery q) {
+ private void testCQLParse(String field, String defaultPrefix, String cql, SpanQuery q) {
MtasCQLParser p = new MtasCQLParser(new BufferedReader(new StringReader(cql)));
try {
- assertEquals(p.parse(field) ,q);
+ assertEquals(p.parse(field, defaultPrefix) ,q);
System.out.println("Tested CQL parsing:\t"+cql);
} catch (ParseException e) {
System.out.println("Error CQL parsing:\t"+cql);
@@ -34,11 +34,11 @@ public class MtasCQLParserTestWord {
}
}
- private void testCQLEquivalent(String field, String cql1, String cql2) {
+ private void testCQLEquivalent(String field, String defaultPrefix, String cql1, String cql2) {
MtasCQLParser p1 = new MtasCQLParser(new BufferedReader(new StringReader(cql1)));
MtasCQLParser p2 = new MtasCQLParser(new BufferedReader(new StringReader(cql2)));
try {
- assertEquals(p1.parse(field) ,p2.parse(field));
+ assertEquals(p1.parse(field, defaultPrefix) ,p2.parse(field, defaultPrefix));
System.out.println("Tested CQL equivalent:\t"+cql1+" and "+cql2);
} catch (ParseException e) {
System.out.println("Error CQL equivalent:\t"+cql1+" and "+cql2);
@@ -67,6 +67,7 @@ public class MtasCQLParserTestWord {
basicTest10();
basicTest11();
basicTest12();
+ basicTest13();
}
private void basicNotTest1() {
@@ -75,14 +76,14 @@ public class MtasCQLParserTestWord {
SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID");
SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","de");
SpanQuery q = new SpanNotQuery(q1,q2);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicNotTest2() {
String field = "testveld";
String cql1 = "[pos=\"LID\" & (!lemma=\"de\")]";
String cql2 = "[pos=\"LID\" & !(lemma=\"de\")]";
- testCQLEquivalent(field, cql1, cql2);
+ testCQLEquivalent(field, null, cql1, cql2);
}
private void basicNotTest3() {
@@ -93,28 +94,28 @@ public class MtasCQLParserTestWord {
SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","een");
SpanQuery q4 = new MtasSpanOrQuery(new SpanQuery[]{q2,q3});
SpanQuery q = new SpanNotQuery(q1,q4);
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicNotTest4() {
String field = "testveld";
String cql1 = "[pos=\"LID\" & !(lemma=\"de\" | lemma=\"een\")]";
String cql2 = "[pos=\"LID\" & (!lemma=\"de\" & !lemma=\"een\")]";
- testCQLEquivalent(field, cql1, cql2);
+ testCQLEquivalent(field, null, cql1, cql2);
}
private void basicNotTest5() {
String field = "testveld";
String cql1 = "[pos=\"LID\" & !(lemma=\"de\" | lemma=\"een\")]";
String cql2 = "[pos=\"LID\" & !lemma=\"de\" & !lemma=\"een\"]";
- testCQLEquivalent(field, cql1, cql2);
+ testCQLEquivalent(field, null, cql1, cql2);
}
private void basicTest1() {
String field = "testveld";
String cql = "[lemma=\"koe\"]";
SpanQuery q = new MtasCQLParserWordQuery(field, "lemma", "koe");
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest2() {
@@ -123,7 +124,7 @@ public class MtasCQLParserTestWord {
SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe");
SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","N");
SpanQuery q = new MtasSpanAndQuery(new SpanQuery[]{q1,q2});
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest3() {
@@ -132,14 +133,14 @@ public class MtasCQLParserTestWord {
SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe");
SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","paard");
SpanQuery q = new MtasSpanOrQuery(new SpanQuery[]{q1,q2});
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest4() {
String field = "testveld";
String cql1 = "[lemma=\"koe\" | lemma=\"paard\"]";
String cql2 = "[(lemma=\"koe\" | lemma=\"paard\")]";
- testCQLEquivalent(field, cql1, cql2);
+ testCQLEquivalent(field, null, cql1, cql2);
}
private void basicTest5() {
@@ -150,7 +151,7 @@ public class MtasCQLParserTestWord {
SpanQuery q3 = new MtasSpanOrQuery(new SpanQuery[]{q1,q2});
SpanQuery q4 = new MtasCQLParserWordQuery(field,"pos","N");
SpanQuery q = new MtasSpanAndQuery(new SpanQuery[]{q3,q4});
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest6() {
@@ -161,7 +162,7 @@ public class MtasCQLParserTestWord {
SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","paard");
SpanQuery q4 = new MtasSpanOrQuery(new SpanQuery[]{q2,q3});
SpanQuery q = new MtasSpanAndQuery(new SpanQuery[]{q1,q4});
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest7() {
@@ -172,7 +173,7 @@ public class MtasCQLParserTestWord {
SpanQuery q3 = new MtasCQLParserWordQuery(field,"pos","N");
SpanQuery q4 = new MtasSpanAndQuery(new SpanQuery[]{q2,q3});
SpanQuery q = new MtasSpanOrQuery(new SpanQuery[]{q1,q4});
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest8() {
@@ -185,7 +186,7 @@ public class MtasCQLParserTestWord {
SpanQuery q5 = new MtasSpanAndQuery(new SpanQuery[]{q1,q2});
SpanQuery q6 = new MtasSpanAndQuery(new SpanQuery[]{q3,q4});
SpanQuery q = new MtasSpanOrQuery(new SpanQuery[]{q5,q6});
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest9() {
@@ -200,7 +201,7 @@ public class MtasCQLParserTestWord {
SpanQuery q7 = new MtasSpanAndQuery(new SpanQuery[]{q6,q3});
SpanQuery q8 = new MtasSpanAndQuery(new SpanQuery[]{q4,q5});
SpanQuery q = new MtasSpanOrQuery(new SpanQuery[]{q7,q8});
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest10() {
@@ -217,22 +218,22 @@ public class MtasCQLParserTestWord {
SpanQuery q9 = new MtasSpanOrQuery(new SpanQuery[]{q4,q5});
SpanQuery q10 = new MtasSpanAndQuery(new SpanQuery[]{q9,q6});
SpanQuery q = new MtasSpanOrQuery(new SpanQuery[]{q8,q10});
- testCQLParse(field, cql, q);
+ testCQLParse(field, null, cql, q);
}
private void basicTest11() {
String field = "testveld";
String cql1 = "[#300]";
SpanQuery q1 = new MtasCQLParserWordPositionQuery(field, 300);
- testCQLParse(field, cql1, q1);
+ testCQLParse(field, null, cql1, q1);
String cql2 = "[#100-110]";
SpanQuery q2 = new MtasCQLParserWordPositionQuery(field, 100, 110);
- testCQLParse(field, cql2, q2);
+ testCQLParse(field, null, cql2, q2);
String cql3 = "[#100-105 | #110]";
SpanQuery q3a = new MtasCQLParserWordPositionQuery(field, 100, 105);
SpanQuery q3b = new MtasCQLParserWordPositionQuery(field, 110);
SpanQuery q3 = new MtasSpanOrQuery(q3a, q3b);
- testCQLParse(field, cql3, q3);
+ testCQLParse(field, null, cql3, q3);
}
private void basicTest12() {
@@ -242,6 +243,13 @@ public class MtasCQLParserTestWord {
SpanQuery q2 = new MtasCQLParserWordQuery(field,"t_lc","het");
SpanQuery q3 = new MtasCQLParserWordQuery(field,"t_lc","paard");
SpanQuery q = new MtasSpanOrQuery(new SpanQuery[]{q1,q2,q3});
- testCQLParse(field, cql, q);
- }
+ testCQLParse(field, null, cql, q);
+ }
+
+ private void basicTest13() {
+ String field = "testveld";
+ String cql = "\"de\"";
+ SpanQuery q = new MtasCQLParserWordQuery(field,"t_lc","de");
+ testCQLParse(field, "t_lc", cql, q);
+ }
}
diff --git a/pom.xml b/pom.xml
index 329fea5..e02aec0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2,11 +2,13 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <currentDevelopmentVersion>6.2.0</currentDevelopmentVersion>
+ <currentDevelopmentRelease>20160802</currentDevelopmentRelease>
</properties>
<modelVersion>4.0.0</modelVersion>
<groupId>dev.meertens.mtas</groupId>
<artifactId>mtas</artifactId>
- <version>6.1.0</version>
+ <version>6.2.0</version>
<packaging>jar</packaging>
<licenses>
<license>
@@ -23,12 +25,12 @@
<developers>
<developer>
<name>Matthijs Brouwer</name>
- <url>https://www.meertens.knaw.nl/cms/nl/medewerkers/144373-matthijsb</url>
+ <url>https://nl.linkedin.com/in/brouwermatthijs/</url>
</developer>
<developer>
- <name>Marc Kemps-Snijders</name>
- <url>https://www.meertens.knaw.nl/cms/nl/medewerkers/143329-marck</url>
- </developer>
+ <name>Marc Kemps-Snijders</name>
+ <url>https://nl.linkedin.com/in/marc-kemps-snijders-1b33753</url>
+ </developer>
</developers>
<build>
<sourceDirectory>src</sourceDirectory>
@@ -39,6 +41,24 @@
</resources>
<plugins>
<plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>3.0.0</version>
+ <configuration>
+ <filesets>
+ <fileset>
+ <directory>gh-pages</directory>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ <excludes>
+ <exclude>**/.git/</exclude>
+ </excludes>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ </filesets>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
@@ -46,7 +66,7 @@
<source>1.8</source>
<target>1.8</target>
</configuration>
- </plugin>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-site-plugin</artifactId>
@@ -145,27 +165,27 @@
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
- <version>6.1.0</version>
+ <version>6.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
- <version>6.1.0</version>
+ <version>6.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
- <version>6.1.0</version>
+ <version>6.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-codecs</artifactId>
- <version>6.1.0</version>
+ <version>6.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
- <version>6.1.0</version>
+ <version>6.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
diff --git a/src/mtas/analysis/MtasTokenizer.java b/src/mtas/analysis/MtasTokenizer.java
index 7891b87..49193a3 100644
--- a/src/mtas/analysis/MtasTokenizer.java
+++ b/src/mtas/analysis/MtasTokenizer.java
@@ -25,8 +25,11 @@ import org.apache.lucene.util.AttributeFactory;
/**
* The Class MtasTokenizer.
+ *
+ * @param <T>
+ * the generic type
*/
-public final class MtasTokenizer extends Tokenizer {
+public final class MtasTokenizer<T> extends Tokenizer {
/** The configuration mtas. */
public static String CONFIGURATION_MTAS = "mtas";
@@ -73,7 +76,8 @@ public final class MtasTokenizer extends Tokenizer {
/**
* Instantiates a new mtas tokenizer.
*
- * @param configFileName the config file name
+ * @param configFileName
+ * the config file name
*/
public MtasTokenizer(String configFileName) {
readConfigurationFile(configFileName);
@@ -82,8 +86,10 @@ public final class MtasTokenizer extends Tokenizer {
/**
* Instantiates a new mtas tokenizer.
*
- * @param config the config
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param config
+ * the config
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasTokenizer(MtasConfiguration config) throws IOException {
processConfiguration(config);
@@ -92,8 +98,10 @@ public final class MtasTokenizer extends Tokenizer {
/**
* Instantiates a new mtas tokenizer.
*
- * @param reader the reader
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param reader
+ * the reader
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasTokenizer(InputStream reader) throws IOException {
processConfiguration(MtasConfiguration.readConfiguration(reader));
@@ -102,9 +110,12 @@ public final class MtasTokenizer extends Tokenizer {
/**
* Instantiates a new mtas tokenizer.
*
- * @param factory the factory
- * @param config the config
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param factory
+ * the factory
+ * @param config
+ * the config
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasTokenizer(AttributeFactory factory, MtasConfiguration config)
throws IOException {
@@ -112,7 +123,9 @@ public final class MtasTokenizer extends Tokenizer {
processConfiguration(config);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.analysis.TokenStream#incrementToken()
*/
@Override
@@ -128,7 +141,7 @@ public final class MtasTokenizer extends Tokenizer {
// compute info
positionIncrement = token.getPositionStart() - currentPosition;
currentPosition = token.getPositionStart();
- payloadEncoder = new MtasPayloadEncoder(token, encodingFlags);
+ payloadEncoder = new MtasPayloadEncoder(token, encodingFlags);
// set info
termAtt.append(token.getValue().toString());
positionIncrementAtt.setPositionIncrement(positionIncrement);
@@ -157,7 +170,6 @@ public final class MtasTokenizer extends Tokenizer {
e.getClass().getSimpleName() + ": " + e.getMessage());
} catch (MtasParserException e) {
tokenCollectionIterator = null;
- e.printStackTrace();
throw new IOException(
e.getClass().getSimpleName() + ": " + e.getMessage());
}
@@ -167,14 +179,19 @@ public final class MtasTokenizer extends Tokenizer {
/**
* Prints the.
*
- * @param r the r
- * @throws IOException Signals that an I/O exception has occurred.
- * @throws MtasParserException the mtas parser exception
+ * @param r
+ * the r
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ * @throws MtasParserException
+ * the mtas parser exception
*/
public void print(Reader r) throws IOException, MtasParserException {
setReader(r);
reset();
- tokenCollection.print();
+ if (tokenCollection != null) {
+ tokenCollection.print();
+ }
end();
close();
}
@@ -182,10 +199,13 @@ public final class MtasTokenizer extends Tokenizer {
/**
* Gets the list.
*
- * @param r the r
+ * @param r
+ * the r
* @return the list
- * @throws IOException Signals that an I/O exception has occurred.
- * @throws MtasParserException the mtas parser exception
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ * @throws MtasParserException
+ * the mtas parser exception
*/
public String[][] getList(Reader r) throws IOException, MtasParserException {
setReader(r);
@@ -199,9 +219,12 @@ public final class MtasTokenizer extends Tokenizer {
/**
* Construct token collection.
*
- * @param reader the reader
- * @throws MtasConfigException the mtas config exception
- * @throws MtasParserException the mtas parser exception
+ * @param reader
+ * the reader
+ * @throws MtasConfigException
+ * the mtas config exception
+ * @throws MtasParserException
+ * the mtas parser exception
*/
private void constructTokenCollection(Reader reader)
throws MtasConfigException, MtasParserException {
@@ -216,29 +239,28 @@ public final class MtasTokenizer extends Tokenizer {
try {
tokenCollection = parser.createTokenCollection(reader);
return;
- } catch (MtasParserException e) {
+ } catch (MtasParserException e) {
tokenCollection = new MtasTokenCollection();
- e.printStackTrace();
throw new MtasParserException(e.getMessage());
}
} else {
throw new MtasConfigException("no instance of MtasParser");
}
} catch (NoSuchMethodException e) {
- throw new MtasConfigException(e.getClass().getName()
- + " : '" + e.getMessage() + "'");
+ throw new MtasConfigException(
+ e.getClass().getName() + " : '" + e.getMessage() + "'");
} catch (InvocationTargetException e) {
- throw new MtasConfigException(e.getClass().getName()
- + " : '" + e.getMessage() + "'");
+ throw new MtasConfigException(
+ e.getClass().getName() + " : '" + e.getMessage() + "'");
} catch (IllegalAccessException e) {
- throw new MtasConfigException(e.getClass().getName()
- + " : '" + e.getMessage() + "'");
+ throw new MtasConfigException(
+ e.getClass().getName() + " : '" + e.getMessage() + "'");
} catch (ClassNotFoundException e) {
- throw new MtasConfigException(e.getClass().getName()
- + " : '" + e.getMessage() + "'");
+ throw new MtasConfigException(
+ e.getClass().getName() + " : '" + e.getMessage() + "'");
} catch (InstantiationException e) {
- throw new MtasConfigException(e.getClass().getName()
- + " : '" + e.getMessage() + "'");
+ throw new MtasConfigException(
+ e.getClass().getName() + " : '" + e.getMessage() + "'");
}
}
@@ -246,7 +268,8 @@ public final class MtasTokenizer extends Tokenizer {
/**
* Read configuration file.
*
- * @param configFile the config file
+ * @param configFile
+ * the config file
*/
private void readConfigurationFile(String configFile) {
InputStream is;
@@ -261,13 +284,13 @@ public final class MtasTokenizer extends Tokenizer {
}
}
-
-
/**
* Process configuration.
*
- * @param config the config
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param config
+ * the config
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private void processConfiguration(MtasConfiguration config)
throws IOException {
diff --git a/src/mtas/analysis/parser/MtasBasicParser.java b/src/mtas/analysis/parser/MtasBasicParser.java
index be9cfff..1765342 100644
--- a/src/mtas/analysis/parser/MtasBasicParser.java
+++ b/src/mtas/analysis/parser/MtasBasicParser.java
@@ -165,7 +165,8 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Instantiates a new mtas basic parser.
*
- * @param config the config
+ * @param config
+ * the config
*/
public MtasBasicParser(MtasConfiguration config) {
this.config = config;
@@ -174,11 +175,16 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Compute mappings from object.
*
- * @param object the object
- * @param currentList the current list
- * @param updateList the update list
- * @throws MtasParserException the mtas parser exception
- * @throws MtasConfigException the mtas config exception
+ * @param object
+ * the object
+ * @param currentList
+ * the current list
+ * @param updateList
+ * the update list
+ * @throws MtasParserException
+ * the mtas parser exception
+ * @throws MtasConfigException
+ * the mtas config exception
*/
protected void computeMappingsFromObject(MtasParserObject object,
HashMap<String, ArrayList<MtasParserObject>> currentList,
@@ -196,7 +202,6 @@ abstract public class MtasBasicParser extends MtasParser {
updateList.get(UPDATE_TYPE_OFFSET).put(tokenId, object.getRefIds());
}
}
-
for (MtasParserMapping<?> mapping : mappings) {
try {
if (mapping.getTokens().size() == 0) {
@@ -271,9 +276,18 @@ abstract public class MtasBasicParser extends MtasParser {
String checkType = object.objectType.getType();
// register id for update when parent is created
if (currentList.get(checkType).size() > 0) {
- currentList.get(checkType)
- .get(currentList.get(checkType).size() - 1)
- .registerUpdateableMappingAtParent(token.getId());
+ if (currentList.get(checkType).contains(object)) {
+ int listPosition = currentList.get(checkType)
+ .indexOf(object);
+ if (listPosition > 0) {
+ currentList.get(checkType).get(listPosition - 1)
+ .registerUpdateableMappingAtParent(token.getId());
+ }
+ } else {
+ currentList.get(checkType)
+ .get(currentList.get(checkType).size() - 1)
+ .registerUpdateableMappingAtParent(token.getId());
+ }
// if no real ancestor, register id update when group
// ancestor is created
} else if (currentList.get(MAPPING_TYPE_GROUP).size() > 0) {
@@ -287,7 +301,9 @@ abstract public class MtasBasicParser extends MtasParser {
}
// update children
for (Integer tmpId : object.getUpdateableMappingsAsParent()) {
- tokenCollection.get(tmpId).setParentId(token.getId());
+ if (tokenCollection.get(tmpId) != null) {
+ tokenCollection.get(tmpId).setParentId(token.getId());
+ }
}
object.resetUpdateableMappingsAsParent();
// use own position
@@ -372,10 +388,20 @@ abstract public class MtasBasicParser extends MtasParser {
}
// copy remaining updateableMappings to new parent
if (currentList.get(objectType.getType()).size() > 0) {
- currentList.get(objectType.getType())
- .get(currentList.get(objectType.getType()).size() - 1)
- .registerUpdateableMappingsAtParent(
- object.getUpdateableMappingsAsParent());
+ if (currentList.get(objectType.getType()).contains(object)) {
+ int listPosition = currentList.get(objectType.getType())
+ .indexOf(object);
+ if (listPosition > 0) {
+ currentList.get(objectType.getType()).get(listPosition - 1)
+ .registerUpdateableMappingsAtParent(
+ object.getUpdateableMappingsAsParent());
+ }
+ } else {
+ currentList.get(objectType.getType())
+ .get(currentList.get(objectType.getType()).size() - 1)
+ .registerUpdateableMappingsAtParent(
+ object.getUpdateableMappingsAsParent());
+ }
} else if (currentList.get(MAPPING_TYPE_GROUP).size() > 0) {
currentList.get(MAPPING_TYPE_GROUP)
.get(currentList.get(MAPPING_TYPE_GROUP).size() - 1)
@@ -392,9 +418,11 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Compute type from mapping source.
*
- * @param source the source
+ * @param source
+ * the source
* @return the string
- * @throws MtasParserException the mtas parser exception
+ * @throws MtasParserException
+ * the mtas parser exception
*/
private String computeTypeFromMappingSource(String source)
throws MtasParserException {
@@ -423,11 +451,15 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Compute object from mapping value.
*
- * @param object the object
- * @param mappingValue the mapping value
- * @param currentList the current list
+ * @param object
+ * the object
+ * @param mappingValue
+ * the mapping value
+ * @param currentList
+ * the current list
* @return the mtas parser object[]
- * @throws MtasParserException the mtas parser exception
+ * @throws MtasParserException
+ * the mtas parser exception
*/
private MtasParserObject[] computeObjectFromMappingValue(
MtasParserObject object, HashMap<String, String> mappingValue,
@@ -469,12 +501,17 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Compute value from mapping values.
*
- * @param object the object
- * @param mappingValues the mapping values
- * @param currentList the current list
+ * @param object
+ * the object
+ * @param mappingValues
+ * the mapping values
+ * @param currentList
+ * the current list
* @return the string[]
- * @throws MtasParserException the mtas parser exception
- * @throws MtasConfigException the mtas config exception
+ * @throws MtasParserException
+ * the mtas parser exception
+ * @throws MtasConfigException
+ * the mtas config exception
*/
private String[] computeValueFromMappingValues(MtasParserObject object,
ArrayList<HashMap<String, String>> mappingValues,
@@ -486,8 +523,8 @@ abstract public class MtasBasicParser extends MtasParser {
if (mappingValue.get("source").equals(MtasParserMapping.SOURCE_STRING)) {
if (mappingValue.get("type")
.equals(MtasParserMapping.PARSER_TYPE_STRING)) {
- String subvalue = computeFilteredPrefixedValue(mappingValue.get("type"),
- mappingValue.get("text"), null, null);
+ String subvalue = computeFilteredPrefixedValue(
+ mappingValue.get("type"), mappingValue.get("text"), null, null);
if (subvalue != null) {
for (int i = 0; i < value.length; i++) {
value[i] = value[i] + subvalue;
@@ -499,7 +536,7 @@ abstract public class MtasBasicParser extends MtasParser {
MtasParserObject[] checkObjects = computeObjectFromMappingValue(object,
mappingValue, currentList);
// create value
- if (checkObjects != null) {
+ if (checkObjects != null && checkObjects.length > 0) {
MtasParserType checkType = checkObjects[0].getType();
// add name to value
if (mappingValue.get("type")
@@ -586,11 +623,15 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Compute payload from mapping payload.
*
- * @param object the object
- * @param mappingPayloads the mapping payloads
- * @param currentList the current list
+ * @param object
+ * the object
+ * @param mappingPayloads
+ * the mapping payloads
+ * @param currentList
+ * the current list
* @return the bytes ref
- * @throws MtasParserException the mtas parser exception
+ * @throws MtasParserException
+ * the mtas parser exception
*/
private BytesRef computePayloadFromMappingPayload(MtasParserObject object,
ArrayList<HashMap<String, String>> mappingPayloads,
@@ -605,7 +646,7 @@ abstract public class MtasBasicParser extends MtasParser {
if (mappingPayload.get("text") != null) {
BytesRef subpayload = computeMaximumFilteredPayload(
mappingPayload.get("text"), payload, null);
- payload = (subpayload != null) ? subpayload : payload;
+ payload = (subpayload != null) ? subpayload : payload;
}
}
// from objects
@@ -637,8 +678,10 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Prevalidate object.
*
- * @param object the object
- * @param currentList the current list
+ * @param object
+ * the object
+ * @param currentList
+ * the current list
* @return the boolean
*/
Boolean prevalidateObject(MtasParserObject object,
@@ -663,10 +706,14 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Precheck mapping conditions.
*
- * @param object the object
- * @param mappingConditions the mapping conditions
- * @param currentList the current list
- * @throws MtasParserException the mtas parser exception
+ * @param object
+ * the object
+ * @param mappingConditions
+ * the mapping conditions
+ * @param currentList
+ * the current list
+ * @throws MtasParserException
+ * the mtas parser exception
*/
void precheckMappingConditions(MtasParserObject object,
ArrayList<HashMap<String, String>> mappingConditions,
@@ -771,7 +818,31 @@ abstract public class MtasBasicParser extends MtasParser {
// condition on text
} else if (mappingCondition.get("type")
.equals(MtasParserMapping.PARSER_TYPE_TEXT)) {
- // can't pre-check this type of condition
+ // can't pre-check this type of condition, only for group
+ if (object.getType().precheckText()) {
+ String textCondition = mappingCondition.get("condition");
+ String textValue = object.getText();
+ if ((textCondition == null)
+ && ((textValue == null) || textValue.equals(""))) {
+ if (!notCondition) {
+ throw new MtasParserException("no text available");
+ }
+ } else if ((textCondition != null) && (textValue == null)) {
+ if (!notCondition) {
+ throw new MtasParserException("condition " + textCondition
+ + " on text not matched (is null)");
+ }
+ } else if (textCondition != null) {
+ if (!notCondition && !textCondition.equals(textValue)) {
+ throw new MtasParserException("condition " + textCondition
+ + " on text not matched (is " + textValue + ")");
+ } else if (notCondition && textCondition.equals(textValue)) {
+ throw new MtasParserException(
+ "condition NOT " + textCondition
+ + " on text not matched (is " + textValue + ")");
+ }
+ }
+ }
}
}
} else if (!notCondition) {
@@ -785,10 +856,14 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Postcheck mapping conditions.
*
- * @param object the object
- * @param mappingConditions the mapping conditions
- * @param currentList the current list
- * @throws MtasParserException the mtas parser exception
+ * @param object
+ * the object
+ * @param mappingConditions
+ * the mapping conditions
+ * @param currentList
+ * the current list
+ * @throws MtasParserException
+ * the mtas parser exception
*/
private void postcheckMappingConditions(MtasParserObject object,
ArrayList<HashMap<String, String>> mappingConditions,
@@ -835,10 +910,13 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Compute filtered split values.
*
- * @param values the values
- * @param filter the filter
+ * @param values
+ * the values
+ * @param filter
+ * the filter
* @return the string[]
- * @throws MtasConfigException the mtas config exception
+ * @throws MtasConfigException
+ * the mtas config exception
*/
private String[] computeFilteredSplitValues(String[] values, String filter)
throws MtasConfigException {
@@ -847,39 +925,42 @@ abstract public class MtasBasicParser extends MtasParser {
boolean[] valuesFilter = new boolean[values.length];
boolean doSplitFilter = false;
for (String item : filters) {
- if (item.trim()
- .matches("^"+Pattern.quote(MAPPING_FILTER_SPLIT) + "\\([0-9\\-]+\\)$")) {
+ if (item.trim().matches(
+ "^" + Pattern.quote(MAPPING_FILTER_SPLIT) + "\\([0-9\\-]+\\)$")) {
doSplitFilter = true;
- Pattern splitContent = Pattern.compile("^"+Pattern.quote(MAPPING_FILTER_SPLIT) + "\\(([0-9]+)(-([0-9]+))?\\)$");
+ Pattern splitContent = Pattern
+ .compile("^" + Pattern.quote(MAPPING_FILTER_SPLIT)
+ + "\\(([0-9]+)(-([0-9]+))?\\)$");
Matcher splitContentMatcher = splitContent.matcher(item.trim());
- while(splitContentMatcher.find()) {
- if(splitContentMatcher.group(3)==null) {
+ while (splitContentMatcher.find()) {
+ if (splitContentMatcher.group(3) == null) {
int i = Integer.parseInt(splitContentMatcher.group(1));
- if(i>=0 && i<values.length) {
+ if (i >= 0 && i < values.length) {
valuesFilter[i] = true;
- }
+ }
} else {
int i1 = Integer.parseInt(splitContentMatcher.group(1));
int i2 = Integer.parseInt(splitContentMatcher.group(3));
- for(int i=Math.max(0, i1); i<Math.min(values.length, i2); i++) {
+ for (int i = Math.max(0, i1); i < Math.min(values.length,
+ i2); i++) {
valuesFilter[i] = true;
}
}
- }
+ }
}
}
- if(doSplitFilter) {
+ if (doSplitFilter) {
int number = 0;
- for(int i=0;i<valuesFilter.length; i++) {
- if(valuesFilter[i]) {
+ for (int i = 0; i < valuesFilter.length; i++) {
+ if (valuesFilter[i]) {
number++;
}
}
- if(number>0) {
+ if (number > 0) {
String[] newValues = new String[number];
number = 0;
- for(int i=0;i<valuesFilter.length; i++) {
- if(valuesFilter[i]) {
+ for (int i = 0; i < valuesFilter.length; i++) {
+ if (valuesFilter[i]) {
newValues[number] = values[i];
number++;
}
@@ -888,7 +969,7 @@ abstract public class MtasBasicParser extends MtasParser {
} else {
return null;
}
- }
+ }
}
return values;
}
@@ -896,12 +977,17 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Compute filtered prefixed value.
*
- * @param type the type
- * @param value the value
- * @param filter the filter
- * @param prefix the prefix
+ * @param type
+ * the type
+ * @param value
+ * the value
+ * @param filter
+ * the filter
+ * @param prefix
+ * the prefix
* @return the string
- * @throws MtasConfigException the mtas config exception
+ * @throws MtasConfigException
+ * the mtas config exception
*/
private String computeFilteredPrefixedValue(String type, String value,
String filter, String prefix) throws MtasConfigException {
@@ -926,7 +1012,7 @@ abstract public class MtasBasicParser extends MtasParser {
}
} else if (item.trim()
.matches(Pattern.quote(MAPPING_FILTER_SPLIT) + "\\([0-9\\-]+\\)")) {
- if(!type.equals(MtasParserMapping.PARSER_TYPE_TEXT_SPLIT)) {
+ if (!type.equals(MtasParserMapping.PARSER_TYPE_TEXT_SPLIT)) {
throw new MtasConfigException(
"split filter not allowed for " + type);
}
@@ -947,9 +1033,12 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Compute maximum filtered payload.
*
- * @param value the value
- * @param payload the payload
- * @param filter the filter
+ * @param value
+ * the value
+ * @param payload
+ * the payload
+ * @param filter
+ * the filter
* @return the bytes ref
*/
private BytesRef computeMaximumFilteredPayload(String value, BytesRef payload,
@@ -981,6 +1070,9 @@ abstract public class MtasBasicParser extends MtasParser {
/** The name. */
private String name;
+ /** The precheck text. */
+ protected boolean precheckText;
+
/** The ref attribute name. */
private String refAttributeName;
@@ -990,23 +1082,34 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Instantiates a new mtas parser type.
*
- * @param type the type
- * @param name the name
+ * @param type
+ * the type
+ * @param name
+ * the name
+ * @param precheckText
+ * the precheck text
*/
- MtasParserType(String type, String name) {
+ MtasParserType(String type, String name, boolean precheckText) {
this.type = type;
this.name = name;
+ this.precheckText = precheckText;
}
/**
* Instantiates a new mtas parser type.
*
- * @param type the type
- * @param name the name
- * @param refAttributeName the ref attribute name
+ * @param type
+ * the type
+ * @param name
+ * the name
+ * @param precheckText
+ * the precheck text
+ * @param refAttributeName
+ * the ref attribute name
*/
- MtasParserType(String type, String name, String refAttributeName) {
- this(type, name);
+ MtasParserType(String type, String name, boolean precheckText,
+ String refAttributeName) {
+ this(type, name, precheckText);
this.refAttributeName = refAttributeName;
}
@@ -1038,9 +1141,19 @@ abstract public class MtasBasicParser extends MtasParser {
}
/**
+ * Precheck text.
+ *
+ * @return true, if successful
+ */
+ public boolean precheckText() {
+ return precheckText;
+ }
+
+ /**
* Adds the mapping.
*
- * @param mapping the mapping
+ * @param mapping
+ * the mapping
*/
public void addMapping(MtasParserMapping<?> mapping) {
mappings.add(mapping);
@@ -1080,7 +1193,8 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Instantiates a new mtas parser mapping token.
*
- * @param tokenType the token type
+ * @param tokenType
+ * the token type
*/
public MtasParserMappingToken(String tokenType) {
type = tokenType;
@@ -1095,7 +1209,8 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Sets the offset.
*
- * @param tokenOffset the new offset
+ * @param tokenOffset
+ * the new offset
*/
public void setOffset(Boolean tokenOffset) {
offset = tokenOffset;
@@ -1104,7 +1219,8 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Sets the real offset.
*
- * @param tokenRealOffset the new real offset
+ * @param tokenRealOffset
+ * the new real offset
*/
public void setRealOffset(Boolean tokenRealOffset) {
realoffset = tokenRealOffset;
@@ -1113,7 +1229,8 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Sets the parent.
*
- * @param tokenParent the new parent
+ * @param tokenParent
+ * the new parent
*/
public void setParent(Boolean tokenParent) {
parent = tokenParent;
@@ -1124,7 +1241,8 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* The Class MtasParserMapping.
*
- * @param <T> the generic type
+ * @param <T>
+ * the generic type
*/
protected abstract class MtasParserMapping<T extends MtasParserMapping<T>> {
@@ -1216,8 +1334,10 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Process config.
*
- * @param config the config
- * @throws MtasConfigException the mtas config exception
+ * @param config
+ * the config
+ * @throws MtasConfigException
+ * the mtas config exception
*/
public void processConfig(MtasConfiguration config)
throws MtasConfigException {
@@ -1581,7 +1701,8 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Condition unknown ancestor.
*
- * @param number the number
+ * @param number
+ * the number
*/
private void conditionUnknownAncestor(String number) {
HashMap<String, String> mapConstructionItem = new HashMap<String, String>();
@@ -1593,9 +1714,12 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Adds the string.
*
- * @param mappingToken the mapping token
- * @param type the type
- * @param text the text
+ * @param mappingToken
+ * the mapping token
+ * @param type
+ * the type
+ * @param text
+ * the text
*/
private void addString(MtasParserMappingToken mappingToken, String type,
String text) {
@@ -1613,8 +1737,10 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Payload string.
*
- * @param mappingToken the mapping token
- * @param text the text
+ * @param mappingToken
+ * the mapping token
+ * @param text
+ * the text
*/
private void payloadString(MtasParserMappingToken mappingToken,
String text) {
@@ -1628,10 +1754,14 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Adds the name.
*
- * @param mappingToken the mapping token
- * @param type the type
- * @param prefix the prefix
- * @param filter the filter
+ * @param mappingToken
+ * the mapping token
+ * @param type
+ * the type
+ * @param prefix
+ * the prefix
+ * @param filter
+ * the filter
*/
private void addName(MtasParserMappingToken mappingToken, String type,
String prefix, String filter) {
@@ -1650,8 +1780,10 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Condition name.
*
- * @param condition the condition
- * @param not the not
+ * @param condition
+ * the condition
+ * @param not
+ * the not
*/
private void conditionName(String condition, String not) {
HashMap<String, String> mapConstructionItem = new HashMap<String, String>();
@@ -1665,10 +1797,14 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Adds the text.
*
- * @param mappingToken the mapping token
- * @param type the type
- * @param prefix the prefix
- * @param filter the filter
+ * @param mappingToken
+ * the mapping token
+ * @param type
+ * the type
+ * @param prefix
+ * the prefix
+ * @param filter
+ * the filter
*/
private void addText(MtasParserMappingToken mappingToken, String type,
String prefix, String filter) {
@@ -1687,11 +1823,16 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Adds the text split.
*
- * @param mappingToken the mapping token
- * @param type the type
- * @param split the split
- * @param prefix the prefix
- * @param filter the filter
+ * @param mappingToken
+ * the mapping token
+ * @param type
+ * the type
+ * @param split
+ * the split
+ * @param prefix
+ * the prefix
+ * @param filter
+ * the filter
*/
private void addTextSplit(MtasParserMappingToken mappingToken, String type,
String split, String prefix, String filter) {
@@ -1711,9 +1852,12 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Condition text.
*
- * @param condition the condition
- * @param filter the filter
- * @param not the not
+ * @param condition
+ * the condition
+ * @param filter
+ * the filter
+ * @param not
+ * the not
*/
private void conditionText(String condition, String filter, String not) {
HashMap<String, String> mapConstructionItem = new HashMap<String, String>();
@@ -1728,8 +1872,10 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Payload text.
*
- * @param mappingToken the mapping token
- * @param filter the filter
+ * @param mappingToken
+ * the mapping token
+ * @param filter
+ * the filter
*/
private void payloadText(MtasParserMappingToken mappingToken,
String filter) {
@@ -1743,11 +1889,16 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Adds the attribute.
*
- * @param mappingToken the mapping token
- * @param type the type
- * @param name the name
- * @param prefix the prefix
- * @param filter the filter
+ * @param mappingToken
+ * the mapping token
+ * @param type
+ * the type
+ * @param name
+ * the name
+ * @param prefix
+ * the prefix
+ * @param filter
+ * the filter
*/
private void addAttribute(MtasParserMappingToken mappingToken, String type,
String name, String prefix, String filter) {
@@ -1769,10 +1920,14 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Condition attribute.
*
- * @param name the name
- * @param condition the condition
- * @param filter the filter
- * @param not the not
+ * @param name
+ * the name
+ * @param condition
+ * the condition
+ * @param filter
+ * the filter
+ * @param not
+ * the not
*/
private void conditionAttribute(String name, String condition,
String filter, String not) {
@@ -1791,9 +1946,12 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Payload attribute.
*
- * @param mappingToken the mapping token
- * @param name the name
- * @param filter the filter
+ * @param mappingToken
+ * the mapping token
+ * @param name
+ * the name
+ * @param filter
+ * the filter
*/
private void payloadAttribute(MtasParserMappingToken mappingToken,
String name, String filter) {
@@ -1808,8 +1966,10 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Condition ancestor.
*
- * @param ancestorType the ancestor type
- * @param number the number
+ * @param ancestorType
+ * the ancestor type
+ * @param number
+ * the number
*/
public void conditionAncestor(String ancestorType, String number) {
if (ancestorType.equals(SOURCE_ANCESTOR_GROUP)
@@ -1829,12 +1989,18 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Adds the ancestor name.
*
- * @param ancestorType the ancestor type
- * @param mappingToken the mapping token
- * @param type the type
- * @param distance the distance
- * @param prefix the prefix
- * @param filter the filter
+ * @param ancestorType
+ * the ancestor type
+ * @param mappingToken
+ * the mapping token
+ * @param type
+ * the type
+ * @param distance
+ * the distance
+ * @param prefix
+ * the prefix
+ * @param filter
+ * the filter
*/
private void addAncestorName(String ancestorType,
MtasParserMappingToken mappingToken, String type, String distance,
@@ -1862,11 +2028,16 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Condition ancestor name.
*
- * @param ancestorType the ancestor type
- * @param distance the distance
- * @param condition the condition
- * @param filter the filter
- * @param not the not
+ * @param ancestorType
+ * the ancestor type
+ * @param distance
+ * the distance
+ * @param condition
+ * the condition
+ * @param filter
+ * the filter
+ * @param not
+ * the not
*/
public void conditionAncestorName(String ancestorType, String distance,
String condition, String filter, String not) {
@@ -1890,13 +2061,20 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Adds the ancestor attribute.
*
- * @param ancestorType the ancestor type
- * @param mappingToken the mapping token
- * @param type the type
- * @param distance the distance
- * @param name the name
- * @param prefix the prefix
- * @param filter the filter
+ * @param ancestorType
+ * the ancestor type
+ * @param mappingToken
+ * the mapping token
+ * @param type
+ * the type
+ * @param distance
+ * the distance
+ * @param name
+ * the name
+ * @param prefix
+ * the prefix
+ * @param filter
+ * the filter
*/
public void addAncestorAttribute(String ancestorType,
MtasParserMappingToken mappingToken, String type, String distance,
@@ -1927,12 +2105,18 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Condition ancestor attribute.
*
- * @param ancestorType the ancestor type
- * @param distance the distance
- * @param name the name
- * @param condition the condition
- * @param filter the filter
- * @param not the not
+ * @param ancestorType
+ * the ancestor type
+ * @param distance
+ * the distance
+ * @param name
+ * the name
+ * @param condition
+ * the condition
+ * @param filter
+ * the filter
+ * @param not
+ * the not
*/
public void conditionAncestorAttribute(String ancestorType, String distance,
String name, String condition, String filter, String not) {
@@ -1959,11 +2143,16 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Payload ancestor attribute.
*
- * @param mappingToken the mapping token
- * @param ancestorType the ancestor type
- * @param distance the distance
- * @param name the name
- * @param filter the filter
+ * @param mappingToken
+ * the mapping token
+ * @param ancestorType
+ * the ancestor type
+ * @param distance
+ * the distance
+ * @param name
+ * the name
+ * @param filter
+ * the filter
*/
private void payloadAncestorAttribute(MtasParserMappingToken mappingToken,
String ancestorType, String distance, String name, String filter) {
@@ -1988,9 +2177,11 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Compute ancestor source type.
*
- * @param type the type
+ * @param type
+ * the type
* @return the string
- * @throws MtasConfigException the mtas config exception
+ * @throws MtasConfigException
+ * the mtas config exception
*/
private String computeAncestorSourceType(String type)
throws MtasConfigException {
@@ -2014,7 +2205,8 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Compute distance.
*
- * @param distance the distance
+ * @param distance
+ * the distance
* @return the string
*/
private String computeDistance(String distance) {
@@ -2033,7 +2225,8 @@ abstract public class MtasBasicParser extends MtasParser {
/**
* Compute number.
*
- * @param number the number
+ * @param number
+ * the number
* @return the string
*/
private String computeNumber(String number) {
diff --git a/src/mtas/analysis/parser/MtasCRMParser.java b/src/mtas/analysis/parser/MtasCRMParser.java
new file mode 100644
index 0000000..3d2ac96
--- /dev/null
+++ b/src/mtas/analysis/parser/MtasCRMParser.java
@@ -0,0 +1,1013 @@
+package mtas.analysis.parser;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.TreeSet;
+import java.util.Map.Entry;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import mtas.analysis.token.MtasToken;
+import mtas.analysis.token.MtasTokenCollection;
+import mtas.analysis.util.MtasBufferedReader;
+import mtas.analysis.util.MtasConfigException;
+import mtas.analysis.util.MtasConfiguration;
+import mtas.analysis.util.MtasParserException;
+
+/**
+ * The Class MtasCRMParser.
+ */
+
+public class MtasCRMParser extends MtasBasicParser {
+
+ /** The word type. */
+ private MtasParserType wordType = null;
+
+ /** The word annotation types. */
+ private HashMap<String, MtasParserType> wordAnnotationTypes = new HashMap<String, MtasParserType>();
+
+ /** The crm sentence types. */
+ private HashMap<String, MtasParserType> crmSentenceTypes = new HashMap<String, MtasParserType>();
+
+ /** The crm clause types. */
+ private HashMap<String, MtasParserType> crmClauseTypes = new HashMap<String, MtasParserType>();
+
+ /** The crm pair types. */
+ private HashMap<String, MtasParserType> crmPairTypes = new HashMap<String, MtasParserType>();
+
+ /** The functions. */
+ private HashMap<String, HashMap<String, MtasCRMParserFunction>> functions = new HashMap<String, HashMap<String, MtasCRMParserFunction>>();
+
+ /** The Constant MAPPING_TYPE_CRM_SENTENCE. */
+ protected final static String MAPPING_TYPE_CRM_SENTENCE = "crmSentence";
+
+ /** The Constant MAPPING_TYPE_CRM_CLAUSE. */
+ protected final static String MAPPING_TYPE_CRM_CLAUSE = "crmClause";
+
+ /** The Constant MAPPING_TYPE_CRM_PAIR. */
+ protected final static String MAPPING_TYPE_CRM_PAIR = "crmPair";
+
+ /** The history pair. */
+ private HashMap<String, HashMap<String, MtasParserObject>> historyPair = new HashMap<String, HashMap<String, MtasParserObject>>();
+
+ /** The pair pattern. */
+ Pattern pairPattern = Pattern.compile("^([b|e])([a-z])([0-9]+)$");
+
+ /**
+ * Instantiates a new mtas crm parser.
+ *
+ * @param config
+ * the config
+ */
+ public MtasCRMParser(MtasConfiguration config) {
+ super(config);
+ try {
+ initParser();
+ // System.out.print(printConfig());
+ } catch (MtasConfigException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.analysis.parser.MtasParser#initParser()
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void initParser() throws MtasConfigException {
+ super.initParser();
+ if (config != null) {
+ // always word, no mappings
+ wordType = new MtasParserType(MAPPING_TYPE_WORD, null, false);
+ for (int i = 0; i < config.children.size(); i++) {
+ MtasConfiguration current = config.children.get(i);
+ if (current.name.equals("mappings")) {
+ for (int j = 0; j < current.children.size(); j++) {
+ if (current.children.get(j).name.equals("mapping")) {
+ MtasConfiguration mapping = current.children.get(j);
+ String typeMapping = mapping.attributes.get("type");
+ String nameMapping = mapping.attributes.get("name");
+ if ((typeMapping != null)) {
+ if (typeMapping.equals(MAPPING_TYPE_WORD)) {
+ MtasCRMParserMappingWordAnnotation m = new MtasCRMParserMappingWordAnnotation();
+ m.processConfig(mapping);
+ wordType.addMapping(m);
+ } else if (typeMapping.equals(MAPPING_TYPE_WORD_ANNOTATION)
+ && (nameMapping != null)) {
+ MtasCRMParserMappingWordAnnotation m = new MtasCRMParserMappingWordAnnotation();
+ m.processConfig(mapping);
+ if (wordAnnotationTypes.containsKey(nameMapping)) {
+ wordAnnotationTypes.get(nameMapping).addMapping(m);
+ } else {
+ MtasParserType t = new MtasParserType(typeMapping,
+ nameMapping, false);
+ t.addMapping(m);
+ wordAnnotationTypes.put(nameMapping, t);
+ }
+ } else if (typeMapping.equals(MAPPING_TYPE_CRM_SENTENCE)) {
+ MtasCRMParserMappingCRMSentence m = new MtasCRMParserMappingCRMSentence();
+ m.processConfig(mapping);
+ if (crmSentenceTypes.containsKey(nameMapping)) {
+ crmSentenceTypes.get(nameMapping).addMapping(m);
+ } else {
+ MtasParserType t = new MtasParserType(MAPPING_TYPE_GROUP,
+ nameMapping, true);
+ t.addMapping(m);
+ crmSentenceTypes.put(nameMapping, t);
+ }
+ } else if (typeMapping.equals(MAPPING_TYPE_CRM_CLAUSE)) {
+ MtasCRMParserMappingCRMSentence m = new MtasCRMParserMappingCRMSentence();
+ m.processConfig(mapping);
+ if (crmClauseTypes.containsKey(nameMapping)) {
+ crmClauseTypes.get(nameMapping).addMapping(m);
+ } else {
+ MtasParserType t = new MtasParserType(MAPPING_TYPE_GROUP,
+ nameMapping, true);
+ t.addMapping(m);
+ crmClauseTypes.put(nameMapping, t);
+ }
+ } else if (typeMapping.equals(MAPPING_TYPE_CRM_PAIR)) {
+ MtasCRMParserMappingCRMPair m = new MtasCRMParserMappingCRMPair();
+ m.processConfig(mapping);
+ if (crmPairTypes.containsKey(nameMapping)) {
+ crmPairTypes.get(nameMapping).addMapping(m);
+ } else {
+ MtasParserType t = new MtasParserType(MAPPING_TYPE_RELATION,
+ nameMapping, true);
+ t.addMapping(m);
+ crmPairTypes.put(nameMapping, t);
+ }
+ } else {
+ throw new MtasConfigException("unknown mapping type "
+ + typeMapping + " or missing name");
+ }
+ }
+ }
+ }
+ } else if (current.name.equals("functions")) {
+ for (int j = 0; j < current.children.size(); j++) {
+ if (current.children.get(j).name.equals("function")) {
+ MtasConfiguration function = current.children.get(j);
+ String nameFunction = function.attributes.get("name");
+ String typeFunction = function.attributes.get("type");
+ String splitFunction = function.attributes.get("split");
+ if (nameFunction != null && typeFunction != null) {
+ MtasCRMParserFunction mtasCRMParserFunction = new MtasCRMParserFunction(
+ typeFunction, splitFunction);
+ if (!functions.containsKey(typeFunction)) {
+ functions.put(typeFunction,
+ new HashMap<String, MtasCRMParserFunction>());
+ }
+ functions.get(typeFunction).put(nameFunction,
+ mtasCRMParserFunction);
+ MtasConfiguration subCurrent = current.children.get(j);
+ for (int k = 0; k < subCurrent.children.size(); k++) {
+ if (subCurrent.children.get(k).name.equals("condition")) {
+ MtasConfiguration subSubCurrent = subCurrent.children
+ .get(k);
+ if (subSubCurrent.attributes.containsKey("value")) {
+ String[] valuesCondition = subSubCurrent.attributes
+ .get("value").split(Pattern.quote(","));
+ ArrayList<MtasCRMParserFunctionOutput> valueOutputList = new ArrayList<MtasCRMParserFunctionOutput>();
+ for (int l = 0; l < subSubCurrent.children.size(); l++) {
+ if (subSubCurrent.children.get(l).name
+ .equals("output")) {
+ String valueOutput = subSubCurrent.children
+ .get(l).attributes.get("value");
+ String nameOutput = subSubCurrent.children
+ .get(l).attributes.get("name");
+ if (nameOutput != null) {
+ MtasCRMParserFunctionOutput o = new MtasCRMParserFunctionOutput(
+ nameOutput, valueOutput);
+ valueOutputList.add(o);
+ }
+ }
+ }
+ if (valueOutputList.size() > 0) {
+ for (String valueCondition : valuesCondition) {
+ if (mtasCRMParserFunction.output
+ .containsKey(valueCondition)) {
+ mtasCRMParserFunction.output.get(valueCondition)
+ .addAll(
+ (Collection<? extends MtasCRMParserFunctionOutput>) valueOutputList
+ .clone());
+ } else {
+ mtasCRMParserFunction.output.put(valueCondition,
+ (ArrayList<MtasCRMParserFunctionOutput>) valueOutputList
+ .clone());
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.analysis.parser.MtasParser#createTokenCollection(java.io.Reader)
+ */
+ @Override
+ public MtasTokenCollection createTokenCollection(Reader reader)
+ throws MtasParserException, MtasConfigException {
+ AtomicInteger position = new AtomicInteger(0);
+ Integer unknownAncestors = 0;
+
+ HashMap<String, TreeSet<Integer>> idPositions = new HashMap<String, TreeSet<Integer>>();
+ HashMap<String, Integer[]> idOffsets = new HashMap<String, Integer[]>();
+
+ HashMap<String, HashMap<Integer, HashSet<String>>> updateList = new HashMap<String, HashMap<Integer, HashSet<String>>>();
+ updateList.put(UPDATE_TYPE_OFFSET, new HashMap<Integer, HashSet<String>>());
+ updateList.put(UPDATE_TYPE_POSITION,
+ new HashMap<Integer, HashSet<String>>());
+
+ HashMap<String, ArrayList<MtasParserObject>> currentList = new HashMap<String, ArrayList<MtasParserObject>>();
+ currentList.put(MAPPING_TYPE_RELATION, new ArrayList<MtasParserObject>());
+ currentList.put(MAPPING_TYPE_RELATION_ANNOTATION,
+ new ArrayList<MtasParserObject>());
+ currentList.put(MAPPING_TYPE_REF, new ArrayList<MtasParserObject>());
+ currentList.put(MAPPING_TYPE_GROUP, new ArrayList<MtasParserObject>());
+ currentList.put(MAPPING_TYPE_GROUP_ANNOTATION,
+ new ArrayList<MtasParserObject>());
+ currentList.put(MAPPING_TYPE_WORD, new ArrayList<MtasParserObject>());
+ currentList.put(MAPPING_TYPE_WORD_ANNOTATION,
+ new ArrayList<MtasParserObject>());
+
+ tokenCollection = new MtasTokenCollection();
+ MtasToken.resetId();
+ try (MtasBufferedReader br = new MtasBufferedReader(reader)) {
+ String line;
+ int currentOffset, previousOffset = br.getPosition();
+ MtasParserObject currentObject;
+ Pattern headerPattern = Pattern.compile("^@ @ @(.*)$");
+ Pattern regularPattern = Pattern.compile(
+ "^([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+)$");
+ Matcher matcherHeader, matcherRegular = null;
+ HashSet<MtasParserObject> newPreviousSentence = new HashSet<MtasParserObject>(),
+ previousSentence = new HashSet<MtasParserObject>();
+ HashSet<MtasParserObject> newPreviousClause = new HashSet<MtasParserObject>(),
+ previousClause = new HashSet<MtasParserObject>();
+ while ((line = br.readLine()) != null) {
+ currentOffset = br.getPosition();
+ matcherHeader = headerPattern.matcher(line.trim());
+ matcherRegular = regularPattern.matcher(line.trim());
+ if (matcherRegular.matches()) {
+ newPreviousSentence.clear();
+ for (int i = 4; i < 8; i++) {
+ ArrayList<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<MtasCRMParserFunctionOutput>();
+ HashSet<MtasParserObject> tmpList = processCRMSentence(
+ String.valueOf(i), matcherRegular.group((i + 1)), currentOffset,
+ functionOutputList, unknownAncestors, currentList, updateList,
+ idPositions, idOffsets, previousSentence, previousClause);
+ if (tmpList != null) {
+ newPreviousSentence.addAll(tmpList);
+ }
+ for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
+ tmpList = processCRMSentence(functionOutput.name,
+ functionOutput.value, currentOffset, functionOutputList,
+ unknownAncestors, currentList, updateList, idPositions,
+ idOffsets, previousSentence, previousClause);
+ if (tmpList != null) {
+ newPreviousSentence.addAll(tmpList);
+ }
+ }
+ }
+ if (newPreviousSentence.size() > 0) {
+ previousSentence.clear();
+ previousSentence.addAll(newPreviousSentence);
+ }
+ newPreviousClause.clear();
+ for (int i = 4; i < 8; i++) {
+ ArrayList<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<MtasCRMParserFunctionOutput>();
+ HashSet<MtasParserObject> tmpList = processCRMClause(
+ String.valueOf(i), matcherRegular.group((i + 1)), currentOffset,
+ functionOutputList, unknownAncestors, currentList, updateList,
+ idPositions, idOffsets, previousClause);
+ if (tmpList != null) {
+ newPreviousClause.addAll(tmpList);
+ }
+ for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
+ tmpList = processCRMClause(functionOutput.name,
+ functionOutput.value, currentOffset, functionOutputList,
+ unknownAncestors, currentList, updateList, idPositions,
+ idOffsets, previousClause);
+ if (tmpList != null) {
+ newPreviousClause.addAll(tmpList);
+ }
+ }
+ }
+ if (newPreviousClause.size() > 0) {
+ previousClause.clear();
+ previousClause.addAll(newPreviousClause);
+ }
+ }
+
+ if (matcherRegular.matches() && !matcherHeader.matches()) {
+ matcherRegular = regularPattern.matcher(line.trim());
+ if (matcherRegular.matches()) {
+ // regular line - start word
+ currentObject = new MtasParserObject(wordType);
+ currentObject.setOffsetStart(previousOffset);
+ currentObject.setRealOffsetStart(previousOffset);
+ currentObject.setUnknownAncestorNumber(unknownAncestors);
+ if (!prevalidateObject(currentObject, currentList)) {
+ unknownAncestors++;
+ } else {
+ int p = position.getAndIncrement();
+ currentObject.addPosition(p);
+ currentObject.objectId = "word_" + String.valueOf(p);
+ currentList.get(MAPPING_TYPE_WORD).add(currentObject);
+ unknownAncestors = 0;
+ // check for crmPair
+ for (int i = 0; i < 8; i++) {
+ ArrayList<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<MtasCRMParserFunctionOutput>();
+ processCRMPair(p, String.valueOf(i),
+ matcherRegular.group((i + 1)), currentOffset,
+ functionOutputList, unknownAncestors, currentList,
+ updateList, idPositions, idOffsets);
+ for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
+ processCRMPair(p, functionOutput.name, functionOutput.value,
+ currentOffset, functionOutputList, unknownAncestors,
+ currentList, updateList, idPositions, idOffsets);
+ }
+ }
+ // compute word annotations
+ for (int i = 0; i < 8; i++) {
+ ArrayList<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<MtasCRMParserFunctionOutput>();
+ functionOutputList.addAll(processWordAnnotation(
+ String.valueOf(i), matcherRegular.group((i + 1)),
+ previousOffset, currentOffset, unknownAncestors,
+ currentList, updateList, idPositions, idOffsets));
+ for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
+ processWordAnnotation(functionOutput.name,
+ functionOutput.value, previousOffset, currentOffset,
+ unknownAncestors, currentList, updateList, idPositions,
+ idOffsets);
+ }
+ }
+ }
+ // finish word
+ if (unknownAncestors > 0) {
+ unknownAncestors--;
+ } else {
+ currentObject = currentList.get(MAPPING_TYPE_WORD)
+ .remove(currentList.get(MAPPING_TYPE_WORD).size() - 1);
+ assert unknownAncestors == 0 : "error in administration "
+ + currentObject.getType().getName();
+ currentObject.setText(null);
+ currentObject.setOffsetEnd(currentOffset - 1);
+ currentObject.setRealOffsetEnd(currentOffset - 1);
+ // update ancestor groups with position and offset
+ for (MtasParserObject currentGroup : currentList
+ .get(MAPPING_TYPE_GROUP)) {
+ currentGroup.addPositions(currentObject.getPositions());
+ currentGroup.addOffsetStart(currentObject.getOffsetStart());
+ currentGroup.addOffsetEnd(currentObject.getOffsetEnd());
+ }
+ idPositions.put(currentObject.getId(),
+ currentObject.getPositions());
+ idOffsets.put(currentObject.getId(), currentObject.getOffset());
+ currentObject.updateMappings(idPositions, idOffsets);
+ unknownAncestors = currentObject.getUnknownAncestorNumber();
+ computeMappingsFromObject(currentObject, currentList, updateList);
+ }
+
+ } else {
+ // System.out.println("PROBLEM: " + line);
+ }
+ }
+ previousOffset = br.getPosition();
+ }
+ closePrevious(previousSentence, previousOffset, unknownAncestors,
+ currentList, updateList, idPositions, idOffsets);
+ closePrevious(previousClause, previousOffset, unknownAncestors,
+ currentList, updateList, idPositions, idOffsets);
+ } catch (IOException e) {
+ throw new MtasParserException(e.getMessage());
+ }
+ // final check
+ tokenCollection.check(autorepair, makeunique);
+ return tokenCollection;
+
+ }
+
+ /**
+ * Process word annotation.
+ *
+ * @param name
+ * the name
+ * @param text
+ * the text
+ * @param previousOffset
+ * the previous offset
+ * @param currentOffset
+ * the current offset
+ * @param unknownAncestors
+ * the unknown ancestors
+ * @param currentList
+ * the current list
+ * @param updateList
+ * the update list
+ * @param idPositions
+ * the id positions
+ * @param idOffsets
+ * the id offsets
+ * @return the array list
+ * @throws MtasParserException
+ * the mtas parser exception
+ * @throws MtasConfigException
+ * the mtas config exception
+ */
+ private ArrayList<MtasCRMParserFunctionOutput> processWordAnnotation(
+ String name, String text, Integer previousOffset, Integer currentOffset,
+ Integer unknownAncestors,
+ HashMap<String, ArrayList<MtasParserObject>> currentList,
+ HashMap<String, HashMap<Integer, HashSet<String>>> updateList,
+ HashMap<String, TreeSet<Integer>> idPositions,
+ HashMap<String, Integer[]> idOffsets)
+ throws MtasParserException, MtasConfigException {
+ MtasParserType tmpCurrentType;
+ MtasParserObject currentObject;
+ ArrayList<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<MtasCRMParserFunctionOutput>();
+ if ((tmpCurrentType = wordAnnotationTypes.get(name)) != null) {
+ // start word annotation
+ currentObject = new MtasParserObject(tmpCurrentType);
+ currentObject.setRealOffsetStart(previousOffset);
+ currentObject.addPositions(currentList.get(MAPPING_TYPE_WORD)
+ .get((currentList.get(MAPPING_TYPE_WORD).size() - 1)).getPositions());
+ currentObject.setUnknownAncestorNumber(unknownAncestors);
+ if (!prevalidateObject(currentObject, currentList)) {
+ unknownAncestors++;
+ } else {
+ currentList.get(MAPPING_TYPE_WORD_ANNOTATION).add(currentObject);
+ unknownAncestors = 0;
+ }
+ // finish word annotation
+ if (unknownAncestors > 0) {
+ unknownAncestors--;
+ } else {
+ currentObject = currentList.get(MAPPING_TYPE_WORD_ANNOTATION)
+ .remove(currentList.get(MAPPING_TYPE_WORD_ANNOTATION).size() - 1);
+ assert unknownAncestors == 0 : "error in administration "
+ + currentObject.getType().getName();
+ if (functions.containsKey(MAPPING_TYPE_WORD_ANNOTATION)
+ && functions.get(MAPPING_TYPE_WORD_ANNOTATION).containsKey(name)
+ && text != null) {
+ MtasCRMParserFunction function = functions
+ .get(MAPPING_TYPE_WORD_ANNOTATION).get(name);
+ String[] value;
+ if (function.split != null) {
+ value = text.split(Pattern.quote(function.split));
+ } else {
+ value = new String[] { text };
+ }
+ for (int c = 0; c < value.length; c++) {
+ if (function.output.containsKey(value[c])) {
+ functionOutputList.addAll(function.output.get(value[c]));
+ }
+ }
+ }
+ currentObject.setText(text);
+ currentObject.setRealOffsetEnd(currentOffset - 1);
+ idPositions.put(currentObject.getId(), currentObject.getPositions());
+ idOffsets.put(currentObject.getId(), currentObject.getOffset());
+ // offset always null, so update later with word (should be possible)
+ if ((currentObject.getId() != null)
+ && (currentList.get(MAPPING_TYPE_WORD).size() > 0)) {
+ currentList.get(MAPPING_TYPE_WORD)
+ .get((currentList.get(MAPPING_TYPE_WORD).size() - 1))
+ .addUpdateableIdWithOffset(currentObject.getId());
+ }
+ currentObject.updateMappings(idPositions, idOffsets);
+ unknownAncestors = currentObject.getUnknownAncestorNumber();
+ computeMappingsFromObject(currentObject, currentList, updateList);
+ }
+ }
+ return functionOutputList;
+ }
+
+ /**
+ * Process crm sentence.
+ *
+ * @param name
+ * the name
+ * @param text
+ * the text
+ * @param currentOffset
+ * the current offset
+ * @param functionOutputList
+ * the function output list
+ * @param unknownAncestors
+ * the unknown ancestors
+ * @param currentList
+ * the current list
+ * @param updateList
+ * the update list
+ * @param idPositions
+ * the id positions
+ * @param idOffsets
+ * the id offsets
+ * @param previous
+ * the previous
+ * @param previousClause
+ * the previous clause
+ * @return the hash set
+ * @throws MtasParserException
+ * the mtas parser exception
+ * @throws MtasConfigException
+ * the mtas config exception
+ */
+ private HashSet<MtasParserObject> processCRMSentence(String name, String text,
+ Integer currentOffset,
+ ArrayList<MtasCRMParserFunctionOutput> functionOutputList,
+ Integer unknownAncestors,
+ HashMap<String, ArrayList<MtasParserObject>> currentList,
+ HashMap<String, HashMap<Integer, HashSet<String>>> updateList,
+ HashMap<String, TreeSet<Integer>> idPositions,
+ HashMap<String, Integer[]> idOffsets, HashSet<MtasParserObject> previous,
+ HashSet<MtasParserObject> previousClause)
+ throws MtasParserException, MtasConfigException {
+ MtasParserType tmpCurrentType;
+ MtasParserObject currentObject;
+ if ((tmpCurrentType = crmSentenceTypes.get(name)) != null) {
+ currentObject = new MtasParserObject(tmpCurrentType);
+ currentObject.setUnknownAncestorNumber(unknownAncestors);
+ currentObject.setRealOffsetStart(currentOffset);
+ currentObject.setText(text);
+ if (!prevalidateObject(currentObject, currentList)) {
+ return null;
+ } else {
+ closePrevious(previousClause, currentOffset, unknownAncestors,
+ currentList, updateList, idPositions, idOffsets);
+ closePrevious(previous, currentOffset, unknownAncestors, currentList,
+ updateList, idPositions, idOffsets);
+ previous.clear();
+ currentList.get(MAPPING_TYPE_GROUP).add(currentObject);
+ unknownAncestors = 0;
+ return new HashSet<MtasParserObject>(Arrays.asList(currentObject));
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Process crm clause.
+ *
+ * @param name
+ * the name
+ * @param text
+ * the text
+ * @param currentOffset
+ * the current offset
+ * @param functionOutputList
+ * the function output list
+ * @param unknownAncestors
+ * the unknown ancestors
+ * @param currentList
+ * the current list
+ * @param updateList
+ * the update list
+ * @param idPositions
+ * the id positions
+ * @param idOffsets
+ * the id offsets
+ * @param previous
+ * the previous
+ * @return the hash set
+ * @throws MtasParserException
+ * the mtas parser exception
+ * @throws MtasConfigException
+ * the mtas config exception
+ */
+ private HashSet<MtasParserObject> processCRMClause(String name, String text,
+ Integer currentOffset,
+ ArrayList<MtasCRMParserFunctionOutput> functionOutputList,
+ Integer unknownAncestors,
+ HashMap<String, ArrayList<MtasParserObject>> currentList,
+ HashMap<String, HashMap<Integer, HashSet<String>>> updateList,
+ HashMap<String, TreeSet<Integer>> idPositions,
+ HashMap<String, Integer[]> idOffsets, HashSet<MtasParserObject> previous)
+ throws MtasParserException, MtasConfigException {
+ MtasParserType tmpCurrentType;
+ MtasParserObject currentObject;
+ if ((tmpCurrentType = crmClauseTypes.get(name)) != null) {
+ currentObject = new MtasParserObject(tmpCurrentType);
+ currentObject.setUnknownAncestorNumber(unknownAncestors);
+ currentObject.setRealOffsetStart(currentOffset);
+ currentObject.setText(text);
+ if (!prevalidateObject(currentObject, currentList)) {
+ return null;
+ } else {
+ closePrevious(previous, currentOffset, unknownAncestors, currentList,
+ updateList, idPositions, idOffsets);
+ previous.clear();
+ currentList.get(MAPPING_TYPE_GROUP).add(currentObject);
+ unknownAncestors = 0;
+ return new HashSet<MtasParserObject>(Arrays.asList(currentObject));
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Close previous.
+ *
+ * @param previous
+ * the previous
+ * @param currentOffset
+ * the current offset
+ * @param unknownAncestors
+ * the unknown ancestors
+ * @param currentList
+ * the current list
+ * @param updateList
+ * the update list
+ * @param idPositions
+ * the id positions
+ * @param idOffsets
+ * the id offsets
+ * @throws MtasParserException
+ * the mtas parser exception
+ * @throws MtasConfigException
+ * the mtas config exception
+ */
+ private void closePrevious(HashSet<MtasParserObject> previous,
+ Integer currentOffset, Integer unknownAncestors,
+ HashMap<String, ArrayList<MtasParserObject>> currentList,
+ HashMap<String, HashMap<Integer, HashSet<String>>> updateList,
+ HashMap<String, TreeSet<Integer>> idPositions,
+ HashMap<String, Integer[]> idOffsets)
+ throws MtasParserException, MtasConfigException {
+ for (MtasParserObject previousObject : previous) {
+ previousObject.setRealOffsetEnd(currentOffset);
+ idPositions.put(previousObject.getId(), previousObject.getPositions());
+ idOffsets.put(previousObject.getId(), previousObject.getOffset());
+ previousObject.updateMappings(idPositions, idOffsets);
+ unknownAncestors = previousObject.getUnknownAncestorNumber();
+ computeMappingsFromObject(previousObject, currentList, updateList);
+ currentList.get(MAPPING_TYPE_GROUP).remove(previousObject);
+ }
+ }
+
+ /**
+ * Process crm pair.
+ *
+ * @param position
+ * the position
+ * @param name
+ * the name
+ * @param text
+ * the text
+ * @param currentOffset
+ * the current offset
+ * @param functionOutputList
+ * the function output list
+ * @param unknownAncestors
+ * the unknown ancestors
+ * @param currentList
+ * the current list
+ * @param updateList
+ * the update list
+ * @param idPositions
+ * the id positions
+ * @param idOffsets
+ * the id offsets
+ * @throws MtasParserException
+ * the mtas parser exception
+ * @throws MtasConfigException
+ * the mtas config exception
+ */
+ private void processCRMPair(int position, String name, String text,
+ Integer currentOffset,
+ ArrayList<MtasCRMParserFunctionOutput> functionOutputList,
+ Integer unknownAncestors,
+ HashMap<String, ArrayList<MtasParserObject>> currentList,
+ HashMap<String, HashMap<Integer, HashSet<String>>> updateList,
+ HashMap<String, TreeSet<Integer>> idPositions,
+ HashMap<String, Integer[]> idOffsets)
+ throws MtasParserException, MtasConfigException {
+
+ MtasParserType tmpCurrentType;
+ MtasParserObject currentObject;
+
+ if ((tmpCurrentType = crmPairTypes.get(name)) != null) {
+ if ((tmpCurrentType = crmPairTypes.get(name)) != null) {
+ // get history
+ HashMap<String, MtasParserObject> currentNamePairHistory;
+ if (!historyPair.containsKey(name)) {
+ currentNamePairHistory = new HashMap<String, MtasParserObject>();
+ historyPair.put(name, currentNamePairHistory);
+ } else {
+ currentNamePairHistory = historyPair.get(name);
+ }
+ Matcher m = pairPattern.matcher(text);
+ if (m.find()) {
+ String thisKey = m.group(1) + m.group(2);
+ String otherKey = (m.group(1).equals("b") ? "e" : "b") + m.group(2);
+ if (currentNamePairHistory.containsKey(otherKey)) {
+ currentObject = currentNamePairHistory.remove(otherKey);
+ currentObject.setText(currentObject.getText() + "+" + text);
+ currentObject.addPosition(position);
+ processFunctions(name, text, MAPPING_TYPE_CRM_PAIR,
+ functionOutputList);
+ currentObject.setRealOffsetEnd(currentOffset + 1);
+ currentObject.setOffsetEnd(currentOffset + 1);
+ idPositions.put(currentObject.getId(),
+ currentObject.getPositions());
+ idOffsets.put(currentObject.getId(), currentObject.getOffset());
+ currentObject.updateMappings(idPositions, idOffsets);
+ unknownAncestors = currentObject.getUnknownAncestorNumber();
+ computeMappingsFromObject(currentObject, currentList, updateList);
+ } else {
+ currentObject = new MtasParserObject(tmpCurrentType);
+ currentObject.setUnknownAncestorNumber(unknownAncestors);
+ currentObject.setRealOffsetStart(currentOffset);
+ currentObject.setOffsetStart(currentOffset);
+ currentObject.setText(text);
+ currentObject.addPosition(position);
+ if (!prevalidateObject(currentObject, currentList)) {
+ unknownAncestors++;
+ } else {
+ currentNamePairHistory.put(thisKey, currentObject);
+ processFunctions(name, text, MAPPING_TYPE_CRM_PAIR,
+ functionOutputList);
+ currentObject.setRealOffsetEnd(currentOffset + 1);
+ currentObject.setOffsetEnd(currentOffset + 1);
+ idPositions.put(currentObject.getId(),
+ currentObject.getPositions());
+ idOffsets.put(currentObject.getId(), currentObject.getOffset());
+ // offset always null, so update later with word (should be
+ // possible)
+ if ((currentObject.getId() != null)
+ && (currentList.get(MAPPING_TYPE_WORD).size() > 0)) {
+ currentList.get(MAPPING_TYPE_WORD)
+ .get((currentList.get(MAPPING_TYPE_WORD).size() - 1))
+ .addUpdateableIdWithOffset(currentObject.getId());
+ }
+
+ }
+ }
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Process functions.
+ *
+ * @param name
+ * the name
+ * @param text
+ * the text
+ * @param type
+ * the type
+ * @param functionOutputList
+ * the function output list
+ */
+ private void processFunctions(String name, String text, String type,
+ ArrayList<MtasCRMParserFunctionOutput> functionOutputList) {
+ if (functions.containsKey(type) && functions.get(type).containsKey(name)
+ && text != null) {
+ if (functions.get(type).containsKey(name)) {
+ MtasCRMParserFunction function = functions.get(type).get(name);
+ String[] value;
+ if (function.split != null) {
+ value = text.split(Pattern.quote(function.split));
+ } else {
+ value = new String[] { text };
+ }
+ for (int c = 0; c < value.length; c++) {
+ boolean checkedEmpty = false;
+ if (value[c].equals("")) {
+ checkedEmpty = true;
+ }
+ if (function.output.containsKey(value[c])) {
+ ArrayList<MtasCRMParserFunctionOutput> list = function.output
+ .get(value[c]);
+ for (MtasCRMParserFunctionOutput listItem : list) {
+ functionOutputList.add(listItem.create(value[c]));
+ }
+ }
+ if (!checkedEmpty && function.output.containsKey("")) {
+ ArrayList<MtasCRMParserFunctionOutput> list = function.output
+ .get("");
+ for (MtasCRMParserFunctionOutput listItem : list) {
+ functionOutputList.add(listItem.create(value[c]));
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.analysis.parser.MtasParser#printConfig()
+ */
+ @Override
+ public String printConfig() {
+ String text = "";
+ text += "=== CONFIGURATION ===\n";
+ text += "type: " + wordAnnotationTypes.size() + " x wordAnnotation";
+ text += printConfigTypes(wordAnnotationTypes);
+ text += "=== CONFIGURATION ===\n";
+ return text;
+ }
+
+ /**
+ * Prints the config types.
+ *
+ * @param types
+ * the types
+ * @return the string
+ */
+ private String printConfigTypes(HashMap<?, MtasParserType> types) {
+ String text = "";
+ for (Entry<?, MtasParserType> entry : types.entrySet()) {
+ text += "- " + entry.getKey() + ": " + entry.getValue().mappings.size()
+ + " mapping(s)\n";
+ for (int i = 0; i < entry.getValue().mappings.size(); i++) {
+ text += "\t" + entry.getValue().mappings.get(i) + "\n";
+ }
+ }
+ return text;
+ }
+
+ /**
+ * The Class MtasCRMParserFunction.
+ */
+ private class MtasCRMParserFunction {
+
+ /** The split. */
+ public String split;
+
+ /** The output. */
+ public HashMap<String, ArrayList<MtasCRMParserFunctionOutput>> output;
+
+ /**
+ * Instantiates a new mtas crm parser function.
+ *
+ * @param type
+ * the type
+ * @param split
+ * the split
+ */
+ public MtasCRMParserFunction(String type, String split) {
+ this.split = split;
+ output = new HashMap<String, ArrayList<MtasCRMParserFunctionOutput>>();
+ }
+
+ }
+
+ /**
+ * The Class MtasCRMParserFunctionOutput.
+ */
+ private class MtasCRMParserFunctionOutput {
+
+ /** The name. */
+ public String name;
+
+ /** The value. */
+ public String value;
+
+ /**
+ * Instantiates a new mtas crm parser function output.
+ *
+ * @param name
+ * the name
+ * @param value
+ * the value
+ */
+ public MtasCRMParserFunctionOutput(String name, String value) {
+ this.name = name;
+ this.value = value;
+ }
+
+ /**
+ * Creates the.
+ *
+ * @param originalValue
+ * the original value
+ * @return the mtas crm parser function output
+ */
+ public MtasCRMParserFunctionOutput create(String originalValue) {
+ if (value != null) {
+ return this;
+ } else {
+ return new MtasCRMParserFunctionOutput(name, originalValue);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ return "MtasCRMParserFunctionOutput[" + name + "," + value + "]";
+ }
+ }
+
+ /**
+ * The Class MtasCRMParserMappingWordAnnotation.
+ */
+ private class MtasCRMParserMappingWordAnnotation
+ extends MtasParserMapping<MtasCRMParserMappingWordAnnotation> {
+
+ /**
+ * Instantiates a new mtas crm parser mapping word annotation.
+ */
+ public MtasCRMParserMappingWordAnnotation() {
+ super();
+ this.position = SOURCE_OWN;
+ this.realOffset = SOURCE_OWN;
+ this.offset = SOURCE_ANCESTOR_WORD;
+ this.type = MAPPING_TYPE_WORD_ANNOTATION;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.analysis.parser.MtasParser.MtasParserMapping#self()
+ */
+ @Override
+ protected MtasCRMParserMappingWordAnnotation self() {
+ return this;
+ }
+ }
+
+ /**
+ * The Class MtasCRMParserMappingCRMSentence.
+ */
+ private class MtasCRMParserMappingCRMSentence
+ extends MtasParserMapping<MtasCRMParserMappingCRMSentence> {
+
+ /**
+ * Instantiates a new mtas crm parser mapping crm sentence.
+ */
+ public MtasCRMParserMappingCRMSentence() {
+ super();
+ this.position = SOURCE_OWN;
+ this.realOffset = SOURCE_OWN;
+ this.offset = SOURCE_OWN;
+ this.type = MAPPING_TYPE_GROUP;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.analysis.parser.MtasBasicParser.MtasParserMapping#self()
+ */
+ @Override
+ protected MtasCRMParserMappingCRMSentence self() {
+ return this;
+ }
+ }
+
+ /**
+ * The Class MtasCRMParserMappingCRMPair.
+ */
+ private class MtasCRMParserMappingCRMPair
+ extends MtasParserMapping<MtasCRMParserMappingCRMPair> {
+
+ /**
+ * Instantiates a new mtas crm parser mapping crm pair.
+ */
+ public MtasCRMParserMappingCRMPair() {
+ super();
+ this.position = SOURCE_OWN;
+ this.realOffset = SOURCE_OWN;
+ this.offset = SOURCE_OWN;
+ this.type = MAPPING_TYPE_RELATION;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.analysis.parser.MtasBasicParser.MtasParserMapping#self()
+ */
+ @Override
+ protected MtasCRMParserMappingCRMPair self() {
+ return this;
+ }
+ }
+
+}
diff --git a/src/mtas/analysis/parser/MtasElanParser.java b/src/mtas/analysis/parser/MtasElanParser.java
index 93d1974..699b0c6 100644
--- a/src/mtas/analysis/parser/MtasElanParser.java
+++ b/src/mtas/analysis/parser/MtasElanParser.java
@@ -17,10 +17,12 @@ final public class MtasElanParser extends MtasXMLParser {
* @param config the config
*/
public MtasElanParser(MtasConfiguration config) {
- super(config);
+ super(config);
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.analysis.parser.MtasXMLParser#initParser()
*/
@Override
diff --git a/src/mtas/analysis/parser/MtasFoliaParser.java b/src/mtas/analysis/parser/MtasFoliaParser.java
index 7f847b0..ff1ef94 100644
--- a/src/mtas/analysis/parser/MtasFoliaParser.java
+++ b/src/mtas/analysis/parser/MtasFoliaParser.java
@@ -17,10 +17,12 @@ final public class MtasFoliaParser extends MtasXMLParser {
* @param config the config
*/
public MtasFoliaParser(MtasConfiguration config) {
- super(config);
+ super(config);
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.analysis.parser.MtasXMLParser#initParser()
*/
@Override
diff --git a/src/mtas/analysis/parser/MtasParser.java b/src/mtas/analysis/parser/MtasParser.java
index 0caae8a..c218812 100644
--- a/src/mtas/analysis/parser/MtasParser.java
+++ b/src/mtas/analysis/parser/MtasParser.java
@@ -15,16 +15,19 @@ import mtas.analysis.util.MtasParserException;
* The Class MtasParser.
*/
abstract public class MtasParser {
-
+
/** The token collection. */
protected MtasTokenCollection tokenCollection;
-
+
/** The config. */
protected MtasConfiguration config;
/** The autorepair. */
protected Boolean autorepair = false;
-
+
+ /** The makeunique. */
+ protected Boolean makeunique = false;
+
/**
* Inits the parser.
*
@@ -38,10 +41,13 @@ abstract public class MtasParser {
if (current.name.equals("autorepair")) {
autorepair = current.attributes.get("value").equals("true");
}
+ if (current.name.equals("makeunique")) {
+ makeunique = current.attributes.get("value").equals("true");
+ }
}
}
}
-
+
/**
* Creates the token collection.
*
@@ -59,7 +65,7 @@ abstract public class MtasParser {
* @return the string
*/
public abstract String printConfig();
-
+
/**
* The Class MtasParserObject.
*/
@@ -259,17 +265,17 @@ abstract public class MtasParser {
public void setText(String text) {
objectText = text;
}
-
+
/**
* Adds the text.
*
* @param text the text
*/
public void addText(String text) {
- if(objectText==null) {
+ if (objectText == null) {
objectText = text;
} else {
- objectText+=text;
+ objectText += text;
}
}
diff --git a/src/mtas/analysis/parser/MtasSketchParser.java b/src/mtas/analysis/parser/MtasSketchParser.java
index 7ddab26..d097cea 100644
--- a/src/mtas/analysis/parser/MtasSketchParser.java
+++ b/src/mtas/analysis/parser/MtasSketchParser.java
@@ -53,7 +53,9 @@ final public class MtasSketchParser extends MtasBasicParser {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.analysis.parser.MtasParser#initParser()
*/
@Override
@@ -62,7 +64,7 @@ final public class MtasSketchParser extends MtasBasicParser {
if (config != null) {
// always word, no mappings
- wordType = new MtasParserType(MAPPING_TYPE_WORD, null);
+ wordType = new MtasParserType(MAPPING_TYPE_WORD, null, false);
for (int i = 0; i < config.children.size(); i++) {
MtasConfiguration current = config.children.get(i);
@@ -74,7 +76,7 @@ final public class MtasSketchParser extends MtasBasicParser {
String nameMapping = mapping.attributes.get("name");
if ((typeMapping != null)) {
if (typeMapping.equals(MAPPING_TYPE_WORD)) {
- MtasSketchParserMappingWordAnnotation m = new MtasSketchParserMappingWordAnnotation();
+ MtasSketchParserMappingWord m = new MtasSketchParserMappingWord();
m.processConfig(mapping);
wordType.addMapping(m);
} else if (typeMapping.equals(MAPPING_TYPE_WORD_ANNOTATION)
@@ -85,7 +87,7 @@ final public class MtasSketchParser extends MtasBasicParser {
wordAnnotationTypes.get(nameMapping).addMapping(m);
} else {
MtasParserType t = new MtasParserType(typeMapping,
- nameMapping);
+ nameMapping, false);
t.addMapping(m);
wordAnnotationTypes.put(Integer.parseInt(nameMapping), t);
}
@@ -97,7 +99,7 @@ final public class MtasSketchParser extends MtasBasicParser {
groupTypes.get(nameMapping).addMapping(m);
} else {
MtasParserType t = new MtasParserType(typeMapping,
- nameMapping);
+ nameMapping, false);
t.addMapping(m);
groupTypes.put(nameMapping, t);
}
@@ -113,7 +115,9 @@ final public class MtasSketchParser extends MtasBasicParser {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.analysis.parser.MtasParser#createTokenCollection(java.io.Reader)
*/
@Override
@@ -337,11 +341,13 @@ final public class MtasSketchParser extends MtasBasicParser {
}
}
// final check
- tokenCollection.check(autorepair);
+ tokenCollection.check(autorepair, makeunique);
return tokenCollection;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.analysis.parser.MtasParser#printConfig()
*/
@Override
@@ -373,6 +379,34 @@ final public class MtasSketchParser extends MtasBasicParser {
}
/**
+ * The Class MtasSketchParserMappingWord.
+ */
+ private class MtasSketchParserMappingWord
+ extends MtasParserMapping<MtasSketchParserMappingWord> {
+
+ /**
+ * Instantiates a new mtas sketch parser mapping word.
+ */
+ public MtasSketchParserMappingWord() {
+ super();
+ this.position = SOURCE_OWN;
+ this.realOffset = SOURCE_OWN;
+ this.offset = SOURCE_OWN;
+ this.type = MAPPING_TYPE_WORD;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.analysis.parser.MtasBasicParser.MtasParserMapping#self()
+ */
+ @Override
+ protected MtasSketchParserMappingWord self() {
+ return this;
+ }
+ }
+
+ /**
* The Class MtasSketchParserMappingWordAnnotation.
*/
private class MtasSketchParserMappingWordAnnotation
diff --git a/src/mtas/analysis/parser/MtasTEIParser.java b/src/mtas/analysis/parser/MtasTEIParser.java
index d92ff61..058a564 100644
--- a/src/mtas/analysis/parser/MtasTEIParser.java
+++ b/src/mtas/analysis/parser/MtasTEIParser.java
@@ -17,10 +17,12 @@ final public class MtasTEIParser extends MtasXMLParser {
* @param config the config
*/
public MtasTEIParser(MtasConfiguration config) {
- super(config);
+ super(config);
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.analysis.parser.MtasXMLParser#initParser()
*/
@Override
diff --git a/src/mtas/analysis/parser/MtasXMLParser.java b/src/mtas/analysis/parser/MtasXMLParser.java
index 32d451c..a10b66a 100644
--- a/src/mtas/analysis/parser/MtasXMLParser.java
+++ b/src/mtas/analysis/parser/MtasXMLParser.java
@@ -173,7 +173,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
String ref = reference.attributes.get("ref");
if ((name != null) && (ref != null)) {
MtasParserType t = new MtasParserType(MAPPING_TYPE_REF, name,
- ref);
+ false, ref);
refTypes.put(getQName(t.getName()), t);
}
}
@@ -193,7 +193,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
relationTypes.get(qn).addMapping(m);
} else {
MtasParserType t = new MtasParserType(typeMapping,
- nameMapping);
+ nameMapping, false);
t.addMapping(m);
relationTypes.put(qn, t);
}
@@ -206,7 +206,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
relationAnnotationTypes.get(qn).addMapping(m);
} else {
MtasParserType t = new MtasParserType(typeMapping,
- nameMapping);
+ nameMapping, false);
t.addMapping(m);
relationAnnotationTypes.put(qn, t);
}
@@ -218,7 +218,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
wordTypes.get(qn).addMapping(m);
} else {
MtasParserType t = new MtasParserType(typeMapping,
- nameMapping);
+ nameMapping, false);
t.addMapping(m);
wordTypes.put(qn, t);
}
@@ -230,7 +230,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
wordAnnotationTypes.get(qn).addMapping(m);
} else {
MtasParserType t = new MtasParserType(typeMapping,
- nameMapping);
+ nameMapping, false);
t.addMapping(m);
wordAnnotationTypes.put(qn, t);
}
@@ -242,7 +242,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
groupTypes.get(qn).addMapping(m);
} else {
MtasParserType t = new MtasParserType(typeMapping,
- nameMapping);
+ nameMapping, false);
t.addMapping(m);
groupTypes.put(qn, t);
}
@@ -254,7 +254,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
groupAnnotationTypes.get(qn).addMapping(m);
} else {
MtasParserType t = new MtasParserType(typeMapping,
- nameMapping);
+ nameMapping, false);
t.addMapping(m);
groupAnnotationTypes.put(qn, t);
}
@@ -321,8 +321,9 @@ abstract class MtasXMLParser extends MtasBasicParser {
String encodingScheme = streamReader.getCharacterEncodingScheme();
if (encodingScheme == null) {
throw new MtasParserException("No encodingScheme found");
- } else if (!encodingScheme.equals("UTF-8")) {
- throw new MtasParserException("XML not UTF-8 encoded");
+ } else if (!encodingScheme.toLowerCase().equals("utf-8")) {
+ throw new MtasParserException(
+ "XML not UTF-8 encoded but '" + encodingScheme + "'");
}
break;
case XMLStreamConstants.END_DOCUMENT:
@@ -488,7 +489,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
relationKeyMap.put(currentRelation.getId(), keyMapList);
}
keyMapList.add(currentObject
- .getAttribute(currentType.getRefAttributeName()));
+ .getAttribute(currentType.getRefAttributeName()));
}
}
}
@@ -649,8 +650,10 @@ abstract class MtasXMLParser extends MtasBasicParser {
assert unknownAncestors == 0 : "error in administration "
+ currentObject.getType().getName();
currentObject.setRealOffsetEnd(lastOffset);
- idPositions.put(currentObject.getId(), currentObject.getPositions());
- idOffsets.put(currentObject.getId(), currentObject.getOffset());
+ idPositions.put(currentObject.getId(),
+ currentObject.getPositions());
+ idOffsets.put(currentObject.getId(),
+ currentObject.getOffset());
currentObject.updateMappings(idPositions, idOffsets);
unknownAncestors = currentObject.getUnknownAncestorNumber();
computeMappingsFromObject(currentObject, currentList,
@@ -671,8 +674,10 @@ abstract class MtasXMLParser extends MtasBasicParser {
+ currentObject.getType().getName();
// ignore text: should not occur
currentObject.setRealOffsetEnd(lastOffset);
- idPositions.put(currentObject.getId(), currentObject.getPositions());
- idOffsets.put(currentObject.getId(), currentObject.getOffset());
+ idPositions.put(currentObject.getId(),
+ currentObject.getPositions());
+ idOffsets.put(currentObject.getId(),
+ currentObject.getOffset());
currentObject.updateMappings(idPositions, idOffsets);
unknownAncestors = currentObject.getUnknownAncestorNumber();
computeMappingsFromObject(currentObject, currentList,
@@ -733,7 +738,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
if (streamReader.hasText()) {
textContent = streamReader.getText();
}
- if(currentObject!=null) {
+ if (currentObject != null) {
currentObject.addText(textContent);
}
break;
@@ -751,19 +756,16 @@ abstract class MtasXMLParser extends MtasBasicParser {
assert hasRoot : "no " + rootTag;
} catch (XMLStreamException e) {
throw new MtasParserException("No valid XML: " + e.getMessage());
- } catch (MtasParserException e) {
- e.printStackTrace();
- throw new MtasParserException("No valid XML");
}
// update tokens with offset
for (Entry<Integer, HashSet<String>> updateItem : updateList
.get(UPDATE_TYPE_OFFSET).entrySet()) {
HashSet<String> refIdList = new HashSet<String>();
- for(String refId : updateItem.getValue()) {
- if(idPositions.containsKey(refId)) {
+ for (String refId : updateItem.getValue()) {
+ if (idPositions.containsKey(refId)) {
refIdList.add(refId);
- }
- if(relationKeyMap.containsKey(refId)) {
+ }
+ if (relationKeyMap.containsKey(refId)) {
refIdList.addAll(recursiveCollect(refId, relationKeyMap, 10));
}
}
@@ -784,11 +786,11 @@ abstract class MtasXMLParser extends MtasBasicParser {
for (Entry<Integer, HashSet<String>> updateItem : updateList
.get(UPDATE_TYPE_POSITION).entrySet()) {
HashSet<String> refIdList = new HashSet<String>();
- for(String refId : updateItem.getValue()) {
- if(idPositions.containsKey(refId)) {
+ for (String refId : updateItem.getValue()) {
+ if (idPositions.containsKey(refId)) {
refIdList.add(refId);
- }
- if(relationKeyMap.containsKey(refId)) {
+ }
+ if (relationKeyMap.containsKey(refId)) {
refIdList.addAll(recursiveCollect(refId, relationKeyMap, 10));
}
}
@@ -799,12 +801,12 @@ abstract class MtasXMLParser extends MtasBasicParser {
if (refPositions != null) {
MtasToken<?> token = tokenCollection.get(tokenId);
token.addPositions(refPositions);
- }
+ }
}
}
}
// final check
- tokenCollection.check(autorepair);
+ tokenCollection.check(autorepair, makeunique);
return tokenCollection;
}
@@ -819,14 +821,15 @@ abstract class MtasXMLParser extends MtasBasicParser {
private Collection<? extends String> recursiveCollect(String refId,
HashMap<String, TreeSet<String>> relationKeyMap, int maxRecursion) {
HashSet<String> list = new HashSet<String>();
- if(maxRecursion>0 && relationKeyMap.containsKey(refId)) {
- TreeSet<String> subList = relationKeyMap.get(refId);
- for(String subRefId : subList) {
+ if (maxRecursion > 0 && relationKeyMap.containsKey(refId)) {
+ TreeSet<String> subList = relationKeyMap.get(refId);
+ for (String subRefId : subList) {
list.add(subRefId);
- list.addAll(recursiveCollect(subRefId, relationKeyMap, maxRecursion-1));
- }
- }
- return list;
+ list.addAll(
+ recursiveCollect(subRefId, relationKeyMap, maxRecursion - 1));
+ }
+ }
+ return list;
}
/**
diff --git a/src/mtas/analysis/token/MtasOffset.java b/src/mtas/analysis/token/MtasOffset.java
index ab40800..a999c29 100644
--- a/src/mtas/analysis/token/MtasOffset.java
+++ b/src/mtas/analysis/token/MtasOffset.java
@@ -7,32 +7,36 @@ public class MtasOffset {
/** The mtas offset start. */
private int mtasOffsetStart;
-
+
/** The mtas offset end. */
private int mtasOffsetEnd;
-
+
/**
* Instantiates a new mtas offset.
*
- * @param start the start
- * @param end the end
+ * @param start
+ * the start
+ * @param end
+ * the end
*/
public MtasOffset(int start, int end) {
mtasOffsetStart = start;
mtasOffsetEnd = end;
}
-
+
/**
* Adds the.
*
- * @param start the start
- * @param end the end
+ * @param start
+ * the start
+ * @param end
+ * the end
*/
public void add(int start, int end) {
- mtasOffsetStart = Math.min(mtasOffsetStart,start);
- mtasOffsetEnd = Math.max(mtasOffsetEnd,end);
+ mtasOffsetStart = Math.min(mtasOffsetStart, start);
+ mtasOffsetEnd = Math.max(mtasOffsetEnd, end);
}
-
+
/**
* Gets the start.
*
@@ -41,7 +45,7 @@ public class MtasOffset {
public int getStart() {
return mtasOffsetStart;
}
-
+
/**
* Gets the end.
*
@@ -50,13 +54,15 @@ public class MtasOffset {
public int getEnd() {
return mtasOffsetEnd;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
- return "["+mtasOffsetStart+"-"+mtasOffsetEnd+"]";
+ return "[" + mtasOffsetStart + "-" + mtasOffsetEnd + "]";
}
}
diff --git a/src/mtas/analysis/token/MtasPosition.java b/src/mtas/analysis/token/MtasPosition.java
index 0e85500..c183dd2 100644
--- a/src/mtas/analysis/token/MtasPosition.java
+++ b/src/mtas/analysis/token/MtasPosition.java
@@ -2,113 +2,126 @@ package mtas.analysis.token;
import java.util.TreeSet;
+import org.apache.commons.lang.ArrayUtils;
+
/**
* The Class MtasPosition.
*/
public class MtasPosition {
-
+
/** The position single. */
public static String POSITION_SINGLE = "single";
-
+
/** The position range. */
public static String POSITION_RANGE = "range";
-
+
/** The position set. */
public static String POSITION_SET = "set";
-
+
/** The mtas position type. */
private String mtasPositionType;
-
+
/** The mtas position start. */
private int mtasPositionStart;
-
+
/** The mtas position end. */
private int mtasPositionEnd;
-
+
/** The mtas position list. */
- TreeSet<Integer> mtasPositionList;
-
+ private int[] mtasPositionList = null;
+
/**
* Instantiates a new mtas position.
*
- * @param position the position
+ * @param position
+ * the position
*/
public MtasPosition(int position) {
mtasPositionType = POSITION_SINGLE;
mtasPositionStart = position;
}
-
+
/**
* Instantiates a new mtas position.
*
- * @param start the start
- * @param end the end
+ * @param start
+ * the start
+ * @param end
+ * the end
*/
public MtasPosition(int start, int end) {
- if(start==end) {
+ if (start == end) {
mtasPositionType = POSITION_SINGLE;
mtasPositionStart = start;
} else {
mtasPositionType = POSITION_RANGE;
mtasPositionStart = start;
mtasPositionEnd = end;
- }
+ }
}
-
+
/**
* Instantiates a new mtas position.
*
- * @param positions the positions
+ * @param positions
+ * the positions
*/
- public MtasPosition(TreeSet<Integer> positions) {
- if(positions.size()==1) {
+ public MtasPosition(int[] positions) {
+ TreeSet<Integer> list = new TreeSet<Integer>();
+ for (int p : positions) {
+ list.add(p);
+ }
+ if (list.size() == 1) {
mtasPositionType = POSITION_SINGLE;
- mtasPositionStart = positions.first();
+ mtasPositionStart = list.first();
} else {
mtasPositionType = POSITION_SET;
- mtasPositionList = new TreeSet<Integer>();
- mtasPositionList.addAll(positions);
- mtasPositionStart = mtasPositionList.first();
- mtasPositionEnd = mtasPositionList.last();
- if(mtasPositionList.size() == (1+mtasPositionEnd - mtasPositionStart)) {
+ mtasPositionList = ArrayUtils
+ .toPrimitive(list.toArray(new Integer[list.size()]));
+ mtasPositionStart = list.first();
+ mtasPositionEnd = list.last();
+ if (mtasPositionList.length == (1 + mtasPositionEnd
+ - mtasPositionStart)) {
mtasPositionType = POSITION_RANGE;
mtasPositionList = null;
}
}
}
-
+
/**
* Check type.
*
- * @param type the type
+ * @param type
+ * the type
* @return the boolean
*/
public Boolean checkType(String type) {
- if(mtasPositionType==null) {
+ if (mtasPositionType == null) {
return false;
} else {
return mtasPositionType.equals(type);
}
}
-
+
/**
* Gets the start.
*
* @return the start
*/
public Integer getStart() {
- return mtasPositionType.equals(null)?null:mtasPositionStart;
+ return mtasPositionType.equals(null) ? null : mtasPositionStart;
}
-
+
/**
* Gets the end.
*
* @return the end
*/
public Integer getEnd() {
- if(mtasPositionType.equals(POSITION_RANGE)||mtasPositionType.equals(POSITION_SET)) {
+ if (mtasPositionType.equals(POSITION_RANGE)
+ || mtasPositionType.equals(POSITION_SET)) {
return mtasPositionEnd;
- } else if(mtasPositionType.equals(POSITION_SINGLE)) {
+ } else if (mtasPositionType.equals(POSITION_SINGLE)) {
return mtasPositionStart;
} else {
return null;
@@ -120,127 +133,137 @@ public class MtasPosition {
*
* @return the positions
*/
- public TreeSet<Integer> getPositions() {
- if(mtasPositionType.equals(POSITION_SET)) {
+ public int[] getPositions() {
+ if (mtasPositionType.equals(POSITION_SET)) {
return mtasPositionList;
} else {
return null;
}
}
-
+
/**
* Gets the length.
*
* @return the length
*/
public Integer getLength() {
- if(mtasPositionType.equals(POSITION_SINGLE)) {
- return 1;
- } else if(mtasPositionType.equals(POSITION_RANGE)||mtasPositionType.equals(POSITION_SET)) {
- return 1+mtasPositionEnd-mtasPositionStart;
+ if (mtasPositionType.equals(POSITION_SINGLE)) {
+ return 1;
+ } else if (mtasPositionType.equals(POSITION_RANGE)
+ || mtasPositionType.equals(POSITION_SET)) {
+ return 1 + mtasPositionEnd - mtasPositionStart;
} else {
return null;
}
}
-
+
/**
* Adds the.
*
- * @param positions the positions
+ * @param positions
+ * the positions
*/
- public void add(TreeSet<Integer> positions) {
- if(mtasPositionType.equals(POSITION_SINGLE)) {
+ public void add(int[] positions) {
+ TreeSet<Integer> list = new TreeSet<Integer>();
+ for (int p : positions) {
+ list.add(p);
+ }
+ if (mtasPositionType.equals(POSITION_SINGLE)) {
mtasPositionType = POSITION_SET;
- mtasPositionList = new TreeSet<Integer>();
- mtasPositionList.add(mtasPositionStart);
- } else if(mtasPositionType.equals(POSITION_RANGE)) {
+ list.add(mtasPositionStart);
+ } else if (mtasPositionType.equals(POSITION_RANGE)) {
mtasPositionType = POSITION_SET;
- mtasPositionList = new TreeSet<Integer>();
- for(int i=mtasPositionStart; i<=mtasPositionEnd; i++) {
- mtasPositionList.add(i);
+ for (int i = mtasPositionStart; i <= mtasPositionEnd; i++) {
+ list.add(i);
}
- }
- if(mtasPositionType.equals(POSITION_SET)) {
- for(int position: positions) {
- if(!mtasPositionList.contains(position)) {
- mtasPositionList.add(position);
- }
- }
- mtasPositionStart = mtasPositionList.first();
- mtasPositionEnd = mtasPositionList.last();
- if(mtasPositionList.size() == 1) {
- mtasPositionType = POSITION_SINGLE;
- mtasPositionList = null;
- } else if(mtasPositionList.size() == (1+mtasPositionEnd - mtasPositionStart)) {
- mtasPositionType = POSITION_RANGE;
- mtasPositionList = null;
+ } else if (mtasPositionType.equals(POSITION_SET)) {
+ for (int p : mtasPositionList) {
+ list.add(p);
}
}
+ mtasPositionList = ArrayUtils
+ .toPrimitive(list.toArray(new Integer[list.size()]));
+ mtasPositionStart = list.first();
+ mtasPositionEnd = list.last();
+ if (list.size() == 1) {
+ mtasPositionType = POSITION_SINGLE;
+ mtasPositionList = null;
+ } else if (list.size() == (1 + mtasPositionEnd - mtasPositionStart)) {
+ mtasPositionType = POSITION_RANGE;
+ mtasPositionList = null;
+ }
}
-
+
/**
* Adds the.
*
- * @param position the position
+ * @param position
+ * the position
*/
public void add(int position) {
- if(mtasPositionType.equals(POSITION_SINGLE)) {
- if(position!=mtasPositionStart) {
- if(position==(mtasPositionStart+1)) {
- mtasPositionType=POSITION_RANGE;
- mtasPositionEnd=position;
- } else if(position==(mtasPositionStart-1)) {
- mtasPositionType=POSITION_RANGE;
- mtasPositionEnd=mtasPositionStart;
- mtasPositionStart=position;
+ if (mtasPositionType.equals(POSITION_SINGLE)) {
+ if (position != mtasPositionStart) {
+ if (position == (mtasPositionStart + 1)) {
+ mtasPositionType = POSITION_RANGE;
+ mtasPositionEnd = position;
+ } else if (position == (mtasPositionStart - 1)) {
+ mtasPositionType = POSITION_RANGE;
+ mtasPositionEnd = mtasPositionStart;
+ mtasPositionStart = position;
} else {
mtasPositionType = POSITION_SET;
- mtasPositionList = new TreeSet<Integer>();
- mtasPositionList.add(position);
- mtasPositionList.add(mtasPositionStart);
- mtasPositionStart = mtasPositionList.first();
- mtasPositionEnd = mtasPositionList.last();
+ TreeSet<Integer> list = new TreeSet<Integer>();
+ list.add(position);
+ list.add(mtasPositionStart);
+ mtasPositionList = ArrayUtils
+ .toPrimitive(list.toArray(new Integer[list.size()]));
+ mtasPositionStart = list.first();
+ mtasPositionEnd = list.last();
}
}
} else {
- if(mtasPositionType.equals(POSITION_RANGE)) {
+ TreeSet<Integer> list = new TreeSet<Integer>();
+ if (mtasPositionType.equals(POSITION_RANGE)) {
mtasPositionType = POSITION_SET;
- mtasPositionList = new TreeSet<Integer>();
- for(int i=mtasPositionStart; i<=mtasPositionEnd; i++) {
- mtasPositionList.add(i);
+ for (int i = mtasPositionStart; i <= mtasPositionEnd; i++) {
+ list.add(i);
}
- }
- if(mtasPositionType.equals(POSITION_SET)) {
- if(!mtasPositionList.contains(position)) {
- mtasPositionList.add(position);
- mtasPositionStart = mtasPositionList.first();
- mtasPositionEnd = mtasPositionList.last();
- if(mtasPositionList.size() == (1+mtasPositionEnd - mtasPositionStart)) {
- mtasPositionType = POSITION_RANGE;
- mtasPositionList = null;
- }
+ list.add(position);
+ } else if (mtasPositionType.equals(POSITION_SET)) {
+ for (int p : mtasPositionList) {
+ list.add(p);
}
+ list.add(position);
+ }
+ mtasPositionList = ArrayUtils
+ .toPrimitive(list.toArray(new Integer[list.size()]));
+ mtasPositionStart = list.first();
+ mtasPositionEnd = list.last();
+ if (list.size() == (1 + mtasPositionEnd - mtasPositionStart)) {
+ mtasPositionType = POSITION_RANGE;
+ mtasPositionList = null;
}
- }
+ }
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
- if(mtasPositionType==null) {
+ if (mtasPositionType == null) {
return "[null]";
- } else if(mtasPositionType.equals(POSITION_SINGLE)) {
- return "["+mtasPositionStart+"]";
- } else if(mtasPositionType.equals(POSITION_RANGE)) {
- return "["+mtasPositionStart+"-"+mtasPositionEnd+"]";
- } else if(mtasPositionType.equals(POSITION_SET)) {
- return mtasPositionList.toArray().toString();
+ } else if (mtasPositionType.equals(POSITION_SINGLE)) {
+ return "[" + mtasPositionStart + "]";
+ } else if (mtasPositionType.equals(POSITION_RANGE)) {
+ return "[" + mtasPositionStart + "-" + mtasPositionEnd + "]";
+ } else if (mtasPositionType.equals(POSITION_SET)) {
+ return mtasPositionList.toString();
} else {
return "[unknown]";
}
}
-
-
+
}
diff --git a/src/mtas/analysis/token/MtasToken.java b/src/mtas/analysis/token/MtasToken.java
index f14d3ed..d07ad27 100644
--- a/src/mtas/analysis/token/MtasToken.java
+++ b/src/mtas/analysis/token/MtasToken.java
@@ -1,66 +1,84 @@
package mtas.analysis.token;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.lucene.analysis.payloads.PayloadHelper;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.automaton.Operations;
+import org.apache.lucene.util.automaton.RegExp;
/**
* The Class MtasToken.
*
- * @param <GenericType> the generic type
+ * @param <GenericType>
+ * the generic type
*/
public abstract class MtasToken<GenericType> {
/** The Constant DELIMITER. */
public static final String DELIMITER = "\u0001";
+ /** The Constant regexpPrePostFix. */
+ public static final String regexpPrePostFix = "(.*)" + DELIMITER
+ + "(.[^\u0000]*)";
+
+ /** The Constant patternPrePostFix. */
+ public static final Pattern patternPrePostFix = Pattern
+ .compile(regexpPrePostFix);
+
/** The Constant MTAS_TOKEN_ID. */
- static final AtomicInteger MTAS_TOKEN_ID = new AtomicInteger(0);
-
+ static final AtomicInteger MTAS_TOKEN_ID = new AtomicInteger(0);
+
/** The token id. */
private Integer tokenId = MTAS_TOKEN_ID.getAndIncrement();
-
+
/** The token ref. */
private Long tokenRef = null;
-
+
/** The term ref. */
private Long termRef = null;
-
+
/** The prefix id. */
private Integer prefixId = null;
-
+
/** The token type. */
protected String tokenType = null;
-
+
/** The token parent id. */
private Integer tokenParentId = null;
-
+
/** The token value. */
private String tokenValue = null;
-
+
/** The token position. */
private MtasPosition tokenPosition = null;
-
+
/** The token offset. */
private MtasOffset tokenOffset = null;
-
+
/** The token real offset. */
private MtasOffset tokenRealOffset = null;
-
+
/** The token payload. */
private BytesRef tokenPayload = null;
-
+
/** The provide offset. */
private Boolean provideOffset = true;
-
+
/** The provide real offset. */
private Boolean provideRealOffset = true;
-
+
/** The provide parent id. */
private Boolean provideParentId = true;
@@ -70,22 +88,25 @@ public abstract class MtasToken<GenericType> {
public static void resetId() {
MTAS_TOKEN_ID.set(0);
}
-
+
/**
* Instantiates a new mtas token.
*
- * @param value the value
+ * @param value
+ * the value
*/
protected MtasToken(String value) {
setType();
setValue(value);
}
-
+
/**
* Instantiates a new mtas token.
*
- * @param value the value
- * @param position the position
+ * @param value
+ * the value
+ * @param position
+ * the position
*/
protected MtasToken(String value, Integer position) {
setType();
@@ -96,7 +117,8 @@ public abstract class MtasToken<GenericType> {
/**
* Sets the token ref.
*
- * @param ref the new token ref
+ * @param ref
+ * the new token ref
*/
final public void setTokenRef(Long ref) {
tokenRef = ref;
@@ -110,11 +132,12 @@ public abstract class MtasToken<GenericType> {
final public Long getTokenRef() {
return tokenRef;
}
-
+
/**
* Sets the term ref.
*
- * @param ref the new term ref
+ * @param ref
+ * the new term ref
*/
final public void setTermRef(Long ref) {
termRef = ref;
@@ -128,11 +151,12 @@ public abstract class MtasToken<GenericType> {
final public Long getTermRef() {
return termRef;
}
-
+
/**
* Sets the prefix id.
*
- * @param id the new prefix id
+ * @param id
+ * the new prefix id
*/
final public void setPrefixId(int id) {
prefixId = id;
@@ -142,10 +166,11 @@ public abstract class MtasToken<GenericType> {
* Gets the prefix id.
*
* @return the prefix id
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
final public int getPrefixId() throws IOException {
- if(prefixId!=null) {
+ if (prefixId != null) {
return prefixId;
} else {
throw new IOException("no prefixId");
@@ -155,7 +180,8 @@ public abstract class MtasToken<GenericType> {
/**
* Sets the id.
*
- * @param id the new id
+ * @param id
+ * the new id
*/
final public void setId(Integer id) {
tokenId = id;
@@ -173,7 +199,8 @@ public abstract class MtasToken<GenericType> {
/**
* Sets the parent id.
*
- * @param id the new parent id
+ * @param id
+ * the new parent id
*/
final public void setParentId(Integer id) {
tokenParentId = id;
@@ -187,16 +214,17 @@ public abstract class MtasToken<GenericType> {
final public Integer getParentId() {
return tokenParentId;
}
-
+
/**
* Sets the provide parent id.
*
- * @param provide the new provide parent id
+ * @param provide
+ * the new provide parent id
*/
final public void setProvideParentId(Boolean provide) {
provideParentId = provide;
}
-
+
/**
* Gets the provide parent id.
*
@@ -225,7 +253,8 @@ public abstract class MtasToken<GenericType> {
/**
* Adds the position.
*
- * @param position the position
+ * @param position
+ * the position
*/
final public void addPosition(int position) {
if (tokenPosition == null) {
@@ -238,16 +267,18 @@ public abstract class MtasToken<GenericType> {
/**
* Adds the position range.
*
- * @param start the start
- * @param end the end
+ * @param start
+ * the start
+ * @param end
+ * the end
*/
final public void addPositionRange(int start, int end) {
if (tokenPosition == null) {
tokenPosition = new MtasPosition(start, end);
} else {
- TreeSet<Integer> positions = new TreeSet<Integer>();
+ int[] positions = new int[end - start + 1];
for (int i = start; i <= end; i++) {
- positions.add(i);
+ positions[i - start] = i;
}
tokenPosition.add(positions);
}
@@ -256,10 +287,11 @@ public abstract class MtasToken<GenericType> {
/**
* Adds the positions.
*
- * @param positions the positions
+ * @param positions
+ * the positions
*/
- final public void addPositions(TreeSet<Integer> positions) {
- if (positions != null && positions.size() > 0) {
+ final public void addPositions(int[] positions) {
+ if (positions != null && positions.length > 0) {
if (tokenPosition == null) {
tokenPosition = new MtasPosition(positions);
} else {
@@ -269,9 +301,22 @@ public abstract class MtasToken<GenericType> {
}
/**
+ * Adds the positions.
+ *
+ * @param list
+ * the list
+ */
+ final public void addPositions(TreeSet<Integer> list) {
+ int[] positions = ArrayUtils
+ .toPrimitive(list.toArray(new Integer[list.size()]));
+ addPositions(positions);
+ }
+
+ /**
* Check position type.
*
- * @param type the type
+ * @param type
+ * the type
* @return the boolean
*/
final public Boolean checkPositionType(String type) {
@@ -310,7 +355,7 @@ public abstract class MtasToken<GenericType> {
*
* @return the positions
*/
- final public TreeSet<Integer> getPositions() {
+ final public int[] getPositions() {
return tokenPosition == null ? null : tokenPosition.getPositions();
}
@@ -320,7 +365,7 @@ public abstract class MtasToken<GenericType> {
* @return the boolean
*/
final public Boolean checkOffset() {
- if ((tokenOffset == null)||!provideOffset) {
+ if ((tokenOffset == null) || !provideOffset) {
return false;
} else {
return true;
@@ -333,7 +378,7 @@ public abstract class MtasToken<GenericType> {
* @return the boolean
*/
final public Boolean checkRealOffset() {
- if ((tokenRealOffset == null)||!provideRealOffset) {
+ if ((tokenRealOffset == null) || !provideRealOffset) {
return false;
} else if (tokenOffset == null) {
return true;
@@ -348,8 +393,10 @@ public abstract class MtasToken<GenericType> {
/**
* Sets the offset.
*
- * @param start the start
- * @param end the end
+ * @param start
+ * the start
+ * @param end
+ * the end
*/
final public void setOffset(Integer start, Integer end) {
if ((start == null) || (end == null)) {
@@ -364,8 +411,10 @@ public abstract class MtasToken<GenericType> {
/**
* Adds the offset.
*
- * @param start the start
- * @param end the end
+ * @param start
+ * the start
+ * @param end
+ * the end
*/
final public void addOffset(Integer start, Integer end) {
if (tokenOffset == null) {
@@ -378,11 +427,12 @@ public abstract class MtasToken<GenericType> {
tokenOffset.add(start, end);
}
}
-
+
/**
* Sets the provide offset.
*
- * @param provide the new provide offset
+ * @param provide
+ * the new provide offset
*/
final public void setProvideOffset(Boolean provide) {
provideOffset = provide;
@@ -391,8 +441,10 @@ public abstract class MtasToken<GenericType> {
/**
* Sets the real offset.
*
- * @param start the start
- * @param end the end
+ * @param start
+ * the start
+ * @param end
+ * the end
*/
final public void setRealOffset(Integer start, Integer end) {
if ((start == null) || (end == null)) {
@@ -404,17 +456,17 @@ public abstract class MtasToken<GenericType> {
tokenRealOffset = new MtasOffset(start, end);
}
}
-
-
+
/**
* Sets the provide real offset.
*
- * @param provide the new provide real offset
+ * @param provide
+ * the new provide real offset
*/
final public void setProvideRealOffset(Boolean provide) {
provideRealOffset = provide;
}
-
+
/**
* Gets the provide offset.
*
@@ -423,7 +475,7 @@ public abstract class MtasToken<GenericType> {
final public boolean getProvideOffset() {
return provideOffset;
}
-
+
/**
* Gets the provide real offset.
*
@@ -472,41 +524,127 @@ public abstract class MtasToken<GenericType> {
/**
* Sets the value.
*
- * @param value the new value
+ * @param value
+ * the new value
*/
public void setValue(String value) {
- tokenValue = value;
+ tokenValue = value;
}
-
+
/**
* Gets the prefix from value.
*
- * @param value the value
+ * @param value
+ * the value
* @return the prefix from value
*/
public static String getPrefixFromValue(String value) {
String prefix = null;
- if(value.contains(DELIMITER)) {
+ if (value.contains(DELIMITER)) {
prefix = value.split(DELIMITER)[0];
} else {
prefix = value;
}
return prefix.replaceAll("\u0000", "");
}
-
+
/**
* Gets the postfix from value.
*
- * @param value the value
+ * @param value
+ * the value
* @return the postfix from value
*/
public static String getPostfixFromValue(String value) {
String postfix = "";
- if(value.contains(DELIMITER)) {
- String[] list = value.split(DELIMITER);
- postfix = StringUtils.join(Arrays.copyOfRange(list,1,list.length),DELIMITER);
- }
- return postfix.replaceAll("\u0000", "");
+ Matcher m = patternPrePostFix.matcher(value);
+ if (m.find()) {
+ postfix = m.group(2);
+
+ }
+ return postfix;
+ }
+
+ /**
+ * Gets the postfix from value.
+ *
+ * @param term
+ * the term
+ * @return the postfix from value
+ */
+ public static String getPostfixFromValue(BytesRef term) {
+ int i = term.offset, length = term.offset + term.length;
+ byte[] postfix = new byte[length];
+ while (i < length) {
+ if ((term.bytes[i] & 0b10000000) == 0b00000000) {
+ if (term.bytes[i] == 0b00000001) {
+ i++;
+ break;
+ } else {
+ i++;
+ }
+ } else if ((term.bytes[i] & 0b11100000) == 0b11000000) {
+ i += 2;
+ } else if ((term.bytes[i] & 0b11110000) == 0b11100000) {
+ i += 3;
+ } else if ((term.bytes[i] & 0b11111000) == 0b11110000) {
+ i += 4;
+ } else if ((term.bytes[i] & 0b11111100) == 0b11111000) {
+ i += 5;
+ } else if ((term.bytes[i] & 0b11111110) == 0b11111100) {
+ i += 6;
+ } else {
+ return "";
+ }
+ }
+ int start = i;
+ while (i < length) {
+ if ((term.bytes[i] & 0b10000000) == 0b00000000) {
+ if (term.bytes[i] == 0b00000000) {
+ break;
+ }
+ postfix[i] = term.bytes[i];
+ i++;
+ } else if ((term.bytes[i] & 0b11100000) == 0b11000000) {
+ postfix[i] = term.bytes[i];
+ postfix[i + 1] = term.bytes[i + 1];
+ i += 2;
+ } else if ((term.bytes[i] & 0b11110000) == 0b11100000) {
+ postfix[i] = term.bytes[i];
+ postfix[i + 1] = term.bytes[i + 1];
+ postfix[i + 2] = term.bytes[i + 2];
+ i += 3;
+ } else if ((term.bytes[i] & 0b11111000) == 0b11110000) {
+ postfix[i] = term.bytes[i];
+ postfix[i + 1] = term.bytes[i + 1];
+ postfix[i + 2] = term.bytes[i + 2];
+ postfix[i + 3] = term.bytes[i + 3];
+ i += 4;
+ } else if ((term.bytes[i] & 0b11111100) == 0b11111000) {
+ postfix[i] = term.bytes[i];
+ postfix[i + 1] = term.bytes[i + 1];
+ postfix[i + 2] = term.bytes[i + 2];
+ postfix[i + 3] = term.bytes[i + 3];
+ postfix[i + 4] = term.bytes[i + 4];
+ i += 5;
+ } else if ((term.bytes[i] & 0b11111110) == 0b11111100) {
+ postfix[i] = term.bytes[i];
+ postfix[i + 1] = term.bytes[i + 1];
+ postfix[i + 2] = term.bytes[i + 2];
+ postfix[i + 3] = term.bytes[i + 3];
+ postfix[i + 4] = term.bytes[i + 4];
+ postfix[i + 5] = term.bytes[i + 5];
+ i += 6;
+ } else {
+ return "";
+ }
+ }
+ try {
+ return new String(Arrays.copyOfRange(postfix, start, i), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ return "";
+ }
+
}
/**
@@ -517,7 +655,7 @@ public abstract class MtasToken<GenericType> {
public String getValue() {
return tokenValue;
}
-
+
/**
* Gets the prefix.
*
@@ -526,7 +664,7 @@ public abstract class MtasToken<GenericType> {
public String getPrefix() {
return getPrefixFromValue(tokenValue);
}
-
+
/**
* Gets the postfix.
*
@@ -542,30 +680,31 @@ public abstract class MtasToken<GenericType> {
* @return the boolean
*/
final public Boolean checkParentId() {
- if ((tokenParentId==null)||!provideParentId) {
+ if ((tokenParentId == null) || !provideParentId) {
return false;
} else {
return true;
}
}
-
+
/**
* Check payload.
*
* @return the boolean
*/
final public Boolean checkPayload() {
- if (tokenPayload == null) {
+ if (tokenPayload == null) {
return false;
} else {
return true;
}
}
-
+
/**
* Sets the payload.
*
- * @param payload the new payload
+ * @param payload
+ * the new payload
*/
public void setPayload(BytesRef payload) {
tokenPayload = payload;
@@ -580,42 +719,78 @@ public abstract class MtasToken<GenericType> {
return tokenPayload;
}
- /* (non-Javadoc)
+ /**
+ * Creates the automata.
+ *
+ * @param prefix
+ * the prefix
+ * @param valueList
+ * the value list
+ * @return the list
+ */
+ public static List<CompiledAutomaton> createAutomata(String prefix,
+ List<String> valueList) {
+ List<CompiledAutomaton> list = new ArrayList<CompiledAutomaton>();
+ int step = 100;
+ for (int i = 0; i < valueList.size(); i += step) {
+ int next = Math.min(valueList.size(), i + step);
+ List<Automaton> listAutomaton = new ArrayList<Automaton>();
+ for (int j = i; j < next; j++) {
+ String value = valueList.get(j);
+ value = value.replaceAll("([\\\"\\)\\(\\<\\>\\.\\@\\#\\]\\[\\{\\}])",
+ "\\\\\\1");
+ listAutomaton
+ .add((new RegExp(prefix + MtasToken.DELIMITER + value + "\u0000*"))
+ .toAutomaton());
+ }
+ Automaton automaton = Operations.union(listAutomaton);
+ CompiledAutomaton compiledAutomaton = new CompiledAutomaton(automaton);
+ list.add(compiledAutomaton);
+ }
+ return list;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
String text = "";
- text+="[" + String.format("%05d", getId()) + "] ";
- text+=((getRealOffsetStart() == null) ? "[-------,-------]"
- : "[" + String.format("%07d", getRealOffsetStart()) + "-"
- + String.format("%07d", getRealOffsetEnd()) + "]");
- text+=(provideRealOffset?" ":"* ");
- text+=((getOffsetStart() == null) ? "[-------,-------]"
+ text += "[" + String.format("%05d", getId()) + "] ";
+ text += ((getRealOffsetStart() == null) ? "[-------,-------]"
+ : "[" + String.format("%07d", getRealOffsetStart()) + "-"
+ + String.format("%07d", getRealOffsetEnd()) + "]");
+ text += (provideRealOffset ? " " : "* ");
+ text += ((getOffsetStart() == null) ? "[-------,-------]"
: "[" + String.format("%07d", getOffsetStart()) + "-"
+ String.format("%07d", getOffsetEnd()) + "]");
- text+=(provideOffset?" ":"* ");
+ text += (provideOffset ? " " : "* ");
if (getPositionLength() == null) {
- text+=String.format("%11s", "");
+ text += String.format("%11s", "");
} else if (getPositionStart().equals(getPositionEnd())) {
- text+=String.format("%11s", "[" + getPositionStart()
- + "]");
- } else if ((getPositions() == null)
- || (getPositions().size() == (1 + getPositionEnd() -getPositionStart()))) {
- text+=String.format("%11s", "[" + getPositionStart()
- + "-" + getPositionEnd() + "]");
+ text += String.format("%11s", "[" + getPositionStart() + "]");
+ } else if ((getPositions() == null) || (getPositions().length == (1
+ + getPositionEnd() - getPositionStart()))) {
+ text += String.format("%11s",
+ "[" + getPositionStart() + "-" + getPositionEnd() + "]");
} else {
- text+=String.format("%11s", getPositions());
+ text += String.format("%11s", Arrays.toString(getPositions()));
}
- text+=((getParentId() == null) ? "[-----]" : "["
- + String.format("%05d", getParentId()) + "]");
- text+=(provideParentId?" ":"* ");
+ text += ((getParentId() == null) ? "[-----]"
+ : "[" + String.format("%05d", getParentId()) + "]");
+ text += (provideParentId ? " " : "* ");
BytesRef payload = getPayload();
- text+=(payload == null) ? "[------] " : "["
- + String.format("%.4f",
- PayloadHelper.decodeFloat(Arrays.copyOfRange(payload.bytes, payload.offset, (payload.offset+payload.length)))) + "] ";
- text+=String.format("%25s", "[" + getPrefix() + "]") + " ";
- text+=((getPostfix()==null)?"---":"[" + getPostfix() + "]") + " ";
+ text += (payload == null) ? "[------] "
+ : "["
+ + String
+ .format("%.4f",
+ PayloadHelper.decodeFloat(Arrays.copyOfRange(payload.bytes,
+ payload.offset, (payload.offset + payload.length))))
+ + "] ";
+ text += String.format("%25s", "[" + getPrefix() + "]") + " ";
+ text += ((getPostfix() == null) ? "---" : "[" + getPostfix() + "]") + " ";
return text;
}
diff --git a/src/mtas/analysis/token/MtasTokenCollection.java b/src/mtas/analysis/token/MtasTokenCollection.java
index ec231d6..05ecf54 100644
--- a/src/mtas/analysis/token/MtasTokenCollection.java
+++ b/src/mtas/analysis/token/MtasTokenCollection.java
@@ -33,7 +33,8 @@ public class MtasTokenCollection {
/**
* Adds the.
*
- * @param token the token
+ * @param token
+ * the token
* @return the integer
*/
public Integer add(MtasToken<?> token) {
@@ -45,7 +46,8 @@ public class MtasTokenCollection {
/**
* Gets the.
*
- * @param id the id
+ * @param id
+ * the id
* @return the mtas token
*/
public MtasToken<?> get(Integer id) {
@@ -56,7 +58,8 @@ public class MtasTokenCollection {
* Iterator.
*
* @return the iterator
- * @throws MtasParserException the mtas parser exception
+ * @throws MtasParserException
+ * the mtas parser exception
*/
public Iterator<MtasToken<?>> iterator() throws MtasParserException {
checkTokenCollectionIndex();
@@ -85,7 +88,8 @@ public class MtasTokenCollection {
/**
* Prints the.
*
- * @throws MtasParserException the mtas parser exception
+ * @throws MtasParserException
+ * the mtas parser exception
*/
public void print() throws MtasParserException {
Iterator<MtasToken<?>> it = this.iterator();
@@ -99,7 +103,8 @@ public class MtasTokenCollection {
* Gets the list.
*
* @return the list
- * @throws MtasParserException the mtas parser exception
+ * @throws MtasParserException
+ * the mtas parser exception
*/
public String[][] getList() throws MtasParserException {
String[][] result = new String[(tokenCollection.size() + 1)][];
@@ -113,15 +118,15 @@ public class MtasTokenCollection {
MtasToken<?> token = it.next();
String[] row = new String[15];
row[0] = token.getId().toString();
- if(token.getRealOffsetStart() != null) {
+ if (token.getRealOffsetStart() != null) {
row[1] = token.getRealOffsetStart().toString();
row[2] = token.getRealOffsetEnd().toString();
- row[3] = token.getProvideRealOffset()?"1":null;
+ row[3] = token.getProvideRealOffset() ? "1" : null;
}
- if(token.getOffsetStart() != null) {
+ if (token.getOffsetStart() != null) {
row[4] = token.getOffsetStart().toString();
row[5] = token.getOffsetEnd().toString();
- row[6] = token.getProvideOffset()?"1":null;
+ row[6] = token.getProvideOffset() ? "1" : null;
}
if (token.getPositionLength() != null) {
if (token.getPositionStart().equals(token.getPositionEnd())) {
@@ -129,7 +134,8 @@ public class MtasTokenCollection {
row[8] = token.getPositionEnd().toString();
row[9] = null;
} else if ((token.getPositions() == null)
- || (token.getPositions().size() == (1 + token.getPositionEnd() - token.getPositionStart()))) {
+ || (token.getPositions().length == (1 + token.getPositionEnd()
+ - token.getPositionStart()))) {
row[7] = token.getPositionStart().toString();
row[8] = token.getPositionEnd().toString();
row[9] = null;
@@ -139,16 +145,17 @@ public class MtasTokenCollection {
row[9] = token.getPositions().toString();
}
}
- if(token.getParentId()!=null) {
+ if (token.getParentId() != null) {
row[10] = token.getParentId().toString();
- row[11] = token.getProvideParentId()?"1":null;
+ row[11] = token.getProvideParentId() ? "1" : null;
}
- if(token.getPayload()!=null) {
- BytesRef payload = token.getPayload();
- row[12] = Float.toString(PayloadHelper.decodeFloat(Arrays.copyOfRange(payload.bytes, payload.offset, (payload.offset+payload.length))));
+ if (token.getPayload() != null) {
+ BytesRef payload = token.getPayload();
+ row[12] = Float.toString(PayloadHelper.decodeFloat(Arrays.copyOfRange(
+ payload.bytes, payload.offset, (payload.offset + payload.length))));
}
row[13] = token.getPrefix();
- row[14] = token.getPostfix();
+ row[14] = token.getPostfix();
result[number] = row;
number++;
}
@@ -158,13 +165,21 @@ public class MtasTokenCollection {
/**
* Check.
*
- * @param autorepair the autorepair
- * @throws MtasParserException the mtas parser exception
+ * @param autoRepair
+ * the auto repair
+ * @param makeUnique
+ * the make unique
+ * @throws MtasParserException
+ * the mtas parser exception
*/
- public void check(Boolean autorepair) throws MtasParserException {
- if (autorepair) {
+ public void check(Boolean autoRepair, Boolean makeUnique)
+ throws MtasParserException {
+ if (autoRepair) {
autoRepair();
}
+ if (makeUnique) {
+ makeUnique();
+ }
checkTokenCollectionIndex();
for (Integer i : tokenCollectionIndex) {
// minimal properties
@@ -179,6 +194,34 @@ public class MtasTokenCollection {
}
/**
+ * Make unique.
+ */
+ private void makeUnique() {
+ HashMap<String, ArrayList<MtasToken<?>>> currentPositionTokens = new HashMap<String, ArrayList<MtasToken<?>>>();
+ ArrayList<MtasToken<?>> currentValueTokens;
+ int currentStartPosition = -1;
+ MtasToken<?> currentToken = null;
+ for (Integer i : tokenCollection.keySet()) {
+ currentToken = tokenCollection.get(i);
+ if (currentToken.getPositionStart() > currentStartPosition) {
+ currentPositionTokens.clear();
+ currentStartPosition = currentToken.getPositionStart();
+ } else {
+ if (currentPositionTokens.containsKey(currentToken.getValue())) {
+ currentValueTokens = currentPositionTokens
+ .get(currentToken.getValue());
+
+ } else {
+ currentValueTokens = new ArrayList<MtasToken<?>>();
+ currentPositionTokens.put(currentToken.getValue(),
+ currentValueTokens);
+ }
+ currentValueTokens.add(currentToken);
+ }
+ }
+ }
+
+ /**
* Auto repair.
*/
private void autoRepair() {
@@ -197,6 +240,8 @@ public class MtasTokenCollection {
trash.add(i);
} else if (token.getValue() == null || (token.getValue().equals(""))) {
trash.add(i);
+ } else if (token.getPrefix() == null || (token.getPrefix().equals(""))) {
+ trash.add(i);
}
}
// check parentId
@@ -256,7 +301,8 @@ public class MtasTokenCollection {
/**
* Check token collection index.
*
- * @throws MtasParserException the mtas parser exception
+ * @throws MtasParserException
+ * the mtas parser exception
*/
private void checkTokenCollectionIndex() throws MtasParserException {
if (tokenCollectionIndex.size() != tokenCollection.size()) {
@@ -277,6 +323,10 @@ public class MtasTokenCollection {
} else if (token.getValue() == null || (token.getValue().equals(""))) {
throw new MtasParserException(
"no value for token with id " + token.getId());
+ } else if (token.getPrefix() == null
+ || (token.getPrefix().equals(""))) {
+ throw new MtasParserException(
+ "no prefix for token with id " + token.getId());
} else if ((token.getParentId() != null)
&& !tokenCollection.containsKey(token.getParentId())) {
throw new MtasParserException(
diff --git a/src/mtas/analysis/token/MtasTokenString.java b/src/mtas/analysis/token/MtasTokenString.java
index 6d9bc6c..a934313 100644
--- a/src/mtas/analysis/token/MtasTokenString.java
+++ b/src/mtas/analysis/token/MtasTokenString.java
@@ -3,37 +3,41 @@ package mtas.analysis.token;
/**
* The Class MtasTokenString.
*/
-public class MtasTokenString extends MtasToken<String> {
+public class MtasTokenString extends MtasToken<String> {
/** The Constant TOKEN_TYPE. */
public static final String TOKEN_TYPE = "string";
-
+
/**
* Instantiates a new mtas token string.
*
- * @param value the value
+ * @param value
+ * the value
*/
public MtasTokenString(String value) {
- super(value);
+ super(value);
}
/**
* Instantiates a new mtas token string.
*
- * @param value the value
- * @param position the position
+ * @param value
+ * the value
+ * @param position
+ * the position
*/
public MtasTokenString(String value, Integer position) {
- super(value, position);
+ super(value, position);
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.analysis.token.MtasToken#setType()
*/
@Override
public void setType() {
tokenType = TOKEN_TYPE;
- }
-
-
+ }
+
}
diff --git a/src/mtas/analysis/util/MtasBufferedReader.java b/src/mtas/analysis/util/MtasBufferedReader.java
index 4833031..e73bc05 100644
--- a/src/mtas/analysis/util/MtasBufferedReader.java
+++ b/src/mtas/analysis/util/MtasBufferedReader.java
@@ -20,10 +20,10 @@ public class MtasBufferedReader extends Reader {
/** The cb. */
private char cb[];
-
+
/** The next char. */
private int nChars, nextChar;
-
+
/** The previous buffer size. */
private int previousBufferSize;
@@ -32,15 +32,17 @@ public class MtasBufferedReader extends Reader {
/** The default char buffer size. */
private static int defaultCharBufferSize = 8192;
-
+
/** The default expected line length. */
private static int defaultExpectedLineLength = 80;
/**
* Instantiates a new mtas buffered reader.
*
- * @param in the in
- * @param sz the sz
+ * @param in
+ * the in
+ * @param sz
+ * the sz
*/
public MtasBufferedReader(Reader in, int sz) {
super(in);
@@ -54,7 +56,8 @@ public class MtasBufferedReader extends Reader {
/**
* Instantiates a new mtas buffered reader.
*
- * @param in the in
+ * @param in
+ * the in
*/
public MtasBufferedReader(Reader in) {
this(in, defaultCharBufferSize);
@@ -63,7 +66,8 @@ public class MtasBufferedReader extends Reader {
/**
* Ensure open.
*
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private void ensureOpen() throws IOException {
if (in == null)
@@ -73,7 +77,8 @@ public class MtasBufferedReader extends Reader {
/**
* Fill.
*
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private void fill() throws IOException {
int n;
@@ -87,7 +92,9 @@ public class MtasBufferedReader extends Reader {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.io.Reader#read()
*/
@Override
@@ -115,11 +122,15 @@ public class MtasBufferedReader extends Reader {
/**
* Read1.
*
- * @param cbuf the cbuf
- * @param off the off
- * @param len the len
+ * @param cbuf
+ * the cbuf
+ * @param off
+ * the off
+ * @param len
+ * the len
* @return the int
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private int read1(char[] cbuf, int off, int len) throws IOException {
if (nextChar >= nChars) {
@@ -152,7 +163,9 @@ public class MtasBufferedReader extends Reader {
return n;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.io.Reader#read(char[], int, int)
*/
@Override
@@ -182,9 +195,11 @@ public class MtasBufferedReader extends Reader {
/**
* Read line.
*
- * @param ignoreLF the ignore lf
+ * @param ignoreLF
+ * the ignore lf
* @return the string
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
String readLine(boolean ignoreLF) throws IOException {
StringBuffer s = null;
@@ -251,7 +266,8 @@ public class MtasBufferedReader extends Reader {
* Read line.
*
* @return the string
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public String readLine() throws IOException {
return readLine(false);
@@ -266,7 +282,9 @@ public class MtasBufferedReader extends Reader {
return previousBufferSize + nextChar;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.io.Reader#skip(long)
*/
@Override
@@ -302,7 +320,9 @@ public class MtasBufferedReader extends Reader {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.io.Reader#ready()
*/
@Override
@@ -332,7 +352,9 @@ public class MtasBufferedReader extends Reader {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.io.Reader#reset()
*/
@Override
@@ -344,7 +366,9 @@ public class MtasBufferedReader extends Reader {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.io.Reader#close()
*/
@Override
diff --git a/src/mtas/analysis/util/MtasCharFilterFactory.java b/src/mtas/analysis/util/MtasCharFilterFactory.java
index 4447a22..5cf7cac 100644
--- a/src/mtas/analysis/util/MtasCharFilterFactory.java
+++ b/src/mtas/analysis/util/MtasCharFilterFactory.java
@@ -13,23 +13,27 @@ import java.util.Map;
/**
* A factory for creating MtasCharFilter objects.
*/
-public class MtasCharFilterFactory extends CharFilterFactory implements ResourceLoaderAware {
+public class MtasCharFilterFactory extends CharFilterFactory
+ implements ResourceLoaderAware {
/** The argument type. */
public static String ARGUMENT_TYPE = "type";
-
+
/** The argument prefix. */
public static String ARGUMENT_PREFIX = "prefix";
-
+
+ /** The argument postfix. */
+ public static String ARGUMENT_POSTFIX = "postfix";
+
/** The argument config. */
public static String ARGUMENT_CONFIG = "config";
-
+
/** The argument default. */
public static String ARGUMENT_DEFAULT = "default";
/** The value type url. */
public static String VALUE_TYPE_URL = "url";
-
+
/** The value type file. */
public static String VALUE_TYPE_FILE = "file";
@@ -44,18 +48,23 @@ public class MtasCharFilterFactory extends CharFilterFactory implements Resource
/** The prefix argument. */
String prefixArgument;
+
+ /** The postfix argument. */
+ String postfixArgument;
/** The configs. */
private HashMap<String, MtasConfiguration> configs = null;
-
+
/** The config. */
private MtasConfiguration config = null;
-
+
/**
* Instantiates a new mtas char filter factory.
*
- * @param args the args
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param args
+ * the args
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasCharFilterFactory(Map<String, String> args) throws IOException {
this(args, null);
@@ -64,15 +73,19 @@ public class MtasCharFilterFactory extends CharFilterFactory implements Resource
/**
* Instantiates a new mtas char filter factory.
*
- * @param args the args
- * @param resourceLoader the resource loader
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param args
+ * the args
+ * @param resourceLoader
+ * the resource loader
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasCharFilterFactory(Map<String, String> args,
SolrResourceLoader resourceLoader) throws IOException {
super(args);
typeArgument = get(args, ARGUMENT_TYPE);
prefixArgument = get(args, ARGUMENT_PREFIX);
+ postfixArgument = get(args, ARGUMENT_POSTFIX);
configArgument = get(args, ARGUMENT_CONFIG);
defaultArgument = get(args, ARGUMENT_DEFAULT);
if (typeArgument != null && configArgument != null) {
@@ -81,6 +94,9 @@ public class MtasCharFilterFactory extends CharFilterFactory implements Resource
} else if (typeArgument == null && prefixArgument != null) {
throw new IOException(this.getClass().getName() + " can't have "
+ ARGUMENT_PREFIX + " without " + ARGUMENT_TYPE);
+ } else if (typeArgument == null && postfixArgument != null) {
+ throw new IOException(this.getClass().getName() + " can't have "
+ + ARGUMENT_POSTFIX + " without " + ARGUMENT_TYPE);
} else if (configArgument == null && defaultArgument != null) {
throw new IOException(this.getClass().getName() + " can't have "
+ ARGUMENT_DEFAULT + " without " + ARGUMENT_CONFIG);
@@ -94,8 +110,10 @@ public class MtasCharFilterFactory extends CharFilterFactory implements Resource
/**
* Inits the.
*
- * @param resourceLoader the resource loader
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param resourceLoader
+ * the resource loader
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private void init(ResourceLoader resourceLoader) throws IOException {
if (config == null && configs == null) {
@@ -109,6 +127,9 @@ public class MtasCharFilterFactory extends CharFilterFactory implements Resource
config.attributes.put(
MtasConfiguration.CHARFILTER_CONFIGURATION_PREFIX,
prefixArgument);
+ config.attributes.put(
+ MtasConfiguration.CHARFILTER_CONFIGURATION_POSTFIX,
+ postfixArgument);
}
if (configArgument != null) {
if (resourceLoader != null) {
@@ -119,14 +140,17 @@ public class MtasCharFilterFactory extends CharFilterFactory implements Resource
throw new IOException("problem loading configurations from "
+ configArgument + ": " + e.getMessage());
}
- }
+ }
}
}
}
}
- /* (non-Javadoc)
- * @see org.apache.lucene.analysis.util.CharFilterFactory#create(java.io.Reader)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.analysis.util.CharFilterFactory#create(java.io.Reader)
*/
@Override
public Reader create(Reader input) {
@@ -141,10 +165,13 @@ public class MtasCharFilterFactory extends CharFilterFactory implements Resource
/**
* Creates the.
*
- * @param input the input
- * @param configuration the configuration
+ * @param input
+ * the input
+ * @param configuration
+ * the configuration
* @return the reader
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public Reader create(Reader input, String configuration) throws IOException {
if (configs != null && configs.size() > 0) {
@@ -187,11 +214,16 @@ public class MtasCharFilterFactory extends CharFilterFactory implements Resource
/**
* Creates the.
*
- * @param input the input
- * @param config the config
+ * @param input
+ * the input
+ * @param config
+ * the config
* @return the reader
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- public Reader create(Reader input, MtasConfiguration config) {
+ public Reader create(Reader input, MtasConfiguration config)
+ throws IOException {
MtasFetchData fetchData = new MtasFetchData(input);
if (config.attributes
.containsKey(MtasConfiguration.CHARFILTER_CONFIGURATION_TYPE)) {
@@ -199,18 +231,20 @@ public class MtasCharFilterFactory extends CharFilterFactory implements Resource
.equals(VALUE_TYPE_URL)) {
try {
return fetchData.getUrl(config.attributes
- .get(MtasConfiguration.CHARFILTER_CONFIGURATION_PREFIX));
+ .get(MtasConfiguration.CHARFILTER_CONFIGURATION_PREFIX), config.attributes
+ .get(MtasConfiguration.CHARFILTER_CONFIGURATION_POSTFIX));
} catch (MtasParserException e) {
- return null;
+ throw new IOException(e.getMessage());
}
} else if (config.attributes
.get(MtasConfiguration.CHARFILTER_CONFIGURATION_TYPE)
.equals(VALUE_TYPE_FILE)) {
try {
return fetchData.getFile(config.attributes
- .get(MtasConfiguration.CHARFILTER_CONFIGURATION_PREFIX));
+ .get(MtasConfiguration.CHARFILTER_CONFIGURATION_PREFIX), config.attributes
+ .get(MtasConfiguration.CHARFILTER_CONFIGURATION_POSTFIX));
} catch (MtasParserException e) {
- return null;
+ throw new IOException(e.getMessage());
}
} else {
return fetchData.getDefault();
@@ -220,8 +254,11 @@ public class MtasCharFilterFactory extends CharFilterFactory implements Resource
}
}
- /* (non-Javadoc)
- * @see org.apache.lucene.analysis.util.ResourceLoaderAware#inform(org.apache.lucene.analysis.util.ResourceLoader)
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.analysis.util.ResourceLoaderAware#inform(org.apache.
+ * lucene.analysis.util.ResourceLoader)
*/
@Override
public void inform(ResourceLoader loader) throws IOException {
diff --git a/src/mtas/analysis/util/MtasConfigException.java b/src/mtas/analysis/util/MtasConfigException.java
index 9974703..0365b2d 100644
--- a/src/mtas/analysis/util/MtasConfigException.java
+++ b/src/mtas/analysis/util/MtasConfigException.java
@@ -14,11 +14,12 @@ public class MtasConfigException extends Exception {
public MtasConfigException() {
super();
}
-
+
/**
* Instantiates a new mtas config exception.
*
- * @param string the string
+ * @param string
+ * the string
*/
public MtasConfigException(String string) {
super(string);
diff --git a/src/mtas/analysis/util/MtasConfiguration.java b/src/mtas/analysis/util/MtasConfiguration.java
index d4a8be8..9319808 100644
--- a/src/mtas/analysis/util/MtasConfiguration.java
+++ b/src/mtas/analysis/util/MtasConfiguration.java
@@ -21,25 +21,28 @@ public class MtasConfiguration {
/** The configurations mtas. */
public static String CONFIGURATIONS_MTAS = "mtas";
-
+
/** The configurations configurations. */
public static String CONFIGURATIONS_CONFIGURATIONS = "configurations";
-
+
/** The configurations configuration. */
public static String CONFIGURATIONS_CONFIGURATION = "configuration";
-
+
/** The configurations configuration name. */
public static String CONFIGURATIONS_CONFIGURATION_NAME = "name";
/** The tokenizer configuration file. */
public static String TOKENIZER_CONFIGURATION_FILE = "file";
-
+
/** The charfilter configuration type. */
public static String CHARFILTER_CONFIGURATION_TYPE = "type";
-
+
/** The charfilter configuration prefix. */
public static String CHARFILTER_CONFIGURATION_PREFIX = "prefix";
+ /** The charfilter configuration postfix. */
+ public static String CHARFILTER_CONFIGURATION_POSTFIX = "postfix";
+
/** The name. */
public String name;
@@ -57,7 +60,7 @@ public class MtasConfiguration {
*/
public MtasConfiguration() {
name = null;
- attributes = new HashMap<String,String>();
+ attributes = new HashMap<String, String>();
children = new ArrayList<MtasConfiguration>();
parent = null;
}
@@ -65,11 +68,15 @@ public class MtasConfiguration {
/**
* Read configurations.
*
- * @param resourceLoader the resource loader
- * @param configFile the config file
- * @param className the class name
+ * @param resourceLoader
+ * the resource loader
+ * @param configFile
+ * the config file
+ * @param className
+ * the class name
* @return the hash map
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static HashMap<String, HashMap<String, String>> readConfigurations(
ResourceLoader resourceLoader, String configFile, String className)
@@ -180,14 +187,17 @@ public class MtasConfiguration {
}
return configs;
}
-
+
/**
* Read mtas char filter configurations.
*
- * @param resourceLoader the resource loader
- * @param configFile the config file
+ * @param resourceLoader
+ * the resource loader
+ * @param configFile
+ * the config file
* @return the hash map
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static HashMap<String, MtasConfiguration> readMtasCharFilterConfigurations(
ResourceLoader resourceLoader, String configFile) throws IOException {
@@ -196,30 +206,37 @@ public class MtasConfiguration {
if (configs == null) {
throw new IOException("no configurations");
} else {
- HashMap<String, MtasConfiguration> result = new HashMap<String, MtasConfiguration>();
+ HashMap<String, MtasConfiguration> result = new HashMap<String, MtasConfiguration>();
for (String name : configs.keySet()) {
HashMap<String, String> config = configs.get(name);
if (config.containsKey(CHARFILTER_CONFIGURATION_TYPE)) {
MtasConfiguration item = new MtasConfiguration();
- item.attributes.put(CHARFILTER_CONFIGURATION_TYPE, config.get(CHARFILTER_CONFIGURATION_TYPE));
- item.attributes.put(CHARFILTER_CONFIGURATION_PREFIX, config.get(CHARFILTER_CONFIGURATION_PREFIX));
+ item.attributes.put(CHARFILTER_CONFIGURATION_TYPE,
+ config.get(CHARFILTER_CONFIGURATION_TYPE));
+ item.attributes.put(CHARFILTER_CONFIGURATION_PREFIX,
+ config.get(CHARFILTER_CONFIGURATION_PREFIX));
+ item.attributes.put(CHARFILTER_CONFIGURATION_POSTFIX,
+ config.get(CHARFILTER_CONFIGURATION_POSTFIX));
result.put(name, item);
} else {
throw new IOException("configuration " + name + " has no "
+ CHARFILTER_CONFIGURATION_TYPE);
- }
+ }
}
return result;
}
- }
+ }
/**
* Read mtas tokenizer configurations.
*
- * @param resourceLoader the resource loader
- * @param configFile the config file
+ * @param resourceLoader
+ * the resource loader
+ * @param configFile
+ * the config file
* @return the hash map
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static HashMap<String, MtasConfiguration> readMtasTokenizerConfigurations(
ResourceLoader resourceLoader, String configFile) throws IOException {
@@ -246,9 +263,11 @@ public class MtasConfiguration {
/**
* Read configuration.
*
- * @param reader the reader
+ * @param reader
+ * the reader
* @return the mtas configuration
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static MtasConfiguration readConfiguration(InputStream reader)
throws IOException {
@@ -316,27 +335,29 @@ public class MtasConfiguration {
}
return null;
}
-
-// public String toString() {
-// return toString(0);
-// }
-//
-// private String toString(int indent) {
-// String text = "";
-// if(name!=null) {
-// text+=(indent>0?String.format("%"+indent+"s", ""):"")+"name: "+name+"\n";
-// }
-// if(attributes!=null) {
-// for(String key : attributes.keySet()) {
-// text+=(indent>0?String.format("%"+indent+"s", ""):"")+key+": "+attributes.get(key)+"\n";
-// }
-// }
-// if(children!=null) {
-// for(MtasConfiguration child : children) {
-// text+=(indent>0?String.format("%"+indent+"s", ""):"")+child.toString(indent+2);
-// }
-// }
-// return text;
-// }
+
+ // public String toString() {
+ // return toString(0);
+ // }
+ //
+ // private String toString(int indent) {
+ // String text = "";
+ // if(name!=null) {
+ // text+=(indent>0?String.format("%"+indent+"s", ""):"")+"name: "+name+"\n";
+ // }
+ // if(attributes!=null) {
+ // for(String key : attributes.keySet()) {
+ // text+=(indent>0?String.format("%"+indent+"s", ""):"")+key+":
+ // "+attributes.get(key)+"\n";
+ // }
+ // }
+ // if(children!=null) {
+ // for(MtasConfiguration child : children) {
+ // text+=(indent>0?String.format("%"+indent+"s",
+ // ""):"")+child.toString(indent+2);
+ // }
+ // }
+ // return text;
+ // }
}
diff --git a/src/mtas/analysis/util/MtasFetchData.java b/src/mtas/analysis/util/MtasFetchData.java
index c557dbd..8f2ddc5 100644
--- a/src/mtas/analysis/util/MtasFetchData.java
+++ b/src/mtas/analysis/util/MtasFetchData.java
@@ -18,85 +18,100 @@ import org.apache.commons.io.IOUtils;
* The Class MtasFetchData.
*/
public class MtasFetchData {
-
+
/** The reader. */
Reader reader;
-
+
/**
* Instantiates a new mtas fetch data.
*
- * @param input the input
+ * @param input
+ * the input
*/
public MtasFetchData(Reader input) {
reader = input;
}
-
+
/**
* Gets the string.
*
* @return the string
- * @throws MtasParserException the mtas parser exception
+ * @throws MtasParserException
+ * the mtas parser exception
*/
private String getString() throws MtasParserException {
String text = null;
BufferedReader bufferedReader = new BufferedReader(reader, 2048);
- try {
+ try {
text = IOUtils.toString(bufferedReader);
bufferedReader.close();
return text;
} catch (IOException e) {
throw new MtasParserException("couldn't read text");
- }
+ }
}
-
+
/**
* Gets the url.
*
- * @param prefix the prefix
+ * @param prefix
+ * the prefix
* @return the url
- * @throws MtasParserException the mtas parser exception
+ * @throws MtasParserException
+ * the mtas parser exception
*/
- public Reader getUrl(String prefix) throws MtasParserException {
+ public Reader getUrl(String prefix, String postfix) throws MtasParserException {
String url = getString();
- if((url!=null) && !url.equals("")) {
- if(prefix!=null) {
- url = prefix+url;
- }
- if(url.startsWith("http://")||url.startsWith("https://")) {
+ if ((url != null) && !url.equals("")) {
+ if (prefix != null) {
+ url = prefix + url;
+ }
+ if (postfix != null) {
+ url = url + postfix;
+ }
+ if (url.startsWith("http://") || url.startsWith("https://")) {
BufferedReader in = null;
try {
- URLConnection connection = new URL(url).openConnection();
+ URLConnection connection = new URL(url).openConnection();
connection.setRequestProperty("Accept-Encoding", "gzip");
- connection.setReadTimeout(10000);
- if (connection.getHeaderField("Content-Encoding")!=null && connection.getHeaderField("Content-Encoding").equals("gzip")){
- in = new BufferedReader(new InputStreamReader(new GZIPInputStream(connection.getInputStream())));
+ connection.setReadTimeout(10000);
+ if (connection.getHeaderField("Content-Encoding") != null
+ && connection.getHeaderField("Content-Encoding").equals("gzip")) {
+ in = new BufferedReader(new InputStreamReader(
+ new GZIPInputStream(connection.getInputStream())));
} else {
- in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
- }
+ in = new BufferedReader(
+ new InputStreamReader(connection.getInputStream()));
+ }
return in;
} catch (IOException ex) {
- throw new MtasParserException("couldn't get "+url);
- }
+ throw new MtasParserException("couldn't get " + url);
+ }
} else {
- throw new MtasParserException("no valid url: "+url);
- }
+ throw new MtasParserException("no valid url: " + url);
+ }
} else {
- throw new MtasParserException("no valid url: "+url);
+ throw new MtasParserException("no valid url: " + url);
}
}
-
+
/**
* Gets the file.
*
- * @param prefix the prefix
+ * @param prefix
+ * the prefix
* @return the file
- * @throws MtasParserException the mtas parser exception
+ * @throws MtasParserException
+ * the mtas parser exception
*/
- public Reader getFile(String prefix) throws MtasParserException {
+ public Reader getFile(String prefix, String postfix) throws MtasParserException {
String file = getString();
- if((file!=null) && !file.equals("")) {
- if(prefix!=null) {
- file = prefix+file;
+ if ((file != null) && !file.equals("")) {
+ if (prefix != null) {
+ file = prefix + file;
+ }
+ if (postfix != null) {
+ file = file + postfix;
}
GZIPInputStream in;
try {
@@ -105,16 +120,16 @@ public class MtasFetchData {
} catch (IOException e1) {
try {
String text = new String(Files.readAllBytes(Paths.get(file)));
- return new StringReader(text);
+ return new StringReader(text);
} catch (IOException e2) {
throw new MtasParserException(e2.getMessage());
}
- }
+ }
} else {
- throw new MtasParserException("no valid file: "+file);
- }
+ throw new MtasParserException("no valid file: " + file);
+ }
}
-
+
/**
* Gets the default.
*
@@ -123,5 +138,5 @@ public class MtasFetchData {
public Reader getDefault() {
return reader;
}
-
+
}
diff --git a/src/mtas/analysis/util/MtasParserException.java b/src/mtas/analysis/util/MtasParserException.java
index c7d26fa..91b3796 100644
--- a/src/mtas/analysis/util/MtasParserException.java
+++ b/src/mtas/analysis/util/MtasParserException.java
@@ -14,11 +14,12 @@ public class MtasParserException extends Exception {
public MtasParserException() {
super();
}
-
+
/**
* Instantiates a new mtas parser exception.
*
- * @param string the string
+ * @param string
+ * the string
*/
public MtasParserException(String string) {
super(string);
diff --git a/src/mtas/analysis/util/MtasPrefixTokenFilterFactory.java b/src/mtas/analysis/util/MtasPrefixTokenFilterFactory.java
index 0033312..deab22e 100644
--- a/src/mtas/analysis/util/MtasPrefixTokenFilterFactory.java
+++ b/src/mtas/analysis/util/MtasPrefixTokenFilterFactory.java
@@ -19,22 +19,26 @@ public class MtasPrefixTokenFilterFactory extends TokenFilterFactory {
/**
* Instantiates a new mtas prefix token filter factory.
*
- * @param args the args
+ * @param args
+ * the args
*/
public MtasPrefixTokenFilterFactory(Map<String, String> args) {
super(args);
prefix = get(args, "prefix");
}
- /* (non-Javadoc)
- * @see org.apache.lucene.analysis.util.TokenFilterFactory#create(org.apache.lucene.analysis.TokenStream)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.analysis.util.TokenFilterFactory#create(org.apache.lucene
+ * .analysis.TokenStream)
*/
@Override
public TokenStream create(TokenStream input) {
return new MtasPrefixTokenFilter(input, prefix);
}
-
/**
* The Class MtasPrefixTokenFilter.
*/
@@ -42,37 +46,42 @@ public class MtasPrefixTokenFilterFactory extends TokenFilterFactory {
/** The prefix. */
private String prefix;
-
+
/** The term att. */
- private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final CharTermAttribute termAtt = addAttribute(
+ CharTermAttribute.class);
/**
* Instantiates a new mtas prefix token filter.
*
- * @param input the input
- * @param prefix the prefix
+ * @param input
+ * the input
+ * @param prefix
+ * the prefix
*/
protected MtasPrefixTokenFilter(TokenStream input, String prefix) {
super(input);
- this.prefix = prefix+MtasToken.DELIMITER;
+ this.prefix = prefix + MtasToken.DELIMITER;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.analysis.TokenStream#incrementToken()
*/
@Override
public final boolean incrementToken() throws IOException {
if (input.incrementToken()) {
- int oldLen = termAtt.length();
- char [] buffer = termAtt.resizeBuffer(oldLen + prefix.length());
-
- for (int i = 0; i < oldLen; i++) {
- buffer[(oldLen + prefix.length() - 1 - i)] = buffer[(oldLen - 1 - i)];
- }
- for(int i=0;i< prefix.length();i++) {
- buffer[i]=prefix.charAt(i);
+ int oldLen = termAtt.length();
+ char[] buffer = termAtt.resizeBuffer(oldLen + prefix.length());
+
+ for (int i = 0; i < oldLen; i++) {
+ buffer[(oldLen + prefix.length() - 1 - i)] = buffer[(oldLen - 1 - i)];
}
- termAtt.copyBuffer(buffer, 0, oldLen + prefix.length());
+ for (int i = 0; i < prefix.length(); i++) {
+ buffer[i] = prefix.charAt(i);
+ }
+ termAtt.copyBuffer(buffer, 0, oldLen + prefix.length());
return true;
} else {
return false;
@@ -81,5 +90,4 @@ public class MtasPrefixTokenFilterFactory extends TokenFilterFactory {
}
-
}
diff --git a/src/mtas/analysis/util/MtasTokenizerFactory.java b/src/mtas/analysis/util/MtasTokenizerFactory.java
index 3c1af04..b857197 100644
--- a/src/mtas/analysis/util/MtasTokenizerFactory.java
+++ b/src/mtas/analysis/util/MtasTokenizerFactory.java
@@ -52,7 +52,7 @@ public class MtasTokenizerFactory extends TokenizerFactory
* Signals that an I/O exception has occurred.
*/
public MtasTokenizerFactory(Map<String, String> args) throws IOException {
- this(args, null);
+ this(args, null);
}
/**
@@ -92,10 +92,10 @@ public class MtasTokenizerFactory extends TokenizerFactory
* util.AttributeFactory)
*/
@Override
- public MtasTokenizer create(AttributeFactory factory) {
- MtasTokenizer tokenizer = null;
+ public MtasTokenizer<?> create(AttributeFactory factory) {
+ MtasTokenizer<?> tokenizer = null;
try {
- tokenizer = create(factory, null);
+ tokenizer = create(factory, null);
} catch (IOException e) {
e.printStackTrace();
}
@@ -111,7 +111,7 @@ public class MtasTokenizerFactory extends TokenizerFactory
* @throws IOException
* Signals that an I/O exception has occurred.
*/
- public MtasTokenizer create(String configuration) throws IOException {
+ public MtasTokenizer<?> create(String configuration) throws IOException {
return create(TokenStream.DEFAULT_TOKEN_ATTRIBUTE_FACTORY, configuration);
}
@@ -126,14 +126,15 @@ public class MtasTokenizerFactory extends TokenizerFactory
* @throws IOException
* Signals that an I/O exception has occurred.
*/
- public MtasTokenizer create(AttributeFactory factory, String configuration)
+ public MtasTokenizer<?> create(AttributeFactory factory, String configuration)
throws IOException {
if (configs != null && configs.size() > 0) {
if (configuration == null && defaultArgument == null) {
throw new IOException("no (default)configuration");
} else if (configuration == null) {
if (configs.get(defaultArgument) != null) {
- return new MtasTokenizer(factory, configs.get(defaultArgument));
+ return new MtasTokenizer<String>(factory,
+ configs.get(defaultArgument));
} else {
throw new IOException(
"default configuration " + defaultArgument + " not available");
@@ -143,7 +144,8 @@ public class MtasTokenizerFactory extends TokenizerFactory
if (config == null) {
if (defaultArgument != null) {
if (configs.get(defaultArgument) != null) {
- return new MtasTokenizer(factory, configs.get(defaultArgument));
+ return new MtasTokenizer<String>(factory,
+ configs.get(defaultArgument));
} else {
throw new IOException("configuration " + configuration
+ " not found and default configuration " + defaultArgument
@@ -154,11 +156,11 @@ public class MtasTokenizerFactory extends TokenizerFactory
+ " not available and no default configuration");
}
} else {
- return new MtasTokenizer(factory, config);
+ return new MtasTokenizer<String>(factory, config);
}
}
} else if (config != null) {
- return new MtasTokenizer(factory, config);
+ return new MtasTokenizer<String>(factory, config);
} else {
throw new IOException("no configuration");
}
@@ -172,7 +174,7 @@ public class MtasTokenizerFactory extends TokenizerFactory
* @throws IOException
* Signals that an I/O exception has occurred.
*/
- private void init(ResourceLoader resourceLoader) throws IOException {
+ private void init(ResourceLoader resourceLoader) throws IOException {
if (config == null && configs == null) {
if (resourceLoader == null) {
return;
@@ -182,7 +184,7 @@ public class MtasTokenizerFactory extends TokenizerFactory
if (configFileArgument != null) {
try {
config = MtasConfiguration.readConfiguration(
- resourceLoader.openResource(configFileArgument));
+ resourceLoader.openResource(configFileArgument));
} catch (IOException e) {
throw new IOException("Problem loading configuration from "
+ configFileArgument + ": " + e.getMessage());
diff --git a/src/mtas/codec/MtasCodec.java b/src/mtas/codec/MtasCodec.java
index ea6ed89..8a5e2ce 100644
--- a/src/mtas/codec/MtasCodec.java
+++ b/src/mtas/codec/MtasCodec.java
@@ -20,66 +20,73 @@ public class MtasCodec extends Codec {
/** The Constant MTAS_CODEC_NAME. */
public static final String MTAS_CODEC_NAME = "MtasCodec";
-
+
/** The delegate. */
Codec delegate;
-
+
/**
* Instantiates a new mtas codec.
*/
- public MtasCodec() {
+ public MtasCodec() {
super(MTAS_CODEC_NAME);
delegate = null;
}
-
+
/**
* Instantiates a new mtas codec.
*
- * @param name the name
- * @param delegate the delegate
+ * @param name
+ * the name
+ * @param delegate
+ * the delegate
*/
protected MtasCodec(String name, Codec delegate) {
super(name);
this.delegate = delegate;
}
-
+
/**
* Inits the delegate.
*/
private void initDelegate() {
- if(delegate == null) {
+ if (delegate == null) {
delegate = Codec.getDefault();
}
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.codecs.Codec#postingsFormat()
*/
@Override
public PostingsFormat postingsFormat() {
initDelegate();
- if(delegate.postingsFormat() instanceof PerFieldPostingsFormat) {
+ if (delegate.postingsFormat() instanceof PerFieldPostingsFormat) {
Codec defaultCodec = Codec.getDefault();
PostingsFormat defaultPostingsFormat = defaultCodec.postingsFormat();
- if(defaultPostingsFormat instanceof PerFieldPostingsFormat) {
- defaultPostingsFormat = ((PerFieldPostingsFormat) defaultPostingsFormat).getPostingsFormatForField(null);
- if((defaultPostingsFormat==null) || (defaultPostingsFormat instanceof PerFieldPostingsFormat)) {
- //fallback option
- return new MtasCodecPostingsFormat(PostingsFormat.forName("Lucene50"));
+ if (defaultPostingsFormat instanceof PerFieldPostingsFormat) {
+ defaultPostingsFormat = ((PerFieldPostingsFormat) defaultPostingsFormat)
+ .getPostingsFormatForField(null);
+ if ((defaultPostingsFormat == null)
+ || (defaultPostingsFormat instanceof PerFieldPostingsFormat)) {
+ // fallback option
+ return new MtasCodecPostingsFormat(
+ PostingsFormat.forName("Lucene60"));
} else {
- return new MtasCodecPostingsFormat(defaultPostingsFormat);
+ return new MtasCodecPostingsFormat(defaultPostingsFormat);
}
} else {
- return new MtasCodecPostingsFormat(defaultPostingsFormat);
+ return new MtasCodecPostingsFormat(defaultPostingsFormat);
}
- } else {
+ } else {
return new MtasCodecPostingsFormat(delegate.postingsFormat());
- }
+ }
}
-
-
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.codecs.Codec#docValuesFormat()
*/
@Override
@@ -88,7 +95,9 @@ public class MtasCodec extends Codec {
return delegate.docValuesFormat();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.codecs.Codec#storedFieldsFormat()
*/
@Override
@@ -97,7 +106,9 @@ public class MtasCodec extends Codec {
return delegate.storedFieldsFormat();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.codecs.Codec#termVectorsFormat()
*/
@Override
@@ -106,7 +117,9 @@ public class MtasCodec extends Codec {
return delegate.termVectorsFormat();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.codecs.Codec#fieldInfosFormat()
*/
@Override
@@ -115,7 +128,9 @@ public class MtasCodec extends Codec {
return delegate.fieldInfosFormat();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.codecs.Codec#segmentInfoFormat()
*/
@Override
@@ -124,7 +139,9 @@ public class MtasCodec extends Codec {
return delegate.segmentInfoFormat();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.codecs.Codec#normsFormat()
*/
@Override
@@ -133,7 +150,9 @@ public class MtasCodec extends Codec {
return delegate.normsFormat();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.codecs.Codec#liveDocsFormat()
*/
@Override
@@ -142,7 +161,9 @@ public class MtasCodec extends Codec {
return delegate.liveDocsFormat();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.codecs.Codec#compoundFormat()
*/
@Override
@@ -151,7 +172,9 @@ public class MtasCodec extends Codec {
return delegate.compoundFormat();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.codecs.Codec#pointsFormat()
*/
@Override
@@ -159,5 +182,5 @@ public class MtasCodec extends Codec {
initDelegate();
return delegate.pointsFormat();
}
-
+
}
diff --git a/src/mtas/codec/MtasCodecPostingsFormat.java b/src/mtas/codec/MtasCodecPostingsFormat.java
index fe973e5..039dba5 100644
--- a/src/mtas/codec/MtasCodecPostingsFormat.java
+++ b/src/mtas/codec/MtasCodecPostingsFormat.java
@@ -1,8 +1,6 @@
package mtas.codec;
import java.io.IOException;
-import java.util.TreeSet;
-
import mtas.analysis.token.MtasToken;
import mtas.analysis.token.MtasTokenString;
import mtas.codec.payload.MtasPayloadDecoder;
@@ -23,10 +21,10 @@ public class MtasCodecPostingsFormat extends PostingsFormat {
/** The Constant VERSION_START. */
public static final int VERSION_START = 1;
-
+
/** The Constant VERSION_OLD_1. */
public static final int VERSION_OLD_1 = 1;
-
+
/** The Constant VERSION_OLD_2. */
public static final int VERSION_OLD_2 = 2;
@@ -50,7 +48,7 @@ public class MtasCodecPostingsFormat extends PostingsFormat {
/** The Constant MTAS_OBJECT_HAS_PAYLOAD. */
static final int MTAS_OBJECT_HAS_PAYLOAD = 32;
-
+
/** The Constant MTAS_STORAGE_BYTE. */
public static final int MTAS_STORAGE_BYTE = 0;
@@ -136,7 +134,8 @@ public class MtasCodecPostingsFormat extends PostingsFormat {
/**
* Instantiates a new mtas codec postings format.
*
- * @param delegate the delegate
+ * @param delegate
+ * the delegate
*/
public MtasCodecPostingsFormat(PostingsFormat delegate) {
super(MtasCodec.MTAS_CODEC_NAME);
@@ -160,7 +159,8 @@ public class MtasCodecPostingsFormat extends PostingsFormat {
/**
* Instantiates a new mtas codec postings format.
*
- * @param codecName the codec name
+ * @param codecName
+ * the codec name
*/
public MtasCodecPostingsFormat(String codecName) {
super(codecName);
@@ -210,21 +210,25 @@ public class MtasCodecPostingsFormat extends PostingsFormat {
delegatePostingsFormat.getName());
} else {
PostingsFormat pf = Codec.forName(delegateCodecName).postingsFormat();
- return new MtasFieldsConsumer(pf.fieldsConsumer(state), state, getName(),
- pf.getName());
+ return pf.fieldsConsumer(state);
}
}
/**
* Gets the token.
*
- * @param inObject the in object
- * @param inTerm the in term
- * @param ref the ref
+ * @param inObject
+ * the in object
+ * @param inTerm
+ * the in term
+ * @param ref
+ * the ref
* @return the token
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static MtasToken<String> getToken(IndexInput inObject,
- IndexInput inTerm, Long ref) {
+ IndexInput inTerm, Long ref) throws IOException {
MtasToken<String> token = null;
try {
inObject.seek(ref);
@@ -232,22 +236,25 @@ public class MtasCodecPostingsFormat extends PostingsFormat {
token.setId(inObject.readVInt());
token.setTokenRef(ref);
int objectFlags = inObject.readVInt();
- TreeSet<Integer> positions = new TreeSet<Integer>();
+ int[] positions = null;
if ((objectFlags & MTAS_OBJECT_HAS_PARENT) == MTAS_OBJECT_HAS_PARENT) {
int parentId = inObject.readVInt();
token.setParentId(parentId);
}
- if ((objectFlags & MTAS_OBJECT_HAS_POSITION_RANGE) == MTAS_OBJECT_HAS_POSITION_RANGE) {
+ if ((objectFlags
+ & MTAS_OBJECT_HAS_POSITION_RANGE) == MTAS_OBJECT_HAS_POSITION_RANGE) {
int positionStart = inObject.readVInt();
int positionEnd = positionStart + inObject.readVInt();
token.addPositionRange(positionStart, positionEnd);
- } else if ((objectFlags & MTAS_OBJECT_HAS_POSITION_SET) == MTAS_OBJECT_HAS_POSITION_SET) {
+ } else if ((objectFlags
+ & MTAS_OBJECT_HAS_POSITION_SET) == MTAS_OBJECT_HAS_POSITION_SET) {
int size = inObject.readVInt();
int tmpPrevious = 0;
+ positions = new int[size];
for (int t = 0; t < size; t++) {
int position = tmpPrevious + inObject.readVInt();
tmpPrevious = position;
- positions.add(position);
+ positions[t] = position;
}
token.addPositions(positions);
} else {
@@ -259,7 +266,8 @@ public class MtasCodecPostingsFormat extends PostingsFormat {
int offsetEnd = offsetStart + inObject.readVInt();
token.setOffset(offsetStart, offsetEnd);
}
- if ((objectFlags & MTAS_OBJECT_HAS_REALOFFSET) == MTAS_OBJECT_HAS_REALOFFSET) {
+ if ((objectFlags
+ & MTAS_OBJECT_HAS_REALOFFSET) == MTAS_OBJECT_HAS_REALOFFSET) {
int realOffsetStart = inObject.readVInt();
int realOffsetEnd = realOffsetStart + inObject.readVInt();
token.setRealOffset(realOffsetStart, realOffsetEnd);
@@ -268,15 +276,14 @@ public class MtasCodecPostingsFormat extends PostingsFormat {
int length = inObject.readVInt();
byte[] mtasPayload = new byte[length];
inObject.readBytes(mtasPayload, 0, length);
- token.setPayload(new BytesRef(mtasPayload));
+ token.setPayload(new BytesRef(mtasPayload));
}
Long termRef = inObject.readVLong();
inTerm.seek(termRef);
token.setTermRef(termRef);
token.setValue(inTerm.readString());
- } catch (IOException e) {
- e.printStackTrace();
- return null;
+ } catch (Exception e) {
+ throw new IOException(e.getMessage());
}
return token;
}
diff --git a/src/mtas/codec/MtasFieldsConsumer.java b/src/mtas/codec/MtasFieldsConsumer.java
index 3877988..077f492 100644
--- a/src/mtas/codec/MtasFieldsConsumer.java
+++ b/src/mtas/codec/MtasFieldsConsumer.java
@@ -1,5 +1,6 @@
package mtas.codec;
+import java.io.Closeable;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
@@ -36,6 +37,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
/**
* The Class MtasFieldsConsumer.
@@ -440,7 +442,7 @@ public class MtasFieldsConsumer extends FieldsConsumer {
prefixIdIndex.put(field, new HashMap<String, Integer>());
}
if (!prefixReferenceIndex.get(field).containsKey(prefix)) {
- int id = prefixReferenceIndex.get(field).size();
+ int id = 1 + prefixReferenceIndex.get(field).size();
prefixReferenceIndex.get(field).put(prefix, outPrefix.getFilePointer());
prefixIdIndex.get(field).put(prefix, id);
outPrefix.writeString(prefix);
@@ -591,8 +593,8 @@ public class MtasFieldsConsumer extends FieldsConsumer {
*/
@Override
public void write(Fields fields) throws IOException {
- write(state.fieldInfos, fields);
delegateFieldsConsumer.write(fields);
+ write(state.fieldInfos, fields);
}
/**
@@ -605,11 +607,12 @@ public class MtasFieldsConsumer extends FieldsConsumer {
* @throws IOException
* Signals that an I/O exception has occurred.
*/
- private void write(FieldInfos fieldInfos, Fields fields) throws IOException {
+ private void write(FieldInfos fieldInfos, Fields fields) {
IndexOutput outField, outDoc, outIndexDocId, outIndexObjectId,
outIndexObjectPosition, outIndexObjectParent, outTerm, outObject,
outPrefix;
IndexOutput outTmpDoc, outTmpField;
+ HashSet<Closeable> closeables = new HashSet<Closeable>();
// temporary temporary index in memory for doc
TreeMap<Integer, Long> memoryIndexTemporaryObject = new TreeMap<Integer, Long>();
@@ -618,437 +621,471 @@ public class MtasFieldsConsumer extends FieldsConsumer {
// list of objectIds and references to objects
TreeMap<Integer, Long> memoryIndexDocList = new TreeMap<Integer, Long>();
- // create file tmpDoc
- outTmpDoc = state.directory.createOutput(mtasTmpDocFileName, state.context);
- // create file tmpField
- outTmpField = state.directory.createOutput(mtasTmpFieldFileName,
- state.context);
- // create file indexDoc
- outDoc = state.directory.createOutput(mtasDocFileName, state.context);
- CodecUtil.writeIndexHeader(outDoc, name,
- MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
- state.segmentSuffix);
- outDoc.writeString(delegatePostingsFormatName);
- // create file indexDocId
- outIndexDocId = state.directory.createOutput(mtasIndexDocIdFileName,
- state.context);
- CodecUtil.writeIndexHeader(outIndexDocId, name,
- MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
- state.segmentSuffix);
- outIndexDocId.writeString(delegatePostingsFormatName);
- // create file indexObjectId
- outIndexObjectId = state.directory.createOutput(mtasIndexObjectIdFileName,
- state.context);
- CodecUtil.writeIndexHeader(outIndexObjectId, name,
- MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
- state.segmentSuffix);
- outIndexObjectId.writeString(delegatePostingsFormatName);
- // create file indexObjectPosition
- outIndexObjectPosition = state.directory
- .createOutput(mtasIndexObjectPositionFileName, state.context);
- CodecUtil.writeIndexHeader(outIndexObjectPosition, name,
- MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
- state.segmentSuffix);
- outIndexObjectPosition.writeString(delegatePostingsFormatName);
- // create file indexObjectParent
- outIndexObjectParent = state.directory
- .createOutput(mtasIndexObjectParentFileName, state.context);
- CodecUtil.writeIndexHeader(outIndexObjectParent, name,
- MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
- state.segmentSuffix);
- outIndexObjectParent.writeString(delegatePostingsFormatName);
- // create file term
- outTerm = state.directory.createOutput(mtasTermFileName, state.context);
- CodecUtil.writeIndexHeader(outTerm, name,
- MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
- state.segmentSuffix);
- outTerm.writeString(delegatePostingsFormatName);
- // create file prefix
- outPrefix = state.directory.createOutput(mtasPrefixFileName, state.context);
- CodecUtil.writeIndexHeader(outPrefix, name,
- MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
- state.segmentSuffix);
- outPrefix.writeString(delegatePostingsFormatName);
- // create file object
- outObject = state.directory.createOutput(mtasObjectFileName, state.context);
- CodecUtil.writeIndexHeader(outObject, name,
- MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
- state.segmentSuffix);
- outObject.writeString(delegatePostingsFormatName);
- // For each field
- for (String field : fields) {
- Terms terms = fields.terms(field);
- if (terms == null) {
- continue;
- } else {
- // new temporary object storage for this field
- IndexOutput outTmpObject = state.directory
- .createOutput(mtasTmpObjectFileName, state.context);
- // new temporary index docs for this field
- IndexOutput outTmpDocs = state.directory
- .createOutput(mtasTmpDocsFileName, state.context);
- // get fieldInfo
- FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
- // get properties terms
- boolean hasPositions = terms.hasPositions();
- boolean hasFreqs = terms.hasFreqs();
- boolean hasPayloads = fieldInfo.hasPayloads();
- boolean hasOffsets = terms.hasOffsets();
- // register references
- Long smallestTermFilepointer = outTerm.getFilePointer();
- Long smallestPrefixFilepointer = outTerm.getFilePointer();
- int termCounter = 0;
- // only if freqs, positions and payload available
- if (hasFreqs && hasPositions && hasPayloads) {
- // compute flags
- int flags = PostingsEnum.POSITIONS | PostingsEnum.PAYLOADS;
- if (hasOffsets) {
- flags = flags | PostingsEnum.OFFSETS;
- }
- // get terms
- TermsEnum termsEnum = terms.iterator();
- PostingsEnum postingsEnum = null;
- // for each term in field
- while (true) {
- BytesRef term = termsEnum.next();
- if (term == null) {
- break;
+ try {
+ // create file tmpDoc
+ closeables.add(outTmpDoc = state.directory
+ .createOutput(mtasTmpDocFileName, state.context));
+ // create file tmpField
+ closeables.add(outTmpField = state.directory
+ .createOutput(mtasTmpFieldFileName, state.context));
+ // create file indexDoc
+ closeables.add(outDoc = state.directory.createOutput(mtasDocFileName,
+ state.context));
+ CodecUtil.writeIndexHeader(outDoc, name,
+ MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
+ state.segmentSuffix);
+ outDoc.writeString(delegatePostingsFormatName);
+ // create file indexDocId
+ closeables.add(outIndexDocId = state.directory
+ .createOutput(mtasIndexDocIdFileName, state.context));
+ CodecUtil.writeIndexHeader(outIndexDocId, name,
+ MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
+ state.segmentSuffix);
+ outIndexDocId.writeString(delegatePostingsFormatName);
+ // create file indexObjectId
+ closeables.add(outIndexObjectId = state.directory
+ .createOutput(mtasIndexObjectIdFileName, state.context));
+ CodecUtil.writeIndexHeader(outIndexObjectId, name,
+ MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
+ state.segmentSuffix);
+ outIndexObjectId.writeString(delegatePostingsFormatName);
+ // create file indexObjectPosition
+ closeables.add(outIndexObjectPosition = state.directory
+ .createOutput(mtasIndexObjectPositionFileName, state.context));
+ CodecUtil.writeIndexHeader(outIndexObjectPosition, name,
+ MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
+ state.segmentSuffix);
+ outIndexObjectPosition.writeString(delegatePostingsFormatName);
+ // create file indexObjectParent
+ closeables.add(outIndexObjectParent = state.directory
+ .createOutput(mtasIndexObjectParentFileName, state.context));
+ CodecUtil.writeIndexHeader(outIndexObjectParent, name,
+ MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
+ state.segmentSuffix);
+ outIndexObjectParent.writeString(delegatePostingsFormatName);
+ // create file term
+ closeables.add(outTerm = state.directory.createOutput(mtasTermFileName,
+ state.context));
+ CodecUtil.writeIndexHeader(outTerm, name,
+ MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
+ state.segmentSuffix);
+ outTerm.writeString(delegatePostingsFormatName);
+ // create file prefix
+ closeables.add(outPrefix = state.directory
+ .createOutput(mtasPrefixFileName, state.context));
+ CodecUtil.writeIndexHeader(outPrefix, name,
+ MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
+ state.segmentSuffix);
+ outPrefix.writeString(delegatePostingsFormatName);
+ // create file object
+ closeables.add(outObject = state.directory
+ .createOutput(mtasObjectFileName, state.context));
+ CodecUtil.writeIndexHeader(outObject, name,
+ MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
+ state.segmentSuffix);
+ outObject.writeString(delegatePostingsFormatName);
+ // For each field
+ for (String field : fields) {
+ Terms terms = fields.terms(field);
+ if (terms == null) {
+ continue;
+ } else {
+ // new temporary object storage for this field
+ IndexOutput outTmpObject = state.directory
+ .createOutput(mtasTmpObjectFileName, state.context);
+ closeables.add(outTmpObject);
+ // new temporary index docs for this field
+ IndexOutput outTmpDocs = state.directory
+ .createOutput(mtasTmpDocsFileName, state.context);
+ closeables.add(outTmpDocs);
+ // get fieldInfo
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ // get properties terms
+ boolean hasPositions = terms.hasPositions();
+ boolean hasFreqs = terms.hasFreqs();
+ boolean hasPayloads = fieldInfo.hasPayloads();
+ boolean hasOffsets = terms.hasOffsets();
+ // register references
+ Long smallestTermFilepointer = outTerm.getFilePointer();
+ Long smallestPrefixFilepointer = outTerm.getFilePointer();
+ int termCounter = 0;
+ // only if freqs, positions and payload available
+ if (hasFreqs && hasPositions && hasPayloads) {
+ // compute flags
+ int flags = PostingsEnum.POSITIONS | PostingsEnum.PAYLOADS;
+ if (hasOffsets) {
+ flags = flags | PostingsEnum.OFFSETS;
}
- // store term and get ref
- Long termRef = outTerm.getFilePointer();
- outTerm.writeString(term.utf8ToString());
- termCounter++;
- // get postings
- postingsEnum = termsEnum.postings(postingsEnum, flags);
- // for each doc in field+term
+ // get terms
+ TermsEnum termsEnum = terms.iterator();
+ PostingsEnum postingsEnum = null;
+ // for each term in field
while (true) {
- Integer doc = postingsEnum.nextDoc();
- if (doc.equals(DocIdSetIterator.NO_MORE_DOCS)) {
+ BytesRef term = termsEnum.next();
+ if (term == null) {
break;
}
- int freq = postingsEnum.freq();
- // temporary storage objects and temporary index in memory for
- // doc
- memoryIndexTemporaryObject.clear();
- Long offsetFilePointerTmpObject = outTmpObject.getFilePointer();
- for (int i = 0; i < freq; i++) {
- Long currentFilePointerTmpObject = outTmpObject
- .getFilePointer();
- Integer mtasId;
- int position = postingsEnum.nextPosition();
- BytesRef payload = postingsEnum.getPayload();
- if (hasOffsets) {
- mtasId = createObjectAndRegisterPrefix(field, outTmpObject,
- term, termRef, position, payload,
- postingsEnum.startOffset(), postingsEnum.endOffset(),
- outPrefix);
- } else {
- mtasId = createObjectAndRegisterPrefix(field, outTmpObject,
- term, termRef, position, payload, outPrefix);
+ // store term and get ref
+ Long termRef = outTerm.getFilePointer();
+ outTerm.writeString(term.utf8ToString());
+ termCounter++;
+ // get postings
+ postingsEnum = termsEnum.postings(postingsEnum, flags);
+ // for each doc in field+term
+ while (true) {
+ Integer doc = postingsEnum.nextDoc();
+ if (doc.equals(DocIdSetIterator.NO_MORE_DOCS)) {
+ break;
}
- if (mtasId != null) {
- assert !memoryIndexTemporaryObject.containsKey(
- mtasId) : "mtasId should be unique in this selection";
- memoryIndexTemporaryObject.put(mtasId,
- currentFilePointerTmpObject);
- }
- } // end loop positions
- // store temporary index for this doc
- if (memoryIndexTemporaryObject.size() > 0) {
- // docId for this part
- outTmpDocs.writeVInt(doc);
- // number of objects/tokens in this part
- outTmpDocs.writeVInt(memoryIndexTemporaryObject.size());
- // offset to be used for references
- outTmpDocs.writeVLong(offsetFilePointerTmpObject);
- // loop over tokens
- for (Entry<Integer, Long> entry : memoryIndexTemporaryObject
- .entrySet()) {
- // mtasId object
- outTmpDocs.writeVInt(entry.getKey());
- // reference object
- outTmpDocs.writeVLong(
- (entry.getValue() - offsetFilePointerTmpObject));
+ int freq = postingsEnum.freq();
+ // temporary storage objects and temporary index in memory for
+ // doc
+ memoryIndexTemporaryObject.clear();
+ Long offsetFilePointerTmpObject = outTmpObject.getFilePointer();
+ for (int i = 0; i < freq; i++) {
+ Long currentFilePointerTmpObject = outTmpObject
+ .getFilePointer();
+ Integer mtasId;
+ int position = postingsEnum.nextPosition();
+ BytesRef payload = postingsEnum.getPayload();
+ if (hasOffsets) {
+ mtasId = createObjectAndRegisterPrefix(field, outTmpObject,
+ term, termRef, position, payload,
+ postingsEnum.startOffset(), postingsEnum.endOffset(),
+ outPrefix);
+ } else {
+ mtasId = createObjectAndRegisterPrefix(field, outTmpObject,
+ term, termRef, position, payload, outPrefix);
+ }
+ if (mtasId != null) {
+ assert !memoryIndexTemporaryObject.containsKey(
+ mtasId) : "mtasId should be unique in this selection";
+ memoryIndexTemporaryObject.put(mtasId,
+ currentFilePointerTmpObject);
+ }
+ } // end loop positions
+ // store temporary index for this doc
+ if (memoryIndexTemporaryObject.size() > 0) {
+ // docId for this part
+ outTmpDocs.writeVInt(doc);
+ // number of objects/tokens in this part
+ outTmpDocs.writeVInt(memoryIndexTemporaryObject.size());
+ // offset to be used for references
+ outTmpDocs.writeVLong(offsetFilePointerTmpObject);
+ // loop over tokens
+ for (Entry<Integer, Long> entry : memoryIndexTemporaryObject
+ .entrySet()) {
+ // mtasId object
+ outTmpDocs.writeVInt(entry.getKey());
+ // reference object
+ outTmpDocs.writeVLong(
+ (entry.getValue() - offsetFilePointerTmpObject));
+ }
}
- }
- // clean up
- memoryIndexTemporaryObject.clear();
- } // end loop docs
- } // end loop terms
- // set fieldInfo
- fieldInfos.fieldInfo(field).putAttribute(
- MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_SINGLE_POSITION,
- getPrefixStatsSinglePositionPrefixAttribute(field));
- fieldInfos.fieldInfo(field).putAttribute(
- MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_MULTIPLE_POSITION,
- getPrefixStatsMultiplePositionPrefixAttribute(field));
- fieldInfos.fieldInfo(field).putAttribute(
- MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_SET_POSITION,
- getPrefixStatsSetPositionPrefixAttribute(field));
+ // clean up
+ memoryIndexTemporaryObject.clear();
+ } // end loop docs
+ } // end loop terms
+ // set fieldInfo
+ fieldInfos.fieldInfo(field).putAttribute(
+ MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_SINGLE_POSITION,
+ getPrefixStatsSinglePositionPrefixAttribute(field));
+ fieldInfos.fieldInfo(field).putAttribute(
+ MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_MULTIPLE_POSITION,
+ getPrefixStatsMultiplePositionPrefixAttribute(field));
+ fieldInfos.fieldInfo(field).putAttribute(
+ MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_SET_POSITION,
+ getPrefixStatsSetPositionPrefixAttribute(field));
- } // end processing field with freqs, positions and payload
- // close temporary object storage and index docs
- outTmpObject.close();
- outTmpDocs.close();
+ } // end processing field with freqs, positions and payload
+ // close temporary object storage and index docs
+ outTmpObject.close();
+ closeables.remove(outTmpObject);
+ outTmpDocs.close();
+ closeables.remove(outTmpDocs);
- // create (backwards) chained new temporary index docs
- IndexInput inTmpDocs = state.directory.openInput(mtasTmpDocsFileName,
- state.context);
- IndexOutput outTmpDocsChained = state.directory
- .createOutput(mtasTmpDocsChainedFileName, state.context);
- memoryTmpDocChainList.clear();
- while (true) {
- try {
- Long currentFilepointer = outTmpDocsChained.getFilePointer();
- // copy docId
- int docId = inTmpDocs.readVInt();
- outTmpDocsChained.writeVInt(docId);
- // copy size
- int size = inTmpDocs.readVInt();
- outTmpDocsChained.writeVInt(size);
- // offset references
- outTmpDocsChained.writeVLong(inTmpDocs.readVLong());
- for (int t = 0; t < size; t++) {
- outTmpDocsChained.writeVInt(inTmpDocs.readVInt());
+ // create (backwards) chained new temporary index docs
+ IndexInput inTmpDocs = state.directory.openInput(mtasTmpDocsFileName,
+ state.context);
+ closeables.add(inTmpDocs);
+ IndexOutput outTmpDocsChained = state.directory
+ .createOutput(mtasTmpDocsChainedFileName, state.context);
+ closeables.add(outTmpDocsChained);
+ memoryTmpDocChainList.clear();
+ while (true) {
+ try {
+ Long currentFilepointer = outTmpDocsChained.getFilePointer();
+ // copy docId
+ int docId = inTmpDocs.readVInt();
+ outTmpDocsChained.writeVInt(docId);
+ // copy size
+ int size = inTmpDocs.readVInt();
+ outTmpDocsChained.writeVInt(size);
+ // offset references
outTmpDocsChained.writeVLong(inTmpDocs.readVLong());
- }
- // set back reference to part with same docId
- if (memoryTmpDocChainList.containsKey(docId)) {
- // reference to previous
- outTmpDocsChained.writeVLong(memoryTmpDocChainList.get(docId));
- } else {
- // self reference indicates end of chain
- outTmpDocsChained.writeVLong(currentFilepointer);
- }
- // update temporary index in memory
- memoryTmpDocChainList.put(docId, currentFilepointer);
- } catch (IOException ex) {
- break;
- }
- }
- outTmpDocsChained.close();
- inTmpDocs.close();
- state.directory.deleteFile(mtasTmpDocsFileName);
-
- // set reference to tmpDoc in Field
- if (memoryTmpDocChainList.size() > 0) {
- outTmpField.writeString(field);
- outTmpField.writeVLong(outTmpDoc.getFilePointer());
- outTmpField.writeVInt(memoryTmpDocChainList.size());
- outTmpField.writeVLong(smallestTermFilepointer);
- outTmpField.writeVInt(termCounter);
- outTmpField.writeVLong(smallestPrefixFilepointer);
- outTmpField.writeVInt(prefixReferenceIndex.get(field).size());
- // fill indexDoc
- IndexInput inTmpDocsChained = state.directory
- .openInput(mtasTmpDocsChainedFileName, state.context);
- IndexInput inTmpObject = state.directory
- .openInput(mtasTmpObjectFileName, state.context);
- for (Entry<Integer, Long> entry : memoryTmpDocChainList.entrySet()) {
- Integer docId = entry.getKey();
- Long currentFilePointer, newFilePointer;
- // list of objectIds and references to objects
- memoryIndexDocList.clear();
- // construct final object + indexObjectId for docId
- currentFilePointer = entry.getValue();
- // collect objects for document
- tokenStatsMinPos = null;
- tokenStatsMaxPos = null;
- tokenStatsNumber = 0;
- while (true) {
- inTmpDocsChained.seek(currentFilePointer);
- Integer docIdPart = inTmpDocsChained.readVInt();
- assert docIdPart.equals(
- docId) : "conflicting docId in reference to temporaryIndexDocsChained";
- // number of objects/tokens in part
- int size = inTmpDocsChained.readVInt();
- long offsetFilePointerTmpObject = inTmpDocsChained.readVLong();
- assert size > 0 : "number of objects/tokens in part cannot be "
- + size;
for (int t = 0; t < size; t++) {
- int mtasId = inTmpDocsChained.readVInt();
- Long tmpObjectRef = inTmpDocsChained.readVLong()
- + offsetFilePointerTmpObject;
- assert !memoryIndexDocList.containsKey(
- mtasId) : "mtasId should be unique in this selection";
- // initially, store ref to tmpObject
- memoryIndexDocList.put(mtasId, tmpObjectRef);
+ outTmpDocsChained.writeVInt(inTmpDocs.readVInt());
+ outTmpDocsChained.writeVLong(inTmpDocs.readVLong());
}
- // reference to next part
- newFilePointer = inTmpDocsChained.readVLong();
- if (newFilePointer.equals(currentFilePointer)) {
- break; // end of chained parts
+ // set back reference to part with same docId
+ if (memoryTmpDocChainList.containsKey(docId)) {
+ // reference to previous
+ outTmpDocsChained.writeVLong(memoryTmpDocChainList.get(docId));
} else {
- currentFilePointer = newFilePointer;
+ // self reference indicates end of chain
+ outTmpDocsChained.writeVLong(currentFilepointer);
}
+ // update temporary index in memory
+ memoryTmpDocChainList.put(docId, currentFilepointer);
+ } catch (IOException ex) {
+ break;
}
- // now create new objects, sorted by mtasId
- Long smallestObjectFilepointer = outObject.getFilePointer();
- for (Entry<Integer, Long> objectEntry : memoryIndexDocList
+ }
+ outTmpDocsChained.close();
+ closeables.remove(outTmpDocsChained);
+ inTmpDocs.close();
+ closeables.remove(inTmpDocs);
+ state.directory.deleteFile(mtasTmpDocsFileName);
+
+ // set reference to tmpDoc in Field
+ if (memoryTmpDocChainList.size() > 0) {
+ outTmpField.writeString(field);
+ outTmpField.writeVLong(outTmpDoc.getFilePointer());
+ outTmpField.writeVInt(memoryTmpDocChainList.size());
+ outTmpField.writeVLong(smallestTermFilepointer);
+ outTmpField.writeVInt(termCounter);
+ outTmpField.writeVLong(smallestPrefixFilepointer);
+ outTmpField.writeVInt(prefixReferenceIndex.get(field).size());
+ // fill indexDoc
+ IndexInput inTmpDocsChained = state.directory
+ .openInput(mtasTmpDocsChainedFileName, state.context);
+ closeables.add(inTmpDocsChained);
+ IndexInput inTmpObject = state.directory
+ .openInput(mtasTmpObjectFileName, state.context);
+ closeables.add(inTmpObject);
+ for (Entry<Integer, Long> entry : memoryTmpDocChainList
.entrySet()) {
- int mtasId = objectEntry.getKey();
- Long tmpObjectRef = objectEntry.getValue();
- Long objectRef = outObject.getFilePointer();
- copyObjectAndUpdateStats(mtasId, inTmpObject, tmpObjectRef,
- outObject);
- // update with new ref
- memoryIndexDocList.put(mtasId, objectRef);
- }
- // check mtasIds properties
- assert memoryIndexDocList.firstKey()
- .equals(0) : "first mtasId should not be "
- + memoryIndexDocList.firstKey();
- assert (1 + memoryIndexDocList.lastKey()
- - memoryIndexDocList.firstKey()) == memoryIndexDocList
- .size() : "missing mtasId";
- assert tokenStatsNumber.equals(memoryIndexDocList
- .size()) : "incorrect number of items in tokenStats";
+ Integer docId = entry.getKey();
+ Long currentFilePointer, newFilePointer;
+ // list of objectIds and references to objects
+ memoryIndexDocList.clear();
+ // construct final object + indexObjectId for docId
+ currentFilePointer = entry.getValue();
+ // collect objects for document
+ tokenStatsMinPos = null;
+ tokenStatsMaxPos = null;
+ tokenStatsNumber = 0;
+ while (true) {
+ inTmpDocsChained.seek(currentFilePointer);
+ Integer docIdPart = inTmpDocsChained.readVInt();
+ assert docIdPart.equals(
+ docId) : "conflicting docId in reference to temporaryIndexDocsChained";
+ // number of objects/tokens in part
+ int size = inTmpDocsChained.readVInt();
+ long offsetFilePointerTmpObject = inTmpDocsChained.readVLong();
+ assert size > 0 : "number of objects/tokens in part cannot be "
+ + size;
+ for (int t = 0; t < size; t++) {
+ int mtasId = inTmpDocsChained.readVInt();
+ Long tmpObjectRef = inTmpDocsChained.readVLong()
+ + offsetFilePointerTmpObject;
+ assert !memoryIndexDocList.containsKey(
+ mtasId) : "mtasId should be unique in this selection";
+ // initially, store ref to tmpObject
+ memoryIndexDocList.put(mtasId, tmpObjectRef);
+ }
+ // reference to next part
+ newFilePointer = inTmpDocsChained.readVLong();
+ if (newFilePointer.equals(currentFilePointer)) {
+ break; // end of chained parts
+ } else {
+ currentFilePointer = newFilePointer;
+ }
+ }
+ // now create new objects, sorted by mtasId
+ Long smallestObjectFilepointer = outObject.getFilePointer();
+ for (Entry<Integer, Long> objectEntry : memoryIndexDocList
+ .entrySet()) {
+ int mtasId = objectEntry.getKey();
+ Long tmpObjectRef = objectEntry.getValue();
+ Long objectRef = outObject.getFilePointer();
+ copyObjectAndUpdateStats(mtasId, inTmpObject, tmpObjectRef,
+ outObject);
+ // update with new ref
+ memoryIndexDocList.put(mtasId, objectRef);
+ }
+ // check mtasIds properties
+ assert memoryIndexDocList.firstKey()
+ .equals(0) : "first mtasId should not be "
+ + memoryIndexDocList.firstKey();
+ assert (1 + memoryIndexDocList.lastKey()
+ - memoryIndexDocList.firstKey()) == memoryIndexDocList
+ .size() : "missing mtasId";
+ assert tokenStatsNumber.equals(memoryIndexDocList
+ .size()) : "incorrect number of items in tokenStats";
- // store item in tmpDoc
- outTmpDoc.writeVInt(docId);
- outTmpDoc.writeVLong(outIndexObjectId.getFilePointer());
+ // store item in tmpDoc
+ outTmpDoc.writeVInt(docId);
+ outTmpDoc.writeVLong(outIndexObjectId.getFilePointer());
- int mtasId = 0;
- // compute linear approximation (least squares method, integer
- // constants)
- long tmpN = memoryIndexDocList.size();
- long tmpSumY = 0, tmpSumXY = 0;
- long tmpSumX = 0, tmpSumXX = 0;
- for (Entry<Integer, Long> objectEntry : memoryIndexDocList
- .entrySet()) {
- assert objectEntry.getKey().equals(mtasId) : "unexpected mtasId";
- tmpSumY += objectEntry.getValue();
- tmpSumX += mtasId;
- tmpSumXY += mtasId * objectEntry.getValue();
- tmpSumXX += mtasId * mtasId;
- mtasId++;
- }
- int objectRefApproxQuotient = (int) (((tmpN * tmpSumXY)
- - (tmpSumX * tmpSumY))
- / ((tmpN * tmpSumXX) - (tmpSumX * tmpSumX)));
- long objectRefApproxOffset = (tmpSumY
- - objectRefApproxQuotient * tmpSumX) / tmpN;
- Long objectRefApproxCorrection;
- long maxAbsObjectRefApproxCorrection = 0;
- // compute maximum correction
- mtasId = 0;
- for (Entry<Integer, Long> objectEntry : memoryIndexDocList
- .entrySet()) {
- objectRefApproxCorrection = (objectEntry.getValue()
- - (objectRefApproxOffset
- + (mtasId * objectRefApproxQuotient)));
- maxAbsObjectRefApproxCorrection = Math.max(
- maxAbsObjectRefApproxCorrection,
- Math.abs(objectRefApproxCorrection));
- mtasId++;
- }
- byte storageFlags;
- if (maxAbsObjectRefApproxCorrection <= Long.valueOf(Byte.MAX_VALUE)
- + 1) {
- storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_BYTE;
- } else if (maxAbsObjectRefApproxCorrection <= Long
- .valueOf(Short.MAX_VALUE) + 1) {
- storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_SHORT;
- } else if (maxAbsObjectRefApproxCorrection <= Long
- .valueOf(Integer.MAX_VALUE) + 1) {
- storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_INTEGER;
- } else {
- storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_LONG;
- }
- // update indexObjectId with correction on approximated ref (assume
- // can be stored as int)
- mtasId = 0;
- for (Entry<Integer, Long> objectEntry : memoryIndexDocList
- .entrySet()) {
- objectRefApproxCorrection = (objectEntry.getValue()
- - (objectRefApproxOffset
- + (mtasId * objectRefApproxQuotient)));
- if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_BYTE) {
- outIndexObjectId
- .writeByte(objectRefApproxCorrection.byteValue());
- } else if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_SHORT) {
- outIndexObjectId
- .writeShort(objectRefApproxCorrection.shortValue());
- } else if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_INTEGER) {
- outIndexObjectId.writeInt(objectRefApproxCorrection.intValue());
+ int mtasId = 0;
+ // compute linear approximation (least squares method, integer
+ // constants)
+ long tmpN = memoryIndexDocList.size();
+ long tmpSumY = 0, tmpSumXY = 0;
+ long tmpSumX = 0, tmpSumXX = 0;
+ for (Entry<Integer, Long> objectEntry : memoryIndexDocList
+ .entrySet()) {
+ assert objectEntry.getKey()
+ .equals(mtasId) : "unexpected mtasId";
+ tmpSumY += objectEntry.getValue();
+ tmpSumX += mtasId;
+ tmpSumXY += mtasId * objectEntry.getValue();
+ tmpSumXX += mtasId * mtasId;
+ mtasId++;
+ }
+ int objectRefApproxQuotient = (int) (((tmpN * tmpSumXY)
+ - (tmpSumX * tmpSumY))
+ / ((tmpN * tmpSumXX) - (tmpSumX * tmpSumX)));
+ long objectRefApproxOffset = (tmpSumY
+ - objectRefApproxQuotient * tmpSumX) / tmpN;
+ Long objectRefApproxCorrection;
+ long maxAbsObjectRefApproxCorrection = 0;
+ // compute maximum correction
+ mtasId = 0;
+ for (Entry<Integer, Long> objectEntry : memoryIndexDocList
+ .entrySet()) {
+ objectRefApproxCorrection = (objectEntry.getValue()
+ - (objectRefApproxOffset
+ + (mtasId * objectRefApproxQuotient)));
+ maxAbsObjectRefApproxCorrection = Math.max(
+ maxAbsObjectRefApproxCorrection,
+ Math.abs(objectRefApproxCorrection));
+ mtasId++;
+ }
+ byte storageFlags;
+ if (maxAbsObjectRefApproxCorrection <= Long
+ .valueOf(Byte.MAX_VALUE) + 1) {
+ storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_BYTE;
+ } else if (maxAbsObjectRefApproxCorrection <= Long
+ .valueOf(Short.MAX_VALUE) + 1) {
+ storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_SHORT;
+ } else if (maxAbsObjectRefApproxCorrection <= Long
+ .valueOf(Integer.MAX_VALUE) + 1) {
+ storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_INTEGER;
} else {
- outIndexObjectId.writeLong(objectRefApproxCorrection);
+ storageFlags = MtasCodecPostingsFormat.MTAS_STORAGE_LONG;
}
- mtasId++;
- }
- outTmpDoc.writeVLong(smallestObjectFilepointer);
- outTmpDoc.writeVInt(objectRefApproxQuotient);
- outTmpDoc.writeZLong(objectRefApproxOffset);
- outTmpDoc.writeByte(storageFlags);
- outTmpDoc.writeVInt(tokenStatsNumber);
- outTmpDoc.writeVInt(tokenStatsMinPos);
- outTmpDoc.writeVInt(tokenStatsMaxPos);
- // clean up
- memoryIndexDocList.clear();
- } // end loop over docs
- inTmpDocsChained.close();
- inTmpObject.close();
- }
- // clean up
- memoryTmpDocChainList.clear();
- // remove temporary files
- state.directory.deleteFile(mtasTmpObjectFileName);
- state.directory.deleteFile(mtasTmpDocsChainedFileName);
- // store references for field
+ // update indexObjectId with correction on approximated ref
+ // (assume
+ // can be stored as int)
+ mtasId = 0;
+ for (Entry<Integer, Long> objectEntry : memoryIndexDocList
+ .entrySet()) {
+ objectRefApproxCorrection = (objectEntry.getValue()
+ - (objectRefApproxOffset
+ + (mtasId * objectRefApproxQuotient)));
+ if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_BYTE) {
+ outIndexObjectId
+ .writeByte(objectRefApproxCorrection.byteValue());
+ } else if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_SHORT) {
+ outIndexObjectId
+ .writeShort(objectRefApproxCorrection.shortValue());
+ } else if (storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_INTEGER) {
+ outIndexObjectId
+ .writeInt(objectRefApproxCorrection.intValue());
+ } else {
+ outIndexObjectId.writeLong(objectRefApproxCorrection);
+ }
+ mtasId++;
+ }
+ outTmpDoc.writeVLong(smallestObjectFilepointer);
+ outTmpDoc.writeVInt(objectRefApproxQuotient);
+ outTmpDoc.writeZLong(objectRefApproxOffset);
+ outTmpDoc.writeByte(storageFlags);
+ outTmpDoc.writeVInt(tokenStatsNumber);
+ outTmpDoc.writeVInt(tokenStatsMinPos);
+ outTmpDoc.writeVInt(tokenStatsMaxPos);
+ // clean up
+ memoryIndexDocList.clear();
+ } // end loop over docs
+ inTmpDocsChained.close();
+ closeables.remove(inTmpDocsChained);
+ inTmpObject.close();
+ closeables.remove(inTmpObject);
+ }
+ // clean up
+ memoryTmpDocChainList.clear();
+ // remove temporary files
+ state.directory.deleteFile(mtasTmpObjectFileName);
+ state.directory.deleteFile(mtasTmpDocsChainedFileName);
+ // store references for field
- } // end processing field
- } // end loop fields
- // close temporary index doc
- outTmpDoc.close();
- // close indexField, indexObjectId and object
- CodecUtil.writeFooter(outTmpField);
- outTmpField.close();
- CodecUtil.writeFooter(outIndexObjectId);
- outIndexObjectId.close();
- CodecUtil.writeFooter(outObject);
- outObject.close();
- CodecUtil.writeFooter(outTerm);
- outTerm.close();
- CodecUtil.writeFooter(outPrefix);
- outPrefix.close();
+ } // end processing field
+ } // end loop fields
+ // close temporary index doc
+ outTmpDoc.close();
+ closeables.remove(outTmpDoc);
+ // close indexField, indexObjectId and object
+ CodecUtil.writeFooter(outTmpField);
+ outTmpField.close();
+ closeables.remove(outTmpField);
+ CodecUtil.writeFooter(outIndexObjectId);
+ outIndexObjectId.close();
+ closeables.remove(outIndexObjectId);
+ CodecUtil.writeFooter(outObject);
+ outObject.close();
+ closeables.remove(outObject);
+ CodecUtil.writeFooter(outTerm);
+ outTerm.close();
+ closeables.remove(outTerm);
+ CodecUtil.writeFooter(outPrefix);
+ outPrefix.close();
+ closeables.remove(outPrefix);
- // create final doc, fill indexObjectPosition, indexObjectParent and
- // indexTermPrefixPosition, create final field
- IndexInput inTmpField = state.directory.openInput(mtasTmpFieldFileName,
- state.context);
- IndexInput inTmpDoc = state.directory.openInput(mtasTmpDocFileName,
- state.context);
- IndexInput inObjectId = state.directory.openInput(mtasIndexObjectIdFileName,
- state.context);
- IndexInput inObject = state.directory.openInput(mtasObjectFileName,
- state.context);
- IndexInput inTerm = state.directory.openInput(mtasTermFileName,
- state.context);
- outField = state.directory.createOutput(mtasIndexFieldFileName,
- state.context);
- CodecUtil.writeIndexHeader(outField, name,
- MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
- state.segmentSuffix);
- outField.writeString(delegatePostingsFormatName);
+ // create final doc, fill indexObjectPosition, indexObjectParent and
+ // indexTermPrefixPosition, create final field
+ IndexInput inTmpField = state.directory.openInput(mtasTmpFieldFileName,
+ state.context);
+ closeables.add(inTmpField);
+ IndexInput inTmpDoc = state.directory.openInput(mtasTmpDocFileName,
+ state.context);
+ closeables.add(inTmpDoc);
+ IndexInput inObjectId = state.directory
+ .openInput(mtasIndexObjectIdFileName, state.context);
+ closeables.add(inObjectId);
+ IndexInput inObject = state.directory.openInput(mtasObjectFileName,
+ state.context);
+ closeables.add(inObject);
+ IndexInput inTerm = state.directory.openInput(mtasTermFileName,
+ state.context);
+ closeables.add(inTerm);
+ closeables.add(outField = state.directory
+ .createOutput(mtasIndexFieldFileName, state.context));
+ CodecUtil.writeIndexHeader(outField, name,
+ MtasCodecPostingsFormat.VERSION_CURRENT, state.segmentInfo.getId(),
+ state.segmentSuffix);
+ outField.writeString(delegatePostingsFormatName);
+
+ while (true) {
+ try {
+ // read from tmpField
+ String field = inTmpField.readString();
+ long fpTmpDoc = inTmpField.readVLong();
+ int numberDocs = inTmpField.readVInt();
+ long fpTerm = inTmpField.readVLong();
+ int numberTerms = inTmpField.readVInt();
+ long fpPrefix = inTmpField.readVLong();
+ int numberPrefixes = inTmpField.readVInt();
+ inTmpDoc.seek(fpTmpDoc);
+ long fpFirstDoc = outDoc.getFilePointer();
+ // get prefixId index
+ HashMap<String, Integer> prefixIdIndexField = prefixIdIndex
+ .get(field);
+ // construct MtasRBTree for indexDocId
+ MtasRBTree mtasDocIdTree = new MtasRBTree(true, false);
+ for (int docCounter = 0; docCounter < numberDocs; docCounter++) {
- while (true) {
- try {
- // read from tmpField
- String field = inTmpField.readString();
- long fpTmpDoc = inTmpField.readVLong();
- int numberDocs = inTmpField.readVInt();
- long fpTerm = inTmpField.readVLong();
- int numberTerms = inTmpField.readVInt();
- long fpPrefix = inTmpField.readVLong();
- int numberPrefixes = inTmpField.readVInt();
- inTmpDoc.seek(fpTmpDoc);
- long fpFirstDoc = outDoc.getFilePointer();
- // get prefixId index
- HashMap<String, Integer> prefixIdIndexField = prefixIdIndex.get(field);
- // construct MtasRBTree for indexDocId
- MtasRBTree mtasDocIdTree = new MtasRBTree(true, false);
- for (int docCounter = 0; docCounter < numberDocs; docCounter++) {
- try {
// get info from tmpDoc
int docId = inTmpDoc.readVInt();
// filePointer indexObjectId
@@ -1085,7 +1122,8 @@ public class MtasFieldsConsumer extends FieldsConsumer {
MtasToken<String> token = MtasCodecPostingsFormat
.getToken(inObject, inTerm, ref);
String prefix = token.getPrefix();
- Integer prefixId = prefixIdIndexField.get(prefix);
+ int prefixId = prefixIdIndexField.containsKey(prefix)
+ ? prefixIdIndexField.get(prefix) : 0;
token.setPrefixId(prefixId);
assert token.getId().equals(mtasId) : "unexpected mtasId "
+ mtasId;
@@ -1119,47 +1157,75 @@ public class MtasFieldsConsumer extends FieldsConsumer {
outDoc.writeVInt(inTmpDoc.readVInt());
// add to tree for indexDocId
mtasDocIdTree.addIdFromDoc(docId, fpDoc);
- } catch (IOException ex) {
- break;
}
+ long fpIndexDocId = storeTree(mtasDocIdTree, outIndexDocId,
+ fpFirstDoc);
+ // store in indexField
+ outField.writeString(field);
+ outField.writeVLong(fpFirstDoc);
+ outField.writeVLong(fpIndexDocId);
+ outField.writeVInt(numberDocs);
+ outField.writeVLong(fpTerm);
+ outField.writeVInt(numberTerms);
+ outField.writeVLong(fpPrefix);
+ outField.writeVInt(numberPrefixes);
+ } catch (EOFException e) {
+ break;
}
- long fpIndexDocId = storeTree(mtasDocIdTree, outIndexDocId, fpFirstDoc);
+ // end loop over fields
+ }
+ inTerm.close();
+ closeables.remove(inTerm);
+ inObject.close();
+ closeables.remove(inObject);
+ inObjectId.close();
+ closeables.remove(inObjectId);
+ inTmpDoc.close();
+ closeables.remove(inTmpDoc);
+ inTmpField.close();
+ closeables.remove(inTmpField);
- // store in indexField
- outField.writeString(field);
- outField.writeVLong(fpFirstDoc);
- outField.writeVLong(fpIndexDocId);
- outField.writeVInt(numberDocs);
- outField.writeVLong(fpTerm);
- outField.writeVInt(numberTerms);
- outField.writeVLong(fpPrefix);
- outField.writeVInt(numberPrefixes);
- } catch (EOFException e) {
- break;
+ // remove temporary files
+ state.directory.deleteFile(mtasTmpDocFileName);
+ state.directory.deleteFile(mtasTmpFieldFileName);
+ // close indexDoc, indexObjectPosition and indexObjectParent
+ CodecUtil.writeFooter(outDoc);
+ outDoc.close();
+ closeables.remove(outDoc);
+ CodecUtil.writeFooter(outIndexObjectPosition);
+ outIndexObjectPosition.close();
+ closeables.remove(outIndexObjectPosition);
+ CodecUtil.writeFooter(outIndexObjectParent);
+ outIndexObjectParent.close();
+ closeables.remove(outIndexObjectParent);
+ CodecUtil.writeFooter(outIndexDocId);
+ outIndexDocId.close();
+ closeables.remove(outIndexDocId);
+ CodecUtil.writeFooter(outField);
+ outField.close();
+ closeables.remove(outField);
+ } catch (IOException e) {
+ // ignore, can happen when merging segment already written by
+ // delegateFieldsConsumer
+ } finally {
+ IOUtils.closeWhileHandlingException(closeables);
+ try {
+ state.directory.deleteFile(mtasTmpDocsFileName);
+ } catch (IOException e) {
+ // ignore
+ }
+ try {
+ state.directory.deleteFile(mtasTmpDocFileName);
+ } catch (IOException e) {
+ // ignore
+ }
+ try {
+ state.directory.deleteFile(mtasTmpFieldFileName);
+ } catch (IOException e) {
+ // ignore
}
- // end loop over fields
}
- inObject.close();
- inObjectId.close();
- inTmpDoc.close();
- inTmpField.close();
-
- // remove temporary files
- state.directory.deleteFile(mtasTmpDocFileName);
- state.directory.deleteFile(mtasTmpFieldFileName);
- // close indexDoc, indexObjectPosition and indexObjectParent
- CodecUtil.writeFooter(outDoc);
- outDoc.close();
- CodecUtil.writeFooter(outIndexObjectPosition);
- outIndexObjectPosition.close();
- CodecUtil.writeFooter(outIndexObjectParent);
- outIndexObjectParent.close();
- CodecUtil.writeFooter(outIndexDocId);
- outIndexDocId.close();
- CodecUtil.writeFooter(outField);
- outField.close();
-
}
/**
@@ -1179,9 +1245,8 @@ public class MtasFieldsConsumer extends FieldsConsumer {
* the payload
* @param outPrefix
* the out prefix
- * @return the integer
* @throws IOException
- * Signals that an I/O exception has occurred.
+ * @return the integer Signals that an I/O exception has occurred.
*/
private Integer createObjectAndRegisterPrefix(String field, IndexOutput out,
BytesRef term, Long termRef, int startPosition, BytesRef payload,
@@ -1219,104 +1284,109 @@ public class MtasFieldsConsumer extends FieldsConsumer {
BytesRef term, Long termRef, int startPosition, BytesRef payload,
Integer startOffset, Integer endOffset, IndexOutput outPrefix)
throws IOException {
- Integer mtasId = null;
- if (payload != null) {
- MtasPayloadDecoder payloadDecoder = new MtasPayloadDecoder();
- payloadDecoder.init(startPosition, Arrays.copyOfRange(payload.bytes,
- payload.offset, (payload.offset + payload.length)));
- mtasId = payloadDecoder.getMtasId();
- Integer mtasParentId = payloadDecoder.getMtasParentId();
- byte[] mtasPayload = payloadDecoder.getMtasPayload();
- MtasPosition mtasPosition = payloadDecoder.getMtasPosition();
- if (mtasPosition == null) {
- if (startOffset != null) {
- mtasPosition = new MtasPosition(startOffset, endOffset);
+ try {
+ Integer mtasId = null;
+ if (payload != null) {
+ MtasPayloadDecoder payloadDecoder = new MtasPayloadDecoder();
+ payloadDecoder.init(startPosition, Arrays.copyOfRange(payload.bytes,
+ payload.offset, (payload.offset + payload.length)));
+ mtasId = payloadDecoder.getMtasId();
+ Integer mtasParentId = payloadDecoder.getMtasParentId();
+ byte[] mtasPayload = payloadDecoder.getMtasPayload();
+ MtasPosition mtasPosition = payloadDecoder.getMtasPosition();
+ MtasOffset mtasOffset = payloadDecoder.getMtasOffset();
+ if (mtasOffset == null) {
+ if (startOffset != null) {
+ mtasOffset = new MtasOffset(startOffset, endOffset);
+ }
}
- }
- MtasOffset mtasOffset = payloadDecoder.getMtasOffset();
- MtasOffset mtasRealOffset = payloadDecoder.getMtasRealOffset();
- // only if really mtas object
- if (mtasId != null) {
- // compute flags
- int objectFlags = 0;
- if (mtasPosition != null) {
- if (mtasPosition.checkType(MtasPosition.POSITION_RANGE)) {
+ MtasOffset mtasRealOffset = payloadDecoder.getMtasRealOffset();
+ // only if really mtas object
+ if (mtasId != null) {
+ // compute flags
+ int objectFlags = 0;
+ if (mtasPosition != null) {
+ if (mtasPosition.checkType(MtasPosition.POSITION_RANGE)) {
+ objectFlags = objectFlags
+ | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE;
+ registerPrefixStatsRangePositionValue(field, term.utf8ToString(),
+ outPrefix);
+ } else if (mtasPosition.checkType(MtasPosition.POSITION_SET)) {
+ objectFlags = objectFlags
+ | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET;
+ registerPrefixStatsSetPositionValue(field, term.utf8ToString(),
+ outPrefix);
+ } else {
+ registerPrefixStatsSinglePositionValue(field, term.utf8ToString(),
+ outPrefix);
+ }
+ } else {
+ throw new IOException("no position");
+ }
+ if (mtasParentId != null) {
+ objectFlags = objectFlags
+ | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT;
+ }
+ if (mtasOffset != null) {
objectFlags = objectFlags
- | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE;
- registerPrefixStatsRangePositionValue(field, term.utf8ToString(),
- outPrefix);
- } else if (mtasPosition.checkType(MtasPosition.POSITION_SET)) {
+ | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET;
+ }
+ if (mtasRealOffset != null) {
objectFlags = objectFlags
- | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET;
- registerPrefixStatsSetPositionValue(field, term.utf8ToString(),
- outPrefix);
+ | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET;
+ }
+ if (mtasPayload != null) {
+ objectFlags = objectFlags
+ | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD;
+ }
+ // create object
+ out.writeVInt(mtasId);
+ out.writeVInt(objectFlags);
+ if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) {
+ out.writeVInt(mtasParentId);
+ }
+ if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) {
+ int tmpStart = mtasPosition.getStart();
+ out.writeVInt(tmpStart);
+ out.writeVInt((mtasPosition.getEnd() - tmpStart));
+ } else if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) {
+ int[] positions = mtasPosition.getPositions();
+ out.writeVInt(positions.length);
+ int tmpPrevious = 0;
+ for (int position : positions) {
+ out.writeVInt((position - tmpPrevious));
+ tmpPrevious = position;
+ }
} else {
- registerPrefixStatsSinglePositionValue(field, term.utf8ToString(),
- outPrefix);
+ out.writeVInt(mtasPosition.getStart());
}
- }
- if (mtasParentId != null) {
- objectFlags = objectFlags
- | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT;
- }
- if (mtasOffset != null) {
- objectFlags = objectFlags
- | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET;
- }
- if (mtasRealOffset != null) {
- objectFlags = objectFlags
- | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET;
- }
- if (mtasPayload != null) {
- objectFlags = objectFlags
- | MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD;
- }
- // create object
- out.writeVInt(mtasId);
- out.writeVInt(objectFlags);
- if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) {
- out.writeVInt(mtasParentId);
- }
- if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) {
- int tmpStart = mtasPosition.getStart();
- out.writeVInt(tmpStart);
- out.writeVInt((mtasPosition.getEnd() - tmpStart));
- } else if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) {
- TreeSet<Integer> positions = mtasPosition.getPositions();
- out.writeVInt(positions.size());
- int tmpPrevious = 0;
- for (int position : positions) {
- out.writeVInt((position - tmpPrevious));
- tmpPrevious = position;
+ if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) {
+ int tmpStart = mtasOffset.getStart();
+ out.writeVInt(mtasOffset.getStart());
+ out.writeVInt((mtasOffset.getEnd() - tmpStart));
}
- } else {
- out.writeVInt(mtasPosition.getStart());
- }
- if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) {
- int tmpStart = mtasOffset.getStart();
- out.writeVInt(mtasOffset.getStart());
- out.writeVInt((mtasOffset.getEnd() - tmpStart));
- }
- if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) {
- int tmpStart = mtasRealOffset.getStart();
- out.writeVInt(mtasRealOffset.getStart());
- out.writeVInt((mtasRealOffset.getEnd() - tmpStart));
- }
- if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) {
- out.writeVInt(mtasPayload.length);
- out.writeBytes(mtasPayload, mtasPayload.length);
- }
- out.writeVLong(termRef);
- } // storage token
- } // payload available
-
- return mtasId;
+ if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) {
+ int tmpStart = mtasRealOffset.getStart();
+ out.writeVInt(mtasRealOffset.getStart());
+ out.writeVInt((mtasRealOffset.getEnd() - tmpStart));
+ }
+ if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) {
+ out.writeVInt(mtasPayload.length);
+ out.writeBytes(mtasPayload, mtasPayload.length);
+ }
+ out.writeVLong(termRef);
+ } // storage token
+ } // payload available
+ return mtasId;
+ } catch (Exception e) {
+ throw new IOException(e.getMessage());
+ }
}
/**
@@ -1360,75 +1430,80 @@ public class MtasFieldsConsumer extends FieldsConsumer {
private Long storeTree(MtasTreeNode<?> node, boolean isSinglePoint,
boolean storeAdditionalInformation, IndexOutput out,
Long nodeRefApproxOffset, long refApproxOffset) throws IOException {
- if (node != null) {
- Boolean isRoot = false;
- if (nodeRefApproxOffset == null) {
- nodeRefApproxOffset = out.getFilePointer();
- isRoot = true;
- }
- Long fpIndexObjectPositionLeftChild, fpIndexObjectPositionRightChild;
- if (node.leftChild != null) {
- fpIndexObjectPositionLeftChild = storeTree(node.leftChild,
- isSinglePoint, storeAdditionalInformation, out, nodeRefApproxOffset,
- refApproxOffset);
- } else {
- fpIndexObjectPositionLeftChild = (long) 0; // tmp
- }
- if (node.rightChild != null) {
- fpIndexObjectPositionRightChild = storeTree(node.rightChild,
- isSinglePoint, storeAdditionalInformation, out, nodeRefApproxOffset,
- refApproxOffset);
- } else {
- fpIndexObjectPositionRightChild = (long) 0; // tmp
- }
- Long fpIndexObjectPosition = out.getFilePointer();
- if (node.leftChild == null) {
- fpIndexObjectPositionLeftChild = fpIndexObjectPosition;
- }
- if (node.rightChild == null) {
- fpIndexObjectPositionRightChild = fpIndexObjectPosition;
- }
- if (isRoot) {
- out.writeVLong(nodeRefApproxOffset);
- byte flag = 0;
- if (isSinglePoint) {
- flag |= MtasTree.SINGLE_POSITION_TREE;
+ try {
+ if (node != null) {
+ Boolean isRoot = false;
+ if (nodeRefApproxOffset == null) {
+ nodeRefApproxOffset = out.getFilePointer();
+ isRoot = true;
}
- if (storeAdditionalInformation) {
- flag |= MtasTree.STORE_ADDITIONAL_ID;
+ Long fpIndexObjectPositionLeftChild, fpIndexObjectPositionRightChild;
+ if (node.leftChild != null) {
+ fpIndexObjectPositionLeftChild = storeTree(node.leftChild,
+ isSinglePoint, storeAdditionalInformation, out,
+ nodeRefApproxOffset, refApproxOffset);
+ } else {
+ fpIndexObjectPositionLeftChild = (long) 0; // tmp
}
- out.writeByte(flag);
- }
- out.writeVInt(node.left);
- out.writeVInt(node.right);
- out.writeVInt(node.max);
- out.writeVLong((fpIndexObjectPositionLeftChild - nodeRefApproxOffset));
- out.writeVLong((fpIndexObjectPositionRightChild - nodeRefApproxOffset));
- if (!isSinglePoint) {
- out.writeVInt(node.ids.size());
- }
- HashMap<Integer, MtasTreeNodeId> ids = node.ids;
- Long objectRefCorrected;
- long objectRefCorrectedPrevious = 0;
- // sort refs
- List<MtasTreeNodeId> nodeIds = new ArrayList<MtasTreeNodeId>(
- ids.values());
- Collections.sort(nodeIds);
- if (isSinglePoint && (nodeIds.size() != 1)) {
- throw new IOException("singlePoint tree, but missing single point...");
- }
- for (MtasTreeNodeId nodeId : nodeIds) {
- objectRefCorrected = (nodeId.ref - refApproxOffset);
- out.writeVLong((objectRefCorrected - objectRefCorrectedPrevious));
- objectRefCorrectedPrevious = objectRefCorrected;
- if (storeAdditionalInformation) {
- out.writeVInt(nodeId.additionalId);
- out.writeVLong(nodeId.additionalRef);
+ if (node.rightChild != null) {
+ fpIndexObjectPositionRightChild = storeTree(node.rightChild,
+ isSinglePoint, storeAdditionalInformation, out,
+ nodeRefApproxOffset, refApproxOffset);
+ } else {
+ fpIndexObjectPositionRightChild = (long) 0; // tmp
+ }
+ Long fpIndexObjectPosition = out.getFilePointer();
+ if (node.leftChild == null) {
+ fpIndexObjectPositionLeftChild = fpIndexObjectPosition;
+ }
+ if (node.rightChild == null) {
+ fpIndexObjectPositionRightChild = fpIndexObjectPosition;
+ }
+ if (isRoot) {
+ out.writeVLong(nodeRefApproxOffset);
+ byte flag = 0;
+ if (isSinglePoint) {
+ flag |= MtasTree.SINGLE_POSITION_TREE;
+ }
+ if (storeAdditionalInformation) {
+ flag |= MtasTree.STORE_ADDITIONAL_ID;
+ }
+ out.writeByte(flag);
+ }
+ out.writeVInt(node.left);
+ out.writeVInt(node.right);
+ out.writeVInt(node.max);
+ out.writeVLong((fpIndexObjectPositionLeftChild - nodeRefApproxOffset));
+ out.writeVLong((fpIndexObjectPositionRightChild - nodeRefApproxOffset));
+ if (!isSinglePoint) {
+ out.writeVInt(node.ids.size());
+ }
+ HashMap<Integer, MtasTreeNodeId> ids = node.ids;
+ Long objectRefCorrected;
+ long objectRefCorrectedPrevious = 0;
+ // sort refs
+ List<MtasTreeNodeId> nodeIds = new ArrayList<MtasTreeNodeId>(
+ ids.values());
+ Collections.sort(nodeIds);
+ if (isSinglePoint && (nodeIds.size() != 1)) {
+ throw new IOException(
+ "singlePoint tree, but missing single point...");
+ }
+ for (MtasTreeNodeId nodeId : nodeIds) {
+ objectRefCorrected = (nodeId.ref - refApproxOffset);
+ out.writeVLong((objectRefCorrected - objectRefCorrectedPrevious));
+ objectRefCorrectedPrevious = objectRefCorrected;
+ if (storeAdditionalInformation) {
+ out.writeVInt(nodeId.additionalId);
+ out.writeVLong(nodeId.additionalRef);
+ }
}
+ return fpIndexObjectPosition;
+ } else {
+ return null;
}
- return fpIndexObjectPosition;
- } else {
- return null;
+ } catch (Exception e) {
+ throw new IOException(e.getMessage());
}
}
@@ -1470,68 +1545,70 @@ public class MtasFieldsConsumer extends FieldsConsumer {
*/
private void copyObjectAndUpdateStats(int id, IndexInput in, Long inRef,
IndexOutput out) throws IOException {
- int mtasId, objectFlags;
- // read
- in.seek(inRef);
- mtasId = in.readVInt();
- assert id == mtasId : "wrong id detected while copying object";
- objectFlags = in.readVInt();
- out.writeVInt(mtasId);
- out.writeVInt(objectFlags);
- if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) {
- out.writeVInt(in.readVInt());
- }
- if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) {
- int minPos = in.readVInt();
- int maxPos = in.readVInt();
- out.writeVInt(minPos);
- out.writeVInt(maxPos);
- tokenStatsAdd(minPos, maxPos);
- } else if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) {
- int size = in.readVInt();
- out.writeVInt(size);
- TreeSet<Integer> list = new TreeSet<Integer>();
- int previousPosition = 0;
- for (int t = 0; t < size; t++) {
+ try {
+ int mtasId, objectFlags;
+ // read
+ in.seek(inRef);
+ mtasId = in.readVInt();
+ assert id == mtasId : "wrong id detected while copying object";
+ objectFlags = in.readVInt();
+ out.writeVInt(mtasId);
+ out.writeVInt(objectFlags);
+ if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) {
+ out.writeVInt(in.readVInt());
+ }
+ if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) {
+ int minPos = in.readVInt();
+ int maxPos = in.readVInt();
+ out.writeVInt(minPos);
+ out.writeVInt(maxPos);
+ tokenStatsAdd(minPos, maxPos);
+ } else if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) {
+ int size = in.readVInt();
+ out.writeVInt(size);
+ TreeSet<Integer> list = new TreeSet<Integer>();
+ int previousPosition = 0;
+ for (int t = 0; t < size; t++) {
+ int pos = in.readVInt();
+ out.writeVInt(pos);
+ previousPosition = (pos + previousPosition);
+ list.add(previousPosition);
+ }
+ assert list
+ .size() == size : "duplicate positions in set are not allowed";
+ tokenStatsAdd(list.first(), list.last());
+ } else {
int pos = in.readVInt();
out.writeVInt(pos);
- previousPosition = (pos + previousPosition);
- list.add(previousPosition);
+ tokenStatsAdd(pos, pos);
}
- assert list.size() == size : "duplicate positions in set are not allowed";
- tokenStatsAdd(list.first(), list.last());
- } else {
- int pos = in.readVInt();
- out.writeVInt(pos);
- tokenStatsAdd(pos, pos);
- }
- if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) {
- out.writeVInt(in.readVInt());
- out.writeVInt(in.readVInt());
- }
- if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) {
- out.writeVInt(in.readVInt());
- out.writeVInt(in.readVInt());
- }
- if ((objectFlags
- & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) {
- int length = in.readVInt();
- out.writeVInt(length);
- byte[] payload = new byte[length];
- in.readBytes(payload, 0, length);
- out.writeBytes(payload, payload.length);
+ if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) {
+ out.writeVInt(in.readVInt());
+ out.writeVInt(in.readVInt());
+ }
+ if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) {
+ out.writeVInt(in.readVInt());
+ out.writeVInt(in.readVInt());
+ }
+ if ((objectFlags
+ & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) {
+ int length = in.readVInt();
+ out.writeVInt(length);
+ byte[] payload = new byte[length];
+ in.readBytes(payload, 0, length);
+ out.writeBytes(payload, payload.length);
+ }
+ out.writeVLong(in.readVLong());
+ } catch (Exception e) {
+ throw new IOException(e.getMessage());
}
- out.writeVLong(in.readVLong());
}
- /** The closed. */
- private boolean closed;
-
/*
* (non-Javadoc)
*
@@ -1539,10 +1616,6 @@ public class MtasFieldsConsumer extends FieldsConsumer {
*/
@Override
public void close() throws IOException {
- if (closed) {
- return;
- }
- closed = true;
delegateFieldsConsumer.close();
}
diff --git a/src/mtas/codec/MtasFieldsProducer.java b/src/mtas/codec/MtasFieldsProducer.java
index 13a405d..aa01bd4 100644
--- a/src/mtas/codec/MtasFieldsProducer.java
+++ b/src/mtas/codec/MtasFieldsProducer.java
@@ -29,19 +29,22 @@ public class MtasFieldsProducer extends FieldsProducer {
/** The index input list. */
private HashMap<String, IndexInput> indexInputList;
-
+
/** The index input offset list. */
private HashMap<String, Long> indexInputOffsetList;
-
+
/** The version. */
private int version;
/**
* Instantiates a new mtas fields producer.
*
- * @param state the state
- * @param name the name
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param state
+ * the state
+ * @param name
+ * the name
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasFieldsProducer(SegmentReadState state, String name)
throws IOException {
@@ -49,15 +52,13 @@ public class MtasFieldsProducer extends FieldsProducer {
indexInputList = new HashMap<String, IndexInput>();
indexInputOffsetList = new HashMap<String, Long>();
version = MtasCodecPostingsFormat.VERSION_CURRENT;
- postingsFormatName = addIndexInputToList("object",
- openMtasFile(state, name, MtasCodecPostingsFormat.MTAS_OBJECT_EXTENSION),
- postingsFormatName);
+ postingsFormatName = addIndexInputToList("object", openMtasFile(state, name,
+ MtasCodecPostingsFormat.MTAS_OBJECT_EXTENSION), postingsFormatName);
addIndexInputToList("term",
openMtasFile(state, name, MtasCodecPostingsFormat.MTAS_TERM_EXTENSION),
postingsFormatName);
- addIndexInputToList("prefix",
- openMtasFile(state, name, MtasCodecPostingsFormat.MTAS_PREFIX_EXTENSION),
- postingsFormatName);
+ addIndexInputToList("prefix", openMtasFile(state, name,
+ MtasCodecPostingsFormat.MTAS_PREFIX_EXTENSION), postingsFormatName);
addIndexInputToList("field",
openMtasFile(state, name, MtasCodecPostingsFormat.MTAS_FIELD_EXTENSION),
postingsFormatName);
@@ -68,50 +69,60 @@ public class MtasFieldsProducer extends FieldsProducer {
addIndexInputToList("indexObjectId",
openMtasFile(state, name,
MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_ID_EXTENSION),
- postingsFormatName);
+ postingsFormatName);
try {
- addIndexInputToList("doc",
- openMtasFile(state, name, MtasCodecPostingsFormat.MTAS_DOC_EXTENSION, version, version),
+ addIndexInputToList(
+ "doc", openMtasFile(state, name,
+ MtasCodecPostingsFormat.MTAS_DOC_EXTENSION, version, version),
postingsFormatName);
addIndexInputToList("indexObjectPosition",
openMtasFile(state, name,
- MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_POSITION_EXTENSION, version, version),
+ MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_POSITION_EXTENSION,
+ version, version),
postingsFormatName);
addIndexInputToList("indexObjectParent",
openMtasFile(state, name,
- MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_PARENT_EXTENSION, version, version),
+ MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_PARENT_EXTENSION,
+ version, version),
postingsFormatName);
- } catch (IndexFormatTooOldException e1) {
+ } catch (IndexFormatTooOldException e1) {
version = MtasCodecPostingsFormat.VERSION_OLD_2;
try {
addIndexInputToList("doc",
- openMtasFile(state, name, MtasCodecPostingsFormat.MTAS_DOC_EXTENSION, version, version),
+ openMtasFile(state, name,
+ MtasCodecPostingsFormat.MTAS_DOC_EXTENSION, version, version),
postingsFormatName);
addIndexInputToList("indexObjectPosition",
openMtasFile(state, name,
- MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_POSITION_EXTENSION, version, version),
+ MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_POSITION_EXTENSION,
+ version, version),
postingsFormatName);
addIndexInputToList("indexObjectParent",
openMtasFile(state, name,
- MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_PARENT_EXTENSION, version, version),
+ MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_PARENT_EXTENSION,
+ version, version),
postingsFormatName);
} catch (IndexFormatTooOldException e2) {
version = MtasCodecPostingsFormat.VERSION_OLD_1;
addIndexInputToList("doc",
- openMtasFile(state, name, MtasCodecPostingsFormat.MTAS_DOC_EXTENSION, version, version),
+ openMtasFile(state, name,
+ MtasCodecPostingsFormat.MTAS_DOC_EXTENSION, version, version),
postingsFormatName);
addIndexInputToList("indexObjectPosition",
openMtasFile(state, name,
- MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_POSITION_EXTENSION, version, version),
+ MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_POSITION_EXTENSION,
+ version, version),
postingsFormatName);
addIndexInputToList("indexObjectParent",
openMtasFile(state, name,
- MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_PARENT_EXTENSION, version, version),
+ MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_PARENT_EXTENSION,
+ version, version),
postingsFormatName);
}
}
- if(version == MtasCodecPostingsFormat.VERSION_OLD_2) {
- throw new IOException("This MTAS doesn't support index version "+version+", please upgrade");
+ if (version == MtasCodecPostingsFormat.VERSION_OLD_2) {
+ throw new IOException("This MTAS doesn't support index version " + version
+ + ", please upgrade");
}
// Load the delegate postingsFormatName from this file
this.delegateFieldsProducer = PostingsFormat.forName(postingsFormatName)
@@ -121,15 +132,19 @@ public class MtasFieldsProducer extends FieldsProducer {
/**
* Adds the index input to list.
*
- * @param name the name
- * @param in the in
- * @param postingsFormatName the postings format name
+ * @param name
+ * the name
+ * @param in
+ * the in
+ * @param postingsFormatName
+ * the postings format name
* @return the string
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private String addIndexInputToList(String name, IndexInput in,
String postingsFormatName) throws IOException {
- if(indexInputList.get(name)!=null) {
+ if (indexInputList.get(name) != null) {
indexInputList.get(name).close();
}
if (postingsFormatName == null) {
@@ -237,16 +252,22 @@ public class MtasFieldsProducer extends FieldsProducer {
/**
* Open mtas file.
*
- * @param state the state
- * @param name the name
- * @param extension the extension
- * @param minimum the minimum
- * @param maximum the maximum
+ * @param state
+ * the state
+ * @param name
+ * the name
+ * @param extension
+ * the extension
+ * @param minimum
+ * the minimum
+ * @param maximum
+ * the maximum
* @return the index input
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private IndexInput openMtasFile(SegmentReadState state, String name,
- String extension, Integer minimum, Integer maximum) throws IOException {
+ String extension, Integer minimum, Integer maximum) throws IOException {
String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
state.segmentSuffix, extension);
IndexInput object;
@@ -263,20 +284,25 @@ public class MtasFieldsProducer extends FieldsProducer {
CodecUtil.checkIndexHeader(object, name, minVersion, maxVersion,
state.segmentInfo.getId(), state.segmentSuffix);
} catch (IndexFormatTooOldException e) {
- object.close();
- throw new IndexFormatTooOldException(e.getMessage(),e.getVersion(), e.getMinVersion(), e.getMaxVersion());
- }
+ object.close();
+ throw new IndexFormatTooOldException(e.getMessage(), e.getVersion(),
+ e.getMinVersion(), e.getMaxVersion());
+ }
return object;
}
/**
* Open mtas file.
*
- * @param state the state
- * @param name the name
- * @param extension the extension
+ * @param state
+ * the state
+ * @param name
+ * the name
+ * @param extension
+ * the extension
* @return the index input
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private IndexInput openMtasFile(SegmentReadState state, String name,
String extension) throws IOException {
diff --git a/src/mtas/codec/MtasSimpleTextCodec.java b/src/mtas/codec/MtasSimpleTextCodec.java
index cac7a58..3721658 100644
--- a/src/mtas/codec/MtasSimpleTextCodec.java
+++ b/src/mtas/codec/MtasSimpleTextCodec.java
@@ -9,11 +9,11 @@ public class MtasSimpleTextCodec extends MtasCodec {
/** The Constant MTAS_CODEC_NAME. */
public static final String MTAS_CODEC_NAME = "MtasSimpleTextCodec";
-
+
/**
* Instantiates a new mtas simple text codec.
*/
- public MtasSimpleTextCodec() {
+ public MtasSimpleTextCodec() {
super(MTAS_CODEC_NAME, new SimpleTextCodec());
}
}
diff --git a/src/mtas/codec/MtasTerms.java b/src/mtas/codec/MtasTerms.java
index 409b6dd..9fc41ca 100644
--- a/src/mtas/codec/MtasTerms.java
+++ b/src/mtas/codec/MtasTerms.java
@@ -15,26 +15,30 @@ public class MtasTerms extends Terms {
/** The index input list. */
HashMap<String, IndexInput> indexInputList;
-
+
/** The index input offset list. */
HashMap<String, Long> indexInputOffsetList;
-
+
/** The version. */
int version;
-
+
/** The delegate terms. */
Terms delegateTerms;
-
/**
* Instantiates a new mtas terms.
*
- * @param terms the terms
- * @param indexInputList the index input list
- * @param indexInputOffsetList the index input offset list
- * @param version the version
+ * @param terms
+ * the terms
+ * @param indexInputList
+ * the index input list
+ * @param indexInputOffsetList
+ * the index input offset list
+ * @param version
+ * the version
*/
- public MtasTerms(Terms terms, HashMap<String, IndexInput> indexInputList, HashMap<String, Long> indexInputOffsetList, int version) {
+ public MtasTerms(Terms terms, HashMap<String, IndexInput> indexInputList,
+ HashMap<String, Long> indexInputOffsetList, int version) {
delegateTerms = terms;
this.indexInputList = indexInputList;
this.indexInputOffsetList = indexInputOffsetList;
@@ -166,7 +170,7 @@ public class MtasTerms extends Terms {
return false;
}
}
-
+
/**
* Gets the version.
*
@@ -175,20 +179,20 @@ public class MtasTerms extends Terms {
public int getVersion() {
return version;
}
-
+
/**
* Gets the index input list.
*
* @return the index input list
*/
public HashMap<String, IndexInput> getIndexInputList() {
- HashMap<String, IndexInput> clonedIndexInputList = new HashMap<String, IndexInput>();
- for(Entry<String, IndexInput> entry : indexInputList.entrySet()) {
+ HashMap<String, IndexInput> clonedIndexInputList = new HashMap<String, IndexInput>();
+ for (Entry<String, IndexInput> entry : indexInputList.entrySet()) {
clonedIndexInputList.put(entry.getKey(), entry.getValue().clone());
}
return clonedIndexInputList;
}
-
+
/**
* Gets the index input offset list.
*
@@ -196,6 +200,6 @@ public class MtasTerms extends Terms {
*/
public HashMap<String, Long> getIndexInputOffsetList() {
return indexInputOffsetList;
- }
+ }
}
diff --git a/src/mtas/codec/payload/MtasBitInputStream.java b/src/mtas/codec/payload/MtasBitInputStream.java
index 6a22b3f..dd9cd8b 100644
--- a/src/mtas/codec/payload/MtasBitInputStream.java
+++ b/src/mtas/codec/payload/MtasBitInputStream.java
@@ -7,17 +7,18 @@ import java.io.IOException;
* The Class MtasBitInputStream.
*/
public class MtasBitInputStream extends ByteArrayInputStream {
-
+
/** The bit buffer. */
private int bitBuffer = 0;
-
+
/** The bit count. */
private int bitCount = 0;
/**
* Instantiates a new mtas bit input stream.
*
- * @param buf the buf
+ * @param buf
+ * the buf
*/
public MtasBitInputStream(byte[] buf) {
super(buf);
@@ -27,7 +28,8 @@ public class MtasBitInputStream extends ByteArrayInputStream {
* Read bit.
*
* @return the int
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public int readBit() throws IOException {
if (bitCount == 0) {
@@ -48,7 +50,8 @@ public class MtasBitInputStream extends ByteArrayInputStream {
* Read remaining bytes.
*
* @return the byte[]
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public byte[] readRemainingBytes() throws IOException {
if (this.available() > 0) {
@@ -64,7 +67,8 @@ public class MtasBitInputStream extends ByteArrayInputStream {
* Read elias gamma coding integer.
*
* @return the int
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public int readEliasGammaCodingInteger() throws IOException {
int value = readEliasGammaCodingPositiveInteger();
@@ -79,7 +83,8 @@ public class MtasBitInputStream extends ByteArrayInputStream {
* Read elias gamma coding non negative integer.
*
* @return the int
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public int readEliasGammaCodingNonNegativeInteger() throws IOException {
int value = readEliasGammaCodingPositiveInteger();
@@ -90,7 +95,8 @@ public class MtasBitInputStream extends ByteArrayInputStream {
* Read elias gamma coding positive integer.
*
* @return the int
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public int readEliasGammaCodingPositiveInteger() throws IOException {
int value;
diff --git a/src/mtas/codec/payload/MtasBitOutputStream.java b/src/mtas/codec/payload/MtasBitOutputStream.java
index 74496fa..8fefa44 100644
--- a/src/mtas/codec/payload/MtasBitOutputStream.java
+++ b/src/mtas/codec/payload/MtasBitOutputStream.java
@@ -7,10 +7,10 @@ import java.io.IOException;
* The Class MtasBitOutputStream.
*/
public class MtasBitOutputStream extends ByteArrayOutputStream {
-
+
/** The bit buffer. */
private int bitBuffer = 0;
-
+
/** The bit count. */
private int bitCount = 0;
@@ -23,75 +23,90 @@ public class MtasBitOutputStream extends ByteArrayOutputStream {
/**
* Write bit.
*
- * @param value the value
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param value
+ * the value
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public void writeBit(int value) throws IOException {
- writeBit(value,1);
+ writeBit(value, 1);
}
-
+
/**
* Write bit.
*
- * @param value the value
- * @param number the number
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param value
+ * the value
+ * @param number
+ * the number
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public void writeBit(int value, int number) throws IOException {
while (number > 0) {
number--;
bitBuffer |= ((value & 1) << bitCount++);
- if (bitCount==8) {
+ if (bitCount == 8) {
createByte();
- }
+ }
}
}
-
+
/**
* Write elias gamma coding integer.
*
- * @param value the value
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param value
+ * the value
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- public void writeEliasGammaCodingInteger(int value) throws IOException {
- if(value>=0) {
- writeEliasGammaCodingPositiveInteger(2*value+1);
+ public void writeEliasGammaCodingInteger(int value) throws IOException {
+ if (value >= 0) {
+ writeEliasGammaCodingPositiveInteger(2 * value + 1);
} else {
- writeEliasGammaCodingPositiveInteger(-2*value);
+ writeEliasGammaCodingPositiveInteger(-2 * value);
}
}
-
+
/**
* Write elias gamma coding non negative integer.
*
- * @param value the value
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param value
+ * the value
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- public void writeEliasGammaCodingNonNegativeInteger(int value) throws IOException {
- if(value>=0) {
- writeEliasGammaCodingPositiveInteger(value+1);
- }
+ public void writeEliasGammaCodingNonNegativeInteger(int value)
+ throws IOException {
+ if (value >= 0) {
+ writeEliasGammaCodingPositiveInteger(value + 1);
+ }
}
-
+
/**
* Write elias gamma coding positive integer.
*
- * @param value the value
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param value
+ * the value
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- public void writeEliasGammaCodingPositiveInteger(int value) throws IOException {
- if(value>0) {
- if(value==1) {
+ public void writeEliasGammaCodingPositiveInteger(int value)
+ throws IOException {
+ if (value > 0) {
+ if (value == 1) {
writeBit(1);
} else {
writeBit(0);
- writeEliasGammaCodingPositiveInteger(value/2);
- writeBit(value%2);
+ writeEliasGammaCodingPositiveInteger(value / 2);
+ writeBit(value % 2);
}
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.io.ByteArrayOutputStream#close()
*/
@Override
@@ -103,7 +118,8 @@ public class MtasBitOutputStream extends ByteArrayOutputStream {
/**
* Creates the byte.
*
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public void createByte() throws IOException {
if (bitCount > 0) {
diff --git a/src/mtas/codec/payload/MtasPayloadDecoder.java b/src/mtas/codec/payload/MtasPayloadDecoder.java
index f8f6bea..af24322 100644
--- a/src/mtas/codec/payload/MtasPayloadDecoder.java
+++ b/src/mtas/codec/payload/MtasPayloadDecoder.java
@@ -16,140 +16,149 @@ public class MtasPayloadDecoder {
/** The mtas position. */
private MtasPosition mtasPosition;
-
+
/** The mtas start position. */
private int mtasStartPosition;
-
+
/** The mtas positions. */
private TreeSet<Integer> mtasPositions;
-
+
/** The mtas id. */
private Integer mtasId = null;
-
+
/** The mtas payload value. */
private byte[] mtasPayloadValue = null;
-
+
/** The mtas parent id. */
private Integer mtasParentId = null;
-
+
/** The mtas payload. */
private Boolean mtasPayload = null;
-
+
/** The mtas parent. */
private Boolean mtasParent = null;
-
+
/** The mtas position type. */
private String mtasPositionType = null;
-
+
/** The mtas offset. */
- private MtasOffset mtasOffset;
-
+ private MtasOffset mtasOffset;
+
/** The mtas real offset. */
- private MtasOffset mtasRealOffset;
-
+ private MtasOffset mtasRealOffset;
+
/**
* Instantiates a new mtas payload decoder.
*/
- public MtasPayloadDecoder() {
+ public MtasPayloadDecoder() {
}
/**
* Inits the.
*
- * @param startPosition the start position
- * @param payload the payload
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param startPosition
+ * the start position
+ * @param payload
+ * the payload
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public void init(int startPosition, byte[] payload) throws IOException {
byteStream = new MtasBitInputStream(payload);
mtasStartPosition = startPosition;
// analyse initial bits - position
Boolean getOffset, getRealOffset;
- if (byteStream.readBit()==1) {
- if (byteStream.readBit()==1) {
+ if (byteStream.readBit() == 1) {
+ if (byteStream.readBit() == 1) {
mtasPositionType = null;
} else {
mtasPositionType = MtasPosition.POSITION_RANGE;
}
} else {
- if (byteStream.readBit()==1) {
+ if (byteStream.readBit() == 1) {
mtasPositionType = MtasPosition.POSITION_SET;
} else {
mtasPositionType = MtasPosition.POSITION_SINGLE;
- }
+ }
}
// analyze initial bits - offset
- if (byteStream.readBit()==1) {
+ if (byteStream.readBit() == 1) {
getOffset = true;
} else {
getOffset = false;
}
// analyze initial bits - realOffset
- if (byteStream.readBit()==1) {
+ if (byteStream.readBit() == 1) {
getRealOffset = true;
} else {
getRealOffset = false;
}
// analyze initial bits - parent
- if (byteStream.readBit()==1) {
+ if (byteStream.readBit() == 1) {
mtasParent = true;
} else {
mtasParent = false;
}
// analyse initial bits - payload
- if (byteStream.readBit()==1) {
+ if (byteStream.readBit() == 1) {
mtasPayload = true;
} else {
mtasPayload = false;
}
- if (byteStream.readBit()==0) {
- //string
+ if (byteStream.readBit() == 0) {
+ // string
} else {
- //other
+ // other
}
- //get id
+ // get id
mtasId = byteStream.readEliasGammaCodingNonNegativeInteger();
- //get position info
- if(mtasPositionType.equals(MtasPosition.POSITION_SINGLE)) {
+ // get position info
+ if (mtasPositionType.equals(MtasPosition.POSITION_SINGLE)) {
mtasPosition = new MtasPosition(mtasStartPosition);
- } else if(mtasPositionType.equals(MtasPosition.POSITION_RANGE)) {
- mtasPosition = new MtasPosition(mtasStartPosition, (mtasStartPosition + byteStream.readEliasGammaCodingPositiveInteger() - 1));
- } else if(mtasPositionType.equals(MtasPosition.POSITION_SET)) {
+ } else if (mtasPositionType.equals(MtasPosition.POSITION_RANGE)) {
+ mtasPosition = new MtasPosition(mtasStartPosition, (mtasStartPosition
+ + byteStream.readEliasGammaCodingPositiveInteger() - 1));
+ } else if (mtasPositionType.equals(MtasPosition.POSITION_SET)) {
mtasPositions = new TreeSet<Integer>();
mtasPositions.add(mtasStartPosition);
int numberOfPoints = byteStream.readEliasGammaCodingPositiveInteger();
- TreeSet<Integer> positionList = new TreeSet<Integer>();
- positionList.add(mtasStartPosition);
+ int[] positionList = new int[numberOfPoints];
+ positionList[0] = mtasStartPosition;
int previousPosition = 0;
int currentPosition = mtasStartPosition;
- for(int i=1; i<numberOfPoints; i++) {
+ for (int i = 1; i < numberOfPoints; i++) {
previousPosition = currentPosition;
- currentPosition = previousPosition+byteStream.readEliasGammaCodingPositiveInteger();
- positionList.add(currentPosition);
+ currentPosition = previousPosition
+ + byteStream.readEliasGammaCodingPositiveInteger();
+ positionList[i] = currentPosition;
}
mtasPosition = new MtasPosition(positionList);
} else {
mtasPosition = null;
}
- //get offset and realOffset info
- if(getOffset) {
+ // get offset and realOffset info
+ if (getOffset) {
int offsetStart = byteStream.readEliasGammaCodingNonNegativeInteger();
- int offsetEnd = offsetStart + byteStream.readEliasGammaCodingPositiveInteger() - 1;
+ int offsetEnd = offsetStart
+ + byteStream.readEliasGammaCodingPositiveInteger() - 1;
mtasOffset = new MtasOffset(offsetStart, offsetEnd);
- if(getRealOffset) {
- int realOffsetStart = byteStream.readEliasGammaCodingInteger() + offsetStart;
- int realOffsetEnd = realOffsetStart + byteStream.readEliasGammaCodingPositiveInteger() - 1;
+ if (getRealOffset) {
+ int realOffsetStart = byteStream.readEliasGammaCodingInteger()
+ + offsetStart;
+ int realOffsetEnd = realOffsetStart
+ + byteStream.readEliasGammaCodingPositiveInteger() - 1;
mtasRealOffset = new MtasOffset(realOffsetStart, realOffsetEnd);
}
- } else if(getRealOffset) {
+ } else if (getRealOffset) {
int realOffsetStart = byteStream.readEliasGammaCodingNonNegativeInteger();
- int realOffsetEnd = realOffsetStart + byteStream.readEliasGammaCodingPositiveInteger() - 1;
- mtasRealOffset = new MtasOffset(realOffsetStart, realOffsetEnd);
+ int realOffsetEnd = realOffsetStart
+ + byteStream.readEliasGammaCodingPositiveInteger() - 1;
+ mtasRealOffset = new MtasOffset(realOffsetStart, realOffsetEnd);
}
- if(mtasParent) {
+ if (mtasParent) {
mtasParentId = byteStream.readEliasGammaCodingInteger() + mtasId;
- }
- if(mtasPayload) {
+ }
+ if (mtasPayload) {
mtasPayloadValue = byteStream.readRemainingBytes();
}
}
@@ -178,7 +187,7 @@ public class MtasPayloadDecoder {
* @return the mtas payload
*/
public byte[] getMtasPayload() {
- return mtasPayload?mtasPayloadValue:null;
+ return mtasPayload ? mtasPayloadValue : null;
}
/**
@@ -198,7 +207,7 @@ public class MtasPayloadDecoder {
public MtasOffset getMtasOffset() {
return mtasOffset;
}
-
+
/**
* Gets the mtas real offset.
*
diff --git a/src/mtas/codec/payload/MtasPayloadEncoder.java b/src/mtas/codec/payload/MtasPayloadEncoder.java
index 81a7a3d..9816e17 100644
--- a/src/mtas/codec/payload/MtasPayloadEncoder.java
+++ b/src/mtas/codec/payload/MtasPayloadEncoder.java
@@ -2,8 +2,6 @@ package mtas.codec.payload;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Iterator;
-import java.util.TreeSet;
import mtas.analysis.token.MtasPosition;
import mtas.analysis.token.MtasToken;
@@ -12,98 +10,7 @@ import mtas.analysis.token.MtasTokenString;
import org.apache.lucene.util.BytesRef;
/**
- * The Class MtasPayloadEncoder encodes position, (real)offset, parent and
- * original payload into a new payload. <br>
- * <b>Initial bits</b><br>
- * <ul>
- * <li>bit 0,1 describe position type:
- * <ul>
- * <li>00 is single</li>
- * <li>10 is range</li>
- * <li>01 is set</li>
- * <li>11 is reserved (sub-position?)</li>
- * </ul>
- * </li>
- * <li>bit 2 describes offset:
- * <ul>
- * <li>0 is no offset</li>
- * <li>1 is offset
- * </ul>
- * </li>
- * <li>bit 3 describes real offset:
- * <ul>
- * <li>0 is follow offset</li>
- * <li>1 is separate real offset</li>
- * </ul>
- * </li>
- * <li>bit 4 describes parent:
- * <ul>
- * <li>0 is no parent</li>
- * <li>1 is parent</li>
- * </ul>
- * </li>
- * <li>bit 5 describes payload:
- * <ul>
- * <li>0 is no payload</li>
- * <li>1 is payload</li>
- * </ul>
- * </li>
- * </ul>
- *
- * <b>Following bits</b><br>
- *
- * <ul>
- * <li>id: add 1 and use Elias Gamma Coding, see {@link mtas.codec.payload.MtasBitOutputStream#writeEliasGammaCodingNonNegativeInteger(int)}</li>
- * <li>if range-position:
- * <ul>
- * <li>range-length: use Elias Gamma Coding, see {@link mtas.codec.payload.MtasBitOutputStream#writeEliasGammaCodingPositiveInteger(int)}</li>
- * </ul>
- * </li>
- * <li>if set-position:
- * <ul>
- * <li>number of positions: use Elias Gamma Coding, see {@link mtas.codec.payload.MtasBitOutputStream#writeEliasGammaCodingPositiveInteger(int)}</li>
- * <li>increments: use Elias Gamma Coding, see {@link mtas.codec.payload.MtasBitOutputStream#writeEliasGammaCodingPositiveInteger(int)}</li>
- * </ul>
- * </li>
- * <li>if offset:
- * <ul>
- * <li>startOffset: add 1, use Elias Gamma Coding, see {@link mtas.codec.payload.MtasBitOutputStream#writeEliasGammaCodingNonNegativeInteger(int)}</li>
- * <li>length: use Elias Gamma Coding, see {@link mtas.codec.payload.MtasBitOutputStream#writeEliasGammaCodingPositiveInteger(int)}</li>
- * </ul>
- * </li>
- *
- * <li>if realoffset:
- * <ul>
- * <li>if offset available:
- * <ul>
- * <li>difference of startRealOffset with startOffset: generalize for negative values, use Elias Gamma Coding, see {@link mtas.codec.payload.MtasBitOutputStream#writeEliasGammaCodingInteger(int)}</li>
- * <li>length: use Elias Gamma Coding, see {@link mtas.codec.payload.MtasBitOutputStream#writeEliasGammaCodingPositiveInteger(int)}</li>
- * </ul>
- * </li>
- * <li>if no offset available:
- * <ul>
- * <li>startOffset: add 1, use Elias Gamma Coding, see {@link mtas.codec.payload.MtasBitOutputStream#writeEliasGammaCodingNonNegativeInteger(int)}</li>
- * <li>length: use Elias Gamma Coding, see {@link mtas.codec.payload.MtasBitOutputStream#writeEliasGammaCodingPositiveInteger(int)}</li>
- * </ul>
- * </li>
- * </ul>
- * </li>
- *
- * <li>if parent:
- * <ul>
- * <li>increment relative from id to parent_id:
- * generalize for negative values, use Elias Gamma Coding, see {@link mtas.codec.payload.MtasBitOutputStream#writeEliasGammaCodingInteger(int)}</li>
- * </ul>
- * </li>
- * </ul>
- *
- *<b>Finally</b><br>
- *<ul>
- * <li>add minimal number of bits
- * to get multiple of 8 bits</li>
- * <li>if payload: add payload bytes</li>
- *</ul>
- *
+ * The Class MtasPayloadEncoder.
*/
/**
@@ -143,8 +50,10 @@ public class MtasPayloadEncoder {
/**
* Instantiates a new mtas payload encoder.
*
- * @param token the token
- * @param flags the flags
+ * @param token
+ * the token
+ * @param flags
+ * the flags
*/
public MtasPayloadEncoder(MtasToken<?> token, int flags) {
mtasToken = token;
@@ -155,7 +64,8 @@ public class MtasPayloadEncoder {
/**
* Instantiates a new mtas payload encoder.
*
- * @param token the token
+ * @param token
+ * the token
*/
public MtasPayloadEncoder(MtasToken<?> token) {
this(token, ENCODE_DEFAULT);
@@ -165,7 +75,8 @@ public class MtasPayloadEncoder {
* Gets the payload.
*
* @return the payload
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public BytesRef getPayload() throws IOException {
@@ -224,22 +135,17 @@ public class MtasPayloadEncoder {
// do nothing
} else if (mtasToken.checkPositionType(MtasPosition.POSITION_RANGE)) {
// write length
- byteStream.writeEliasGammaCodingPositiveInteger(1
- + mtasToken.getPositionEnd() - mtasToken.getPositionStart());
+ byteStream.writeEliasGammaCodingPositiveInteger(
+ 1 + mtasToken.getPositionEnd() - mtasToken.getPositionStart());
} else if (mtasToken.checkPositionType(MtasPosition.POSITION_SET)) {
// write number of positions
- TreeSet<Integer> positionList = mtasToken.getPositions();
- byteStream.writeEliasGammaCodingPositiveInteger(positionList.size());
- int previousPosition = positionList.first();
- Iterator<Integer> it = positionList.iterator();
- if (it.hasNext()) {
- it.next(); // skip start position
- while (it.hasNext()) {
- int currentPosition = it.next();
- byteStream.writeEliasGammaCodingPositiveInteger(currentPosition
- - previousPosition);
- previousPosition = currentPosition;
- }
+ int[] positionList = mtasToken.getPositions();
+ byteStream.writeEliasGammaCodingPositiveInteger(positionList.length);
+ int previousPosition = positionList[0];
+ for (int i = 1; i < positionList.length; i++) {
+ byteStream.writeEliasGammaCodingPositiveInteger(
+ positionList[i] - previousPosition);
+ previousPosition = positionList[i];
}
} else {
// do nothing
@@ -247,32 +153,32 @@ public class MtasPayloadEncoder {
// add offset info (EliasGammaCoding)
if ((encodingFlags & ENCODE_OFFSET) == ENCODE_OFFSET
&& mtasToken.checkOffset()) {
- byteStream.writeEliasGammaCodingNonNegativeInteger(mtasToken
- .getOffsetStart());
- byteStream.writeEliasGammaCodingPositiveInteger(1
- + mtasToken.getOffsetEnd() - mtasToken.getOffsetStart());
+ byteStream
+ .writeEliasGammaCodingNonNegativeInteger(mtasToken.getOffsetStart());
+ byteStream.writeEliasGammaCodingPositiveInteger(
+ 1 + mtasToken.getOffsetEnd() - mtasToken.getOffsetStart());
}
// add realOffset info (EliasGammaCoding)
if ((encodingFlags & ENCODE_REALOFFSET) == ENCODE_REALOFFSET
&& mtasToken.checkRealOffset()) {
if ((encodingFlags & ENCODE_OFFSET) == ENCODE_OFFSET
&& mtasToken.checkOffset()) {
- byteStream.writeEliasGammaCodingInteger(mtasToken.getRealOffsetStart()
- - mtasToken.getOffsetStart());
- byteStream.writeEliasGammaCodingPositiveInteger(1
- + mtasToken.getRealOffsetEnd() - mtasToken.getRealOffsetStart());
+ byteStream.writeEliasGammaCodingInteger(
+ mtasToken.getRealOffsetStart() - mtasToken.getOffsetStart());
+ byteStream.writeEliasGammaCodingPositiveInteger(
+ 1 + mtasToken.getRealOffsetEnd() - mtasToken.getRealOffsetStart());
} else {
- byteStream.writeEliasGammaCodingNonNegativeInteger(mtasToken
- .getRealOffsetStart());
- byteStream.writeEliasGammaCodingPositiveInteger(1
- + mtasToken.getRealOffsetEnd() - mtasToken.getRealOffsetStart());
+ byteStream.writeEliasGammaCodingNonNegativeInteger(
+ mtasToken.getRealOffsetStart());
+ byteStream.writeEliasGammaCodingPositiveInteger(
+ 1 + mtasToken.getRealOffsetEnd() - mtasToken.getRealOffsetStart());
}
}
// add parent info (EliasGammaCoding)
if ((encodingFlags & ENCODE_PARENT) == ENCODE_PARENT
&& mtasToken.checkParentId()) {
- byteStream.writeEliasGammaCodingInteger(mtasToken.getParentId()
- - mtasToken.getId());
+ byteStream.writeEliasGammaCodingInteger(
+ mtasToken.getParentId() - mtasToken.getId());
}
// add minimal number of zero-bits to get round number of bytes
byteStream.createByte();
@@ -281,8 +187,8 @@ public class MtasPayloadEncoder {
&& mtasToken.getPayload() != null) {
BytesRef payload = mtasToken.getPayload();
byteStream.write(Arrays.copyOfRange(payload.bytes, payload.offset,
- (payload.offset + payload.length)));
- }
+ (payload.offset + payload.length)));
+ }
// construct new payload
return new BytesRef(byteStream.toByteArray());
}
diff --git a/src/mtas/codec/tree/IntervalRBTree.java b/src/mtas/codec/tree/IntervalRBTree.java
index 307be05..fb65680 100644
--- a/src/mtas/codec/tree/IntervalRBTree.java
+++ b/src/mtas/codec/tree/IntervalRBTree.java
@@ -7,6 +7,9 @@ import mtas.codec.util.CodecSearchTree.MtasTreeHit;
/**
* The Class IntervalRBTree.
+ *
+ * @param <T>
+ * the generic type
*/
public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
@@ -24,17 +27,20 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
/**
* Instantiates a new interval rb tree.
*
- * @param positionsHits the positions hits
+ * @param positionsHits
+ * the positions hits
*/
public IntervalRBTree(ArrayList<IntervalTreeNodeData<T>> positionsHits) {
- this();
+ this();
for (IntervalTreeNodeData<T> positionsHit : positionsHits) {
addRange(positionsHit.start, positionsHit.end, positionsHit.list);
}
close();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.tree.IntervalTree#addRangeEmpty(int, int)
*/
@Override
@@ -49,7 +55,9 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.tree.IntervalTree#addSinglePoint(int, java.util.ArrayList)
*/
@Override
@@ -58,7 +66,9 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
addRange(position, position, list);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.tree.IntervalTree#addRange(int, int, java.util.ArrayList)
*/
@Override
@@ -73,15 +83,18 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
root.color = IntervalRBTreeNode.BLACK;
}
}
-
/**
* Adds the range.
*
- * @param n the n
- * @param left the left
- * @param right the right
- * @param list the list
+ * @param n
+ * the n
+ * @param left
+ * the left
+ * @param right
+ * the right
+ * @param list
+ * the list
* @return the interval rb tree node
*/
private IntervalRBTreeNode<T> addRange(IntervalRBTreeNode<T> n, Integer left,
@@ -116,8 +129,10 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
/**
* Update max min.
*
- * @param n the n
- * @param c the c
+ * @param n
+ * the n
+ * @param c
+ * the c
*/
private void updateMaxMin(IntervalRBTreeNode<T> n, IntervalRBTreeNode<T> c) {
if (c != null) {
@@ -134,7 +149,8 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
/**
* Rotate right.
*
- * @param n the n
+ * @param n
+ * the n
* @return the interval rb tree node
*/
private IntervalRBTreeNode<T> rotateRight(IntervalRBTreeNode<T> n) {
@@ -155,7 +171,8 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
/**
* Rotate left.
*
- * @param n the n
+ * @param n
+ * the n
* @return the interval rb tree node
*/
private IntervalRBTreeNode<T> rotateLeft(IntervalRBTreeNode<T> n) {
@@ -176,7 +193,8 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
/**
* Flip colors.
*
- * @param n the n
+ * @param n
+ * the n
*/
private void flipColors(IntervalRBTreeNode<T> n) {
// n must have opposite color of its two children
@@ -191,7 +209,8 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
/**
* Checks if is red.
*
- * @param n the n
+ * @param n
+ * the n
* @return true, if is red
*/
private boolean isRed(IntervalRBTreeNode<T> n) {
@@ -204,7 +223,8 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
/**
* Size.
*
- * @param n the n
+ * @param n
+ * the n
* @return the int
*/
private int size(IntervalRBTreeNode<T> n) {
@@ -216,7 +236,8 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
/**
* Sets the max min.
*
- * @param n the new max min
+ * @param n
+ * the new max min
*/
private void setMaxMin(IntervalRBTreeNode<T> n) {
n.min = n.left;
@@ -231,6 +252,4 @@ public class IntervalRBTree<T> extends IntervalTree<T, IntervalRBTreeNode<T>> {
}
}
-
-
}
diff --git a/src/mtas/codec/tree/IntervalRBTreeNode.java b/src/mtas/codec/tree/IntervalRBTreeNode.java
index c9ec2ae..ca56c6e 100644
--- a/src/mtas/codec/tree/IntervalRBTreeNode.java
+++ b/src/mtas/codec/tree/IntervalRBTreeNode.java
@@ -2,42 +2,52 @@ package mtas.codec.tree;
/**
* The Class IntervalRBTreeNode.
+ *
+ * @param <T>
+ * the generic type
*/
-public class IntervalRBTreeNode<T> extends IntervalTreeNode<T, IntervalRBTreeNode<T>> {
-
- /** The Constant BLACK. */
- static final int BLACK = 1;
-
- /** The Constant RED. */
- static final int RED = 0;
-
- /** The color. */
- public int color;
-
- /** The n. */
- public int n;
-
- // node with start and end position
- /**
- * Instantiates a new interval rb tree node.
- *
- * @param left the left
- * @param right the right
- * @param color the color
- * @param n the n
- */
- public IntervalRBTreeNode(int left, int right, int color, int n) {
- super(left, right);
- this.color = color;
- this.n = n;
- }
-
- /* (non-Javadoc)
- * @see mtas.codec.tree.MtasTreeNode#self()
- */
- @Override
- protected IntervalRBTreeNode<T> self() {
- return self();
- }
+public class IntervalRBTreeNode<T>
+ extends IntervalTreeNode<T, IntervalRBTreeNode<T>> {
+ /** The Constant BLACK. */
+ static final int BLACK = 1;
+
+ /** The Constant RED. */
+ static final int RED = 0;
+
+ /** The color. */
+ public int color;
+
+ /** The n. */
+ public int n;
+
+ // node with start and end position
+ /**
+ * Instantiates a new interval rb tree node.
+ *
+ * @param left
+ * the left
+ * @param right
+ * the right
+ * @param color
+ * the color
+ * @param n
+ * the n
+ */
+ public IntervalRBTreeNode(int left, int right, int color, int n) {
+ super(left, right);
+ this.color = color;
+ this.n = n;
}
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.tree.MtasTreeNode#self()
+ */
+ @Override
+ protected IntervalRBTreeNode<T> self() {
+ return self();
+ }
+
+}
diff --git a/src/mtas/codec/tree/IntervalTree.java b/src/mtas/codec/tree/IntervalTree.java
index ee172cd..23eb48a 100644
--- a/src/mtas/codec/tree/IntervalTree.java
+++ b/src/mtas/codec/tree/IntervalTree.java
@@ -6,55 +6,66 @@ import mtas.codec.util.CodecSearchTree.MtasTreeHit;
/**
* The Class IntervalTree.
*
- * @param <N> the number type
+ * @param <T>
+ * the generic type
+ * @param <N>
+ * the number type
*/
abstract public class IntervalTree<T, N extends IntervalTreeNode<T, N>> {
-
+
/** The current. */
protected N root, current;
-
-
+
/**
* Instantiates a new interval tree.
*/
public IntervalTree() {
root = null;
}
-
+
/**
* Close.
*
* @return the n
*/
final public N close() {
- if(root==null) {
- addRangeEmpty(0,0);
+ if (root == null) {
+ addRangeEmpty(0, 0);
}
return root;
- }
-
+ }
+
/**
* Adds the single point.
*
- * @param position the position
- * @param list the list
+ * @param position
+ * the position
+ * @param list
+ * the list
*/
- abstract protected void addSinglePoint(int position, ArrayList<MtasTreeHit<T>> list);
-
+ abstract protected void addSinglePoint(int position,
+ ArrayList<MtasTreeHit<T>> list);
+
/**
* Adds the range.
*
- * @param left the left
- * @param right the right
- * @param list the list
+ * @param left
+ * the left
+ * @param right
+ * the right
+ * @param list
+ * the list
*/
- abstract protected void addRange(int left, int right, ArrayList<MtasTreeHit<T>> list);
-
+ abstract protected void addRange(int left, int right,
+ ArrayList<MtasTreeHit<T>> list);
+
/**
* Adds the range empty.
*
- * @param left the left
- * @param right the right
+ * @param left
+ * the left
+ * @param right
+ * the right
*/
abstract protected void addRangeEmpty(int left, int right);
@@ -68,22 +79,26 @@ abstract public class IntervalTree<T, N extends IntervalTreeNode<T, N>> {
/**
* Prints the balance.
*
- * @param p the p
- * @param n the n
+ * @param p
+ * the p
+ * @param n
+ * the n
*/
- final private void printBalance(Integer p, N n) {
- if(n!=null) {
- printBalance((p+1), n.leftChild);
- System.out.print(String.format("%"+(3*p)+"s", ""));
- if(n.left==n.right) {
- System.out.println("["+n.left+"] ("+n.max+") : "+n.lists.size()+" lists");
+ final private void printBalance(Integer p, N n) {
+ if (n != null) {
+ printBalance((p + 1), n.leftChild);
+ System.out.print(String.format("%" + (3 * p) + "s", ""));
+ if (n.left == n.right) {
+ System.out.println(
+ "[" + n.left + "] (" + n.max + ") : " + n.lists.size() + " lists");
} else {
- System.out.println("["+n.left+"-"+n.right+"] ("+n.max+") : "+n.lists.size()+" lists");
+ System.out.println("[" + n.left + "-" + n.right + "] (" + n.max + ") : "
+ + n.lists.size() + " lists");
}
- printBalance((p+1), n.rightChild);
+ printBalance((p + 1), n.rightChild);
}
}
-
+
/**
* Gets the root.
*
@@ -92,7 +107,7 @@ abstract public class IntervalTree<T, N extends IntervalTreeNode<T, N>> {
final public N getRoot() {
return root;
}
-
+
/**
* Gets the current.
*
@@ -101,11 +116,12 @@ abstract public class IntervalTree<T, N extends IntervalTreeNode<T, N>> {
final public N getCurrent() {
return current;
}
-
+
/**
* Sets the current.
*
- * @param node the new current
+ * @param node
+ * the new current
*/
final public void setCurrent(N node) {
current = node;
diff --git a/src/mtas/codec/tree/IntervalTreeNode.java b/src/mtas/codec/tree/IntervalTreeNode.java
index 0121ebe..7a32c09 100644
--- a/src/mtas/codec/tree/IntervalTreeNode.java
+++ b/src/mtas/codec/tree/IntervalTreeNode.java
@@ -7,7 +7,10 @@ import mtas.codec.util.CodecSearchTree.MtasTreeHit;
/**
* The Class IntervalTreeNode.
*
- * @param <N> the number type
+ * @param <T>
+ * the generic type
+ * @param <N>
+ * the number type
*/
abstract public class IntervalTreeNode<T, N extends IntervalTreeNode<T, N>> {
@@ -17,60 +20,56 @@ abstract public class IntervalTreeNode<T, N extends IntervalTreeNode<T, N>> {
* @return the n
*/
protected abstract N self();
-
- /** The left. */
- public int left;
-
- /** The right. */
- public int right;
-
- /** The max. */
- public int max;
-
- /** The min. */
- public int min;
-
- /** The left child. */
- public N leftChild;
-
- /** The right child. */
- public N rightChild;
-
-
- /** The lists. */
- public ArrayList<ArrayList<MtasTreeHit<T>>> lists;
- // node with start and end position
- /**
- * Instantiates a new interval tree node.
- *
- * @param left the left
- * @param right the right
- */
- public IntervalTreeNode(int left, int right) {
- this.left = left;
- this.right = right;
- min = left;
- max = right;
- lists = new ArrayList<ArrayList<MtasTreeHit<T>>>();
- }
+ /** The left. */
+ public int left;
- // add id to node
- /**
- * Adds the list.
- *
- * @param list the list
- */
- final public void addList(ArrayList<MtasTreeHit<T>> list) {
- if(list!=null) {
- lists.add(list);
- }
- }
-
-
+ /** The right. */
+ public int right;
- }
+ /** The max. */
+ public int max;
+
+ /** The min. */
+ public int min;
+
+ /** The left child. */
+ public N leftChild;
+ /** The right child. */
+ public N rightChild;
+ /** The lists. */
+ public ArrayList<ArrayList<MtasTreeHit<T>>> lists;
+
+ // node with start and end position
+ /**
+ * Instantiates a new interval tree node.
+ *
+ * @param left
+ * the left
+ * @param right
+ * the right
+ */
+ public IntervalTreeNode(int left, int right) {
+ this.left = left;
+ this.right = right;
+ min = left;
+ max = right;
+ lists = new ArrayList<ArrayList<MtasTreeHit<T>>>();
+ }
+ // add id to node
+ /**
+ * Adds the list.
+ *
+ * @param list
+ * the list
+ */
+ final public void addList(ArrayList<MtasTreeHit<T>> list) {
+ if (list != null) {
+ lists.add(list);
+ }
+ }
+}
diff --git a/src/mtas/codec/tree/IntervalTreeNodeData.java b/src/mtas/codec/tree/IntervalTreeNodeData.java
index dd32a86..7770e7e 100644
--- a/src/mtas/codec/tree/IntervalTreeNodeData.java
+++ b/src/mtas/codec/tree/IntervalTreeNodeData.java
@@ -7,23 +7,28 @@ import mtas.codec.util.CodecSearchTree.MtasTreeHit;
/**
* The Class IntervalTreeNodeData.
*
- * @param <T> the generic type
+ * @param <T>
+ * the generic type
*/
public class IntervalTreeNodeData<T> {
-
+
/** The hit end. */
public int start, end, hitStart, hitEnd;
-
+
/** The list. */
public ArrayList<MtasTreeHit<T>> list;
-
+
/**
* Instantiates a new interval tree node data.
*
- * @param start the start
- * @param end the end
- * @param hitStart the hit start
- * @param hitEnd the hit end
+ * @param start
+ * the start
+ * @param end
+ * the end
+ * @param hitStart
+ * the hit start
+ * @param hitEnd
+ * the hit end
*/
public IntervalTreeNodeData(int start, int end, int hitStart, int hitEnd) {
this.start = start;
@@ -32,5 +37,5 @@ public class IntervalTreeNodeData<T> {
this.hitEnd = hitEnd;
list = new ArrayList<MtasTreeHit<T>>();
}
-
+
}
diff --git a/src/mtas/codec/tree/MtasAVLTree.java b/src/mtas/codec/tree/MtasAVLTree.java
index bba593a..1d13d3a 100644
--- a/src/mtas/codec/tree/MtasAVLTree.java
+++ b/src/mtas/codec/tree/MtasAVLTree.java
@@ -15,8 +15,10 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
/**
* Instantiates a new mtas avl tree.
*
- * @param singlePoint the single point
- * @param storePrefixId the store prefix id
+ * @param singlePoint
+ * the single point
+ * @param storePrefixId
+ * the store prefix id
*/
public MtasAVLTree(boolean singlePoint, boolean storePrefixId) {
super(singlePoint, storePrefixId);
@@ -29,7 +31,8 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
* @see mtas.codec.tree.MtasTree#addTokenRangeEmpty(int, int)
*/
@Override
- final protected void addRangeEmpty(int left, int right, int additionalId, long additionalRef) {
+ final protected void addRangeEmpty(int left, int right, int additionalId,
+ long additionalRef) {
String key = ((Integer) left).toString() + "_"
+ ((Integer) right).toString();
if (index.containsKey(key)) {
@@ -46,7 +49,8 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
* java.lang.Long)
*/
@Override
- final protected void addSinglePoint(int position, int additionalId, long additionalRef, Integer id, Long ref) {
+ final protected void addSinglePoint(int position, int additionalId,
+ long additionalRef, Integer id, Long ref) {
addRange(position, position, additionalId, additionalRef, id, ref);
}
@@ -57,7 +61,8 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
* java.lang.Long)
*/
@Override
- final protected void addRange(int left, int right, int additionalId, long additionalRef, Integer id, Long ref) {
+ final protected void addRange(int left, int right, int additionalId,
+ long additionalRef, Integer id, Long ref) {
String key = ((Integer) left).toString() + "_"
+ ((Integer) right).toString();
if (index.containsKey(key)) {
@@ -98,8 +103,10 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
/**
* Update max.
*
- * @param n the n
- * @param max the max
+ * @param n
+ * the n
+ * @param max
+ * the max
*/
private void updateMax(MtasAVLTreeNode n, int max) {
if (n != null) {
@@ -113,7 +120,8 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
/**
* Rebalance.
*
- * @param n the n
+ * @param n
+ * the n
*/
private void rebalance(MtasAVLTreeNode n) {
setBalance(n);
@@ -140,7 +148,8 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
/**
* Rotate left.
*
- * @param a the a
+ * @param a
+ * the a
* @return the mtas avl tree node
*/
private MtasAVLTreeNode rotateLeft(MtasAVLTreeNode a) {
@@ -168,7 +177,8 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
/**
* Rotate right.
*
- * @param a the a
+ * @param a
+ * the a
* @return the mtas avl tree node
*/
private MtasAVLTreeNode rotateRight(MtasAVLTreeNode a) {
@@ -196,7 +206,8 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
/**
* Rotate left then right.
*
- * @param n the n
+ * @param n
+ * the n
* @return the mtas avl tree node
*/
private MtasAVLTreeNode rotateLeftThenRight(MtasAVLTreeNode n) {
@@ -207,7 +218,8 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
/**
* Rotate right then left.
*
- * @param n the n
+ * @param n
+ * the n
* @return the mtas avl tree node
*/
private MtasAVLTreeNode rotateRightThenLeft(MtasAVLTreeNode n) {
@@ -218,7 +230,8 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
/**
* Height.
*
- * @param n the n
+ * @param n
+ * the n
* @return the int
*/
private int height(MtasAVLTreeNode n) {
@@ -232,7 +245,8 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
/**
* Sets the balance.
*
- * @param nodes the new balance
+ * @param nodes
+ * the new balance
*/
private void setBalance(MtasAVLTreeNode... nodes) {
for (MtasAVLTreeNode n : nodes) {
@@ -243,7 +257,8 @@ public class MtasAVLTree extends MtasTree<MtasAVLTreeNode> {
/**
* Sets the max.
*
- * @param n the new max
+ * @param n
+ * the new max
*/
private void setMax(MtasAVLTreeNode n) {
n.max = n.right;
diff --git a/src/mtas/codec/tree/MtasAVLTreeNode.java b/src/mtas/codec/tree/MtasAVLTreeNode.java
index 7f2174f..c01c1e3 100644
--- a/src/mtas/codec/tree/MtasAVLTreeNode.java
+++ b/src/mtas/codec/tree/MtasAVLTreeNode.java
@@ -7,31 +7,36 @@ import mtas.codec.tree.MtasTreeNode;
*/
public class MtasAVLTreeNode extends MtasTreeNode<MtasAVLTreeNode> {
- /** The balance. */
- public int balance;
-
- /** The parent. */
- public MtasAVLTreeNode parent;
-
- // node with start and end position
- /**
- * Instantiates a new mtas avl tree node.
- *
- * @param left the left
- * @param right the right
- * @param parent the parent
- */
- public MtasAVLTreeNode(int left, int right, MtasAVLTreeNode parent) {
- super(left, right);
- this.parent = parent;
- }
+ /** The balance. */
+ public int balance;
- /* (non-Javadoc)
- * @see mtas.codec.tree.MtasTreeNode#self()
- */
- @Override
- protected MtasAVLTreeNode self() {
- return self();
- }
+ /** The parent. */
+ public MtasAVLTreeNode parent;
+ // node with start and end position
+ /**
+ * Instantiates a new mtas avl tree node.
+ *
+ * @param left
+ * the left
+ * @param right
+ * the right
+ * @param parent
+ * the parent
+ */
+ public MtasAVLTreeNode(int left, int right, MtasAVLTreeNode parent) {
+ super(left, right);
+ this.parent = parent;
}
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.tree.MtasTreeNode#self()
+ */
+ @Override
+ protected MtasAVLTreeNode self() {
+ return self();
+ }
+
+}
diff --git a/src/mtas/codec/tree/MtasRBTree.java b/src/mtas/codec/tree/MtasRBTree.java
index b05e299..0f72477 100644
--- a/src/mtas/codec/tree/MtasRBTree.java
+++ b/src/mtas/codec/tree/MtasRBTree.java
@@ -15,64 +15,87 @@ public class MtasRBTree extends MtasTree<MtasRBTreeNode> {
/**
* Instantiates a new mtas rb tree.
*
- * @param singlePoint the single point
- * @param storePrefixId the store prefix id
+ * @param singlePoint
+ * the single point
+ * @param storePrefixId
+ * the store prefix id
*/
public MtasRBTree(boolean singlePoint, boolean storePrefixId) {
super(singlePoint, storePrefixId);
index = new HashMap<>();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.tree.MtasTree#addRangeEmpty(int, int)
*/
@Override
- final protected void addRangeEmpty(int left, int right, int additionalId, long additionalRef) {
- String key = ((Integer) left).toString() + "_" + ((Integer) right).toString();
+ final protected void addRangeEmpty(int left, int right, int additionalId,
+ long additionalRef) {
+ String key = ((Integer) left).toString() + "_"
+ + ((Integer) right).toString();
if (index.containsKey(key)) {
- //do nothing (empty...)
+ // do nothing (empty...)
} else {
- root = addRange(root, left, right, additionalId, additionalRef, null, null);
+ root = addRange(root, left, right, additionalId, additionalRef, null,
+ null);
root.color = MtasRBTreeNode.BLACK;
}
}
-
- /* (non-Javadoc)
- * @see mtas.codec.tree.MtasTree#addSinglePoint(int, java.lang.Integer, java.lang.Long)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.tree.MtasTree#addSinglePoint(int, java.lang.Integer,
+ * java.lang.Long)
*/
@Override
- final protected void addSinglePoint(int position, int additionalId, long additionalRef, Integer id, Long ref) {
+ final protected void addSinglePoint(int position, int additionalId,
+ long additionalRef, Integer id, Long ref) {
addRange(position, position, additionalId, additionalRef, id, ref);
}
- /* (non-Javadoc)
- * @see mtas.codec.tree.MtasTree#addRange(int, int, java.lang.Integer, java.lang.Long)
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.tree.MtasTree#addRange(int, int, java.lang.Integer,
+ * java.lang.Long)
*/
@Override
- final protected void addRange(int left, int right, int additionalId, long additionalRef, Integer id,
- Long ref) {
- String key = ((Integer) left).toString() + "_" + ((Integer) right).toString();
+ final protected void addRange(int left, int right, int additionalId,
+ long additionalRef, Integer id, Long ref) {
+ String key = ((Integer) left).toString() + "_"
+ + ((Integer) right).toString();
if (index.containsKey(key)) {
- index.get(key).addIdAndRef(id, ref, additionalId, additionalRef);
+ index.get(key).addIdAndRef(id, ref, additionalId, additionalRef);
} else {
root = addRange(root, left, right, additionalId, additionalRef, id, ref);
root.color = MtasRBTreeNode.BLACK;
- }
+ }
}
/**
* Adds the range.
*
- * @param n the n
- * @param left the left
- * @param right the right
- * @param additionalId the additional id
- * @param id the id
- * @param ref the ref
+ * @param n
+ * the n
+ * @param left
+ * the left
+ * @param right
+ * the right
+ * @param additionalId
+ * the additional id
+ * @param additionalRef
+ * the additional ref
+ * @param id
+ * the id
+ * @param ref
+ * the ref
* @return the mtas rb tree node
*/
- private MtasRBTreeNode addRange(MtasRBTreeNode n, Integer left,
- Integer right, int additionalId, long additionalRef, Integer id, Long ref) {
+ private MtasRBTreeNode addRange(MtasRBTreeNode n, Integer left, Integer right,
+ int additionalId, long additionalRef, Integer id, Long ref) {
if (n == null) {
String key = left.toString() + "_" + right.toString();
n = new MtasRBTreeNode(left, right, MtasRBTreeNode.RED, 1);
@@ -80,10 +103,12 @@ public class MtasRBTree extends MtasTree<MtasRBTreeNode> {
index.put(key, n);
} else {
if (left <= n.left) {
- n.leftChild = addRange(n.leftChild, left, right, additionalId, additionalRef, id, ref);
+ n.leftChild = addRange(n.leftChild, left, right, additionalId,
+ additionalRef, id, ref);
updateMax(n, n.leftChild);
} else {
- n.rightChild = addRange(n.rightChild, left, right, additionalId, additionalRef, id, ref);
+ n.rightChild = addRange(n.rightChild, left, right, additionalId,
+ additionalRef, id, ref);
updateMax(n, n.rightChild);
}
if (isRed(n.rightChild) && !isRed(n.leftChild)) {
@@ -103,8 +128,10 @@ public class MtasRBTree extends MtasTree<MtasRBTreeNode> {
/**
* Update max.
*
- * @param n the n
- * @param c the c
+ * @param n
+ * the n
+ * @param c
+ * the c
*/
private void updateMax(MtasRBTreeNode n, MtasRBTreeNode c) {
if (c != null) {
@@ -118,7 +145,8 @@ public class MtasRBTree extends MtasTree<MtasRBTreeNode> {
/**
* Rotate right.
*
- * @param n the n
+ * @param n
+ * the n
* @return the mtas rb tree node
*/
private MtasRBTreeNode rotateRight(MtasRBTreeNode n) {
@@ -139,7 +167,8 @@ public class MtasRBTree extends MtasTree<MtasRBTreeNode> {
/**
* Rotate left.
*
- * @param n the n
+ * @param n
+ * the n
* @return the mtas rb tree node
*/
private MtasRBTreeNode rotateLeft(MtasRBTreeNode n) {
@@ -160,7 +189,8 @@ public class MtasRBTree extends MtasTree<MtasRBTreeNode> {
/**
* Flip colors.
*
- * @param n the n
+ * @param n
+ * the n
*/
private void flipColors(MtasRBTreeNode n) {
// n must have opposite color of its two children
@@ -175,7 +205,8 @@ public class MtasRBTree extends MtasTree<MtasRBTreeNode> {
/**
* Checks if is red.
*
- * @param n the n
+ * @param n
+ * the n
* @return true, if is red
*/
private boolean isRed(MtasRBTreeNode n) {
@@ -188,7 +219,8 @@ public class MtasRBTree extends MtasTree<MtasRBTreeNode> {
/**
* Size.
*
- * @param n the n
+ * @param n
+ * the n
* @return the int
*/
private int size(MtasRBTreeNode n) {
@@ -196,18 +228,19 @@ public class MtasRBTree extends MtasTree<MtasRBTreeNode> {
return 0;
return n.n;
}
-
+
/**
* Sets the max.
*
- * @param n the new max
+ * @param n
+ * the new max
*/
private void setMax(MtasRBTreeNode n) {
n.max = n.right;
- if(n.leftChild!=null) {
+ if (n.leftChild != null) {
n.max = Math.max(n.max, n.leftChild.max);
}
- if(n.rightChild!=null) {
+ if (n.rightChild != null) {
n.max = Math.max(n.max, n.rightChild.max);
}
}
diff --git a/src/mtas/codec/tree/MtasRBTreeNode.java b/src/mtas/codec/tree/MtasRBTreeNode.java
index 9a8adc4..fae9360 100644
--- a/src/mtas/codec/tree/MtasRBTreeNode.java
+++ b/src/mtas/codec/tree/MtasRBTreeNode.java
@@ -7,39 +7,45 @@ import mtas.codec.tree.MtasTreeNode;
*/
public class MtasRBTreeNode extends MtasTreeNode<MtasRBTreeNode> {
- /** The Constant BLACK. */
- static final int BLACK = 1;
-
- /** The Constant RED. */
- static final int RED = 0;
-
- /** The color. */
- public int color;
-
- /** The n. */
- public int n;
-
- // node with start and end position
- /**
- * Instantiates a new mtas rb tree node.
- *
- * @param left the left
- * @param right the right
- * @param color the color
- * @param n the n
- */
- public MtasRBTreeNode(int left, int right, int color, int n) {
- super(left, right);
- this.color = color;
- this.n = n;
- }
-
- /* (non-Javadoc)
- * @see mtas.codec.tree.MtasTreeNode#self()
- */
- @Override
- protected MtasRBTreeNode self() {
- return self();
- }
+ /** The Constant BLACK. */
+ static final int BLACK = 1;
+ /** The Constant RED. */
+ static final int RED = 0;
+
+ /** The color. */
+ public int color;
+
+ /** The n. */
+ public int n;
+
+ // node with start and end position
+ /**
+ * Instantiates a new mtas rb tree node.
+ *
+ * @param left
+ * the left
+ * @param right
+ * the right
+ * @param color
+ * the color
+ * @param n
+ * the n
+ */
+ public MtasRBTreeNode(int left, int right, int color, int n) {
+ super(left, right);
+ this.color = color;
+ this.n = n;
}
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.tree.MtasTreeNode#self()
+ */
+ @Override
+ protected MtasRBTreeNode self() {
+ return self();
+ }
+
+}
diff --git a/src/mtas/codec/tree/MtasTree.java b/src/mtas/codec/tree/MtasTree.java
index c992593..8ecf0bf 100644
--- a/src/mtas/codec/tree/MtasTree.java
+++ b/src/mtas/codec/tree/MtasTree.java
@@ -1,7 +1,6 @@
package mtas.codec.tree;
import java.util.TreeMap;
-import java.util.TreeSet;
import java.io.IOException;
import java.util.Map.Entry;
@@ -11,170 +10,214 @@ import mtas.analysis.token.MtasToken;
/**
* The Class MtasTree.
*
- * @param <N> the number type
+ * @param <N>
+ * the number type
*/
abstract public class MtasTree<N extends MtasTreeNode<N>> {
-
+
/** The Constant SINGLE_POSITION_TREE. */
final public static byte SINGLE_POSITION_TREE = 1;
-
+
/** The Constant STORE_ADDITIONAL_ID. */
final public static byte STORE_ADDITIONAL_ID = 2;
/** The root. */
protected N root;
-
+
/** The closed. */
private Boolean closed;
-
+
/** The single point. */
- protected Boolean singlePoint;
-
- /** The store prefix id. */
- protected Boolean storePrefixAndTermRef;
+ protected Boolean singlePoint;
+
+ /** The store prefix and term ref. */
+ protected Boolean storePrefixAndTermRef;
/**
* Instantiates a new mtas tree.
*
- * @param singlePoint the single point
- * @param storePrefixId the store prefix id
+ * @param singlePoint
+ * the single point
+ * @param storePrefixAndTermRef
+ * the store prefix and term ref
*/
public MtasTree(boolean singlePoint, boolean storePrefixAndTermRef) {
root = null;
closed = false;
- this.singlePoint = singlePoint;
- this.storePrefixAndTermRef = storePrefixAndTermRef;
+ this.singlePoint = singlePoint;
+ this.storePrefixAndTermRef = storePrefixAndTermRef;
}
-
+
/**
* Adds the id from doc.
*
- * @param docId the doc id
- * @param reference the reference
+ * @param docId
+ * the doc id
+ * @param reference
+ * the reference
*/
- final public void addIdFromDoc(Integer docId, Long reference) {
- if(!closed && (docId!=null)) {
+ final public void addIdFromDoc(Integer docId, Long reference) {
+ if (!closed && (docId != null)) {
addSinglePoint(docId, 0, 0, docId, reference);
}
}
-
+
/**
* Adds the parent from token.
*
- * @param <T> the generic type
- * @param token the token
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param <T>
+ * the generic type
+ * @param token
+ * the token
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- final public <T> void addParentFromToken(MtasToken<T> token) throws IOException {
- if(!closed && (token!=null)) {
- if(token.checkParentId()) {
- addSinglePoint(token.getParentId(), token.getPrefixId(), token.getTermRef(), token.getId(), token.getTokenRef());
+ final public <T> void addParentFromToken(MtasToken<T> token)
+ throws IOException {
+ if (!closed && (token != null)) {
+ if (token.checkParentId()) {
+ addSinglePoint(token.getParentId(), token.getPrefixId(),
+ token.getTermRef(), token.getId(), token.getTokenRef());
}
}
}
-
+
/**
* Adds the position and object from token.
*
- * @param <T> the generic type
- * @param token the token
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param <T>
+ * the generic type
+ * @param token
+ * the token
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- final public <T> void addPositionAndObjectFromToken(MtasToken<T> token) throws IOException {
+ final public <T> void addPositionAndObjectFromToken(MtasToken<T> token)
+ throws IOException {
addPositionFromToken(token, token.getTokenRef());
}
-
-// final public <T> void addPositionAndTermFromToken(MtasToken<T> token) {
-// addPositionFromToken(token, token.getTermRef());
-// }
-
+
+ // final public <T> void addPositionAndTermFromToken(MtasToken<T> token) {
+ // addPositionFromToken(token, token.getTermRef());
+ // }
+
/**
- * Adds the position from token.
- *
- * @param <T> the generic type
- * @param token the token
- * @param ref the ref
- * @throws IOException Signals that an I/O exception has occurred.
- */
- final private <T> void addPositionFromToken(MtasToken<T> token, Long ref) throws IOException {
- int prefixId = storePrefixAndTermRef?token.getPrefixId():0;
- if(!closed && (token!=null)) {
- if(token.checkPositionType(MtasPosition.POSITION_SINGLE)) {
- addSinglePoint(token.getPositionStart(), prefixId, token.getTermRef(), token.getId(), ref);
- } else if(token.checkPositionType(MtasPosition.POSITION_RANGE)) {
- addRange(token.getPositionStart(), token.getPositionEnd(), prefixId, token.getTermRef(), token.getId(), ref);
- } else if(token.checkPositionType(MtasPosition.POSITION_SET)) {
- //split set into minimum number of single points and ranges
- TreeMap<Integer,Integer> list = new TreeMap<Integer,Integer>();
- TreeSet<Integer> positions = token.getPositions();
+ * Adds the position from token.
+ *
+ * @param <T>
+ * the generic type
+ * @param token
+ * the token
+ * @param ref
+ * the ref
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ final private <T> void addPositionFromToken(MtasToken<T> token, Long ref)
+ throws IOException {
+ int prefixId = storePrefixAndTermRef ? token.getPrefixId() : 0;
+ if (!closed && (token != null)) {
+ if (token.checkPositionType(MtasPosition.POSITION_SINGLE)) {
+ addSinglePoint(token.getPositionStart(), prefixId, token.getTermRef(),
+ token.getId(), ref);
+ } else if (token.checkPositionType(MtasPosition.POSITION_RANGE)) {
+ addRange(token.getPositionStart(), token.getPositionEnd(), prefixId,
+ token.getTermRef(), token.getId(), ref);
+ } else if (token.checkPositionType(MtasPosition.POSITION_SET)) {
+ // split set into minimum number of single points and ranges
+ TreeMap<Integer, Integer> list = new TreeMap<Integer, Integer>();
+ int[] positions = token.getPositions();
Integer lastPoint = null;
- Integer startPoint = null;
+ Integer startPoint = null;
for (int position : positions) {
- if(lastPoint==null) {
+ if (lastPoint == null) {
startPoint = position;
lastPoint = position;
- } else if((position-lastPoint)!=1) {
+ } else if ((position - lastPoint) != 1) {
list.put(startPoint, lastPoint);
- startPoint = position;
+ startPoint = position;
}
lastPoint = position;
}
- if(lastPoint!=null) {
+ if (lastPoint != null) {
list.put(startPoint, lastPoint);
}
for (Entry<Integer, Integer> entry : list.entrySet()) {
- if(entry.getKey().equals(entry.getValue())) {
- addSinglePoint(entry.getKey(), prefixId, token.getTermRef(), token.getId(), ref);
+ if (entry.getKey().equals(entry.getValue())) {
+ addSinglePoint(entry.getKey(), prefixId, token.getTermRef(),
+ token.getId(), ref);
} else {
- addRange(entry.getKey(), entry.getValue(), prefixId, token.getTermRef(), token.getId(), ref);
+ addRange(entry.getKey(), entry.getValue(), prefixId,
+ token.getTermRef(), token.getId(), ref);
}
}
}
}
}
-
+
/**
* Close.
*
* @return the n
*/
final public N close() {
- if(root==null) {
- addRangeEmpty(0,0,0,0);
+ if (root == null) {
+ addRangeEmpty(0, 0, 0, 0);
}
closed = true;
return root;
- }
+ }
/**
* Adds the single point.
*
- * @param position the position
- * @param additionalId the additional id
- * @param id the id
- * @param ref the ref
+ * @param position
+ * the position
+ * @param additionalId
+ * the additional id
+ * @param additionalRef
+ * the additional ref
+ * @param id
+ * the id
+ * @param ref
+ * the ref
*/
- abstract protected void addSinglePoint(int position, int additionalId, long additionalRef, Integer id, Long ref);
-
+ abstract protected void addSinglePoint(int position, int additionalId,
+ long additionalRef, Integer id, Long ref);
+
/**
* Adds the range.
*
- * @param left the left
- * @param right the right
- * @param additionalId the additional id
- * @param id the id
- * @param ref the ref
+ * @param left
+ * the left
+ * @param right
+ * the right
+ * @param additionalId
+ * the additional id
+ * @param additionalRef
+ * the additional ref
+ * @param id
+ * the id
+ * @param ref
+ * the ref
*/
- abstract protected void addRange(int left, int right, int additionalId, long additionalRef, Integer id, Long ref);
-
+ abstract protected void addRange(int left, int right, int additionalId,
+ long additionalRef, Integer id, Long ref);
+
/**
* Adds the range empty.
*
- * @param left the left
- * @param right the right
- * @param additionalId the additional id
+ * @param left
+ * the left
+ * @param right
+ * the right
+ * @param additionalId
+ * the additional id
+ * @param additionalRef
+ * the additional ref
*/
- abstract protected void addRangeEmpty(int left, int right, int additionalId, long additionalRef);
+ abstract protected void addRangeEmpty(int left, int right, int additionalId,
+ long additionalRef);
/**
* Checks if is single point.
@@ -183,17 +226,17 @@ abstract public class MtasTree<N extends MtasTreeNode<N>> {
*/
final public boolean isSinglePoint() {
return singlePoint;
- }
-
+ }
+
/**
- * Checks if is store prefix id.
+ * Checks if is store prefix and term ref.
*
- * @return true, if is store prefix id
+ * @return true, if is store prefix and term ref
*/
final public boolean isStorePrefixAndTermRef() {
- return storePrefixAndTermRef;
+ return storePrefixAndTermRef;
}
-
+
/**
* Prints the balance.
*/
@@ -204,19 +247,23 @@ abstract public class MtasTree<N extends MtasTreeNode<N>> {
/**
* Prints the balance.
*
- * @param p the p
- * @param n the n
+ * @param p
+ * the p
+ * @param n
+ * the n
*/
- final private void printBalance(Integer p, N n) {
- if(n!=null) {
- printBalance((p+1), n.leftChild);
- System.out.print(String.format("%"+(3*p)+"s", ""));
- if(n.left==n.right) {
- System.out.println("["+n.left+"] ("+n.max+") : "+n.ids.size()+" tokens");
+ final private void printBalance(Integer p, N n) {
+ if (n != null) {
+ printBalance((p + 1), n.leftChild);
+ System.out.print(String.format("%" + (3 * p) + "s", ""));
+ if (n.left == n.right) {
+ System.out.println(
+ "[" + n.left + "] (" + n.max + ") : " + n.ids.size() + " tokens");
} else {
- System.out.println("["+n.left+"-"+n.right+"] ("+n.max+") : "+n.ids.size()+" tokens");
+ System.out.println("[" + n.left + "-" + n.right + "] (" + n.max + ") : "
+ + n.ids.size() + " tokens");
}
- printBalance((p+1), n.rightChild);
+ printBalance((p + 1), n.rightChild);
}
}
diff --git a/src/mtas/codec/tree/MtasTreeNode.java b/src/mtas/codec/tree/MtasTreeNode.java
index 01c8368..fc6df51 100644
--- a/src/mtas/codec/tree/MtasTreeNode.java
+++ b/src/mtas/codec/tree/MtasTreeNode.java
@@ -53,7 +53,7 @@ abstract public class MtasTreeNode<N extends MtasTreeNode<N>> {
// add id to node
/**
- * Adds the id.
+ * Adds the id and ref.
*
* @param id
* the id
@@ -61,6 +61,8 @@ abstract public class MtasTreeNode<N extends MtasTreeNode<N>> {
* the ref
* @param additionalId
* the additional id
+ * @param additionalRef
+ * the additional ref
*/
final public void addIdAndRef(Integer id, Long ref, int additionalId,
long additionalRef) {
@@ -71,6 +73,4 @@ abstract public class MtasTreeNode<N extends MtasTreeNode<N>> {
}
}
-
-
}
diff --git a/src/mtas/codec/tree/MtasTreeNodeId.java b/src/mtas/codec/tree/MtasTreeNodeId.java
index 42c49d4..6198422 100644
--- a/src/mtas/codec/tree/MtasTreeNodeId.java
+++ b/src/mtas/codec/tree/MtasTreeNodeId.java
@@ -1,5 +1,8 @@
package mtas.codec.tree;
+/**
+ * The Class MtasTreeNodeId.
+ */
public class MtasTreeNodeId implements Comparable<MtasTreeNodeId> {
/** The ref. */
@@ -18,6 +21,8 @@ public class MtasTreeNodeId implements Comparable<MtasTreeNodeId> {
* the ref
* @param additionalId
* the additional id
+ * @param additionalRef
+ * the additional ref
*/
public MtasTreeNodeId(long ref, int additionalId, long additionalRef) {
this.ref = ref;
diff --git a/src/mtas/codec/util/CodecCollector.java b/src/mtas/codec/util/CodecCollector.java
index dfe15b5..3142dfd 100644
--- a/src/mtas/codec/util/CodecCollector.java
+++ b/src/mtas/codec/util/CodecCollector.java
@@ -12,6 +12,7 @@ import java.util.List;
import java.util.ListIterator;
import java.util.TreeMap;
import java.util.regex.Pattern;
+
import mtas.analysis.token.MtasToken;
import mtas.codec.MtasCodecPostingsFormat;
import mtas.codec.tree.IntervalTreeNodeData;
@@ -30,9 +31,12 @@ import mtas.codec.util.CodecComponent.KwicToken;
import mtas.codec.util.CodecComponent.ListHit;
import mtas.codec.util.CodecComponent.ListToken;
import mtas.codec.util.CodecComponent.Match;
-import mtas.codec.util.DataCollector.MtasDataCollector;
+import mtas.codec.util.CodecComponent.SubComponentFunction;
import mtas.codec.util.CodecInfo.IndexDoc;
import mtas.codec.util.CodecSearchTree.MtasTreeHit;
+import mtas.codec.util.collector.MtasDataCollector;
+import mtas.parser.function.ParseException;
+import mtas.parser.function.util.MtasFunctionParserFunction;
import org.apache.commons.lang.ArrayUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.FieldInfo;
@@ -49,29 +53,44 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanWeight;
import org.apache.lucene.search.spans.Spans;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LegacyNumericUtils;
+import org.apache.lucene.util.automaton.CompiledAutomaton;
/**
* The Class CodecCollector.
*/
+@SuppressWarnings("deprecation")
public class CodecCollector {
/**
* Collect.
*
- * @param field the field
- * @param searcher the searcher
- * @param reader the reader
- * @param rawReader the raw reader
- * @param fullDocList the full doc list
- * @param fullDocSet the full doc set
- * @param fieldInfo the field info
- * @param spansQueryWeight the spans query weight
- * @throws IllegalAccessException the illegal access exception
- * @throws IllegalArgumentException the illegal argument exception
- * @throws InvocationTargetException the invocation target exception
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param field
+ * the field
+ * @param searcher
+ * the searcher
+ * @param reader
+ * the reader
+ * @param rawReader
+ * the raw reader
+ * @param fullDocList
+ * the full doc list
+ * @param fullDocSet
+ * the full doc set
+ * @param fieldInfo
+ * the field info
+ * @param spansQueryWeight
+ * the spans query weight
+ * @throws IllegalAccessException
+ * the illegal access exception
+ * @throws IllegalArgumentException
+ * the illegal argument exception
+ * @throws InvocationTargetException
+ * the invocation target exception
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static void collect(String field, IndexSearcher searcher,
IndexReader reader, IndexReader rawReader, ArrayList<Integer> fullDocList,
@@ -95,11 +114,15 @@ public class CodecCollector {
docSets.put(lrc.ord, docSet);
Iterator<Integer> docSetIterator = fullDocSet.iterator();
Integer docSetId = null;
+ Bits liveDocs = lrc.reader().getLiveDocs();
while (docSetIterator.hasNext()) {
docSetId = docSetIterator.next();
if ((docSetId >= lrc.docBase)
&& (docSetId < lrc.docBase + lrc.reader().maxDoc())) {
- docSet.add(docSetId);
+ // just to make sure to ignore deleted documents
+ if (liveDocs == null || liveDocs.get((docSetId - lrc.docBase))) {
+ docSet.add(docSetId);
+ }
}
}
Collections.sort(docSet);
@@ -135,10 +158,8 @@ public class CodecCollector {
boolean needPositions = false;
if (fieldInfo.termVectorList.size() > 0) {
for (ComponentTermVector ctv : fieldInfo.termVectorList) {
- needPositions = !needPositions
- ? (ctv.functionParser != null
- ? ctv.functionParser.needPositions() : needPositions)
- : needPositions;
+ needPositions = !needPositions ? (ctv.functions != null
+ ? ctv.functionNeedPositions() : needPositions) : needPositions;
}
}
HashMap<Integer, Integer> positionsData = null;
@@ -166,17 +187,28 @@ public class CodecCollector {
/**
* Collect spans positions and tokens.
*
- * @param spansQueryWeight the spans query weight
- * @param searcher the searcher
- * @param mtasCodecInfo the mtas codec info
- * @param r the r
- * @param lrc the lrc
- * @param field the field
- * @param t the t
- * @param docSet the doc set
- * @param docList the doc list
- * @param fieldInfo the field info
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param spansQueryWeight
+ * the spans query weight
+ * @param searcher
+ * the searcher
+ * @param mtasCodecInfo
+ * the mtas codec info
+ * @param r
+ * the r
+ * @param lrc
+ * the lrc
+ * @param field
+ * the field
+ * @param t
+ * the t
+ * @param docSet
+ * the doc set
+ * @param docList
+ * the doc list
+ * @param fieldInfo
+ * the field info
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void collectSpansPositionsAndTokens(
HashMap<SpanQuery, SpanWeight> spansQueryWeight, IndexSearcher searcher,
@@ -206,10 +238,9 @@ public class CodecCollector {
}
if (fieldInfo.termVectorList.size() > 0) {
for (ComponentTermVector ctv : fieldInfo.termVectorList) {
- needPositions = !needPositions
- ? (ctv.functionParser == null ? ctv.defaultParser.needPositions()
- : ctv.functionParser.needPositions())
- : needPositions;
+ needPositions = !needPositions ? (ctv.functions == null
+ ? ctv.subComponentFunction.parserFunction.needPositions()
+ : ctv.functionNeedPositions()) : needPositions;
}
}
@@ -223,11 +254,14 @@ public class CodecCollector {
// spans
if (fieldInfo.statsSpanList.size() > 0) {
for (ComponentSpan cs : fieldInfo.statsSpanList) {
- needPositions = (!needPositions) ? cs.functionParser.needPositions()
+ needPositions = (!needPositions) ? cs.parser.needPositions()
+ : needPositions;
+ needPositions = (!needPositions) ? cs.functionNeedPositions()
: needPositions;
- needSpans = (!needSpans) ? cs.functionParser.needArgumentsNumber() > 0
+ needSpans = (!needSpans) ? cs.parser.needArgumentsNumber() > 0
: needSpans;
- Integer[] arguments = cs.functionParser.needArguments();
+ HashSet<Integer> arguments = cs.parser.needArgument();
+ arguments.addAll(cs.functionNeedArguments());
for (int a : arguments) {
if (cs.queries.length > a) {
SpanQuery q = cs.queries[a];
@@ -282,13 +316,14 @@ public class CodecCollector {
// facet
if (fieldInfo.facetList.size() > 0) {
for (ComponentFacet cf : fieldInfo.facetList) {
+ needPositions = !needPositions ? cf.baseParserNeedPositions()
+ : needPositions;
+ needPositions = !needPositions ? cf.functionNeedPositions()
+ : needPositions;
for (int i = 0; i < cf.baseFields.length; i++) {
- needPositions = (!needPositions)
- ? cf.baseFunctionParsers[i].needPositions() : needPositions;
- needSpans = (!needSpans)
- ? cf.baseFunctionParsers[i].needArgumentsNumber() > 0
+ needSpans = !needSpans ? cf.baseParsers[i].needArgumentsNumber() > 0
: needSpans;
- Integer[] arguments = cf.baseFunctionParsers[i].needArguments();
+ HashSet<Integer> arguments = cf.baseParsers[i].needArgument();
for (int a : arguments) {
if (cf.spanQueries.length > a) {
SpanQuery q = cf.spanQueries[a];
@@ -297,6 +332,19 @@ public class CodecCollector {
}
}
}
+ for (MtasFunctionParserFunction function : cf.baseFunctionParserFunctions[i]) {
+ needSpans = !needSpans ? function.needArgumentsNumber() > 0
+ : needSpans;
+ arguments = function.needArgument();
+ for (int a : arguments) {
+ if (cf.spanQueries.length > a) {
+ SpanQuery q = cf.spanQueries[a];
+ if (!spansNumberData.containsKey(q)) {
+ spansNumberData.put(q, new HashMap<Integer, Integer>());
+ }
+ }
+ }
+ }
if (!facetData.containsKey(cf.baseFields[i])) {
facetData.put(cf.baseFields[i], new TreeMap<String, int[]>());
facetDataType.put(cf.baseFields[i], cf.baseFieldTypes[i]);
@@ -307,7 +355,9 @@ public class CodecCollector {
// termvector
if (fieldInfo.termVectorList.size() > 0) {
for (ComponentTermVector ctv : fieldInfo.termVectorList) {
- if (ctv.functionParser.needPositions()) {
+ if ((ctv.subComponentFunction.parserFunction != null
+ && ctv.subComponentFunction.parserFunction.needPositions())
+ || (ctv.functions != null && ctv.functionNeedPositions())) {
needPositions = true;
}
}
@@ -555,10 +605,14 @@ public class CodecCollector {
/**
* Collect prefixes.
*
- * @param fieldInfos the field infos
- * @param field the field
- * @param fieldInfo the field info
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param fieldInfos
+ * the field infos
+ * @param field
+ * the field
+ * @param fieldInfo
+ * the field info
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void collectPrefixes(FieldInfos fieldInfos, String field,
ComponentField fieldInfo) throws IOException {
@@ -569,6 +623,8 @@ public class CodecCollector {
MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_SINGLE_POSITION);
String multiplePositionPrefixes = fi.getAttribute(
MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_MULTIPLE_POSITION);
+ String setPositionPrefixes = fi.getAttribute(
+ MtasCodecPostingsFormat.MTAS_FIELDINFO_ATTRIBUTE_PREFIX_SET_POSITION);
if (singlePositionPrefixes != null) {
String[] prefixes = singlePositionPrefixes
.split(Pattern.quote(MtasToken.DELIMITER));
@@ -583,6 +639,13 @@ public class CodecCollector {
fieldInfo.prefix.addMultiplePosition(prefixes[i]);
}
}
+ if (setPositionPrefixes != null) {
+ String[] prefixes = setPositionPrefixes
+ .split(Pattern.quote(MtasToken.DELIMITER));
+ for (int i = 0; i < prefixes.length; i++) {
+ fieldInfo.prefix.addSetPosition(prefixes[i]);
+ }
+ }
}
}
}
@@ -590,14 +653,21 @@ public class CodecCollector {
/**
* Compute positions.
*
- * @param mtasCodecInfo the mtas codec info
- * @param r the r
- * @param lrc the lrc
- * @param field the field
- * @param t the t
- * @param docSet the doc set
+ * @param mtasCodecInfo
+ * the mtas codec info
+ * @param r
+ * the r
+ * @param lrc
+ * the lrc
+ * @param field
+ * the field
+ * @param t
+ * the t
+ * @param docSet
+ * the doc set
* @return the hash map
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static HashMap<Integer, Integer> computePositions(
CodecInfo mtasCodecInfo, LeafReader r, LeafReaderContext lrc,
@@ -633,9 +703,12 @@ public class CodecCollector {
/**
* Compute arguments.
*
- * @param spansNumberData the spans number data
- * @param queries the queries
- * @param docSet the doc set
+ * @param spansNumberData
+ * the spans number data
+ * @param queries
+ * the queries
+ * @param docSet
+ * the doc set
* @return the hash map
*/
private static HashMap<Integer, long[]> computeArguments(
@@ -666,8 +739,10 @@ public class CodecCollector {
/**
* Intersected doc list.
*
- * @param facetDocList the facet doc list
- * @param docSet the doc set
+ * @param facetDocList
+ * the facet doc list
+ * @param docSet
+ * the doc set
* @return the integer[]
*/
private static Integer[] intersectedDocList(int[] facetDocList,
@@ -698,10 +773,14 @@ public class CodecCollector {
/**
* Creates the positions.
*
- * @param statsPositionList the stats position list
- * @param positionsData the positions data
- * @param docSet the doc set
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param statsPositionList
+ * the stats position list
+ * @param positionsData
+ * the positions data
+ * @param docSet
+ * the doc set
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void createPositions(List<ComponentPosition> statsPositionList,
HashMap<Integer, Integer> positionsData, List<Integer> docSet)
@@ -734,10 +813,14 @@ public class CodecCollector {
/**
* Creates the tokens.
*
- * @param statsTokenList the stats token list
- * @param tokensData the tokens data
- * @param docSet the doc set
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param statsTokenList
+ * the stats token list
+ * @param tokensData
+ * the tokens data
+ * @param docSet
+ * the doc set
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void createTokens(List<ComponentToken> statsTokenList,
HashMap<Integer, Integer> tokensData, List<Integer> docSet)
@@ -771,11 +854,16 @@ public class CodecCollector {
/**
* Creates the stats.
*
- * @param statsSpanList the stats span list
- * @param positionsData the positions data
- * @param spansNumberData the spans number data
- * @param docSet the doc set
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param statsSpanList
+ * the stats span list
+ * @param positionsData
+ * the positions data
+ * @param spansNumberData
+ * the spans number data
+ * @param docSet
+ * the doc set
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void createStats(List<ComponentSpan> statsSpanList,
HashMap<Integer, Integer> positionsData,
@@ -783,28 +871,27 @@ public class CodecCollector {
Integer[] docSet) throws IOException {
if (statsSpanList != null) {
for (ComponentSpan span : statsSpanList) {
- if (span.functionParser.needArgumentsNumber() > span.queries.length) {
+ if (span.parser.needArgumentsNumber() > span.queries.length) {
throw new IOException(
- "function " + span.functionParser + " expects (at least) "
- + span.functionParser.needArgumentsNumber() + " queries");
+ "function " + span.parser + " expects (at least) "
+ + span.parser.needArgumentsNumber() + " queries");
}
// collect
HashMap<Integer, long[]> args = computeArguments(spansNumberData,
span.queries, docSet);
- if (span.dataType.equals(CodecUtil.DATA_TYPE_LONG)
- || span.dataType.equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ if (span.dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
// try to call functionParser as little as possible
if (span.statsType.equals(CodecUtil.STATS_BASIC)
- && span.functionParser.sumRule() && (span.minimumLong == null)
- && (span.maximumLong == null)) {
+ && (span.minimumLong == null) && (span.maximumLong == null)
+ && (span.functions == null || (span.functionSumRule()
+ && !span.functionNeedPositions()))) {
// initialise
- int length = span.functionParser.needArgumentsNumber();
+ int length = span.parser.needArgumentsNumber();
long[] valueSum = new long[length];
long valuePositions = 0;
// collect
if (docSet.length > 0) {
long[] tmpArgs;
- span.dataCollector.initNewList(1);
for (int docId : docSet) {
tmpArgs = args.get(docId);
valuePositions += (positionsData == null) ? 0
@@ -815,74 +902,101 @@ public class CodecCollector {
}
}
}
- if (span.dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
- long value;
- try {
- value = span.functionParser.getValueLong(valueSum,
- valuePositions);
- span.dataCollector.add(value, docSet.length);
- } catch (IOException e) {
- span.dataCollector.error(e.getMessage());
- }
- } else {
- double value;
- try {
- value = span.functionParser.getValueDouble(valueSum,
- valuePositions);
- span.dataCollector.add(value, docSet.length);
- } catch (IOException e) {
- span.dataCollector.error(e.getMessage());
+ long valueLong;
+ span.dataCollector.initNewList(1);
+ try {
+ valueLong = span.parser.getValueLong(valueSum, valuePositions);
+ span.dataCollector.add(valueLong, docSet.length);
+ } catch (IOException e) {
+ span.dataCollector.error(e.getMessage());
+ }
+ if (span.functions != null) {
+ for (SubComponentFunction function : span.functions) {
+ function.dataCollector.initNewList(1);
+ if (function.dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
+ try {
+ valueLong = function.parserFunction.getValueLong(valueSum,
+ valuePositions);
+ function.dataCollector.add(valueLong, docSet.length);
+ } catch (IOException e) {
+ function.dataCollector.error(e.getMessage());
+ }
+ } else if (function.dataType
+ .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ try {
+ double valueDouble = function.parserFunction
+ .getValueDouble(valueSum, valuePositions);
+ function.dataCollector.add(valueDouble, docSet.length);
+ } catch (IOException e) {
+ function.dataCollector.error(e.getMessage());
+ }
+ } else {
+ throw new IOException(
+ "can't handle function dataType " + function.dataType);
+ }
+ function.dataCollector.closeNewList();
}
}
span.dataCollector.closeNewList();
}
- } else if (span.dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
+ } else {
// collect
if (docSet.length > 0) {
int number = 0, positions;
- long value;
+ long valueLong;
+ double valueDouble;
long values[] = new long[docSet.length];
+ long functionValuesLong[][] = null;
+ double functionValuesDouble[][] = null;
span.dataCollector.initNewList(1);
- for (int docId : docSet) {
- positions = (positionsData == null) ? 0
- : positionsData.get(docId);
- try {
- value = span.functionParser.getValueLong(args.get(docId),
- positions);
- if (((span.minimumLong == null)
- || (value >= span.minimumLong))
- && ((span.maximumLong == null)
- || (value <= span.maximumLong))) {
- values[number] = value;
- number++;
+ if (span.functions != null) {
+ functionValuesLong = new long[span.functions.size()][];
+ functionValuesDouble = new double[span.functions.size()][];
+ for (int i = 0; i < span.functions.size(); i++) {
+ SubComponentFunction function = span.functions.get(i);
+ if (function.dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
+ functionValuesLong[i] = new long[docSet.length];
+ functionValuesDouble[i] = null;
+ } else if (function.dataType
+ .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ functionValuesLong[i] = null;
+ functionValuesDouble[i] = new double[docSet.length];
}
- } catch (IOException e) {
- span.dataCollector.error(e.getMessage());
+ function.dataCollector.initNewList(1);
}
}
- if (number > 0) {
- span.dataCollector.add(values, number);
- }
- span.dataCollector.closeNewList();
- }
- } else if (span.dataType.equals(CodecUtil.DATA_TYPE_DOUBLE)) {
- // collect
- if (docSet.length > 0) {
- int number = 0, positions;
- double value;
- double values[] = new double[docSet.length];
- span.dataCollector.initNewList(1);
for (int docId : docSet) {
positions = (positionsData == null) ? 0
- : positionsData.get(docId);
+ : (positionsData.get(docId) == null ? 0
+ : positionsData.get(docId));
try {
- value = span.functionParser.getValueDouble(args.get(docId),
+ valueLong = span.parser.getValueLong(args.get(docId),
positions);
if (((span.minimumLong == null)
- || (value >= span.minimumLong))
+ || (valueLong >= span.minimumLong))
&& ((span.maximumLong == null)
- || (value <= span.maximumLong))) {
- values[number] = value;
+ || (valueLong <= span.maximumLong))) {
+ values[number] = valueLong;
+ if (span.functions != null) {
+ for (int i = 0; i < span.functions.size(); i++) {
+ SubComponentFunction function = span.functions.get(i);
+ try {
+ if (function.dataType
+ .equals(CodecUtil.DATA_TYPE_LONG)) {
+ valueLong = function.parserFunction
+ .getValueLong(args.get(docId), positions);
+ functionValuesLong[i][number] = valueLong;
+ } else if (function.dataType
+ .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ valueDouble = function.parserFunction
+ .getValueDouble(args.get(docId), positions);
+ functionValuesDouble[i][number] = valueDouble;
+ }
+ } catch (IOException e) {
+ function.dataCollector.error(e.getMessage());
+ }
+ }
+ }
number++;
}
} catch (IOException e) {
@@ -891,12 +1005,29 @@ public class CodecCollector {
}
if (number > 0) {
span.dataCollector.add(values, number);
+ if (span.functions != null) {
+ for (int i = 0; i < span.functions.size(); i++) {
+ SubComponentFunction function = span.functions.get(i);
+ if (function.dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
+ function.dataCollector.add(functionValuesLong[i], number);
+ } else if (function.dataType
+ .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ function.dataCollector.add(functionValuesDouble[i],
+ number);
+ }
+ }
+ }
}
span.dataCollector.closeNewList();
+ if (span.functions != null) {
+ for (SubComponentFunction function : span.functions) {
+ function.dataCollector.closeNewList();
+ }
+ }
}
}
} else {
- throw new IOException("unknown dataType " + span.dataType);
+ throw new IOException("unexpected dataType " + span.dataType);
}
}
}
@@ -905,23 +1036,32 @@ public class CodecCollector {
/**
* Creates the list.
*
- * @param listList the list list
- * @param spansNumberData the spans number data
- * @param spansMatchData the spans match data
- * @param docSet the doc set
- * @param field the field
- * @param docBase the doc base
- * @param uniqueKeyField the unique key field
- * @param mtasCodecInfo the mtas codec info
- * @param searcher the searcher
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param listList
+ * the list list
+ * @param spansNumberData
+ * the spans number data
+ * @param spansMatchData
+ * the spans match data
+ * @param docSet
+ * the doc set
+ * @param field
+ * the field
+ * @param docBase
+ * the doc base
+ * @param uniqueKeyField
+ * the unique key field
+ * @param mtasCodecInfo
+ * the mtas codec info
+ * @param searcher
+ * the searcher
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void createList(List<ComponentList> listList,
HashMap<SpanQuery, HashMap<Integer, Integer>> spansNumberData,
HashMap<SpanQuery, HashMap<Integer, ArrayList<Match>>> spansMatchData,
List<Integer> docSet, String field, int docBase, String uniqueKeyField,
CodecInfo mtasCodecInfo, IndexSearcher searcher) throws IOException {
- // System.out.println(Thread.currentThread().getId()+" createList");
if (listList != null) {
for (ComponentList list : listList) {
// collect not only stats
@@ -1078,13 +1218,20 @@ public class CodecCollector {
/**
* Creates the group.
*
- * @param groupList the group list
- * @param spansMatchData the spans match data
- * @param docSet the doc set
- * @param field the field
- * @param docBase the doc base
- * @param mtasCodecInfo the mtas codec info
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param groupList
+ * the group list
+ * @param spansMatchData
+ * the spans match data
+ * @param docSet
+ * the doc set
+ * @param field
+ * the field
+ * @param docBase
+ * the doc base
+ * @param mtasCodecInfo
+ * the mtas codec info
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void createGroup(List<ComponentGroup> groupList,
HashMap<SpanQuery, HashMap<Integer, ArrayList<Match>>> spansMatchData,
@@ -1171,13 +1318,18 @@ public class CodecCollector {
m.startPosition, m.endPosition - 1));
}
// if(1>2) {
- // for(IntervalTreeNodeData positionHit : positionsHits) {
- // ArrayList<MtasTreeHit<String>> list = mtasCodecInfo
+ // for(IntervalTreeNodeData positionHit :
+ // positionsHits) {
+ // ArrayList<MtasTreeHit<String>> list =
+ // mtasCodecInfo
// .getPositionedTermsByPrefixesAndPositionRange(field,
- // (docId - docBase), group.prefixes, positionHit.start,
+ // (docId - docBase), group.prefixes,
+ // positionHit.start,
// positionHit.end);
- // GroupHit hit = new GroupHit(list, positionHit.start,
- // positionHit.end, positionHit.hitStart, positionHit.hitEnd,
+ // GroupHit hit = new GroupHit(list,
+ // positionHit.start,
+ // positionHit.end, positionHit.hitStart,
+ // positionHit.hitEnd,
// group);
// String key = hit.toString();
// if (key != null) {
@@ -1190,7 +1342,8 @@ public class CodecCollector {
// }
// for (String key : occurences.keySet()) {
// group.dataCollector.add(new String[] { key },
- // ArrayUtils.toPrimitive(new Long[] { occurences.get(key) }),
+ // ArrayUtils.toPrimitive(new Long[] {
+ // occurences.get(key) }),
// 1);
// }
// } else {
@@ -1228,15 +1381,24 @@ public class CodecCollector {
/**
* Creates the kwic.
*
- * @param kwicList the kwic list
- * @param spansMatchData the spans match data
- * @param docList the doc list
- * @param field the field
- * @param docBase the doc base
- * @param uniqueKeyField the unique key field
- * @param mtasCodecInfo the mtas codec info
- * @param searcher the searcher
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param kwicList
+ * the kwic list
+ * @param spansMatchData
+ * the spans match data
+ * @param docList
+ * the doc list
+ * @param field
+ * the field
+ * @param docBase
+ * the doc base
+ * @param uniqueKeyField
+ * the unique key field
+ * @param mtasCodecInfo
+ * the mtas codec info
+ * @param searcher
+ * the searcher
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void createKwic(List<ComponentKwic> kwicList,
HashMap<SpanQuery, HashMap<Integer, ArrayList<Match>>> spansMatchData,
@@ -1354,14 +1516,22 @@ public class CodecCollector {
/**
* Creates the facet base.
*
- * @param cf the cf
- * @param level the level
- * @param dataCollector the data collector
- * @param positionsData the positions data
- * @param spansNumberData the spans number data
- * @param facetData the facet data
- * @param docSet the doc set
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param cf
+ * the cf
+ * @param level
+ * the level
+ * @param dataCollector
+ * the data collector
+ * @param positionsData
+ * the positions data
+ * @param spansNumberData
+ * the spans number data
+ * @param facetData
+ * the facet data
+ * @param docSet
+ * the doc set
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void createFacetBase(ComponentFacet cf, int level,
MtasDataCollector<?, ?> dataCollector,
@@ -1369,16 +1539,40 @@ public class CodecCollector {
HashMap<SpanQuery, HashMap<Integer, Integer>> spansNumberData,
HashMap<String, TreeMap<String, int[]>> facetData, Integer[] docSet)
throws IOException {
- if (cf.baseFunctionParsers[level]
- .needArgumentsNumber() > cf.spanQueries.length) {
- throw new IOException("function " + cf.baseFunctionParsers[level]
- + " expects (at least) "
- + cf.baseFunctionParsers[level].needArgumentsNumber() + " queries");
+ for (MtasFunctionParserFunction function : cf.baseFunctionParserFunctions[level]) {
+ if (function.needArgumentsNumber() > cf.spanQueries.length) {
+ throw new IOException("function " + function + " expects (at least) "
+ + function.needArgumentsNumber() + " queries");
+ }
}
TreeMap<String, int[]> list = facetData.get(cf.baseFields[level]);
if (dataCollector != null) {
MtasDataCollector<?, ?>[] subDataCollectors = null;
- dataCollector.initNewList(cf.baseFields.length - level);
+ dataCollector.initNewList(1);
+ if (cf.baseFunctionList[level] != null) {
+ SubComponentFunction[] tmpList;
+ if (!cf.baseFunctionList[level].containsKey(dataCollector)) {
+ tmpList = new SubComponentFunction[cf.baseFunctionParserFunctions[level].length];
+ cf.baseFunctionList[level].put(dataCollector, tmpList);
+ for (int i = 0; i < cf.baseFunctionParserFunctions[level].length; i++) {
+ try {
+ tmpList[i] = new SubComponentFunction(
+ DataCollector.COLLECTOR_TYPE_LIST,
+ cf.baseFunctionKeys[level][i], cf.baseFunctionTypes[level][i],
+ cf.baseFunctionParserFunctions[level][i], null, null, 0,
+ Integer.MAX_VALUE, null, null);
+
+ } catch (ParseException e) {
+ throw new IOException(e.getMessage());
+ }
+ }
+ } else {
+ tmpList = cf.baseFunctionList[level].get(dataCollector);
+ }
+ for (SubComponentFunction function : tmpList) {
+ function.dataCollector.initNewList(1);
+ }
+ }
// check type
if (dataCollector.getCollectorType()
.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
@@ -1399,11 +1593,10 @@ public class CodecCollector {
if (documentsInFacets) {
HashMap<Integer, long[]> args = computeArguments(spansNumberData,
cf.spanQueries, docSet);
- if (cf.baseDataTypes[level].equals(CodecUtil.DATA_TYPE_LONG)
- || cf.baseDataTypes[level].equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ if (cf.baseDataTypes[level].equals(CodecUtil.DATA_TYPE_LONG)) {
// sumrule
if (cf.baseStatsTypes[level].equals(CodecUtil.STATS_BASIC)
- && cf.baseFunctionParsers[level].sumRule()
+ && cf.baseParsers[level].sumRule()
&& (cf.baseMinimumLongs[level] == null)
&& (cf.baseMaximumLongs[level] == null)) {
for (String key : list.keySet()) {
@@ -1411,8 +1604,7 @@ public class CodecCollector {
// initialise
String[] keys = new String[] { key };
Integer[] subDocSet = docLists.get(key);
- int length = cf.baseFunctionParsers[level]
- .needArgumentsNumber();
+ int length = cf.baseParsers[level].needArgumentsNumber();
long[] valueSum = new long[length];
long valuePositions = 0;
// collect
@@ -1428,56 +1620,59 @@ public class CodecCollector {
}
}
}
- if (cf.baseDataTypes[level]
- .equals(CodecUtil.DATA_TYPE_LONG)) {
- long value;
- try {
- value = cf.baseFunctionParsers[level]
- .getValueLong(valueSum, valuePositions);
- subDataCollectors = dataCollector.add(keys, value,
- subDocSet.length);
- } catch (IOException e) {
- dataCollector.error(keys, e.getMessage());
- subDataCollectors = null;
- }
- if (subDataCollectors != null) {
- for (MtasDataCollector<?, ?> subDataCollector : subDataCollectors) {
- if (subDataCollector != null) {
- createFacetBase(cf, (level + 1), subDataCollector,
- positionsData, spansNumberData, facetData,
- subDocSet);
+ long value;
+ try {
+ value = cf.baseParsers[level].getValueLong(valueSum,
+ valuePositions);
+ subDataCollectors = dataCollector.add(keys, value,
+ subDocSet.length);
+ } catch (IOException e) {
+ dataCollector.error(keys, e.getMessage());
+ subDataCollectors = null;
+ }
+ if (cf.baseFunctionList[level] != null
+ && cf.baseFunctionList[level]
+ .containsKey(dataCollector)) {
+ SubComponentFunction[] functionList = cf.baseFunctionList[level]
+ .get(dataCollector);
+ for (SubComponentFunction function : functionList) {
+ if (function.dataType
+ .equals(CodecUtil.DATA_TYPE_LONG)) {
+ try {
+ long valueLong = function.parserFunction
+ .getValueLong(valueSum, valuePositions);
+ function.dataCollector.add(keys, valueLong,
+ subDocSet.length);
+ } catch (IOException e) {
+ function.dataCollector.error(keys,
+ e.getMessage());
+ }
+ } else if (function.dataType
+ .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ try {
+ double valueDouble = function.parserFunction
+ .getValueDouble(valueSum, valuePositions);
+ function.dataCollector.add(keys, valueDouble,
+ subDocSet.length);
+ } catch (IOException e) {
+ function.dataCollector.error(keys,
+ e.getMessage());
}
}
}
- } else if (cf.baseDataTypes[level]
- .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
- double value;
- try {
- value = cf.baseFunctionParsers[level]
- .getValueDouble(valueSum, valuePositions);
- subDataCollectors = dataCollector.add(keys, value,
- subDocSet.length);
- } catch (IOException e) {
- dataCollector.error(keys, e.getMessage());
- subDataCollectors = null;
- }
- if (subDataCollectors != null) {
- for (MtasDataCollector<?, ?> subDataCollector : subDataCollectors) {
- if (subDataCollector != null) {
- createFacetBase(cf, (level + 1), subDataCollector,
- positionsData, spansNumberData, facetData,
- subDocSet);
- }
+ }
+ if (subDataCollectors != null) {
+ for (MtasDataCollector<?, ?> subDataCollector : subDataCollectors) {
+ if (subDataCollector != null) {
+ createFacetBase(cf, (level + 1), subDataCollector,
+ positionsData, spansNumberData, facetData,
+ subDocSet);
}
}
- } else {
- throw new IOException(
- "unrecognized dataType " + cf.baseDataTypes[level]);
}
}
}
}
- // normal
} else {
for (String key : list.keySet()) {
if (docLists.get(key).length > 0) {
@@ -1488,18 +1683,34 @@ public class CodecCollector {
if (subDocSet.length > 0) {
if (cf.baseDataTypes[level]
.equals(CodecUtil.DATA_TYPE_LONG)) {
+ // check for functions
+ long[][] functionValuesLong = null;
+ double[][] functionValuesDouble = null;
+ int[] functionNumber = null;
+ SubComponentFunction[] functionList = null;
+ if (cf.baseFunctionList[level] != null
+ && cf.baseFunctionList[level]
+ .containsKey(dataCollector)) {
+ functionList = cf.baseFunctionList[level]
+ .get(dataCollector);
+ functionValuesLong = new long[functionList.length][];
+ functionValuesDouble = new double[functionList.length][];
+ functionNumber = new int[functionList.length];
+ for (int i = 0; i < functionList.length; i++) {
+ functionValuesLong[i] = new long[subDocSet.length];
+ functionValuesDouble[i] = new double[subDocSet.length];
+ }
+ }
+ // check main
int number = 0;
Integer[] restrictedSubDocSet = new Integer[subDocSet.length];
- long value;
long[] values = new long[subDocSet.length];
- long[] tmpArgs;
- int tmpPositions;
for (int docId : subDocSet) {
try {
- tmpArgs = args.get(docId);
- tmpPositions = (positionsData == null) ? 0
+ long[] tmpArgs = args.get(docId);
+ int tmpPositions = (positionsData == null) ? 0
: positionsData.get(docId);
- value = cf.baseFunctionParsers[level]
+ long value = cf.baseParsers[level]
.getValueLong(tmpArgs, tmpPositions);
if ((cf.baseMinimumLongs[level] == null
|| value >= cf.baseMinimumLongs[level])
@@ -1508,6 +1719,33 @@ public class CodecCollector {
values[number] = value;
restrictedSubDocSet[number] = docId;
number++;
+ if (functionList != null) {
+ for (int i = 0; i < functionList.length; i++) {
+ SubComponentFunction function = functionList[i];
+ if (function.dataType
+ .equals(CodecUtil.DATA_TYPE_LONG)) {
+ try {
+ functionValuesLong[i][functionNumber[i]] = function.parserFunction
+ .getValueLong(tmpArgs, tmpPositions);
+ functionNumber[i]++;
+ } catch (IOException e) {
+ function.dataCollector.error(keys,
+ e.getMessage());
+ }
+ } else if (function.dataType
+ .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ try {
+ functionValuesDouble[i][functionNumber[i]] = function.parserFunction
+ .getValueDouble(tmpArgs,
+ tmpPositions);
+ functionNumber[i]++;
+ } catch (IOException e) {
+ function.dataCollector.error(keys,
+ e.getMessage());
+ }
+ }
+ }
+ }
}
} catch (IOException e) {
dataCollector.error(keys, e.getMessage());
@@ -1516,48 +1754,22 @@ public class CodecCollector {
if (number > 0) {
subDataCollectors = dataCollector.add(keys, values,
number);
- if (subDataCollectors != null) {
- for (MtasDataCollector<?, ?> subDataCollector : subDataCollectors) {
- if (subDataCollector != null) {
- createFacetBase(cf, (level + 1),
- subDataCollector, positionsData,
- spansNumberData, facetData,
- Arrays.copyOfRange(restrictedSubDocSet, 0,
- number));
+ if (cf.baseFunctionList[level] != null
+ && cf.baseFunctionList[level]
+ .containsKey(dataCollector)) {
+ for (int i = 0; i < functionList.length; i++) {
+ SubComponentFunction function = functionList[i];
+ if (function.dataType
+ .equals(CodecUtil.DATA_TYPE_LONG)) {
+ function.dataCollector.add(keys,
+ functionValuesLong[i], functionNumber[i]);
+ } else if (function.dataType
+ .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ function.dataCollector.add(keys,
+ functionValuesDouble[i], functionNumber[i]);
}
}
}
- }
- } else if (cf.baseDataTypes[level]
- .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
- int number = 0;
- Integer[] restrictedSubDocSet = new Integer[subDocSet.length];
- double value;
- double[] values = new double[subDocSet.length];
- long[] tmpArgs;
- int tmpPositions;
- for (int docId : subDocSet) {
- try {
- tmpArgs = args.get(docId);
- tmpPositions = (positionsData == null) ? 0
- : positionsData.get(docId);
- value = cf.baseFunctionParsers[level]
- .getValueDouble(tmpArgs, tmpPositions);
- if ((cf.baseMinimumDoubles[level] == null
- || value >= cf.baseMinimumDoubles[level])
- && (cf.baseMaximumDoubles[level] == null
- || value <= cf.baseMaximumDoubles[level])) {
- values[number] = value;
- restrictedSubDocSet[number] = docId;
- number++;
- }
- } catch (IOException e) {
- dataCollector.error(keys, e.getMessage());
- }
- }
- if (number > 0) {
- subDataCollectors = dataCollector.add(keys, values,
- number);
if (subDataCollectors != null) {
for (MtasDataCollector<?, ?> subDataCollector : subDataCollectors) {
if (subDataCollector != null) {
@@ -1577,32 +1789,52 @@ public class CodecCollector {
}
} else {
throw new IOException(
- "unknown dataType " + cf.baseDataTypes[level]);
+ "unexpected dataType " + cf.baseDataTypes[level]);
}
}
}
} else {
throw new IOException(
- "unknown type " + dataCollector.getCollectorType());
+ "unexpected type " + dataCollector.getCollectorType());
}
dataCollector.closeNewList();
+ if (cf.baseFunctionList[level] != null
+ && cf.baseFunctionList[level].containsKey(dataCollector)) {
+ SubComponentFunction[] tmpList = cf.baseFunctionList[level]
+ .get(dataCollector);
+ for (SubComponentFunction function : tmpList) {
+ function.dataCollector.closeNewList();
+ }
+ }
}
+
}
/**
* Creates the facet.
*
- * @param facetList the facet list
- * @param positionsData the positions data
- * @param spansNumberData the spans number data
- * @param facetData the facet data
- * @param docSet the doc set
- * @param field the field
- * @param docBase the doc base
- * @param uniqueKeyField the unique key field
- * @param mtasCodecInfo the mtas codec info
- * @param searcher the searcher
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param facetList
+ * the facet list
+ * @param positionsData
+ * the positions data
+ * @param spansNumberData
+ * the spans number data
+ * @param facetData
+ * the facet data
+ * @param docSet
+ * the doc set
+ * @param field
+ * the field
+ * @param docBase
+ * the doc base
+ * @param uniqueKeyField
+ * the unique key field
+ * @param mtasCodecInfo
+ * the mtas codec info
+ * @param searcher
+ * the searcher
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void createFacet(List<ComponentFacet> facetList,
HashMap<Integer, Integer> positionsData,
@@ -1625,14 +1857,22 @@ public class CodecCollector {
/**
* Creates the termvector.
*
- * @param termVectorList the term vector list
- * @param positionsData the positions data
- * @param docSet the doc set
- * @param field the field
- * @param t the t
- * @param r the r
- * @param lrc the lrc
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param termVectorList
+ * the term vector list
+ * @param positionsData
+ * the positions data
+ * @param docSet
+ * the doc set
+ * @param field
+ * the field
+ * @param t
+ * the t
+ * @param r
+ * the r
+ * @param lrc
+ * the lrc
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void createTermvector(List<ComponentTermVector> termVectorList,
HashMap<Integer, Integer> positionsData, List<Integer> docSet,
@@ -1645,14 +1885,22 @@ public class CodecCollector {
/**
* Creates the termvector first round.
*
- * @param termVectorList the term vector list
- * @param positionsData the positions data
- * @param docSet the doc set
- * @param field the field
- * @param t the t
- * @param r the r
- * @param lrc the lrc
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param termVectorList
+ * the term vector list
+ * @param positionsData
+ * the positions data
+ * @param docSet
+ * the doc set
+ * @param field
+ * the field
+ * @param t
+ * the t
+ * @param r
+ * the r
+ * @param lrc
+ * the lrc
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void createTermvectorFirstRound(
List<ComponentTermVector> termVectorList,
@@ -1668,104 +1916,96 @@ public class CodecCollector {
// loop over termvectors
for (ComponentTermVector termVector : termVectorList) {
termsEnum = t.intersect(termVector.compiledAutomaton, null);
- termVector.dataDefaultCollector.initNewList((int) t.size(), segmentName,
- segmentNumber);
- if (termVector.dataFunctionCollector != null) {
- termVector.dataFunctionCollector.initNewList((int) t.size(),
- segmentName, segmentNumber);
+ termVector.subComponentFunction.dataCollector.initNewList(
+ (int) t.size(), segmentName, segmentNumber, termVector.boundary);
+ if (termVector.functions != null) {
+ for (SubComponentFunction function : termVector.functions) {
+ function.dataCollector.initNewList((int) t.size());
+ }
}
// only if documents
if (docSet.size() > 0) {
int termDocId;
- String key;
- String[] keys;
- int termNumberMaximum = (termVector.start + termVector.number);
+ int termNumberMaximum = termVector.number;
+ HashMap<BytesRef, RegisterStatus> computeFullList = new HashMap<BytesRef, RegisterStatus>();
+ RegisterStatus registerStatus;
// basic, don't need full values
- if (termVector.defaultStatsType.equals(CodecUtil.STATS_BASIC)
- && (termVector.functionParser == null
- || (termVector.functionParser.sumRule()
- && !termVector.functionParser.needPositions()))) {
- if ((termVector.sortType.equals(CodecUtil.SORT_TERM)
- && termVector.sortDirection.equals(CodecUtil.SORT_ASC))
- || (termVector.sortDirection.equals(CodecUtil.SORT_DESC)
- && (termVector.sortType.equals(CodecUtil.STATS_TYPE_SUM)
- || termVector.sortType
- .equals(CodecUtil.STATS_TYPE_N)))) {
- int termCounter = 0;
- // loop over terms
- while ((term = termsEnum.next()) != null) {
- termDocId = -1;
- key = MtasToken.getPostfixFromValue(term.utf8ToString());
- keys = new String[] { key };
- // compute numbers;
- TermvectorNumberBasic numberBasic = computeTermvectorNumberBasic(
- docSet, termDocId, termsEnum, r, lrc, postingsEnum);
- // register
- if (numberBasic.docNumber > 0) {
- termCounter++;
- registerBasicValue(keys, termVector, numberBasic,
- termNumberMaximum, segmentNumber);
- }
- // stop after termCounterMaximum
- if (termVector.sortType.equals(CodecUtil.SORT_TERM)
- && termCounter >= termNumberMaximum) {
- break;
+ if (termVector.subComponentFunction.sortType
+ .equals(CodecUtil.SORT_TERM)
+ || termVector.subComponentFunction.sortType
+ .equals(CodecUtil.STATS_TYPE_SUM)
+ || termVector.subComponentFunction.sortType
+ .equals(CodecUtil.STATS_TYPE_N)) {
+ int termCounter = 0;
+ // loop over terms
+ while ((term = termsEnum.next()) != null) {
+ termDocId = -1;
+ // compute numbers;
+ TermvectorNumberBasic numberBasic = computeTermvectorNumberBasic(
+ docSet, termDocId, termsEnum, r, lrc, postingsEnum);
+ // register
+ if (numberBasic.docNumber > 0) {
+ termCounter++;
+ registerStatus = registerValue(term, termVector, numberBasic,
+ termNumberMaximum, segmentNumber, false);
+ if (registerStatus != null) {
+ computeFullList.put(BytesRef.deepCopyOf(term),
+ registerStatus);
}
}
- } else {
- throw new IOException("sort '" + termVector.sortType + " "
- + termVector.sortDirection + "' not supported");
- }
- // finish if segments are used
- termVector.dataDefaultCollector.closeSegmentKeyValueRegistration();
- if (termVector.dataFunctionCollector != null) {
- termVector.dataFunctionCollector
- .closeSegmentKeyValueRegistration();
+ // stop after termCounterMaximum
+ if (termVector.subComponentFunction.sortType.equals(
+ CodecUtil.SORT_TERM) && termCounter >= termNumberMaximum) {
+ break;
+ }
}
- } else {
- if ((termVector.sortType.equals(CodecUtil.SORT_TERM)
- && termVector.sortDirection.equals(CodecUtil.SORT_ASC))
- || (termVector.sortDirection.equals(CodecUtil.SORT_DESC)
- && (termVector.sortType.equals(CodecUtil.STATS_TYPE_SUM)
- || termVector.sortType
- .equals(CodecUtil.STATS_TYPE_N)))) {
- int termCounter = 0;
+ // rerun for full
+ if (computeFullList.size() > 0) {
+ termsEnum = t.intersect(termVector.compiledAutomaton, null);
while ((term = termsEnum.next()) != null) {
termDocId = -1;
- termCounter++;
- key = MtasToken.getPostfixFromValue(term.utf8ToString());
- keys = new String[] { key };
- // compute numbers;
- TermvectorNumberFull numberFull = computeTermvectorNumberFull(
- docSet, termDocId, termsEnum, r, lrc, postingsEnum,
- positionsData);
- // register
- if (numberFull.docNumber > 0) {
- termCounter++;
- registerFullValues(keys, termVector, numberFull,
- termNumberMaximum, segmentNumber);
- }
- // stop after termCounterMaximum
- if (termVector.sortType.equals(CodecUtil.SORT_TERM)
- && termCounter >= termNumberMaximum) {
- break;
+ // only if (probably) needed
+ if (computeFullList.containsKey(term)) {
+ registerStatus = computeFullList.get(term);
+ if (termVector.subComponentFunction.sortType
+ .equals(CodecUtil.SORT_TERM) || termVector.list != null
+ || termVector.boundaryRegistration || registerStatus.force
+ || termVector.subComponentFunction.dataCollector
+ .validateSegmentBoundary(registerStatus.sortValue)) {
+ TermvectorNumberFull numberFull = computeTermvectorNumberFull(
+ docSet, termDocId, termsEnum, r, lrc, postingsEnum,
+ positionsData);
+ if (numberFull.docNumber > 0) {
+ termCounter++;
+ registerValue(term, termVector, numberFull,
+ termNumberMaximum, segmentNumber);
+ }
+ }
}
}
- } else {
- throw new IOException("sort '" + termVector.sortType + " "
- + termVector.sortDirection + "' not supported");
+ computeFullList.clear();
}
- // finish if segments are used
- termVector.dataDefaultCollector.closeSegmentKeyValueRegistration();
- if (termVector.dataFunctionCollector != null) {
- termVector.dataFunctionCollector
- .closeSegmentKeyValueRegistration();
+ } else {
+ throw new IOException(
+ "sort '" + termVector.subComponentFunction.sortType + " "
+ + termVector.subComponentFunction.sortDirection
+ + "' not supported");
+ }
+ // finish if segments are used
+ termVector.subComponentFunction.dataCollector
+ .closeSegmentKeyValueRegistration();
+ if (termVector.functions != null) {
+ for (SubComponentFunction function : termVector.functions) {
+ function.dataCollector.closeSegmentKeyValueRegistration();
}
}
+
}
- termVector.dataDefaultCollector.closeNewList();
- if (termVector.dataFunctionCollector != null) {
- termVector.dataFunctionCollector.closeNewList();
+ termVector.subComponentFunction.dataCollector.closeNewList();
+ if (termVector.functions != null) {
+ for (SubComponentFunction function : termVector.functions) {
+ function.dataCollector.closeNewList();
+ }
}
}
}
@@ -1775,14 +2015,22 @@ public class CodecCollector {
/**
* Creates the termvector second round.
*
- * @param termVectorList the term vector list
- * @param positionsData the positions data
- * @param docSet the doc set
- * @param field the field
- * @param t the t
- * @param r the r
- * @param lrc the lrc
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param termVectorList
+ * the term vector list
+ * @param positionsData
+ * the positions data
+ * @param docSet
+ * the doc set
+ * @param field
+ * the field
+ * @param t
+ * the t
+ * @param r
+ * the r
+ * @param lrc
+ * the lrc
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void createTermvectorSecondRound(
List<ComponentTermVector> termVectorList,
@@ -1796,83 +2044,98 @@ public class CodecCollector {
String segmentName = "segment" + lrc.ord;
int segmentNumber = lrc.parent.leaves().size();
for (ComponentTermVector termVector : termVectorList) {
- if (termVector.dataDefaultCollector.segmentRecomputeKeyList
- .containsKey(segmentName)) {
- termsEnum = t.intersect(termVector.compiledAutomaton, null);
- termVector.dataDefaultCollector.initNewList(
- termVector.dataDefaultCollector.segmentKeys.size(), segmentName,
- segmentNumber);
- if (termVector.dataFunctionCollector != null) {
- termVector.dataFunctionCollector.initNewList((int) t.size(),
- segmentName, segmentNumber);
- }
- if (docSet.size() > 0) {
- int termDocId;
- String key;
- String[] keys;
- HashSet<String> recomputeKeyList = termVector.dataDefaultCollector.segmentRecomputeKeyList
- .get(segmentName);
- if (termVector.defaultStatsType.equals(CodecUtil.STATS_BASIC)
- && (termVector.functionParser == null
- || (termVector.functionParser.sumRule()
- && !termVector.functionParser.needPositions()))) {
- while ((term = termsEnum.next()) != null) {
- termDocId = -1;
- key = MtasToken.getPostfixFromValue(term.utf8ToString());
- if (recomputeKeyList.contains(key)) {
- keys = new String[] { key };
+ if (termVector.subComponentFunction.dataCollector.segmentRecomputeKeyList != null
+ && termVector.subComponentFunction.dataCollector.segmentRecomputeKeyList
+ .containsKey(segmentName)) {
+ HashSet<String> recomputeKeyList = termVector.subComponentFunction.dataCollector.segmentRecomputeKeyList
+ .get(segmentName);
+ if (recomputeKeyList.size() > 0) {
+ List<CompiledAutomaton> listCompiledAutomata = MtasToken
+ .createAutomata(termVector.prefix,
+ new ArrayList<String>(recomputeKeyList));
+ for (CompiledAutomaton compiledAutomaton : listCompiledAutomata) {
+ termsEnum = t.intersect(compiledAutomaton, null);
+ termVector.subComponentFunction.dataCollector
+ .initNewList(
+ termVector.subComponentFunction.dataCollector.segmentKeys
+ .size(),
+ segmentName, segmentNumber, termVector.boundary);
+ RegisterStatus registerStatus = null;
+ if (termVector.functions != null) {
+ for (SubComponentFunction function : termVector.functions) {
+ function.dataCollector.initNewList((int) t.size(),
+ segmentName, segmentNumber, null);
+ }
+ }
+ if (docSet.size() > 0) {
+ int termDocId;
+ while ((term = termsEnum.next()) != null) {
+ termDocId = -1;
// compute numbers;
TermvectorNumberBasic numberBasic = computeTermvectorNumberBasic(
docSet, termDocId, termsEnum, r, lrc, postingsEnum);
if (numberBasic.docNumber > 0) {
- registerBasicValue(keys, termVector, numberBasic, 0,
- segmentNumber);
+ registerStatus = registerValue(term, termVector,
+ numberBasic, 0, segmentNumber, true);
+ if (registerStatus != null) {
+ TermvectorNumberFull numberFull = computeTermvectorNumberFull(
+ docSet, termDocId, termsEnum, r, lrc, postingsEnum,
+ positionsData);
+ if (numberFull.docNumber > 0) {
+ registerValue(term, termVector, numberFull, 0,
+ segmentNumber);
+ }
+ }
}
}
}
- } else {
- while ((term = termsEnum.next()) != null) {
- termDocId = -1;
- key = MtasToken.getPostfixFromValue(term.utf8ToString());
- if (recomputeKeyList.contains(key)) {
- keys = new String[] { key };
- // compute numbers;
- TermvectorNumberFull numberFull = computeTermvectorNumberFull(
- docSet, termDocId, termsEnum, r, lrc, postingsEnum,
- positionsData);
- // register
- if (numberFull.docNumber > 0) {
- registerFullValues(keys, termVector, numberFull, 0,
- segmentNumber);
- }
+ termVector.subComponentFunction.dataCollector.closeNewList();
+ if (termVector.functions != null) {
+ for (SubComponentFunction function : termVector.functions) {
+ function.dataCollector.closeNewList();
}
}
}
}
- termVector.dataDefaultCollector.closeNewList();
- if (termVector.dataFunctionCollector != null) {
- termVector.dataFunctionCollector.closeNewList();
- }
}
}
-
}
+
}
/**
* Need second round termvector.
*
- * @param termVectorList the term vector list
+ * @param termVectorList
+ * the term vector list
* @return true, if successful
+ * @throws IOException
*/
private static boolean needSecondRoundTermvector(
- List<ComponentTermVector> termVectorList) {
+ List<ComponentTermVector> termVectorList) throws IOException {
boolean needSecondRound = false;
for (ComponentTermVector termVector : termVectorList) {
- if (termVector.dataDefaultCollector.segmentRegistration
+ if (termVector.subComponentFunction.dataCollector.segmentRegistration != null
+ && (termVector.subComponentFunction.dataCollector.segmentRegistration
+ .equals(MtasDataCollector.SEGMENT_SORT_ASC)
+ || termVector.subComponentFunction.dataCollector.segmentRegistration
+ .equals(MtasDataCollector.SEGMENT_SORT_DESC))
&& termVector.number > 0) {
- termVector.dataDefaultCollector.recomputeSegmentKeys();
- if (!termVector.dataDefaultCollector.checkExistenceNecessaryKeys()) {
+ termVector.subComponentFunction.dataCollector.recomputeSegmentKeys();
+ if (!termVector.subComponentFunction.dataCollector
+ .checkExistenceNecessaryKeys()) {
+ needSecondRound = true;
+ }
+ termVector.subComponentFunction.dataCollector.reduceToSegmentKeys();
+ } else if (termVector.subComponentFunction.dataCollector.segmentRegistration != null
+ && (termVector.subComponentFunction.dataCollector.segmentRegistration
+ .equals(MtasDataCollector.SEGMENT_BOUNDARY_ASC)
+ || termVector.subComponentFunction.dataCollector.segmentRegistration
+ .equals(MtasDataCollector.SEGMENT_BOUNDARY_DESC))
+ && termVector.number > 0) {
+ termVector.subComponentFunction.dataCollector.recomputeSegmentKeys();
+ if (!termVector.subComponentFunction.dataCollector
+ .checkExistenceNecessaryKeys()) {
needSecondRound = true;
}
}
@@ -1884,10 +2147,10 @@ public class CodecCollector {
* The Class TermvectorNumberBasic.
*/
private static class TermvectorNumberBasic {
-
+
/** The value sum. */
public long[] valueSum;
-
+
/** The doc number. */
public int docNumber;
@@ -1904,20 +2167,21 @@ public class CodecCollector {
* The Class TermvectorNumberFull.
*/
private static class TermvectorNumberFull {
-
+
/** The args. */
public long[] args;
-
+
/** The positions. */
public int[] positions;
-
+
/** The doc number. */
public int docNumber;
/**
* Instantiates a new termvector number full.
*
- * @param maxSize the max size
+ * @param maxSize
+ * the max size
*/
TermvectorNumberFull(int maxSize) {
args = new long[maxSize];
@@ -1926,245 +2190,260 @@ public class CodecCollector {
}
}
+ private static class RegisterStatus {
+ public long sortValue;
+ public boolean force;
+
+ RegisterStatus(long sortValue, boolean force) {
+ this.sortValue = sortValue;
+ this.force = force;
+ }
+ }
+
/**
* Register basic value.
*
- * @param keys the keys
- * @param termVector the term vector
- * @param numbers the numbers
- * @param termNumberMaximum the term number maximum
- * @param segmentNumber the segment number
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param keys
+ * the keys
+ * @param termVector
+ * the term vector
+ * @param number
+ * the numbers
+ * @param termNumberMaximum
+ * the term number maximum
+ * @param segmentNumber
+ * the segment number
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- private static void registerBasicValue(String[] keys,
- ComponentTermVector termVector, TermvectorNumberBasic numbers,
- Integer termNumberMaximum, Integer segmentNumber) throws IOException {
- if (termVector.dataFunctionCollector == null
- || termVector.dataFunctionCollector.getDataType()
- .equals(CodecUtil.DATA_TYPE_LONG)) {
- long valueFunction = 0;
- long valueDefault = termVector.defaultParser
- .getValueLong(numbers.valueSum, 0);
- boolean addItem;
- MtasDataCollector<Long, ?> dataFunctionCollector = null;
- MtasDataCollector<Long, ?> dataDefaultCollector = (MtasDataCollector<Long, ?>) termVector.dataDefaultCollector;
- for (String key : keys) {
- if (termVector.sortType.equals(CodecUtil.SORT_TERM)) {
- addItem = true;
- } else if (termVector.sortType.equals(CodecUtil.STATS_TYPE_SUM)) {
- addItem = dataDefaultCollector.validateSegmentValue(key, valueDefault,
- termNumberMaximum, segmentNumber);
- } else if (termVector.sortType.equals(CodecUtil.STATS_TYPE_N)) {
- addItem = dataDefaultCollector.validateSegmentValue(key,
- Long.valueOf(numbers.docNumber), termNumberMaximum,
- segmentNumber);
- } else {
- addItem = false;
- }
+ @SuppressWarnings("unchecked")
+ private static RegisterStatus registerValue(BytesRef term,
+ ComponentTermVector termVector, TermvectorNumberBasic number,
+ Integer termNumberMaximum, Integer segmentNumber, boolean forceAccept)
+ throws IOException {
+ long value = termVector.subComponentFunction.parserFunction
+ .getValueLong(number.valueSum, 0);
+ long sortValue = 0;
+ if (termVector.subComponentFunction.sortType
+ .equals(CodecUtil.STATS_TYPE_SUM)) {
+ sortValue = value;
+ } else if (termVector.subComponentFunction.sortType
+ .equals(CodecUtil.STATS_TYPE_N)) {
+ sortValue = Long.valueOf(number.docNumber);
+ }
+ boolean addItem = false, addItemForced = false;
+ String key = null;
+ MtasDataCollector<Long, ?> dataCollector = (MtasDataCollector<Long, ?>) termVector.subComponentFunction.dataCollector;
+ if (termVector.subComponentFunction.sortType.equals(CodecUtil.SORT_TERM)) {
+ if (forceAccept) {
+ addItem = true;
+ } else if (termVector.list != null) {
+ key = MtasToken.getPostfixFromValue(term);
+ addItem = termVector.list.contains(key);
+ } else {
+ addItem = true;
+ }
+ addItemForced = addItem;
+ } else if (termVector.subComponentFunction.sortType
+ .equals(CodecUtil.STATS_TYPE_SUM)
+ || termVector.subComponentFunction.sortType
+ .equals(CodecUtil.STATS_TYPE_N)) {
+ if (forceAccept) {
+ addItem = true;
+ addItemForced = addItem;
+ } else if (termVector.list != null) {
+ key = MtasToken.getPostfixFromValue(term);
+ addItem = termVector.list.contains(key);
+ addItemForced = addItem;
+ } else if (termVector.boundaryRegistration) {
+ addItem = dataCollector.validateSegmentBoundary(sortValue);
if (addItem) {
- dataDefaultCollector.add(new String[] { key }, valueDefault,
- numbers.docNumber);
- if (termVector.functionParser != null) {
- valueFunction = termVector.functionParser
- .getValueLong(numbers.valueSum, 0);
- dataFunctionCollector = (MtasDataCollector<Long, ?>) termVector.dataFunctionCollector;
- dataFunctionCollector.add(new String[] { key }, valueFunction,
- numbers.docNumber);
+ key = MtasToken.getPostfixFromValue(term);
+ String segmentStatus = dataCollector.validateSegmentValue(key,
+ sortValue, termNumberMaximum, segmentNumber);
+ if (segmentStatus != null) {
+ addItem = true;
+ if (segmentStatus.equals(MtasDataCollector.SEGMENT_KEY)) {
+ addItemForced = addItem;
+ }
+ } else {
+ // shouldn't happen
+ }
+ }
+ } else {
+ key = MtasToken.getPostfixFromValue(term);
+ String segmentStatus = dataCollector.validateSegmentValue(key,
+ sortValue, termNumberMaximum, segmentNumber);
+ if (segmentStatus != null) {
+ addItem = true;
+ if (segmentStatus.equals(MtasDataCollector.SEGMENT_KEY)) {
+ addItemForced = addItem;
}
}
}
- } else if (termVector.dataFunctionCollector.getDataType()
- .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
- long valueDefault = termVector.defaultParser
- .getValueLong(numbers.valueSum, 0);
- double valueFunction = termVector.functionParser
- .getValueDouble(numbers.valueSum, 0);
- MtasDataCollector<Long, ?> dataDefaultCollector = (MtasDataCollector<Long, ?>) termVector.dataFunctionCollector;
- MtasDataCollector<Double, ?> dataFunctionCollector = (MtasDataCollector<Double, ?>) termVector.dataFunctionCollector;
- for (String key : keys) {
- if (dataDefaultCollector.validateSegmentValue(key, valueDefault,
- termNumberMaximum, segmentNumber)) {
- dataDefaultCollector.add(new String[] { key }, valueDefault,
- numbers.docNumber);
- dataFunctionCollector.add(new String[] { key }, valueFunction,
- numbers.docNumber);
+ } else {
+ addItem = false;
+ }
+ if (addItem) {
+ boolean computeFull = false;
+ if (key == null) {
+ key = MtasToken.getPostfixFromValue(term);
+ }
+ if (termVector.subComponentFunction.statsType
+ .equals(CodecUtil.STATS_BASIC)) {
+ dataCollector.add(new String[] { key }, value, number.docNumber);
+ } else {
+ computeFull = true;
+ }
+ if (termVector.functions != null) {
+ for (SubComponentFunction function : termVector.functions) {
+ if (function.parserFunction.sumRule()
+ && !function.parserFunction.needPositions()
+ && function.statsType.equals(CodecUtil.STATS_BASIC)) {
+ if (function.dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
+ long valueFunction = function.parserFunction
+ .getValueLong(number.valueSum, 0);
+ function.dataCollector.add(new String[] { key }, valueFunction,
+ number.docNumber);
+ } else if (function.dataType.equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ double valueFunction = function.parserFunction
+ .getValueDouble(number.valueSum, 0);
+ function.dataCollector.add(new String[] { key }, valueFunction,
+ number.docNumber);
+ }
+ } else {
+ computeFull = true;
+ }
}
}
+ return computeFull ? new RegisterStatus(sortValue, addItemForced) : null;
} else {
- throw new IOException(
- "unknown dataType " + termVector.dataFunctionCollector.getDataType());
+ return null;
}
}
/**
* Register full values.
*
- * @param keys the keys
- * @param termVector the term vector
- * @param numbers the numbers
- * @param termNumberMaximum the term number maximum
- * @param segmentNumber the segment number
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param keys
+ * the keys
+ * @param termVector
+ * the term vector
+ * @param number
+ * the numbers
+ * @param termNumberMaximum
+ * the term number maximum
+ * @param segmentNumber
+ * the segment number
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- private static void registerFullValues(String[] keys,
- ComponentTermVector termVector, TermvectorNumberFull numbers,
+ @SuppressWarnings("unchecked")
+ private static void registerValue(BytesRef term,
+ ComponentTermVector termVector, TermvectorNumberFull number,
Integer termNumberMaximum, Integer segmentNumber) throws IOException {
- if (numbers.docNumber > 0) {
- long valueDefault, valueSum = 0;
- boolean[] addItems = new boolean[keys.length];
- boolean addItem = false;
- MtasDataCollector<Long, ?> dataDefaultCollector = (MtasDataCollector<Long, ?>) termVector.dataDefaultCollector;
- //check for each key
- for (int i = 0; i < numbers.docNumber; i++) {
- valueSum += numbers.args[i];
- }
- valueDefault = termVector.defaultParser
- .getValueLong(new long[] { valueSum }, 0);
- for (int i=0; i<keys.length;i++) {
- if (termVector.sortType.equals(CodecUtil.SORT_TERM)) {
- addItems[i] = true;
- addItem = true;
- } else if (termVector.sortType.equals(CodecUtil.STATS_TYPE_SUM)) {
- addItems[i] = dataDefaultCollector.validateSegmentValue(keys[i], valueDefault,
- termNumberMaximum, segmentNumber);
- addItem = addItem?addItem:addItems[i];
- } else if (termVector.sortType.equals(CodecUtil.STATS_TYPE_N)) {
- addItems[i] = dataDefaultCollector.validateSegmentValue(keys[i],
- Long.valueOf(numbers.docNumber), termNumberMaximum,
- segmentNumber);
- addItem = addItem?addItem:addItems[i];
- } else {
- addItems[i] = false;
- addItem = addItem?addItem:addItems[i];
+ if (number.docNumber > 0) {
+ String key = MtasToken.getPostfixFromValue(term);
+ MtasDataCollector<Long, ?> dataCollector = (MtasDataCollector<Long, ?>) termVector.subComponentFunction.dataCollector;
+ long[] valuesLong = new long[number.docNumber];
+ for (int i = 0; i < number.docNumber; i++) {
+ try {
+ valuesLong[i] = termVector.subComponentFunction.parserFunction
+ .getValueLong(new long[] { number.args[i] }, number.positions[i]);
+ } catch (IOException e) {
+ dataCollector.error(new String[] { key }, e.getMessage());
}
}
- if(addItem) {
- if (termVector.dataFunctionCollector==null) {
- long[] values = new long[numbers.docNumber];
- for (int k=0;k<keys.length;k++) {
- if (addItems[k]) {
- for (int i = 0; i < numbers.docNumber; i++) {
- try {
- values[i] = termVector.defaultParser.getValueLong(
- new long[] { numbers.args[i] }, numbers.positions[i]);
- } catch (IOException e) {
- dataDefaultCollector.error(keys, e.getMessage());
- }
- }
- dataDefaultCollector.add(new String[] { keys[k] }, values,
- values.length);
- }
- }
- } else if (termVector.dataFunctionCollector.getDataType()
- .equals(CodecUtil.DATA_TYPE_LONG) && termVector.defaultStatsType.equals(CodecUtil.STATS_BASIC)) {
- long[] values = new long[numbers.docNumber];
- MtasDataCollector<Long, ?> dataFunctionCollector = (MtasDataCollector<Long, ?>) termVector.dataFunctionCollector;
- for (String key : keys) {
- if (termVector.sortType.equals(CodecUtil.SORT_TERM)) {
- addItem = true;
- } else if (termVector.sortType.equals(CodecUtil.STATS_TYPE_SUM)) {
- addItem = dataDefaultCollector.validateSegmentValue(key, valueDefault,
- termNumberMaximum, segmentNumber);
- } else if (termVector.sortType.equals(CodecUtil.STATS_TYPE_N)) {
- addItem = dataDefaultCollector.validateSegmentValue(key,
- Long.valueOf(numbers.docNumber), termNumberMaximum,
- segmentNumber);
- } else {
- addItem = false;
- }
- if (addItem) {
- for (int i = 0; i < numbers.docNumber; i++) {
- try {
- values[i] = termVector.functionParser.getValueLong(
- new long[] { numbers.args[i] }, numbers.positions[i]);
- } catch (IOException e) {
- dataFunctionCollector.error(keys, e.getMessage());
- }
+ if (!termVector.subComponentFunction.statsType
+ .equals(CodecUtil.STATS_BASIC)) {
+ dataCollector.add(new String[] { key }, valuesLong, valuesLong.length);
+ }
+ for (SubComponentFunction function : termVector.functions) {
+ if (!function.parserFunction.sumRule()
+ || function.parserFunction.needPositions()
+ || !function.statsType.equals(CodecUtil.STATS_BASIC)) {
+ if (function.dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
+ valuesLong = new long[number.docNumber];
+ for (int i = 0; i < number.docNumber; i++) {
+ try {
+ valuesLong[i] = function.parserFunction.getValueLong(
+ new long[] { number.args[i] }, number.positions[i]);
+ } catch (IOException e) {
+ function.dataCollector.error(new String[] { key },
+ e.getMessage());
}
- dataDefaultCollector.add(new String[] { key }, valueDefault,
- numbers.docNumber);
- dataFunctionCollector.add(new String[] { key }, values,
- values.length);
- }
- }
- } else if (termVector.dataFunctionCollector.getDataType()
- .equals(CodecUtil.DATA_TYPE_DOUBLE) && termVector.defaultStatsType.equals(CodecUtil.STATS_BASIC)) {
- double[] values = new double[numbers.docNumber];
- MtasDataCollector<Double, ?> dataFunctionCollector = (MtasDataCollector<Double, ?>) termVector.dataFunctionCollector;
- for (String key : keys) {
- if (termVector.sortType.equals(CodecUtil.SORT_TERM)) {
- addItem = true;
- } else if (termVector.sortType.equals(CodecUtil.STATS_TYPE_SUM)) {
- addItem = dataDefaultCollector.validateSegmentValue(key, valueDefault,
- termNumberMaximum, segmentNumber);
- } else if (termVector.sortType.equals(CodecUtil.STATS_TYPE_N)) {
- addItem = dataDefaultCollector.validateSegmentValue(key,
- Long.valueOf(numbers.docNumber), termNumberMaximum,
- segmentNumber);
- } else {
- addItem = false;
}
- if (addItem) {
- for (int i = 0; i < numbers.docNumber; i++) {
- try {
- values[i] = termVector.functionParser.getValueDouble(
- new long[] { numbers.args[i] }, numbers.positions[i]);
- } catch (IOException e) {
- dataFunctionCollector.error(keys, e.getMessage());
- }
+ function.dataCollector.add(new String[] { key }, valuesLong,
+ valuesLong.length);
+ } else if (function.dataType.equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ double[] valuesDouble = new double[number.docNumber];
+ for (int i = 0; i < number.docNumber; i++) {
+ try {
+ valuesDouble[i] = function.parserFunction.getValueDouble(
+ new long[] { number.args[i] }, number.positions[i]);
+ } catch (IOException e) {
+ function.dataCollector.error(new String[] { key },
+ e.getMessage());
}
- dataDefaultCollector.add(new String[] { key }, valueDefault,
- numbers.docNumber);
- dataFunctionCollector.add(new String[] { key }, values,
- values.length);
}
+ function.dataCollector.add(new String[] { key }, valuesDouble,
+ valuesDouble.length);
}
- } else {
- throw new IOException("unknown dataType "
- + termVector.dataFunctionCollector.getDataType() + " or problem with default statsType");
}
}
+
}
+
}
/**
* Compute termvector number basic.
*
- * @param docSet the doc set
- * @param termDocId the term doc id
- * @param termsEnum the terms enum
- * @param r the r
- * @param lrc the lrc
- * @param postingsEnum the postings enum
+ * @param docSet
+ * the doc set
+ * @param termDocId
+ * the term doc id
+ * @param termsEnum
+ * the terms enum
+ * @param r
+ * the r
+ * @param lrc
+ * the lrc
+ * @param postingsEnum
+ * the postings enum
* @return the termvector number basic
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static TermvectorNumberBasic computeTermvectorNumberBasic(
List<Integer> docSet, int termDocId, TermsEnum termsEnum, LeafReader r,
LeafReaderContext lrc, PostingsEnum postingsEnum) throws IOException {
TermvectorNumberBasic result = new TermvectorNumberBasic();
+ boolean hasDeletedDocuments = (r.getLiveDocs() != null);
boolean subtractValue = false;
- if (docSet.size() == r.numDocs()) {
+ if ((docSet.size() == r.numDocs()) && !hasDeletedDocuments) {
result.valueSum[0] = termsEnum.totalTermFreq();
result.docNumber = termsEnum.docFreq();
- } else {
- if (docSet.size() > Math.round(r.numDocs() / 2)) {
- result.valueSum[0] = termsEnum.totalTermFreq();
- subtractValue = true;
+ if (result.valueSum[0] > -1) {
+ return result;
}
- Iterator<Integer> docIterator = docSet.iterator();
- postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.FREQS);
- while (docIterator.hasNext()) {
- int docId = docIterator.next() - lrc.docBase;
- if (docId >= termDocId) {
- if ((docId == termDocId)
- || ((termDocId = postingsEnum.advance(docId)) == docId)) {
- result.docNumber++;
- if (subtractValue) {
- result.valueSum[0] -= postingsEnum.freq();
- } else {
- result.valueSum[0] += postingsEnum.freq();
- }
+ }
+ if (docSet.size() > Math.round(r.numDocs() / 2) && !hasDeletedDocuments) {
+ result.valueSum[0] = termsEnum.totalTermFreq();
+ subtractValue = true;
+ }
+ Iterator<Integer> docIterator = docSet.iterator();
+ postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.FREQS);
+ while (docIterator.hasNext()) {
+ int docId = docIterator.next() - lrc.docBase;
+ if (docId >= termDocId) {
+ if ((docId == termDocId)
+ || ((termDocId = postingsEnum.advance(docId)) == docId)) {
+ result.docNumber++;
+ if (subtractValue) {
+ result.valueSum[0] -= postingsEnum.freq();
+ } else {
+ result.valueSum[0] += postingsEnum.freq();
}
}
}
@@ -2175,15 +2454,23 @@ public class CodecCollector {
/**
* Compute termvector number full.
*
- * @param docSet the doc set
- * @param termDocId the term doc id
- * @param termsEnum the terms enum
- * @param r the r
- * @param lrc the lrc
- * @param postingsEnum the postings enum
- * @param positionsData the positions data
+ * @param docSet
+ * the doc set
+ * @param termDocId
+ * the term doc id
+ * @param termsEnum
+ * the terms enum
+ * @param r
+ * the r
+ * @param lrc
+ * the lrc
+ * @param postingsEnum
+ * the postings enum
+ * @param positionsData
+ * the positions data
* @return the termvector number full
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static TermvectorNumberFull computeTermvectorNumberFull(
List<Integer> docSet, int termDocId, TermsEnum termsEnum, LeafReader r,
@@ -2208,153 +2495,4 @@ public class CodecCollector {
return result;
}
- /**
- * Creates the termvector classic.
- *
- * @param termVectorList the term vector list
- * @param positionsData the positions data
- * @param docSet the doc set
- * @param field the field
- * @param t the t
- * @param r the r
- * @param lrc the lrc
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private static void createTermvectorClassic(
- List<ComponentTermVector> termVectorList,
- HashMap<Integer, Integer> positionsData, List<Integer> docSet,
- String field, Terms t, LeafReader r, LeafReaderContext lrc)
- throws IOException {
- if (t != null) {
- BytesRef term;
- TermsEnum termsEnum;
- PostingsEnum postingsEnum = null;
- for (ComponentTermVector termVector : termVectorList) {
- termsEnum = t.intersect(termVector.compiledAutomaton, null);
- termVector.dataFunctionCollector.initNewList((int) t.size(),
- "segment" + lrc.ord, 0);
- if (docSet.size() > 0) {
- while ((term = termsEnum.next()) != null) {
- int termDocId = -1;
- String key = MtasToken.getPostfixFromValue(term.utf8ToString());
- String[] keys = new String[] { key };
- if (termVector.defaultStatsType.equals(CodecUtil.STATS_BASIC)
- && termVector.functionParser.sumRule()
- && !termVector.functionParser.needPositions()) {
- long[] valueSum = new long[] { 0 };
- int docNumber = 0;
- boolean subtractValue = false;
- if (docSet.size() == r.numDocs()) {
- valueSum[0] = termsEnum.totalTermFreq();
- docNumber = termsEnum.docFreq();
- } else {
- if (docSet.size() > Math.round(r.numDocs() / 2)) {
- valueSum[0] = termsEnum.totalTermFreq();
- subtractValue = true;
- }
- Iterator<Integer> docIterator = docSet.iterator();
- postingsEnum = termsEnum.postings(postingsEnum,
- PostingsEnum.FREQS);
- while (docIterator.hasNext()) {
- int docId = docIterator.next() - lrc.docBase;
- if (docId >= termDocId) {
- if ((docId == termDocId) || ((termDocId = postingsEnum
- .advance(docId)) == docId)) {
- docNumber++;
- if (subtractValue) {
- valueSum[0] -= postingsEnum.freq();
- } else {
- valueSum[0] += postingsEnum.freq();
- }
- }
- }
- }
- }
- if (docNumber > 0) {
- if (termVector.dataFunctionCollector.getDataType()
- .equals(CodecUtil.DATA_TYPE_LONG)) {
- long value = termVector.functionParser.getValueLong(valueSum,
- 0);
- termVector.dataFunctionCollector.add(keys, value, docNumber);
- } else if (termVector.dataFunctionCollector.getDataType()
- .equals(CodecUtil.DATA_TYPE_LONG)) {
- double value = termVector.functionParser
- .getValueDouble(valueSum, 0);
- termVector.dataFunctionCollector.add(keys, value, docNumber);
- } else {
- throw new IOException("unknown dataType "
- + termVector.dataFunctionCollector.getDataType());
- }
- }
- } else {
- Iterator<Integer> docIterator = docSet.iterator();
- postingsEnum = termsEnum.postings(postingsEnum,
- PostingsEnum.FREQS);
- if (termVector.dataFunctionCollector.getDataType()
- .equals(CodecUtil.DATA_TYPE_LONG)) {
- long[] args = new long[1];
- long value;
- long[] values = new long[docSet.size()];
- int positions, number = 0;
- while (docIterator.hasNext()) {
- int docId = docIterator.next() - lrc.docBase;
- if (docId >= termDocId) {
- if ((docId == termDocId) || ((termDocId = postingsEnum
- .advance(docId)) == docId)) {
- args[0] = postingsEnum.freq();
- positions = (positionsData == null) ? 0
- : positionsData.get(docId + lrc.docBase);
- try {
- value = termVector.functionParser.getValueLong(args,
- positions);
- values[number] = value;
- } catch (IOException e) {
- termVector.dataFunctionCollector.error(keys,
- e.getMessage());
- }
- number++;
- }
- }
- }
- if (number > 0) {
- termVector.dataFunctionCollector.add(keys, values, number);
- }
- } else if (termVector.dataFunctionCollector.getDataType()
- .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
- long[] args = new long[1];
- double value;
- double[] values = new double[docSet.size()];
- int positions, number = 0;
- while (docIterator.hasNext()) {
- int docId = docIterator.next() - lrc.docBase;
- if (docId >= termDocId) {
- if ((docId == termDocId) || ((termDocId = postingsEnum
- .advance(docId)) == docId)) {
- args[0] = postingsEnum.freq();
- positions = (positionsData == null) ? 0
- : positionsData.get(docId + lrc.docBase);
- try {
- value = termVector.functionParser.getValueDouble(args,
- positions);
- values[number] = value;
- } catch (IOException e) {
- termVector.dataFunctionCollector.error(keys,
- e.getMessage());
- }
- number++;
- }
- }
- }
- if (number > 0) {
- termVector.dataFunctionCollector.add(keys, values, number);
- }
- }
- }
- }
- }
- termVector.dataFunctionCollector.closeNewList();
- }
- }
- }
-
}
diff --git a/src/mtas/codec/util/CodecComponent.java b/src/mtas/codec/util/CodecComponent.java
index 6ad6611..1049617 100644
--- a/src/mtas/codec/util/CodecComponent.java
+++ b/src/mtas/codec/util/CodecComponent.java
@@ -16,7 +16,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import mtas.analysis.token.MtasToken;
import mtas.codec.util.CodecSearchTree.MtasTreeHit;
-import mtas.codec.util.DataCollector.MtasDataCollector;
+import mtas.codec.util.collector.MtasDataCollector;
import mtas.parser.function.MtasFunctionParser;
import mtas.parser.function.ParseException;
import mtas.parser.function.util.MtasFunctionParserFunction;
@@ -29,14 +29,6 @@ import org.apache.lucene.util.automaton.RegExp;
/**
* The Class CodecComponent.
*/
-/**
- * @author matthijs
- *
- */
-/**
- * @author matthijs
- *
- */
public class CodecComponent {
/**
@@ -139,8 +131,10 @@ public class CodecComponent {
/**
* Instantiates a new component field.
*
- * @param field the field
- * @param uniqueKeyField the unique key field
+ * @param field
+ * the field
+ * @param uniqueKeyField
+ * the unique key field
*/
public ComponentField(String field, String uniqueKeyField) {
this.field = field;
@@ -172,42 +166,72 @@ public class CodecComponent {
/** The multiple position list. */
public TreeSet<String> multiplePositionList;
+ /** The set position list. */
+ public TreeSet<String> setPositionList;
+
/**
* Instantiates a new component prefix.
*
- * @param key the key
+ * @param key
+ * the key
*/
public ComponentPrefix(String key) {
this.key = key;
singlePositionList = new TreeSet<String>();
multiplePositionList = new TreeSet<String>();
+ setPositionList = new TreeSet<String>();
}
/**
* Adds the single position.
*
- * @param prefix the prefix
+ * @param prefix
+ * the prefix
*/
public void addSinglePosition(String prefix) {
- if (!singlePositionList.contains(prefix)
- && !multiplePositionList.contains(prefix)) {
- singlePositionList.add(prefix);
+ if (!prefix.trim().equals("")) {
+ if (!singlePositionList.contains(prefix)
+ && !multiplePositionList.contains(prefix)) {
+ singlePositionList.add(prefix);
+ }
}
}
/**
* Adds the multiple position.
*
- * @param prefix the prefix
+ * @param prefix
+ * the prefix
*/
public void addMultiplePosition(String prefix) {
- if (!singlePositionList.contains(prefix)) {
- if (!multiplePositionList.contains(prefix)) {
+ if (!prefix.trim().equals("")) {
+ if (!singlePositionList.contains(prefix)) {
+ if (!multiplePositionList.contains(prefix)) {
+ multiplePositionList.add(prefix);
+ }
+ } else {
+ singlePositionList.remove(prefix);
multiplePositionList.add(prefix);
}
- } else {
- singlePositionList.remove(prefix);
- multiplePositionList.add(prefix);
+ }
+ }
+
+ /**
+ * Adds the set position.
+ *
+ * @param prefix
+ * the prefix
+ */
+ public void addSetPosition(String prefix) {
+ if (!prefix.trim().equals("")) {
+ if (!singlePositionList.contains(prefix)) {
+ if (!setPositionList.contains(prefix)) {
+ setPositionList.add(prefix);
+ }
+ } else {
+ singlePositionList.remove(prefix);
+ setPositionList.add(prefix);
+ }
}
}
}
@@ -262,15 +286,24 @@ public class CodecComponent {
/**
* Instantiates a new component kwic.
*
- * @param query the query
- * @param key the key
- * @param prefixes the prefixes
- * @param number the number
- * @param start the start
- * @param left the left
- * @param right the right
- * @param output the output
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param query
+ * the query
+ * @param key
+ * the key
+ * @param prefixes
+ * the prefixes
+ * @param number
+ * the number
+ * @param start
+ * the start
+ * @param left
+ * the left
+ * @param right
+ * the right
+ * @param output
+ * the output
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public ComponentKwic(SpanQuery query, String key, String prefixes,
Integer number, int start, int left, int right, String output)
@@ -360,18 +393,30 @@ public class CodecComponent {
/**
* Instantiates a new component list.
*
- * @param spanQuery the span query
- * @param field the field
- * @param queryValue the query value
- * @param queryType the query type
- * @param key the key
- * @param prefix the prefix
- * @param start the start
- * @param number the number
- * @param left the left
- * @param right the right
- * @param output the output
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param spanQuery
+ * the span query
+ * @param field
+ * the field
+ * @param queryValue
+ * the query value
+ * @param queryType
+ * the query type
+ * @param key
+ * the key
+ * @param prefix
+ * the prefix
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param left
+ * the left
+ * @param right
+ * the right
+ * @param output
+ * the output
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public ComponentList(SpanQuery spanQuery, String field, String queryValue,
String queryType, String key, String prefix, int start, int number,
@@ -454,25 +499,44 @@ public class CodecComponent {
/**
* Instantiates a new component group.
*
- * @param spanQuery the span query
- * @param field the field
- * @param queryValue the query value
- * @param queryType the query type
- * @param key the key
- * @param groupingHitInsidePrefixes the grouping hit inside prefixes
- * @param groupingHitInsideLeftPosition the grouping hit inside left position
- * @param groupingHitInsideLeftPrefixes the grouping hit inside left prefixes
- * @param groupingHitInsideRightPosition the grouping hit inside right position
- * @param groupingHitInsideRightPrefixes the grouping hit inside right prefixes
- * @param groupingHitLeftPosition the grouping hit left position
- * @param groupingHitLeftPrefixes the grouping hit left prefixes
- * @param groupingHitRightPosition the grouping hit right position
- * @param groupingHitRightPrefixes the grouping hit right prefixes
- * @param groupingLeftPosition the grouping left position
- * @param groupingLeftPrefixes the grouping left prefixes
- * @param groupingRightPosition the grouping right position
- * @param groupingRightPrefixes the grouping right prefixes
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param spanQuery
+ * the span query
+ * @param field
+ * the field
+ * @param queryValue
+ * the query value
+ * @param queryType
+ * the query type
+ * @param key
+ * the key
+ * @param groupingHitInsidePrefixes
+ * the grouping hit inside prefixes
+ * @param groupingHitInsideLeftPosition
+ * the grouping hit inside left position
+ * @param groupingHitInsideLeftPrefixes
+ * the grouping hit inside left prefixes
+ * @param groupingHitInsideRightPosition
+ * the grouping hit inside right position
+ * @param groupingHitInsideRightPrefixes
+ * the grouping hit inside right prefixes
+ * @param groupingHitLeftPosition
+ * the grouping hit left position
+ * @param groupingHitLeftPrefixes
+ * the grouping hit left prefixes
+ * @param groupingHitRightPosition
+ * the grouping hit right position
+ * @param groupingHitRightPrefixes
+ * the grouping hit right prefixes
+ * @param groupingLeftPosition
+ * the grouping left position
+ * @param groupingLeftPrefixes
+ * the grouping left prefixes
+ * @param groupingRightPosition
+ * the grouping right position
+ * @param groupingRightPrefixes
+ * the grouping right prefixes
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public ComponentGroup(SpanQuery spanQuery, String field, String queryValue,
String queryType, String key, String groupingHitInsidePrefixes,
@@ -529,7 +593,7 @@ public class CodecComponent {
dataCollector = DataCollector.getCollector(
DataCollector.COLLECTOR_TYPE_LIST, this.dataType, this.statsType,
this.statsItems, this.sortType, this.sortDirection, this.start,
- this.number, null, null, null, null, null, null, null, null, false);
+ this.number, null, null);
}
}
@@ -537,11 +601,15 @@ public class CodecComponent {
/**
* Creates the positioned prefixes.
*
- * @param prefixList the prefix list
- * @param position the position
- * @param prefixes the prefixes
+ * @param prefixList
+ * the prefix list
+ * @param position
+ * the position
+ * @param prefixes
+ * the prefixes
* @return the hash set[]
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static HashSet<String>[] createPositionedPrefixes(
HashSet<String> prefixList, String[] position, String[] prefixes)
@@ -582,6 +650,7 @@ public class CodecComponent {
throw new IOException("incorrect position " + position[i]);
}
}
+ @SuppressWarnings("unchecked")
HashSet<String>[] result = new HashSet[maxPosition + 1];
Arrays.fill(result, null);
List<String> tmpPrefixList;
@@ -633,6 +702,9 @@ public class CodecComponent {
/** The data collector. */
public MtasDataCollector<?, ?> dataCollector;
+ /** The base function list. */
+ public HashMap<MtasDataCollector<?, ?>, SubComponentFunction[]>[] baseFunctionList;
+
/** The base numbers. */
public Integer[] baseNumbers;
@@ -642,8 +714,20 @@ public class CodecComponent {
/** The base maximum doubles. */
public Double[] baseMinimumDoubles, baseMaximumDoubles;
- /** The base function parsers. */
- public MtasFunctionParserFunction[] baseFunctionParsers;
+ /** The base parsers. */
+ public MtasFunctionParserFunction[] baseParsers;
+
+ /** The base function keys. */
+ public String[][] baseFunctionKeys;
+
+ /** The base function expressions. */
+ public String[][] baseFunctionExpressions;
+
+ /** The base function types. */
+ public String[][] baseFunctionTypes;
+
+ /** The base function parser functions. */
+ public MtasFunctionParserFunction[][] baseFunctionParserFunctions;
/** The Constant TYPE_INTEGER. */
public static final String TYPE_INTEGER = "integer";
@@ -663,26 +747,46 @@ public class CodecComponent {
/**
* Instantiates a new component facet.
*
- * @param spanQueries the span queries
- * @param field the field
- * @param key the key
- * @param baseFields the base fields
- * @param baseFieldTypes the base field types
- * @param baseTypes the base types
- * @param baseSortTypes the base sort types
- * @param baseSortDirections the base sort directions
- * @param baseNumbers the base numbers
- * @param baseMinimumDoubles the base minimum doubles
- * @param baseMaximumDoubles the base maximum doubles
- * @param baseFunctions the base functions
- * @throws IOException Signals that an I/O exception has occurred.
- * @throws ParseException the parse exception
+ * @param spanQueries
+ * the span queries
+ * @param field
+ * the field
+ * @param key
+ * the key
+ * @param baseFields
+ * the base fields
+ * @param baseFieldTypes
+ * the base field types
+ * @param baseTypes
+ * the base types
+ * @param baseSortTypes
+ * the base sort types
+ * @param baseSortDirections
+ * the base sort directions
+ * @param baseNumbers
+ * the base numbers
+ * @param baseMinimumDoubles
+ * the base minimum doubles
+ * @param baseMaximumDoubles
+ * the base maximum doubles
+ * @param baseFunctionKeys
+ * the base function keys
+ * @param baseFunctionExpressions
+ * the base function expressions
+ * @param baseFunctionTypes
+ * the base function types
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ * @throws ParseException
+ * the parse exception
*/
+ @SuppressWarnings("unchecked")
public ComponentFacet(SpanQuery[] spanQueries, String field, String key,
String[] baseFields, String[] baseFieldTypes, String[] baseTypes,
String[] baseSortTypes, String[] baseSortDirections,
Integer[] baseNumbers, Double[] baseMinimumDoubles,
- Double[] baseMaximumDoubles, String[] baseFunctions)
+ Double[] baseMaximumDoubles, String[][] baseFunctionKeys,
+ String[][] baseFunctionExpressions, String[][] baseFunctionTypes)
throws IOException, ParseException {
this.spanQueries = spanQueries;
this.field = field;
@@ -702,8 +806,9 @@ public class CodecComponent {
this.baseStatsItems = new TreeSet[baseFields.length];
this.baseStatsTypes = new String[baseFields.length];
this.baseDataTypes = new String[baseFields.length];
- this.baseFunctionParsers = new MtasFunctionParserFunction[baseFields.length];
- this.baseDataTypes = new String[baseFields.length];
+ this.baseParsers = new MtasFunctionParserFunction[baseFields.length];
+ this.baseFunctionList = new HashMap[baseFields.length];
+ this.baseFunctionParserFunctions = new MtasFunctionParserFunction[baseFields.length][];
for (int i = 0; i < baseFields.length; i++) {
if (baseMinimumDoubles[i] != null) {
this.baseMinimumLongs[i] = baseMinimumDoubles[i].longValue();
@@ -715,13 +820,11 @@ public class CodecComponent {
} else {
this.baseMaximumLongs[i] = null;
}
- if (baseFunctions[i] != null) {
- baseFunctionParsers[i] = new MtasFunctionParser(
- new BufferedReader(new StringReader(baseFunctions[i]))).parse();
- } else {
- baseFunctionParsers[i] = new MtasFunctionParserFunctionDefault(
- spanQueries.length);
- }
+ baseDataTypes[i] = CodecUtil.DATA_TYPE_LONG;
+ baseFunctionList[i] = new HashMap<MtasDataCollector<?, ?>, SubComponentFunction[]>();
+ baseFunctionParserFunctions[i] = null;
+ baseParsers[i] = new MtasFunctionParserFunctionDefault(
+ this.spanQueries.length);
if (this.baseSortDirections[i] == null) {
this.baseSortDirections[i] = CodecUtil.SORT_ASC;
} else if (!this.baseSortDirections[i].equals(CodecUtil.SORT_ASC)
@@ -736,19 +839,50 @@ public class CodecComponent {
throw new IOException(
"unrecognized sortType " + this.baseSortTypes[i]);
}
- this.baseDataTypes[i] = baseFunctionParsers[i].getType();
this.baseCollectorTypes[i] = DataCollector.COLLECTOR_TYPE_LIST;
this.baseStatsItems[i] = CodecUtil.createStatsItems(baseTypes[i]);
this.baseStatsTypes[i] = CodecUtil.createStatsType(baseStatsItems[i],
- this.baseSortTypes[i], this.baseFunctionParsers[i]);
+ this.baseSortTypes[i], new MtasFunctionParserFunctionDefault(1));
+ }
+ if (baseFunctionKeys != null && baseFunctionExpressions != null
+ && baseFunctionTypes != null) {
+ if (baseFunctionKeys.length == baseFields.length
+ && baseFunctionExpressions.length == baseFields.length
+ && baseFunctionTypes.length == baseFields.length) {
+ this.baseFunctionKeys = new String[baseFields.length][];
+ this.baseFunctionExpressions = new String[baseFields.length][];
+ this.baseFunctionTypes = new String[baseFields.length][];
+ for (int i = 0; i < baseFields.length; i++) {
+ if (baseFunctionKeys[i].length == baseFunctionExpressions[i].length
+ && baseFunctionKeys[i].length == baseFunctionTypes[i].length) {
+ this.baseFunctionKeys[i] = new String[baseFunctionKeys[i].length];
+ this.baseFunctionExpressions[i] = new String[baseFunctionExpressions[i].length];
+ this.baseFunctionTypes[i] = new String[baseFunctionTypes[i].length];
+ baseFunctionParserFunctions[i] = new MtasFunctionParserFunction[baseFunctionExpressions[i].length];
+ for (int j = 0; j < baseFunctionKeys[i].length; j++) {
+ this.baseFunctionKeys[i][j] = baseFunctionKeys[i][j];
+ this.baseFunctionExpressions[i][j] = baseFunctionExpressions[i][j];
+ this.baseFunctionTypes[i][j] = baseFunctionTypes[i][j];
+ baseFunctionParserFunctions[i][j] = new MtasFunctionParser(
+ new BufferedReader(
+ new StringReader(baseFunctionExpressions[i][j])))
+ .parse();
+ }
+ } else {
+ this.baseFunctionKeys[i] = new String[0];
+ this.baseFunctionExpressions[i] = new String[0];
+ this.baseFunctionTypes[i] = new String[0];
+ baseFunctionParserFunctions[i] = new MtasFunctionParserFunction[0];
+ }
+ }
+ }
}
if (baseFields.length > 0) {
if (baseFields.length == 1) {
dataCollector = DataCollector.getCollector(this.baseCollectorTypes[0],
this.baseDataTypes[0], this.baseStatsTypes[0],
this.baseStatsItems[0], this.baseSortTypes[0],
- this.baseSortDirections[0], 0, this.baseNumbers[0], null, null,
- null, null, null, null, null, null, false);
+ this.baseSortDirections[0], 0, this.baseNumbers[0], null, null);
} else {
String[] subBaseCollectorTypes = Arrays
.copyOfRange(baseCollectorTypes, 1, baseDataTypes.length);
@@ -771,13 +905,77 @@ public class CodecComponent {
this.baseSortDirections[0], 0, this.baseNumbers[0],
subBaseCollectorTypes, subBaseDataTypes, subBaseStatsTypes,
subBaseStatsItems, subBaseSortTypes, subBaseSortDirections,
- subStarts, subNumbers, false);
+ subStarts, subNumbers, null, null);
}
} else {
throw new IOException("no baseFields");
}
}
+ /**
+ * Function sum rule.
+ *
+ * @return true, if successful
+ */
+ public boolean functionSumRule() {
+ if (baseFunctionParserFunctions != null) {
+ for (int i = 0; i < baseFields.length; i++) {
+ for (MtasFunctionParserFunction function : baseFunctionParserFunctions[i]) {
+ if (!function.sumRule()) {
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Function need positions.
+ *
+ * @return true, if successful
+ */
+ public boolean functionNeedPositions() {
+ if (baseFunctionParserFunctions != null) {
+ for (int i = 0; i < baseFields.length; i++) {
+ for (MtasFunctionParserFunction function : baseFunctionParserFunctions[i]) {
+ if (function.needPositions()) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Base parser sum rule.
+ *
+ * @return true, if successful
+ */
+ public boolean baseParserSumRule() {
+ for (int i = 0; i < baseFields.length; i++) {
+ if (!baseParsers[i].sumRule()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Base parser need positions.
+ *
+ * @return true, if successful
+ */
+ public boolean baseParserNeedPositions() {
+ for (int i = 0; i < baseFields.length; i++) {
+ if (baseParsers[i].needPositions()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
}
/**
@@ -785,15 +983,17 @@ public class CodecComponent {
*/
public static class ComponentTermVector {
- /** The sort direction. */
- public String key, defaultDataType, functionDataType, defaultStatsType,
- functionStatsType, prefix, regexp, sortType, sortDirection;
+ /** The boundary. */
+ public String key, prefix, regexp, boundary;
- /** The default stats items. */
- public TreeSet<String> functionStatsItems, defaultStatsItems;
+ /** The list. */
+ public HashSet<String> list;
+
+ /** The functions. */
+ public ArrayList<SubComponentFunction> functions;
/** The number. */
- public int start, number;
+ public int number;
/** The start value. */
public String startValue;
@@ -801,106 +1001,124 @@ public class CodecComponent {
/** The compiled automaton. */
public CompiledAutomaton compiledAutomaton;
- /** The data function collector. */
- public MtasDataCollector<?, ?> dataFunctionCollector;
+ /** The sub component function. */
+ public SubComponentFunction subComponentFunction;
- /** The data default collector. */
- public MtasDataCollector<?, ?> dataDefaultCollector;
-
- /** The function parser. */
- public MtasFunctionParserFunction functionParser;
-
- /** The default parser. */
- public MtasFunctionParserFunction defaultParser;
+ /** The boundary registration. */
+ public boolean boundaryRegistration;
/**
* Instantiates a new component term vector.
*
- * @param key the key
- * @param prefix the prefix
- * @param regexp the regexp
- * @param statsType the stats type
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param startValue the start value
- * @param number the number
- * @param function the function
- * @throws IOException Signals that an I/O exception has occurred.
- * @throws ParseException the parse exception
+ * @param key
+ * the key
+ * @param prefix
+ * the prefix
+ * @param regexp
+ * the regexp
+ * @param type
+ * the type
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param startValue
+ * the start value
+ * @param number
+ * the number
+ * @param functionKey
+ * the function key
+ * @param functionExpression
+ * the function expression
+ * @param functionType
+ * the function type
+ * @param boundary
+ * the boundary
+ * @param list
+ * the list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ * @throws ParseException
+ * the parse exception
*/
+ @SuppressWarnings({ "unchecked", "rawtypes" })
public ComponentTermVector(String key, String prefix, String regexp,
- String statsType, String sortType, String sortDirection,
- String startValue, int number, String function)
+ String type, String sortType, String sortDirection, String startValue,
+ int number, String[] functionKey, String[] functionExpression,
+ String[] functionType, String boundary, String[] list)
throws IOException, ParseException {
- this.key = key;
- this.defaultParser = new MtasFunctionParserFunctionDefault(1);
- if (function != null) {
- functionParser = new MtasFunctionParser(
- new BufferedReader(new StringReader(function))).parse();
- functionDataType = functionParser.getType();
- functionStatsItems = CodecUtil.createStatsItems(statsType);
- functionStatsType = CodecUtil.createStatsType(
- this.functionStatsItems, this.sortType, this.functionParser);
- defaultDataType = defaultParser.getType();
- defaultStatsItems = CodecUtil.createStatsItems(null);
- defaultStatsType = CodecUtil.createStatsType(
- this.defaultStatsItems, this.sortType, this.defaultParser);
- } else {
- functionParser = null;
- functionDataType = null;
- functionStatsItems = null;
- functionStatsType = null;
- defaultDataType = defaultParser.getType();
- defaultStatsItems = CodecUtil.createStatsItems(statsType);
- defaultStatsType = CodecUtil.createStatsType(
- this.defaultStatsItems, this.sortType, this.defaultParser);
- }
+ this.key = key;
this.prefix = prefix;
this.regexp = regexp;
- this.sortType = sortType == null ? CodecUtil.SORT_TERM : sortType;
- this.sortDirection = sortDirection == null
- ? (this.sortType == CodecUtil.SORT_TERM) ? CodecUtil.SORT_ASC
- : CodecUtil.SORT_DESC
- : sortDirection;
- this.startValue = startValue;
- this.start = 0;
- this.number = number;
- if (!this.sortType.equals(CodecUtil.SORT_TERM)
- && !CodecUtil.STATS_TYPES.contains(this.sortType)) {
- throw new IOException("unknown sortType '" + this.sortType + "'");
- } else if (!this.sortType.equals(CodecUtil.SORT_TERM)) {
- if (!(this.sortType.equals(CodecUtil.STATS_TYPE_SUM)
- || this.sortType.equals(CodecUtil.STATS_TYPE_N))) {
+ sortType = sortType == null ? CodecUtil.SORT_TERM : sortType;
+ sortDirection = sortDirection == null ? (sortType == CodecUtil.SORT_TERM)
+ ? CodecUtil.SORT_ASC : CodecUtil.SORT_DESC : sortDirection;
+ if (list != null && list.length > 0) {
+ this.list = new HashSet(Arrays.asList(list));
+ this.boundary = null;
+ this.number = Integer.MAX_VALUE;
+ this.startValue = null;
+ sortType = CodecUtil.SORT_TERM;
+ sortDirection = CodecUtil.SORT_ASC;
+ } else {
+ this.list = null;
+ this.startValue = startValue;
+ if (boundary == null) {
+ this.boundary = null;
+ this.number = number;
+ } else {
+ this.boundary = boundary;
+ this.number = Integer.MAX_VALUE;
+ }
+ }
+ functions = new ArrayList<SubComponentFunction>();
+ if (functionKey != null && functionExpression != null
+ && functionType != null) {
+ if (functionKey.length == functionExpression.length
+ && functionKey.length == functionType.length) {
+ for (int i = 0; i < functionKey.length; i++) {
+ functions
+ .add(new SubComponentFunction(DataCollector.COLLECTOR_TYPE_LIST,
+ functionKey[i], functionExpression[i], functionType[i]));
+ }
+ }
+ }
+ if (!sortType.equals(CodecUtil.SORT_TERM)
+ && !CodecUtil.STATS_TYPES.contains(sortType)) {
+ throw new IOException("unknown sortType '" + sortType + "'");
+ } else if (!sortType.equals(CodecUtil.SORT_TERM)) {
+ if (!(sortType.equals(CodecUtil.STATS_TYPE_SUM)
+ || sortType.equals(CodecUtil.STATS_TYPE_N))) {
throw new IOException(
- "sortType '" + this.sortType + "' not supported for termVector");
+ "sortType '" + sortType + "' not supported for termVector");
}
}
- if (!this.sortDirection.equals(CodecUtil.SORT_ASC)
- && !this.sortDirection.equals(CodecUtil.SORT_DESC)) {
+ if (!sortDirection.equals(CodecUtil.SORT_ASC)
+ && !sortDirection.equals(CodecUtil.SORT_DESC)) {
throw new IOException(
- "unrecognized sortDirection '" + this.sortDirection + "'");
+ "unrecognized sortDirection '" + sortDirection + "'");
}
- boolean segmentRegistration = !this.sortType.equals(CodecUtil.SORT_TERM);
- //create datacollectors
- if (functionParser != null) {
- dataFunctionCollector = DataCollector.getCollector(
- DataCollector.COLLECTOR_TYPE_LIST, this.functionDataType,
- this.functionStatsType, this.functionStatsItems, this.sortType,
- this.sortDirection, this.start, this.number, null, null, null, null,
- null, null, null, null, false);
- dataDefaultCollector = DataCollector.getCollector(
- DataCollector.COLLECTOR_TYPE_LIST, this.defaultDataType,
- this.defaultStatsType, this.defaultStatsItems, this.sortType,
- this.sortDirection, this.start, this.number, null, null, null, null,
- null, null, null, null, segmentRegistration);
- } else {
- dataFunctionCollector = null;
- dataDefaultCollector = DataCollector.getCollector(
- DataCollector.COLLECTOR_TYPE_LIST, this.defaultDataType,
- this.defaultStatsType, this.defaultStatsItems, this.sortType,
- this.sortDirection, this.start, this.number, null, null, null, null,
- null, null, null, null, segmentRegistration);
+ boundaryRegistration = this.boundary != null;
+ String segmentRegistration = null;
+ if (this.boundary != null) {
+ if (sortDirection.equals(CodecUtil.SORT_ASC)) {
+ segmentRegistration = MtasDataCollector.SEGMENT_BOUNDARY_ASC;
+ } else if (sortDirection.equals(CodecUtil.SORT_DESC)) {
+ segmentRegistration = MtasDataCollector.SEGMENT_BOUNDARY_DESC;
+ }
+ } else if (!sortType.equals(CodecUtil.SORT_TERM)) {
+ if (sortDirection.equals(CodecUtil.SORT_ASC)) {
+ segmentRegistration = MtasDataCollector.SEGMENT_SORT_ASC;
+ } else if (sortDirection.equals(CodecUtil.SORT_DESC)) {
+ segmentRegistration = MtasDataCollector.SEGMENT_SORT_DESC;
+ }
}
+ // create main subComponentFunction
+ this.subComponentFunction = new SubComponentFunction(
+ DataCollector.COLLECTOR_TYPE_LIST, key, type,
+ new MtasFunctionParserFunctionDefault(1), sortType, sortDirection, 0,
+ this.number, segmentRegistration, boundary);
+
if ((regexp == null) || (regexp.isEmpty())) {
RegExp re = new RegExp(prefix + MtasToken.DELIMITER + ".*");
compiledAutomaton = new CompiledAutomaton(re.toAutomaton());
@@ -910,6 +1128,39 @@ public class CodecComponent {
compiledAutomaton = new CompiledAutomaton(re.toAutomaton());
}
}
+
+ /**
+ * Function sum rule.
+ *
+ * @return true, if successful
+ */
+ public boolean functionSumRule() {
+ if (functions != null) {
+ for (SubComponentFunction function : functions) {
+ if (!function.parserFunction.sumRule()) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Function need positions.
+ *
+ * @return true, if successful
+ */
+ public boolean functionNeedPositions() {
+ if (functions != null) {
+ for (SubComponentFunction function : functions) {
+ if (function.parserFunction.needPositions()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
}
/**
@@ -941,38 +1192,58 @@ public class CodecComponent {
/** The data collector. */
public MtasDataCollector<?, ?> dataCollector;
- /** The function parser. */
- public MtasFunctionParserFunction functionParser;
+ /** The functions. */
+ public ArrayList<SubComponentFunction> functions;
+
+ /** The parser. */
+ public MtasFunctionParserFunction parser;
/**
* Instantiates a new component span.
*
- * @param queries the queries
- * @param key the key
- * @param minimumDouble the minimum double
- * @param maximumDouble the maximum double
- * @param type the type
- * @param function the function
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param queries
+ * the queries
+ * @param key
+ * the key
+ * @param minimumDouble
+ * the minimum double
+ * @param maximumDouble
+ * the maximum double
+ * @param type
+ * the type
+ * @param functionKey
+ * the function key
+ * @param functionExpression
+ * the function expression
+ * @param functionType
+ * the function type
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ * @throws ParseException
+ * the parse exception
*/
public ComponentSpan(SpanQuery[] queries, String key, Double minimumDouble,
- Double maximumDouble, String type, String function) throws IOException {
+ Double maximumDouble, String type, String[] functionKey,
+ String[] functionExpression, String[] functionType)
+ throws IOException, ParseException {
this.queries = queries;
this.key = key;
- if (function != null) {
- try {
- functionParser = new MtasFunctionParser(
- new BufferedReader(new StringReader(function))).parse();
- } catch (ParseException e) {
- throw new IOException("couldn't parse function " + function);
+ functions = new ArrayList<SubComponentFunction>();
+ if (functionKey != null && functionExpression != null
+ && functionType != null) {
+ if (functionKey.length == functionExpression.length
+ && functionKey.length == functionType.length) {
+ for (int i = 0; i < functionKey.length; i++) {
+ functions
+ .add(new SubComponentFunction(DataCollector.COLLECTOR_TYPE_DATA,
+ functionKey[i], functionExpression[i], functionType[i]));
+ }
}
- } else {
- functionParser = new MtasFunctionParserFunctionDefault(queries.length);
}
- dataType = functionParser.getType();
+ parser = new MtasFunctionParserFunctionDefault(queries.length);
+ dataType = parser.getType();
statsItems = CodecUtil.createStatsItems(type);
- statsType = CodecUtil.createStatsType(this.statsItems, null,
- functionParser);
+ statsType = CodecUtil.createStatsType(this.statsItems, null, parser);
if (minimumDouble != null) {
this.minimumDouble = minimumDouble;
this.minimumLong = minimumDouble.longValue();
@@ -989,9 +1260,56 @@ public class CodecComponent {
}
dataCollector = DataCollector.getCollector(
DataCollector.COLLECTOR_TYPE_DATA, dataType, this.statsType,
- this.statsItems, null, null, null, null, null, null, null, null, null,
- null, null, null, false);
+ this.statsItems, null, null, null, null, null, null);
+ }
+
+ /**
+ * Function sum rule.
+ *
+ * @return true, if successful
+ */
+ public boolean functionSumRule() {
+ if (functions != null) {
+ for (SubComponentFunction function : functions) {
+ if (!function.parserFunction.sumRule()) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Function need positions.
+ *
+ * @return true, if successful
+ */
+ public boolean functionNeedPositions() {
+ if (functions != null) {
+ for (SubComponentFunction function : functions) {
+ if (function.parserFunction.needPositions()) {
+ return true;
+ }
+ }
+ }
+ return false;
}
+
+ /**
+ * Function need arguments.
+ *
+ * @return the hash set
+ */
+ public HashSet<Integer> functionNeedArguments() {
+ HashSet<Integer> list = new HashSet<Integer>();
+ if (functions != null) {
+ for (SubComponentFunction function : functions) {
+ list.addAll(function.parserFunction.needArgument());
+ }
+ }
+ return list;
+ }
+
}
/**
@@ -1026,13 +1344,20 @@ public class CodecComponent {
/**
* Instantiates a new component position.
*
- * @param field the field
- * @param key the key
- * @param minimumDouble the minimum double
- * @param maximumDouble the maximum double
- * @param statsType the stats type
- * @throws IOException Signals that an I/O exception has occurred.
- * @throws ParseException the parse exception
+ * @param field
+ * the field
+ * @param key
+ * the key
+ * @param minimumDouble
+ * the minimum double
+ * @param maximumDouble
+ * the maximum double
+ * @param statsType
+ * the stats type
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ * @throws ParseException
+ * the parse exception
*/
public ComponentPosition(String field, String key, Double minimumDouble,
Double maximumDouble, String statsType)
@@ -1059,8 +1384,7 @@ public class CodecComponent {
}
dataCollector = DataCollector.getCollector(
DataCollector.COLLECTOR_TYPE_DATA, dataType, this.statsType,
- this.statsItems, null, null, null, null, null, null, null, null, null,
- null, null, null, false);
+ this.statsItems, null, null, null, null, null, null);
}
}
@@ -1096,13 +1420,20 @@ public class CodecComponent {
/**
* Instantiates a new component token.
*
- * @param field the field
- * @param key the key
- * @param minimumDouble the minimum double
- * @param maximumDouble the maximum double
- * @param statsType the stats type
- * @throws IOException Signals that an I/O exception has occurred.
- * @throws ParseException the parse exception
+ * @param field
+ * the field
+ * @param key
+ * the key
+ * @param minimumDouble
+ * the minimum double
+ * @param maximumDouble
+ * the maximum double
+ * @param statsType
+ * the stats type
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ * @throws ParseException
+ * the parse exception
*/
public ComponentToken(String field, String key, Double minimumDouble,
Double maximumDouble, String statsType)
@@ -1129,8 +1460,123 @@ public class CodecComponent {
}
dataCollector = DataCollector.getCollector(
DataCollector.COLLECTOR_TYPE_DATA, dataType, this.statsType,
- this.statsItems, null, null, null, null, null, null, null, null, null,
- null, null, null, false);
+ this.statsItems, null, null, null, null, null, null);
+ }
+ }
+
+ /**
+ * The Class SubComponentFunction.
+ */
+ public static class SubComponentFunction {
+
+ /** The type. */
+ public String key, expression, type;
+
+ /** The parser function. */
+ public MtasFunctionParserFunction parserFunction;
+
+ /** The sort direction. */
+ public String statsType, dataType, sortType, sortDirection;
+
+ /** The stats items. */
+ public TreeSet<String> statsItems;
+
+ /** The data collector. */
+ public MtasDataCollector<?, ?> dataCollector;
+
+ /**
+ * Instantiates a new sub component function.
+ *
+ * @param collectorType
+ * the collector type
+ * @param key
+ * the key
+ * @param type
+ * the type
+ * @param parserFunction
+ * the parser function
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws ParseException
+ * the parse exception
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public SubComponentFunction(String collectorType, String key, String type,
+ MtasFunctionParserFunction parserFunction, String sortType,
+ String sortDirection, Integer start, Integer number,
+ String segmentRegistration, String boundary)
+ throws ParseException, IOException {
+ this.key = key;
+ this.expression = null;
+ this.type = type;
+ this.parserFunction = parserFunction;
+ this.sortType = sortType;
+ this.sortDirection = sortDirection;
+ this.dataType = parserFunction.getType();
+ this.statsItems = CodecUtil.createStatsItems(this.type);
+ this.statsType = CodecUtil.createStatsType(statsItems, null,
+ parserFunction);
+ if (collectorType.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ dataCollector = DataCollector.getCollector(
+ DataCollector.COLLECTOR_TYPE_LIST, dataType, statsType, statsItems,
+ sortType, sortDirection, start, number, null, null, null, null,
+ null, null, null, null, segmentRegistration, boundary);
+ } else if (collectorType.equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ dataCollector = DataCollector.getCollector(
+ DataCollector.COLLECTOR_TYPE_DATA, dataType, statsType, statsItems,
+ sortType, sortDirection, start, number, segmentRegistration,
+ boundary);
+ }
+ }
+
+ /**
+ * Instantiates a new sub component function.
+ *
+ * @param collectorType
+ * the collector type
+ * @param key
+ * the key
+ * @param expression
+ * the expression
+ * @param type
+ * the type
+ * @throws ParseException
+ * the parse exception
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public SubComponentFunction(String collectorType, String key,
+ String expression, String type) throws ParseException, IOException {
+ this.key = key;
+ this.expression = expression;
+ this.type = type;
+ this.sortType = null;
+ this.sortDirection = null;
+ parserFunction = new MtasFunctionParser(
+ new BufferedReader(new StringReader(this.expression))).parse();
+ dataType = parserFunction.getType();
+ statsItems = CodecUtil.createStatsItems(this.type);
+ statsType = CodecUtil.createStatsType(statsItems, null, parserFunction);
+ if (collectorType.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ dataCollector = DataCollector.getCollector(
+ DataCollector.COLLECTOR_TYPE_LIST, dataType, statsType, statsItems,
+ sortType, sortDirection, 0, Integer.MAX_VALUE, null, null);
+ } else if (collectorType.equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ dataCollector = DataCollector.getCollector(
+ DataCollector.COLLECTOR_TYPE_DATA, dataType, statsType, statsItems,
+ sortType, sortDirection, null, null, null, null);
+ }
}
}
@@ -1151,8 +1597,10 @@ public class CodecComponent {
/**
* Instantiates a new kwic token.
*
- * @param match the match
- * @param tokens the tokens
+ * @param match
+ * the match
+ * @param tokens
+ * the tokens
*/
public KwicToken(Match match, ArrayList<MtasToken<String>> tokens) {
startPosition = match.startPosition;
@@ -1178,8 +1626,10 @@ public class CodecComponent {
/**
* Instantiates a new kwic hit.
*
- * @param match the match
- * @param hits the hits
+ * @param match
+ * the match
+ * @param hits
+ * the hits
*/
public KwicHit(Match match, HashMap<Integer, ArrayList<String>> hits) {
startPosition = match.startPosition;
@@ -1202,7 +1652,8 @@ public class CodecComponent {
/**
* Sort.
*
- * @param data the data
+ * @param data
+ * the data
* @return the array list
*/
private ArrayList<MtasTreeHit<String>> sort(
@@ -1217,30 +1668,22 @@ public class CodecComponent {
}
/**
- * Creates the hash.
- *
- * @param data the data
- * @return the string
- */
- private String createHash(ArrayList<MtasTreeHit<String>> data) {
- String hash = "";
- for (MtasTreeHit<String> item : data) {
- hash += CodecUtil.termValue(item.data) + " ";
- }
- hash = hash.trim();
- return hash;
- }
-
- /**
* Instantiates a new group hit.
*
- * @param list the list
- * @param start the start
- * @param end the end
- * @param hitStart the hit start
- * @param hitEnd the hit end
- * @param group the group
+ * @param list
+ * the list
+ * @param start
+ * the start
+ * @param end
+ * the end
+ * @param hitStart
+ * the hit start
+ * @param hitEnd
+ * the hit end
+ * @param group
+ * the group
*/
+ @SuppressWarnings("unchecked")
public GroupHit(ArrayList<MtasTreeHit<String>> list, int start, int end,
int hitStart, int hitEnd, ComponentGroup group) {
int leftRangeStart = start;
@@ -1266,7 +1709,7 @@ public class CodecComponent {
if (leftRange > 0) {
dataLeft = (ArrayList<String>[]) new ArrayList[leftRange];
for (int p = 0; p < leftRange; p++) {
- dataLeft[p] = new ArrayList();
+ dataLeft[p] = new ArrayList<String>();
}
} else {
dataLeft = null;
@@ -1274,7 +1717,7 @@ public class CodecComponent {
if (hitRange > 0) {
dataHit = (ArrayList<String>[]) new ArrayList[hitRange];
for (int p = 0; p < hitRange; p++) {
- dataHit[p] = new ArrayList();
+ dataHit[p] = new ArrayList<String>();
}
} else {
dataHit = null;
@@ -1282,7 +1725,7 @@ public class CodecComponent {
if (rightRange > 0) {
dataRight = (ArrayList<String>[]) new ArrayList[rightRange];
for (int p = 0; p < rightRange; p++) {
- dataRight[p] = new ArrayList();
+ dataRight[p] = new ArrayList<String>();
}
} else {
dataRight = null;
@@ -1290,8 +1733,6 @@ public class CodecComponent {
ArrayList<MtasTreeHit<String>> sortedList = sort(list);
for (MtasTreeHit<String> hit : sortedList) {
String prefix = CodecUtil.termPrefix(hit.data);
- String value = CodecUtil.termValue(hit.data);
- List<String> positionList;
// inside hit
if (group.hitInside != null && group.hitInside.contains(prefix)) {
for (int p = Math.max(hitStart, hit.startPosition); p <= Math
@@ -1392,7 +1833,8 @@ public class CodecComponent {
/**
* Data to string.
*
- * @param data the data
+ * @param data
+ * the data
* @return the string
*/
private String dataToString(ArrayList<String>[] data) {
@@ -1443,10 +1885,14 @@ public class CodecComponent {
/**
* Instantiates a new list token.
*
- * @param docId the doc id
- * @param docPosition the doc position
- * @param match the match
- * @param tokens the tokens
+ * @param docId
+ * the doc id
+ * @param docPosition
+ * the doc position
+ * @param match
+ * the match
+ * @param tokens
+ * the tokens
*/
public ListToken(Integer docId, Integer docPosition, Match match,
ArrayList<MtasToken<String>> tokens) {
@@ -1475,10 +1921,14 @@ public class CodecComponent {
/**
* Instantiates a new list hit.
*
- * @param docId the doc id
- * @param docPosition the doc position
- * @param match the match
- * @param hits the hits
+ * @param docId
+ * the doc id
+ * @param docPosition
+ * the doc position
+ * @param match
+ * the match
+ * @param hits
+ * the hits
*/
public ListHit(Integer docId, Integer docPosition, Match match,
HashMap<Integer, ArrayList<String>> hits) {
@@ -1504,8 +1954,10 @@ public class CodecComponent {
/**
* Instantiates a new match.
*
- * @param startPosition the start position
- * @param endPosition the end position
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
*/
public Match(int startPosition, int endPosition) {
this.startPosition = startPosition;
diff --git a/src/mtas/codec/util/CodecInfo.java b/src/mtas/codec/util/CodecInfo.java
index c62c0f5..ce6d75a 100644
--- a/src/mtas/codec/util/CodecInfo.java
+++ b/src/mtas/codec/util/CodecInfo.java
@@ -21,26 +21,30 @@ public class CodecInfo {
/** The index input list. */
HashMap<String, IndexInput> indexInputList;
-
+
/** The index input offset list. */
HashMap<String, Long> indexInputOffsetList;
-
+
/** The version. */
int version;
-
+
/** The field references. */
private HashMap<String, FieldReferences> fieldReferences;
-
+
/** The prefix references. */
private HashMap<String, LinkedHashMap<String, Long>> prefixReferences;
/**
* Instantiates a new codec info.
*
- * @param indexInputList the index input list
- * @param indexInputOffsetList the index input offset list
- * @param version the version
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param indexInputList
+ * the index input list
+ * @param indexInputOffsetList
+ * the index input offset list
+ * @param version
+ * the version
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public CodecInfo(HashMap<String, IndexInput> indexInputList,
HashMap<String, Long> indexInputOffsetList, int version)
@@ -54,10 +58,13 @@ public class CodecInfo {
/**
* Gets the codec info from terms.
*
- * @param t the t
+ * @param t
+ * the t
* @return the codec info from terms
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
+ @SuppressWarnings("unchecked")
public static CodecInfo getCodecInfoFromTerms(Terms t) throws IOException {
try {
HashMap<String, IndexInput> indexInputList = null;
@@ -89,7 +96,8 @@ public class CodecInfo {
/**
* Inits the.
*
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private void init() throws IOException {
// move to begin
@@ -110,7 +118,7 @@ public class CodecInfo {
fieldReferences.put(field,
new FieldReferences(refIndexDoc, refIndexDocId, numberOfDocs,
refTerm, numberOfTerms, refPrefix, numberOfPrefixes));
- } catch (IOException ex) {
+ } catch (Exception ex) {
break;
}
}
@@ -121,45 +129,57 @@ public class CodecInfo {
/**
* Gets the object by id.
*
- * @param field the field
- * @param docId the doc id
- * @param mtasId the mtas id
+ * @param field
+ * the field
+ * @param docId
+ * the doc id
+ * @param mtasId
+ * the mtas id
* @return the object by id
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasToken<?> getObjectById(String field, int docId, int mtasId)
throws IOException {
- Long ref, objectRefApproxCorrection;
- IndexDoc doc = getDoc(field, docId);
- IndexInput inObjectId = indexInputList.get("indexObjectId");
- IndexInput inObject = indexInputList.get("object");
- IndexInput inTerm = indexInputList.get("term");
- if (doc.storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_BYTE) {
- inObjectId.seek((doc.fpIndexObjectId + (mtasId * 1)));
- objectRefApproxCorrection = Long.valueOf(inObjectId.readByte());
- } else if (doc.storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_SHORT) {
- inObjectId.seek((doc.fpIndexObjectId + (mtasId * 2)));
- objectRefApproxCorrection = Long.valueOf(inObjectId.readShort());
- } else if (doc.storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_INTEGER) {
- inObjectId.seek((doc.fpIndexObjectId + (mtasId * 4)));
- objectRefApproxCorrection = Long.valueOf(inObjectId.readInt());
- } else {
- inObjectId.seek((doc.fpIndexObjectId + (mtasId * 8)));
- objectRefApproxCorrection = Long.valueOf(inObjectId.readLong());
+ try {
+ Long ref, objectRefApproxCorrection;
+ IndexDoc doc = getDoc(field, docId);
+ IndexInput inObjectId = indexInputList.get("indexObjectId");
+ IndexInput inObject = indexInputList.get("object");
+ IndexInput inTerm = indexInputList.get("term");
+ if (doc.storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_BYTE) {
+ inObjectId.seek((doc.fpIndexObjectId + (mtasId * 1)));
+ objectRefApproxCorrection = Long.valueOf(inObjectId.readByte());
+ } else if (doc.storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_SHORT) {
+ inObjectId.seek((doc.fpIndexObjectId + (mtasId * 2)));
+ objectRefApproxCorrection = Long.valueOf(inObjectId.readShort());
+ } else if (doc.storageFlags == MtasCodecPostingsFormat.MTAS_STORAGE_INTEGER) {
+ inObjectId.seek((doc.fpIndexObjectId + (mtasId * 4)));
+ objectRefApproxCorrection = Long.valueOf(inObjectId.readInt());
+ } else {
+ inObjectId.seek((doc.fpIndexObjectId + (mtasId * 8)));
+ objectRefApproxCorrection = Long.valueOf(inObjectId.readLong());
+ }
+ ref = objectRefApproxCorrection + doc.objectRefApproxOffset
+ + (mtasId * doc.objectRefApproxQuotient);
+ return MtasCodecPostingsFormat.getToken(inObject, inTerm, ref);
+ } catch (Exception e) {
+ throw new IOException(e.getMessage());
}
- ref = objectRefApproxCorrection + doc.objectRefApproxOffset
- + (mtasId * doc.objectRefApproxQuotient);
- return MtasCodecPostingsFormat.getToken(inObject, inTerm, ref);
}
/**
* Gets the objects by parent id.
*
- * @param field the field
- * @param docId the doc id
- * @param position the position
+ * @param field
+ * the field
+ * @param docId
+ * the doc id
+ * @param position
+ * the position
* @return the objects by parent id
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public List<MtasToken<String>> getObjectsByParentId(String field, int docId,
int position) throws IOException {
@@ -174,11 +194,15 @@ public class CodecInfo {
/**
* Gets the objects by position.
*
- * @param field the field
- * @param docId the doc id
- * @param position the position
+ * @param field
+ * the field
+ * @param docId
+ * the doc id
+ * @param position
+ * the position
* @return the objects by position
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public ArrayList<MtasToken<String>> getObjectsByPosition(String field,
int docId, int position) throws IOException {
@@ -205,13 +229,19 @@ public class CodecInfo {
/**
* Gets the prefix filtered objects by positions.
*
- * @param field the field
- * @param docId the doc id
- * @param prefixes the prefixes
- * @param startPosition the start position
- * @param endPosition the end position
+ * @param field
+ * the field
+ * @param docId
+ * the doc id
+ * @param prefixes
+ * the prefixes
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
* @return the prefix filtered objects by positions
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public ArrayList<MtasToken<String>> getPrefixFilteredObjectsByPositions(
String field, int docId, ArrayList<String> prefixes, int startPosition,
@@ -228,12 +258,17 @@ public class CodecInfo {
/**
* Gets the prefix filtered objects.
*
- * @param hits the hits
- * @param prefixes the prefixes
+ * @param hits
+ * the hits
+ * @param prefixes
+ * the prefixes
* @return the prefix filtered objects
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private ArrayList<MtasToken<String>> getPrefixFilteredObjects(
- List<MtasTreeHit<?>> hits, ArrayList<String> prefixes) {
+ List<MtasTreeHit<?>> hits, ArrayList<String> prefixes)
+ throws IOException {
ArrayList<MtasToken<String>> tokens = new ArrayList<MtasToken<String>>();
IndexInput inObject = indexInputList.get("object");
IndexInput inTerm = indexInputList.get("term");
@@ -256,12 +291,17 @@ public class CodecInfo {
/**
* Gets the positioned terms by prefixes and position.
*
- * @param field the field
- * @param docId the doc id
- * @param prefixes the prefixes
- * @param position the position
+ * @param field
+ * the field
+ * @param docId
+ * the doc id
+ * @param prefixes
+ * the prefixes
+ * @param position
+ * the position
* @return the positioned terms by prefixes and position
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public ArrayList<MtasTreeHit<String>> getPositionedTermsByPrefixesAndPosition(
String field, int docId, ArrayList<String> prefixes, int position)
@@ -273,13 +313,19 @@ public class CodecInfo {
/**
* Gets the positioned terms by prefixes and position range.
*
- * @param field the field
- * @param docId the doc id
- * @param prefixes the prefixes
- * @param startPosition the start position
- * @param endPosition the end position
+ * @param field
+ * the field
+ * @param docId
+ * the doc id
+ * @param prefixes
+ * the prefixes
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
* @return the positioned terms by prefixes and position range
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public ArrayList<MtasTreeHit<String>> getPositionedTermsByPrefixesAndPositionRange(
String field, int docId, ArrayList<String> prefixes, int startPosition,
@@ -328,22 +374,29 @@ public class CodecInfo {
/**
* Collect terms by prefixes for list of hit positions.
*
- * @param field the field
- * @param docId the doc id
- * @param prefixes the prefixes
- * @param positionsHits the positions hits
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param field
+ * the field
+ * @param docId
+ * the doc id
+ * @param prefixes
+ * the prefixes
+ * @param positionsHits
+ * the positions hits
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public void collectTermsByPrefixesForListOfHitPositions(String field,
int docId, ArrayList<String> prefixes,
- ArrayList<IntervalTreeNodeData<String>> positionsHits) throws IOException {
+ ArrayList<IntervalTreeNodeData<String>> positionsHits)
+ throws IOException {
IndexDoc doc = getDoc(field, docId);
IndexInput inIndexObjectPosition = indexInputList
.get("indexObjectPosition");
IndexInput inObject = indexInputList.get("object");
- IndexInput inTerm = indexInputList.get("term");
+ IndexInput inTerm = indexInputList.get("term");
// create tree interval hits
- IntervalRBTree<String> positionTree = new IntervalRBTree<String>(positionsHits);
+ IntervalRBTree<String> positionTree = new IntervalRBTree<String>(
+ positionsHits);
if (version == MtasCodecPostingsFormat.VERSION_OLD_1) {
CodecSearchTree.searchMtasTreeWithIntervalTree(null, positionTree,
inIndexObjectPosition, doc.fpIndexObjectPosition,
@@ -352,29 +405,33 @@ public class CodecInfo {
// find prefixIds
HashMap<String, Integer> prefixIds = getPrefixesIds(field, prefixes);
// search matching tokens
- CodecSearchTree.searchMtasTreeWithIntervalTree(prefixIds.values(), positionTree,
- inIndexObjectPosition, doc.fpIndexObjectPosition,
- doc.smallestObjectFilepointer);
+ CodecSearchTree.searchMtasTreeWithIntervalTree(prefixIds.values(),
+ positionTree, inIndexObjectPosition, doc.fpIndexObjectPosition,
+ doc.smallestObjectFilepointer);
}
- for(IntervalTreeNodeData<String> positionHit : positionsHits) {
- for(MtasTreeHit<String> hit : positionHit.list) {
- if(hit.data==null) {
+ for (IntervalTreeNodeData<String> positionHit : positionsHits) {
+ for (MtasTreeHit<String> hit : positionHit.list) {
+ if (hit.data == null) {
MtasToken<String> token = MtasCodecPostingsFormat.getToken(inObject,
inTerm, hit.ref);
hit.data = token.getValue();
}
- }
- }
+ }
+ }
}
/**
* Gets the objects.
*
- * @param hits the hits
+ * @param hits
+ * the hits
* @return the objects
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- public ArrayList<MtasToken<String>> getObjects(List<MtasTreeHit<?>> hits) {
+ public ArrayList<MtasToken<String>> getObjects(List<MtasTreeHit<?>> hits)
+ throws IOException {
ArrayList<MtasToken<String>> tokens = new ArrayList<MtasToken<String>>();
IndexInput inObject = indexInputList.get("object");
IndexInput inTerm = indexInputList.get("term");
@@ -391,55 +448,37 @@ public class CodecInfo {
/**
* Gets the terms.
*
- * @param refs the refs
+ * @param refs
+ * the refs
* @return the terms
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public ArrayList<MtasTreeHit<String>> getTerms(ArrayList<MtasTreeHit<?>> refs)
throws IOException {
- ArrayList<MtasTreeHit<String>> terms = new ArrayList<MtasTreeHit<String>>();
- IndexInput inTerm = indexInputList.get("term");
- for (MtasTreeHit<?> hit : refs) {
- inTerm.seek(hit.ref);
- String term = inTerm.readString();
- MtasTreeHit<String> newHit = new MtasTreeHit<String>(hit.startPosition,
- hit.endPosition, hit.ref, hit.additionalId, term);
- terms.add(newHit);
- }
- return terms;
- }
-
- /**
- * Gets the prefixes references.
- *
- * @param field the field
- * @param prefixes the prefixes
- * @return the prefixes references
- */
- private ArrayList<Long> getPrefixesReferences(String field,
- ArrayList<String> prefixes) {
- LinkedHashMap<String, Long> refs = getPrefixes(field);
- if (refs != null) {
- ArrayList<Long> result = new ArrayList<Long>();
- for (String prefix : prefixes) {
- if (refs.containsKey(prefix)) {
- Long ref = prefixReferences.get(field).get(prefix);
- if (!result.contains(ref)) {
- result.add(ref);
- }
- }
+ try {
+ ArrayList<MtasTreeHit<String>> terms = new ArrayList<MtasTreeHit<String>>();
+ IndexInput inTerm = indexInputList.get("term");
+ for (MtasTreeHit<?> hit : refs) {
+ inTerm.seek(hit.ref);
+ String term = inTerm.readString();
+ MtasTreeHit<String> newHit = new MtasTreeHit<String>(hit.startPosition,
+ hit.endPosition, hit.ref, hit.additionalId, term);
+ terms.add(newHit);
}
- return result;
- } else {
- return null;
+ return terms;
+ } catch (Exception e) {
+ throw new IOException(e.getMessage());
}
}
/**
* Gets the prefixes ids.
*
- * @param field the field
- * @param prefixes the prefixes
+ * @param field
+ * the field
+ * @param prefixes
+ * the prefixes
* @return the prefixes ids
*/
HashMap<String, Integer> getPrefixesIds(String field,
@@ -451,7 +490,7 @@ public class CodecInfo {
for (String prefix : prefixes) {
int id = list.indexOf(prefix);
if (id >= 0) {
- result.put(prefix, id);
+ result.put(prefix, id + 1);
}
}
return result;
@@ -463,7 +502,8 @@ public class CodecInfo {
/**
* Gets the prefixes.
*
- * @param field the field
+ * @param field
+ * the field
* @return the prefixes
*/
private LinkedHashMap<String, Long> getPrefixes(String field) {
@@ -478,7 +518,7 @@ public class CodecInfo {
Long ref = inPrefix.getFilePointer();
refs.put(inPrefix.readString(), ref);
}
- } catch (IOException e) {
+ } catch (Exception e) {
refs.clear();
}
prefixReferences.put(field, refs);
@@ -494,8 +534,10 @@ public class CodecInfo {
/**
* Gets the doc.
*
- * @param field the field
- * @param docId the doc id
+ * @param field
+ * the field
+ * @param docId
+ * the doc id
* @return the doc
*/
public IndexDoc getDoc(String field, int docId) {
@@ -518,8 +560,10 @@ public class CodecInfo {
/**
* Gets the next doc.
*
- * @param field the field
- * @param previousDocId the previous doc id
+ * @param field
+ * the field
+ * @param previousDocId
+ * the previous doc id
* @return the next doc
*/
public IndexDoc getNextDoc(String field, int previousDocId) {
@@ -549,8 +593,10 @@ public class CodecInfo {
/**
* Gets the number of positions.
*
- * @param field the field
- * @param docId the doc id
+ * @param field
+ * the field
+ * @param docId
+ * the doc id
* @return the number of positions
*/
public Integer getNumberOfPositions(String field, int docId) {
@@ -566,10 +612,13 @@ public class CodecInfo {
/**
* Gets the all number of positions.
*
- * @param field the field
- * @param docBase the doc base
+ * @param field
+ * the field
+ * @param docBase
+ * the doc base
* @return the all number of positions
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public HashMap<Integer, Integer> getAllNumberOfPositions(String field,
int docBase) throws IOException {
@@ -587,12 +636,14 @@ public class CodecInfo {
}
return numbers;
}
-
+
/**
* Gets the number of tokens.
*
- * @param field the field
- * @param docId the doc id
+ * @param field
+ * the field
+ * @param docId
+ * the doc id
* @return the number of tokens
*/
public Integer getNumberOfTokens(String field, int docId) {
@@ -604,14 +655,17 @@ public class CodecInfo {
}
return null;
}
-
+
/**
* Gets the all number of tokens.
*
- * @param field the field
- * @param docBase the doc base
+ * @param field
+ * the field
+ * @param docBase
+ * the doc base
* @return the all number of tokens
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public HashMap<Integer, Integer> getAllNumberOfTokens(String field,
int docBase) throws IOException {
@@ -623,8 +677,7 @@ public class CodecInfo {
IndexDoc doc;
for (int i = 0; i < fr.numberOfDocs; i++) {
doc = new IndexDoc(null);
- numbers.put((doc.docId + docBase),
- doc.size);
+ numbers.put((doc.docId + docBase), doc.size);
}
}
return numbers;
@@ -634,53 +687,60 @@ public class CodecInfo {
* The Class IndexDoc.
*/
public class IndexDoc {
-
+
/** The doc id. */
public int docId;
-
+
/** The fp index object parent. */
public long fpIndexObjectId, fpIndexObjectPosition, fpIndexObjectParent;
-
+
/** The object ref approx offset. */
public long smallestObjectFilepointer, objectRefApproxOffset;
-
+
/** The object ref approx quotient. */
public int objectRefApproxQuotient;
-
+
/** The offset. */
public long offset;
-
+
/** The storage flags. */
public byte storageFlags;
-
+
/** The max position. */
public int size, minPosition, maxPosition;
/**
* Instantiates a new index doc.
*
- * @param ref the ref
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param ref
+ * the ref
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public IndexDoc(Long ref) throws IOException {
- IndexInput inIndexDoc = indexInputList.get("doc");
- if (ref != null) {
- inIndexDoc.seek(ref);
- }
- docId = inIndexDoc.readVInt(); // docId
- fpIndexObjectId = inIndexDoc.readVLong(); // ref indexObjectId
- fpIndexObjectPosition = inIndexDoc.readVLong(); // ref indexObjectPosition
- fpIndexObjectParent = inIndexDoc.readVLong(); // ref indexObjectParent
- if (version == MtasCodecPostingsFormat.VERSION_OLD_1) {
- inIndexDoc.readVLong(); // fpIndexTermPrefixPosition ref
+ try {
+ IndexInput inIndexDoc = indexInputList.get("doc");
+ if (ref != null) {
+ inIndexDoc.seek(ref);
+ }
+ docId = inIndexDoc.readVInt(); // docId
+ fpIndexObjectId = inIndexDoc.readVLong(); // ref indexObjectId
+ fpIndexObjectPosition = inIndexDoc.readVLong(); // ref
+ // indexObjectPosition
+ fpIndexObjectParent = inIndexDoc.readVLong(); // ref indexObjectParent
+ if (version == MtasCodecPostingsFormat.VERSION_OLD_1) {
+ inIndexDoc.readVLong(); // fpIndexTermPrefixPosition ref
+ }
+ smallestObjectFilepointer = inIndexDoc.readVLong(); // offset
+ objectRefApproxQuotient = inIndexDoc.readVInt(); // slope
+ objectRefApproxOffset = inIndexDoc.readZLong(); // offset
+ storageFlags = inIndexDoc.readByte(); // flag
+ size = inIndexDoc.readVInt(); // number of objects
+ minPosition = inIndexDoc.readVInt(); // minimum position
+ maxPosition = inIndexDoc.readVInt(); // maximum position
+ } catch (Exception e) {
+ throw new IOException(e.getMessage());
}
- smallestObjectFilepointer = inIndexDoc.readVLong(); // offset
- objectRefApproxQuotient = inIndexDoc.readVInt(); // slope
- objectRefApproxOffset = inIndexDoc.readZLong(); // offset
- storageFlags = inIndexDoc.readByte(); // flag
- size = inIndexDoc.readVInt(); // number of objects
- minPosition = inIndexDoc.readVInt(); // minimum position
- maxPosition = inIndexDoc.readVInt(); // maximum position
}
}
@@ -688,23 +748,30 @@ public class CodecInfo {
* The Class FieldReferences.
*/
public class FieldReferences {
-
+
/** The ref prefix. */
public long refIndexDoc, refIndexDocId, refTerm, refPrefix;
-
+
/** The number of prefixes. */
public int numberOfDocs, numberOfTerms, numberOfPrefixes;
/**
* Instantiates a new field references.
*
- * @param refIndexDoc the ref index doc
- * @param refIndexDocId the ref index doc id
- * @param numberOfDocs the number of docs
- * @param refTerm the ref term
- * @param numberOfTerms the number of terms
- * @param refPrefix the ref prefix
- * @param numberOfPrefixes the number of prefixes
+ * @param refIndexDoc
+ * the ref index doc
+ * @param refIndexDocId
+ * the ref index doc id
+ * @param numberOfDocs
+ * the number of docs
+ * @param refTerm
+ * the ref term
+ * @param numberOfTerms
+ * the number of terms
+ * @param refPrefix
+ * the ref prefix
+ * @param numberOfPrefixes
+ * the number of prefixes
*/
public FieldReferences(long refIndexDoc, long refIndexDocId,
int numberOfDocs, long refTerm, int numberOfTerms, long refPrefix,
diff --git a/src/mtas/codec/util/CodecSearchTree.java b/src/mtas/codec/util/CodecSearchTree.java
index 1be4d2d..d283cc8 100644
--- a/src/mtas/codec/util/CodecSearchTree.java
+++ b/src/mtas/codec/util/CodecSearchTree.java
@@ -18,12 +18,17 @@ public class CodecSearchTree {
/**
* Advance mtas tree.
*
- * @param position the position
- * @param in the in
- * @param ref the ref
- * @param objectRefApproxOffset the object ref approx offset
+ * @param position
+ * the position
+ * @param in
+ * the in
+ * @param ref
+ * the ref
+ * @param objectRefApproxOffset
+ * the object ref approx offset
* @return the array list
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static ArrayList<MtasTreeHit<?>> advanceMtasTree(int position,
IndexInput in, long ref, long objectRefApproxOffset) throws IOException {
@@ -52,16 +57,26 @@ public class CodecSearchTree {
/**
* Advance mtas tree.
*
- * @param treeItem the tree item
- * @param position the position
- * @param in the in
- * @param isSinglePoint the is single point
- * @param isStoreAdditionalId the is store additional id
- * @param objectRefApproxOffset the object ref approx offset
- * @param list the list
- * @param nodeRefApproxOffset the node ref approx offset
- * @param checkList the check list
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param treeItem
+ * the tree item
+ * @param position
+ * the position
+ * @param in
+ * the in
+ * @param isSinglePoint
+ * the is single point
+ * @param isStoreAdditionalId
+ * the is store additional id
+ * @param objectRefApproxOffset
+ * the object ref approx offset
+ * @param list
+ * the list
+ * @param nodeRefApproxOffset
+ * the node ref approx offset
+ * @param checkList
+ * the check list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void advanceMtasTree(MtasTreeItem treeItem, int position,
IndexInput in, AtomicBoolean isSinglePoint,
@@ -109,12 +124,17 @@ public class CodecSearchTree {
/**
* Search mtas tree.
*
- * @param position the position
- * @param in the in
- * @param ref the ref
- * @param objectRefApproxOffset the object ref approx offset
+ * @param position
+ * the position
+ * @param in
+ * the in
+ * @param ref
+ * the ref
+ * @param objectRefApproxOffset
+ * the object ref approx offset
* @return the array list
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static ArrayList<MtasTreeHit<?>> searchMtasTree(int position,
IndexInput in, long ref, long objectRefApproxOffset) throws IOException {
@@ -124,13 +144,19 @@ public class CodecSearchTree {
/**
* Search mtas tree.
*
- * @param startPosition the start position
- * @param endPosition the end position
- * @param in the in
- * @param ref the ref
- * @param objectRefApproxOffset the object ref approx offset
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
+ * @param in
+ * the in
+ * @param ref
+ * the ref
+ * @param objectRefApproxOffset
+ * the object ref approx offset
* @return the array list
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static ArrayList<MtasTreeHit<?>> searchMtasTree(int startPosition,
int endPosition, IndexInput in, long ref, long objectRefApproxOffset)
@@ -160,17 +186,28 @@ public class CodecSearchTree {
/**
* Search mtas tree.
*
- * @param treeItem the tree item
- * @param startPosition the start position
- * @param endPosition the end position
- * @param in the in
- * @param isSinglePoint the is single point
- * @param isStoreAdditionalId the is store additional id
- * @param objectRefApproxOffset the object ref approx offset
- * @param list the list
- * @param nodeRefApproxOffset the node ref approx offset
- * @param checkList the check list
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param treeItem
+ * the tree item
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
+ * @param in
+ * the in
+ * @param isSinglePoint
+ * the is single point
+ * @param isStoreAdditionalId
+ * the is store additional id
+ * @param objectRefApproxOffset
+ * the object ref approx offset
+ * @param list
+ * the list
+ * @param nodeRefApproxOffset
+ * the node ref approx offset
+ * @param checkList
+ * the check list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static void searchMtasTree(MtasTreeItem treeItem, int startPosition,
int endPosition, IndexInput in, AtomicBoolean isSinglePoint,
@@ -213,101 +250,120 @@ public class CodecSearchTree {
/**
* Gets the mtas tree item.
*
- * @param ref the ref
- * @param isSinglePoint the is single point
- * @param isStoreAdditionalIdAndRef the is store additional id and ref
- * @param nodeRefApproxOffset the node ref approx offset
- * @param in the in
- * @param objectRefApproxOffset the object ref approx offset
+ * @param ref
+ * the ref
+ * @param isSinglePoint
+ * the is single point
+ * @param isStoreAdditionalIdAndRef
+ * the is store additional id and ref
+ * @param nodeRefApproxOffset
+ * the node ref approx offset
+ * @param in
+ * the in
+ * @param objectRefApproxOffset
+ * the object ref approx offset
* @return the mtas tree item
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private static MtasTreeItem getMtasTreeItem(Long ref,
AtomicBoolean isSinglePoint, AtomicBoolean isStoreAdditionalIdAndRef,
AtomicLong nodeRefApproxOffset, IndexInput in, long objectRefApproxOffset)
throws IOException {
- Boolean isRoot = false;
- if (nodeRefApproxOffset.get() < 0) {
- isRoot = true;
- }
- in.seek(ref);
- if (isRoot) {
- nodeRefApproxOffset.set(in.readVLong());
- Byte flag = in.readByte();
- if ((flag
- & MtasTree.SINGLE_POSITION_TREE) == MtasTree.SINGLE_POSITION_TREE) {
- isSinglePoint.set(true);
+ try {
+ Boolean isRoot = false;
+ if (nodeRefApproxOffset.get() < 0) {
+ isRoot = true;
}
- if ((flag
- & MtasTree.STORE_ADDITIONAL_ID) == MtasTree.STORE_ADDITIONAL_ID) {
- isStoreAdditionalIdAndRef.set(true);
+ in.seek(ref);
+ if (isRoot) {
+ nodeRefApproxOffset.set(in.readVLong());
+ Byte flag = in.readByte();
+ if ((flag
+ & MtasTree.SINGLE_POSITION_TREE) == MtasTree.SINGLE_POSITION_TREE) {
+ isSinglePoint.set(true);
+ }
+ if ((flag
+ & MtasTree.STORE_ADDITIONAL_ID) == MtasTree.STORE_ADDITIONAL_ID) {
+ isStoreAdditionalIdAndRef.set(true);
+ }
}
- }
- int left = in.readVInt();
- int right = in.readVInt();
- int max = in.readVInt();
- Long leftChild = in.readVLong() + nodeRefApproxOffset.get();
- Long rightChild = in.readVLong() + nodeRefApproxOffset.get();
- int size = 1;
- if (!isSinglePoint.get()) {
- size = in.readVInt();
- }
- // initialize
- long[] objectRefs = new long[size];
- int[] objectAdditionalIds = null;
- long[] objectAdditionalRefs = null;
- // get first
- long objectRef = in.readVLong();
- long objectRefPrevious = objectRef + objectRefApproxOffset;
- objectRefs[0] = objectRefPrevious;
- if (isStoreAdditionalIdAndRef.get()) {
- objectAdditionalIds = new int[size];
- objectAdditionalRefs = new long[size];
- objectAdditionalIds[0] = in.readVInt();
- objectAdditionalRefs[0] = in.readVLong();
- }
- // get others
- for (int t = 1; t < size; t++) {
- objectRef = objectRefPrevious + in.readVLong();
- objectRefs[t] = objectRef;
- objectRefPrevious = objectRef;
+ int left = in.readVInt();
+ int right = in.readVInt();
+ int max = in.readVInt();
+ Long leftChild = in.readVLong() + nodeRefApproxOffset.get();
+ Long rightChild = in.readVLong() + nodeRefApproxOffset.get();
+ int size = 1;
+ if (!isSinglePoint.get()) {
+ size = in.readVInt();
+ }
+ // initialize
+ long[] objectRefs = new long[size];
+ int[] objectAdditionalIds = null;
+ long[] objectAdditionalRefs = null;
+ // get first
+ long objectRef = in.readVLong();
+ long objectRefPrevious = objectRef + objectRefApproxOffset;
+ objectRefs[0] = objectRefPrevious;
if (isStoreAdditionalIdAndRef.get()) {
- objectAdditionalIds[t] = in.readVInt();
- objectAdditionalRefs[t] = in.readVLong();
+ objectAdditionalIds = new int[size];
+ objectAdditionalRefs = new long[size];
+ objectAdditionalIds[0] = in.readVInt();
+ objectAdditionalRefs[0] = in.readVLong();
+ }
+ // get others
+ for (int t = 1; t < size; t++) {
+ objectRef = objectRefPrevious + in.readVLong();
+ objectRefs[t] = objectRef;
+ objectRefPrevious = objectRef;
+ if (isStoreAdditionalIdAndRef.get()) {
+ objectAdditionalIds[t] = in.readVInt();
+ objectAdditionalRefs[t] = in.readVLong();
+ }
}
+ return new MtasTreeItem(left, right, max, objectRefs, objectAdditionalIds,
+ ref, leftChild, rightChild);
+ } catch (Exception e) {
+ throw new IOException(e.getMessage());
}
- return new MtasTreeItem(left, right, max, objectRefs, objectAdditionalIds, ref,
- leftChild, rightChild);
}
/**
* The Class MtasTreeItem.
*/
private static class MtasTreeItem {
-
+
/** The max. */
public int left, right, max;
-
+
/** The object refs. */
public long[] objectRefs;
-
+
/** The object ids. */
public int[] objectIds;
-
+
/** The right child. */
public Long ref, leftChild, rightChild;
/**
* Instantiates a new mtas tree item.
*
- * @param left the left
- * @param right the right
- * @param max the max
- * @param objectRefs the object refs
- * @param objectIds the object ids
- * @param ref the ref
- * @param leftChild the left child
- * @param rightChild the right child
+ * @param left
+ * the left
+ * @param right
+ * the right
+ * @param max
+ * the max
+ * @param objectRefs
+ * the object refs
+ * @param objectIds
+ * the object ids
+ * @param ref
+ * the ref
+ * @param leftChild
+ * the left child
+ * @param rightChild
+ * the right child
*/
public MtasTreeItem(int left, int right, int max, long[] objectRefs,
int[] objectIds, Long ref, Long leftChild, Long rightChild) {
@@ -325,32 +381,37 @@ public class CodecSearchTree {
/**
* The Class MtasTreeHit.
*
- * @param <T> the generic type
+ * @param <T>
+ * the generic type
*/
public static class MtasTreeHit<T> {
-
+
/** The start position. */
public int startPosition;
-
+
/** The end position. */
public int endPosition;
-
+
/** The ref. */
public long ref;
-
+
/** The additional id. */
public int additionalId;
-
+
/** The data. */
public T data;
/**
* Instantiates a new mtas tree hit.
*
- * @param startPosition the start position
- * @param endPosition the end position
- * @param ref the ref
- * @param additionalId the additional id
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
+ * @param ref
+ * the ref
+ * @param additionalId
+ * the additional id
*/
public MtasTreeHit(int startPosition, int endPosition, long ref,
int additionalId) {
@@ -363,11 +424,16 @@ public class CodecSearchTree {
/**
* Instantiates a new mtas tree hit.
*
- * @param startPosition the start position
- * @param endPosition the end position
- * @param ref the ref
- * @param additionalId the additional id
- * @param data the data
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
+ * @param ref
+ * the ref
+ * @param additionalId
+ * the additional id
+ * @param data
+ * the data
*/
public MtasTreeHit(int startPosition, int endPosition, long ref,
int additionalId, T data) {
@@ -375,7 +441,9 @@ public class CodecSearchTree {
this.data = data;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#toString()
*/
@Override
@@ -388,56 +456,76 @@ public class CodecSearchTree {
/**
* Search mtas tree with interval tree.
*
- * @param <T> the generic type
- * @param <N> the number type
- * @param additionalIds the additional ids
- * @param intervalTree the interval tree
- * @param in the in
- * @param ref the ref
- * @param objectRefApproxOffset the object ref approx offset
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param <T>
+ * the generic type
+ * @param <N>
+ * the number type
+ * @param additionalIds
+ * the additional ids
+ * @param intervalTree
+ * the interval tree
+ * @param in
+ * the in
+ * @param ref
+ * the ref
+ * @param objectRefApproxOffset
+ * the object ref approx offset
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static <T, N extends IntervalTreeNode<T, N>> void searchMtasTreeWithIntervalTree(
- Collection<Integer> additionalIds, IntervalTree<T,N> intervalTree, IndexInput in, long ref,
- long objectRefApproxOffset) throws IOException {
- ArrayList<IntervalItem<T,N>> checkList = new ArrayList<IntervalItem<T,N>>();
+ Collection<Integer> additionalIds, IntervalTree<T, N> intervalTree,
+ IndexInput in, long ref, long objectRefApproxOffset) throws IOException {
+ ArrayList<IntervalItem<T, N>> checkList = new ArrayList<IntervalItem<T, N>>();
AtomicBoolean isSinglePoint = new AtomicBoolean(false);
AtomicBoolean isStoreAdditionalId = new AtomicBoolean(false);
AtomicLong nodeRefApproxOffset = new AtomicLong(-1);
- checkList.add(new IntervalItem<T,N>(
+ checkList.add(new IntervalItem<T, N>(
getMtasTreeItem(ref, isSinglePoint, isStoreAdditionalId,
nodeRefApproxOffset, in, objectRefApproxOffset),
intervalTree.getRoot()));
do {
- IntervalItem<T,N> checkItem = checkList.remove(checkList.size() - 1);
- searchMtasTreeWithIntervalTree(additionalIds, checkItem, in, isSinglePoint,
- isStoreAdditionalId, objectRefApproxOffset, nodeRefApproxOffset,
- checkList);
+ IntervalItem<T, N> checkItem = checkList.remove(checkList.size() - 1);
+ searchMtasTreeWithIntervalTree(additionalIds, checkItem, in,
+ isSinglePoint, isStoreAdditionalId, objectRefApproxOffset,
+ nodeRefApproxOffset, checkList);
} while (checkList.size() > 0);
}
/**
* Search mtas tree with interval tree.
*
- * @param <T> the generic type
- * @param <N> the number type
- * @param additionalIds the additional ids
- * @param checkItem the check item
- * @param in the in
- * @param isSinglePoint the is single point
- * @param isStoreAdditionalId the is store additional id
- * @param objectRefApproxOffset the object ref approx offset
- * @param nodeRefApproxOffset the node ref approx offset
- * @param checkList the check list
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param <T>
+ * the generic type
+ * @param <N>
+ * the number type
+ * @param additionalIds
+ * the additional ids
+ * @param checkItem
+ * the check item
+ * @param in
+ * the in
+ * @param isSinglePoint
+ * the is single point
+ * @param isStoreAdditionalId
+ * the is store additional id
+ * @param objectRefApproxOffset
+ * the object ref approx offset
+ * @param nodeRefApproxOffset
+ * the node ref approx offset
+ * @param checkList
+ * the check list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- private static <T, N extends IntervalTreeNode<T, N>> void searchMtasTreeWithIntervalTree(Collection<Integer> additionalIds, IntervalItem<T,N> checkItem,
+ private static <T, N extends IntervalTreeNode<T, N>> void searchMtasTreeWithIntervalTree(
+ Collection<Integer> additionalIds, IntervalItem<T, N> checkItem,
IndexInput in, AtomicBoolean isSinglePoint,
AtomicBoolean isStoreAdditionalId, long objectRefApproxOffset,
- AtomicLong nodeRefApproxOffset, ArrayList<IntervalItem<T,N>> checkList)
+ AtomicLong nodeRefApproxOffset, ArrayList<IntervalItem<T, N>> checkList)
throws IOException {
MtasTreeItem treeItem = checkItem.mtasTreeItem;
- IntervalTreeNode<T,N> intervalTreeNode = checkItem.intervalTreeNode;
+ IntervalTreeNode<T, N> intervalTreeNode = checkItem.intervalTreeNode;
if (intervalTreeNode.min <= treeItem.max) {
// advance intervalTree
while (intervalTreeNode.left > treeItem.max) {
@@ -458,20 +546,21 @@ public class CodecSearchTree {
}
}
// find intervals matching current node
- searchMtasTreeItemWithIntervalTree(additionalIds, treeItem, intervalTreeNode);
+ searchMtasTreeItemWithIntervalTree(additionalIds, treeItem,
+ intervalTreeNode);
// check leftChild
if (!treeItem.leftChild.equals(treeItem.ref)) {
MtasTreeItem treeItemLeft = getMtasTreeItem(treeItem.leftChild,
isSinglePoint, isStoreAdditionalId, nodeRefApproxOffset, in,
objectRefApproxOffset);
- checkList.add(new IntervalItem<T,N>(treeItemLeft, intervalTreeNode));
+ checkList.add(new IntervalItem<T, N>(treeItemLeft, intervalTreeNode));
}
// check rightChild
if (!treeItem.rightChild.equals(treeItem.ref)) {
MtasTreeItem treeItemRight = getMtasTreeItem(treeItem.rightChild,
isSinglePoint, isStoreAdditionalId, nodeRefApproxOffset, in,
objectRefApproxOffset);
- checkList.add(new IntervalItem<T,N>(treeItemRight, intervalTreeNode));
+ checkList.add(new IntervalItem<T, N>(treeItemRight, intervalTreeNode));
}
}
}
@@ -479,66 +568,83 @@ public class CodecSearchTree {
/**
* Search mtas tree item with interval tree.
*
- * @param <T> the generic type
- * @param <N> the number type
- * @param additionalIds the additional ids
- * @param treeItem the tree item
- * @param intervalTreeNode the interval tree node
+ * @param <T>
+ * the generic type
+ * @param <N>
+ * the number type
+ * @param additionalIds
+ * the additional ids
+ * @param treeItem
+ * the tree item
+ * @param intervalTreeNode
+ * the interval tree node
*/
- private static <T, N extends IntervalTreeNode<T, N>> void searchMtasTreeItemWithIntervalTree(Collection<Integer> additionalIds, MtasTreeItem treeItem,
- IntervalTreeNode<T,N> intervalTreeNode) {
- ArrayList<IntervalTreeNode<T,N>> checkList = new ArrayList<IntervalTreeNode<T,N>>();
+ private static <T, N extends IntervalTreeNode<T, N>> void searchMtasTreeItemWithIntervalTree(
+ Collection<Integer> additionalIds, MtasTreeItem treeItem,
+ IntervalTreeNode<T, N> intervalTreeNode) {
+ ArrayList<IntervalTreeNode<T, N>> checkList = new ArrayList<IntervalTreeNode<T, N>>();
checkList.add(intervalTreeNode);
do {
- IntervalTreeNode<T,N> checkItem = checkList.remove(checkList.size() - 1);
- searchMtasTreeItemWithIntervalTree(additionalIds, checkItem, treeItem.left,
- treeItem.right, treeItem.objectRefs, treeItem.objectIds, checkList);
+ IntervalTreeNode<T, N> checkItem = checkList.remove(checkList.size() - 1);
+ searchMtasTreeItemWithIntervalTree(additionalIds, checkItem,
+ treeItem.left, treeItem.right, treeItem.objectRefs,
+ treeItem.objectIds, checkList);
} while (checkList.size() > 0);
}
/**
* Search mtas tree item with interval tree.
*
- * @param <T> the generic type
- * @param <N> the number type
- * @param requiredAdditionalIds the required additional ids
- * @param intervalTreeItem the interval tree item
- * @param startPosition the start position
- * @param endPosition the end position
- * @param refs the refs
- * @param additionalIds the additional ids
- * @param checkList the check list
+ * @param <T>
+ * the generic type
+ * @param <N>
+ * the number type
+ * @param requiredAdditionalIds
+ * the required additional ids
+ * @param intervalTreeItem
+ * the interval tree item
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
+ * @param refs
+ * the refs
+ * @param additionalIds
+ * the additional ids
+ * @param checkList
+ * the check list
*/
- private static <T, N extends IntervalTreeNode<T, N>> void searchMtasTreeItemWithIntervalTree(Collection<Integer> requiredAdditionalIds,
- IntervalTreeNode<T,N> intervalTreeItem, int startPosition, int endPosition,
- long[] refs, int[] additionalIds,
- ArrayList<IntervalTreeNode<T,N>> checkList) {
+ private static <T, N extends IntervalTreeNode<T, N>> void searchMtasTreeItemWithIntervalTree(
+ Collection<Integer> requiredAdditionalIds,
+ IntervalTreeNode<T, N> intervalTreeItem, int startPosition,
+ int endPosition, long[] refs, int[] additionalIds,
+ ArrayList<IntervalTreeNode<T, N>> checkList) {
if (startPosition <= intervalTreeItem.max) {
// match current node
if ((endPosition >= intervalTreeItem.left)
&& (startPosition <= intervalTreeItem.right)) {
- //System.out.print("[" + startPosition + "-" + endPosition + "] ");
- if(requiredAdditionalIds==null || additionalIds==null) {
+ // System.out.print("[" + startPosition + "-" + endPosition + "] ");
+ if (requiredAdditionalIds == null || additionalIds == null) {
for (int i = 0; i < refs.length; i++) {
- MtasTreeHit<T> hit = new MtasTreeHit<T>(startPosition,
- endPosition, refs[i], 0, null);
+ MtasTreeHit<T> hit = new MtasTreeHit<T>(startPosition, endPosition,
+ refs[i], 0, null);
for (ArrayList<MtasTreeHit<T>> list : intervalTreeItem.lists) {
list.add(hit);
}
}
} else {
for (int i = 0; i < refs.length; i++) {
- MtasTreeHit<T> hit = new MtasTreeHit<T>(startPosition,
- endPosition, refs[i], additionalIds[i], null);
+ MtasTreeHit<T> hit = new MtasTreeHit<T>(startPosition, endPosition,
+ refs[i], additionalIds[i], null);
for (ArrayList<MtasTreeHit<T>> list : intervalTreeItem.lists) {
list.add(hit);
- }
+ }
}
- }
+ }
}
// check leftChild
if (intervalTreeItem.leftChild != null) {
- IntervalTreeNode<T,N> treeItemLeft = intervalTreeItem.leftChild;
+ IntervalTreeNode<T, N> treeItemLeft = intervalTreeItem.leftChild;
if (treeItemLeft.max >= startPosition) {
checkList.add(treeItemLeft);
}
@@ -546,7 +652,7 @@ public class CodecSearchTree {
// check rightChild
if (intervalTreeItem.left < endPosition) {
if (intervalTreeItem.rightChild != null) {
- IntervalTreeNode<T,N> treeItemRight = intervalTreeItem.rightChild;
+ IntervalTreeNode<T, N> treeItemRight = intervalTreeItem.rightChild;
if ((treeItemRight.left >= endPosition)
|| (treeItemRight.max >= startPosition)) {
checkList.add(treeItemRight);
@@ -560,25 +666,29 @@ public class CodecSearchTree {
/**
* The Class IntervalItem.
*
- * @param <T> the generic type
- * @param <N> the number type
+ * @param <T>
+ * the generic type
+ * @param <N>
+ * the number type
*/
private static class IntervalItem<T, N extends IntervalTreeNode<T, N>> {
-
+
/** The mtas tree item. */
public MtasTreeItem mtasTreeItem;
-
+
/** The interval tree node. */
- public IntervalTreeNode<T,N> intervalTreeNode;
+ public IntervalTreeNode<T, N> intervalTreeNode;
/**
* Instantiates a new interval item.
*
- * @param mtasTreeItem the mtas tree item
- * @param intervalTreeNode the interval tree node
+ * @param mtasTreeItem
+ * the mtas tree item
+ * @param intervalTreeNode
+ * the interval tree node
*/
public IntervalItem(MtasTreeItem mtasTreeItem,
- IntervalTreeNode<T,N> intervalTreeNode) {
+ IntervalTreeNode<T, N> intervalTreeNode) {
this.mtasTreeItem = mtasTreeItem;
this.intervalTreeNode = intervalTreeNode;
}
diff --git a/src/mtas/codec/util/CodecUtil.java b/src/mtas/codec/util/CodecUtil.java
index 9902fd0..19cbcda 100644
--- a/src/mtas/codec/util/CodecUtil.java
+++ b/src/mtas/codec/util/CodecUtil.java
@@ -26,46 +26,46 @@ public class CodecUtil {
/** The Constant STATS_TYPE_GEOMETRICMEAN. */
public static final String STATS_TYPE_GEOMETRICMEAN = "geometricmean";
-
+
/** The Constant STATS_TYPE_KURTOSIS. */
public static final String STATS_TYPE_KURTOSIS = "kurtosis";
-
+
/** The Constant STATS_TYPE_MAX. */
public static final String STATS_TYPE_MAX = "max";
-
+
/** The Constant STATS_TYPE_MEAN. */
public static final String STATS_TYPE_MEAN = "mean";
-
+
/** The Constant STATS_TYPE_MIN. */
public static final String STATS_TYPE_MIN = "min";
-
+
/** The Constant STATS_TYPE_N. */
public static final String STATS_TYPE_N = "n";
-
+
/** The Constant STATS_TYPE_MEDIAN. */
public static final String STATS_TYPE_MEDIAN = "median";
-
+
/** The Constant STATS_TYPE_POPULATIONVARIANCE. */
public static final String STATS_TYPE_POPULATIONVARIANCE = "populationvariance";
-
+
/** The Constant STATS_TYPE_QUADRATICMEAN. */
public static final String STATS_TYPE_QUADRATICMEAN = "quadraticmean";
-
+
/** The Constant STATS_TYPE_SKEWNESS. */
public static final String STATS_TYPE_SKEWNESS = "skewness";
-
+
/** The Constant STATS_TYPE_STANDARDDEVIATION. */
public static final String STATS_TYPE_STANDARDDEVIATION = "standarddeviation";
-
+
/** The Constant STATS_TYPE_SUM. */
public static final String STATS_TYPE_SUM = "sum";
-
+
/** The Constant STATS_TYPE_SUMSQ. */
public static final String STATS_TYPE_SUMSQ = "sumsq";
-
+
/** The Constant STATS_TYPE_SUMOFLOGS. */
public static final String STATS_TYPE_SUMOFLOGS = "sumoflogs";
-
+
/** The Constant STATS_TYPE_VARIANCE. */
public static final String STATS_TYPE_VARIANCE = "variance";
@@ -77,10 +77,10 @@ public class CodecUtil {
/** The Constant SORT_TERM. */
public static final String SORT_TERM = "term";
-
+
/** The Constant SORT_ASC. */
public static final String SORT_ASC = "asc";
-
+
/** The Constant SORT_DESC. */
public static final String SORT_DESC = "desc";
@@ -113,32 +113,37 @@ public class CodecUtil {
/** The Constant STATS_BASIC. */
public static final String STATS_BASIC = "basic";
-
+
/** The Constant STATS_ADVANCED. */
public static final String STATS_ADVANCED = "advanced";
-
+
/** The Constant STATS_FULL. */
public static final String STATS_FULL = "full";
/** The Constant DATA_TYPE_LONG. */
public static final String DATA_TYPE_LONG = "long";
-
+
/** The Constant DATA_TYPE_DOUBLE. */
public static final String DATA_TYPE_DOUBLE = "double";
-
+
/** The fp stats items. */
- private static Pattern fpStatsItems = Pattern.compile("(([^\\(,]+)(\\([^\\)]*\\))?)");
-
+ private static Pattern fpStatsItems = Pattern
+ .compile("(([^\\(,]+)(\\([^\\)]*\\))?)");
+
/** The fp stats function items. */
- private static Pattern fpStatsFunctionItems = Pattern.compile("(([^\\(,]+)(\\(([^\\)]*)\\)))");
+ private static Pattern fpStatsFunctionItems = Pattern
+ .compile("(([^\\(,]+)(\\(([^\\)]*)\\)))");
/**
* Checks if is single position prefix.
*
- * @param fieldInfo the field info
- * @param prefix the prefix
+ * @param fieldInfo
+ * the field info
+ * @param prefix
+ * the prefix
* @return true, if is single position prefix
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static boolean isSinglePositionPrefix(FieldInfo fieldInfo,
String prefix) throws IOException {
@@ -160,7 +165,8 @@ public class CodecUtil {
/**
* Term value.
*
- * @param term the term
+ * @param term
+ * the term
* @return the string
*/
public static String termValue(String term) {
@@ -176,7 +182,8 @@ public class CodecUtil {
/**
* Term prefix.
*
- * @param term the term
+ * @param term
+ * the term
* @return the string
*/
public static String termPrefix(String term) {
@@ -186,31 +193,42 @@ public class CodecUtil {
prefix = term.substring(0, i);
}
return (prefix == null) ? null : prefix.replace("\u0000", "");
- }
-
+ }
+
/**
* Term prefix value.
*
- * @param term the term
+ * @param term
+ * the term
* @return the string
*/
public static String termPrefixValue(String term) {
- return (term==null)?null:term.replace("\u0000", "");
+ return (term == null) ? null : term.replace("\u0000", "");
}
/**
* Collect.
*
- * @param field the field
- * @param searcher the searcher
- * @param rawReader the raw reader
- * @param fullDocList the full doc list
- * @param fullDocSet the full doc set
- * @param fieldStats the field stats
- * @throws IllegalAccessException the illegal access exception
- * @throws IllegalArgumentException the illegal argument exception
- * @throws InvocationTargetException the invocation target exception
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param field
+ * the field
+ * @param searcher
+ * the searcher
+ * @param rawReader
+ * the raw reader
+ * @param fullDocList
+ * the full doc list
+ * @param fullDocSet
+ * the full doc set
+ * @param fieldStats
+ * the field stats
+ * @throws IllegalAccessException
+ * the illegal access exception
+ * @throws IllegalArgumentException
+ * the illegal argument exception
+ * @throws InvocationTargetException
+ * the invocation target exception
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static void collect(String field, IndexSearcher searcher,
IndexReader rawReader, ArrayList<Integer> fullDocList,
@@ -236,14 +254,16 @@ public class CodecUtil {
/**
* Creates the stats items.
*
- * @param statsType the stats type
+ * @param statsType
+ * the stats type
* @return the tree set
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
static TreeSet<String> createStatsItems(String statsType) throws IOException {
TreeSet<String> statsItems = new TreeSet<String>();
TreeSet<String> functionItems = new TreeSet<String>();
- if (statsType != null) {
+ if (statsType != null) {
Matcher m = fpStatsItems.matcher(statsType.trim());
while (m.find()) {
String tmpStatsItem = m.group(2).trim();
@@ -254,19 +274,18 @@ public class CodecUtil {
statsItems.add(type);
}
} else if (STATS_FUNCTIONS.contains(tmpStatsItem)) {
- if(m.group(3)==null) {
- throw new IOException(
- "'" + tmpStatsItem + "' should be called as '" + tmpStatsItem + "()' with an optional argument");
+ if (m.group(3) == null) {
+ throw new IOException("'" + tmpStatsItem + "' should be called as '"
+ + tmpStatsItem + "()' with an optional argument");
} else {
functionItems.add(m.group(1).trim());
- }
+ }
} else {
- throw new IOException(
- "unknown statsType '" + tmpStatsItem + "'");
+ throw new IOException("unknown statsType '" + tmpStatsItem + "'");
}
}
}
- if (statsItems.size() == 0) {
+ if (statsItems.size() == 0 && functionItems.size() == 0) {
statsItems.add(STATS_TYPE_SUM);
statsItems.add(STATS_TYPE_N);
statsItems.add(STATS_TYPE_MEAN);
@@ -280,9 +299,12 @@ public class CodecUtil {
/**
* Creates the stats type.
*
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param functionParser the function parser
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param functionParser
+ * the function parser
* @return the string
*/
static String createStatsType(TreeSet<String> statsItems, String sortType,
@@ -299,11 +321,11 @@ public class CodecUtil {
statsType = STATS_BASIC;
} else {
Matcher m = fpStatsFunctionItems.matcher(statsItem.trim());
- if(m.find()) {
- if(STATS_FUNCTIONS.contains(m.group(2).trim())) {
+ if (m.find()) {
+ if (STATS_FUNCTIONS.contains(m.group(2).trim())) {
statsType = STATS_FULL;
break;
- }
+ }
}
}
}
diff --git a/src/mtas/codec/util/DataCollector.java b/src/mtas/codec/util/DataCollector.java
index 9c5af8a..def123f 100644
--- a/src/mtas/codec/util/DataCollector.java
+++ b/src/mtas/codec/util/DataCollector.java
@@ -1,23 +1,12 @@
package mtas.codec.util;
import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.Map.Entry;
-import java.util.SortedMap;
-import java.util.TreeMap;
import java.util.TreeSet;
+
+import mtas.codec.util.collector.MtasDataCollector;
import mtas.codec.util.collector.MtasDataDoubleAdvanced;
import mtas.codec.util.collector.MtasDataDoubleBasic;
import mtas.codec.util.collector.MtasDataDoubleFull;
-import mtas.codec.util.collector.MtasDataItem;
import mtas.codec.util.collector.MtasDataLongAdvanced;
import mtas.codec.util.collector.MtasDataLongBasic;
import mtas.codec.util.collector.MtasDataLongFull;
@@ -36,25 +25,81 @@ public class DataCollector {
/**
* Gets the collector.
*
- * @param collectorType the collector type
- * @param dataType the data type
- * @param statsType the stats type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param subCollectorTypes the sub collector types
- * @param subDataTypes the sub data types
- * @param subStatsTypes the sub stats types
- * @param subStatsItems the sub stats items
- * @param subSortTypes the sub sort types
- * @param subSortDirections the sub sort directions
- * @param subStart the sub start
- * @param subNumber the sub number
- * @param segmentRegistration the segment registration
+ * @param collectorType
+ * the collector type
+ * @param dataType
+ * the data type
+ * @param statsType
+ * the stats type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @return the collector
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public static MtasDataCollector<?, ?> getCollector(String collectorType,
+ String dataType, String statsType, TreeSet<String> statsItems,
+ String sortType, String sortDirection, Integer start, Integer number,
+ String segmentRegistration, String boundary) throws IOException {
+ return getCollector(collectorType, dataType, statsType, statsItems,
+ sortType, sortDirection, start, number, null, null, null, null, null,
+ null, null, null, segmentRegistration, boundary);
+ }
+
+ /**
+ * Gets the collector.
+ *
+ * @param collectorType
+ * the collector type
+ * @param dataType
+ * the data type
+ * @param statsType
+ * the stats type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param subCollectorTypes
+ * the sub collector types
+ * @param subDataTypes
+ * the sub data types
+ * @param subStatsTypes
+ * the sub stats types
+ * @param subStatsItems
+ * the sub stats items
+ * @param subSortTypes
+ * the sub sort types
+ * @param subSortDirections
+ * the sub sort directions
+ * @param subStart
+ * the sub start
+ * @param subNumber
+ * the sub number
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
* @return the collector
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public static MtasDataCollector<?, ?> getCollector(String collectorType,
String dataType, String statsType, TreeSet<String> statsItems,
@@ -62,23 +107,23 @@ public class DataCollector {
String[] subCollectorTypes, String[] subDataTypes, String[] subStatsTypes,
TreeSet<String>[] subStatsItems, String[] subSortTypes,
String[] subSortDirections, Integer[] subStart, Integer[] subNumber,
- boolean segmentRegistration) throws IOException {
+ String segmentRegistration, String boundary) throws IOException {
if (dataType != null && dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
if (statsType.equals(CodecUtil.STATS_BASIC)) {
return new MtasDataLongBasic(collectorType, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, segmentRegistration);
+ subStart, subNumber, segmentRegistration, boundary);
} else if (statsType.equals(CodecUtil.STATS_ADVANCED)) {
return new MtasDataLongAdvanced(collectorType, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, segmentRegistration);
+ subStart, subNumber, segmentRegistration, boundary);
} else if (statsType.equals(CodecUtil.STATS_FULL)) {
return new MtasDataLongFull(collectorType, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, segmentRegistration);
+ subStart, subNumber, segmentRegistration, boundary);
} else {
throw new IOException("unknown statsType " + statsType);
}
@@ -88,17 +133,17 @@ public class DataCollector {
return new MtasDataDoubleBasic(collectorType, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, segmentRegistration);
+ subStart, subNumber, segmentRegistration, boundary);
} else if (statsType.equals(CodecUtil.STATS_ADVANCED)) {
return new MtasDataDoubleAdvanced(collectorType, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, segmentRegistration);
+ subStart, subNumber, segmentRegistration, boundary);
} else if (statsType.equals(CodecUtil.STATS_FULL)) {
return new MtasDataDoubleFull(collectorType, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, segmentRegistration);
+ subStart, subNumber, segmentRegistration, boundary);
} else {
throw new IOException("unknown statsType " + statsType);
}
@@ -106,1147 +151,4 @@ public class DataCollector {
throw new IOException("unknown dataType " + dataType);
}
}
-
-
- /**
- * The Class MtasDataCollector.
- *
- * @param <T1> the generic type
- * @param <T2> the generic type
- */
- public abstract static class MtasDataCollector<T1 extends Number, T2 extends MtasDataItem<T1>>
- implements Serializable {
-
- /** The Constant serialVersionUID. */
- private static final long serialVersionUID = 1L;
-
- /** The size. */
- // size and position current level
- private int size;
-
- /** The position. */
- protected int position;
-
- /** The collector type. */
- // properties collector
- protected String collectorType;
-
- /** The stats type. */
- protected String statsType;
-
- /** The data type. */
- protected String dataType;
-
- /** The stats items. */
- protected TreeSet<String> statsItems;
-
- /** The sort type. */
- protected String sortType;
-
- /** The sort direction. */
- protected String sortDirection;
-
- /** The start. */
- protected Integer start;
-
- /** The number. */
- protected Integer number;
-
- /** The error number. */
- // error
- protected int[] errorNumber;
-
- /** The error list. */
- protected HashMap<String, Integer>[] errorList;
-
- /** The key list. */
- // administration keys
- protected String[] keyList;
-
- /** The segment registration. */
- protected boolean segmentRegistration;
-
- /** The segment key value list. */
- protected LinkedHashMap<String, HashMap<String, T1>> segmentKeyValueList;
-
- /** The segment recompute key list. */
- protected LinkedHashMap<String, HashSet<String>> segmentRecomputeKeyList;
-
- /** The segment keys. */
- protected HashSet<String> segmentKeys;
-
- /** The segment value boundary. */
- protected LinkedHashMap<String, T1> segmentValueBoundary;
-
- /** The segment value max list min. */
- protected LinkedHashMap<String, T1> segmentValueMaxListMin;
-
- /** The segment value max list. */
- protected ArrayList<T1> segmentValueMaxList;
-
- /** The segment name. */
- protected String segmentName;
-
- /** The segment number. */
- protected int segmentNumber;
-
- /** The has sub. */
- // subcollectors properties
- private boolean hasSub;
-
- /** The sub collector types. */
- private String[] subCollectorTypes;
-
- /** The sub data types. */
- private String[] subDataTypes;
-
- /** The sub stats types. */
- private String[] subStatsTypes;
-
- /** The sub stats items. */
- private TreeSet<String>[] subStatsItems;
-
- /** The sub sort types. */
- private String[] subSortTypes;
-
- /** The sub sort directions. */
- private String[] subSortDirections;
-
- /** The sub start. */
- private Integer[] subStart;
-
- /** The sub number. */
- private Integer[] subNumber;
-
- /** The sub collector list next level. */
- // subcollectors next level
- protected MtasDataCollector<?, ?>[] subCollectorListNextLevel = null;
-
- /** The sub collector next level. */
- protected MtasDataCollector<?, ?> subCollectorNextLevel = null;
-
- /** The new current position. */
- // administration for adding
- protected int newSize, newPosition, newCurrentPosition;
-
- /** The new current existing. */
- protected boolean newCurrentExisting;
-
- /** The new key list. */
- protected String[] newKeyList = null;
-
- /** The new error number. */
- protected int[] newErrorNumber;
-
- /** The new error list. */
- protected HashMap<String, Integer>[] newErrorList;
-
- /** The new sub collector types. */
- // subcollectors properties for adding
- private String[] newSubCollectorTypes;
-
- /** The new sub data types. */
- private String[] newSubDataTypes;
-
- /** The new sub stats types. */
- private String[] newSubStatsTypes;
-
- /** The new sub stats items. */
- private TreeSet<String>[] newSubStatsItems;
-
- /** The new sub sort types. */
- private String[] newSubSortTypes;
-
- /** The new sub sort directions. */
- private String[] newSubSortDirections;
-
- /** The new sub start. */
- private Integer[] newSubStart;
-
- /** The new sub number. */
- private Integer[] newSubNumber;
-
- /** The new sub collector list next level. */
- // subcollectors next level for adding
- protected MtasDataCollector<?, ?>[] newSubCollectorListNextLevel = null;
-
- /** The new sub collector next level. */
- protected MtasDataCollector<?, ?> newSubCollectorNextLevel = null;
-
- /**
- * Instantiates a new mtas data collector.
- *
- * @param collectorType the collector type
- * @param dataType the data type
- * @param statsType the stats type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param segmentRegistration the segment registration
- */
- protected MtasDataCollector(String collectorType, String dataType,
- String statsType, TreeSet<String> statsItems, String sortType,
- String sortDirection, Integer start, Integer number,
- boolean segmentRegistration) {
- // set properties
- this.collectorType = collectorType; // data or list
- this.dataType = dataType; // long or double
- this.statsType = statsType; // basic, advanced or full
- this.statsItems = statsItems; // sum, n, all, ...
- this.sortType = sortType;
- this.sortDirection = sortDirection;
- this.start = start;
- this.number = number;
- this.segmentRegistration = segmentRegistration;
- if (segmentRegistration) {
- segmentKeys = new HashSet<String>();
- segmentKeyValueList = new LinkedHashMap<String, HashMap<String, T1>>();
- segmentValueBoundary = new LinkedHashMap<String, T1>();
- segmentValueMaxListMin = new LinkedHashMap<String, T1>();
- }
- // initialize administration
- keyList = new String[0];
- errorNumber = new int[0];
- errorList = (HashMap<String, Integer>[]) new HashMap<?, ?>[0];
- size = 0;
- position = 0;
- // subCollectors properties
- hasSub = false;
- subCollectorTypes = null;
- subDataTypes = null;
- subStatsTypes = null;
- subStatsItems = null;
- subSortTypes = null;
- subSortDirections = null;
- subStart = null;
- subNumber = null;
- subCollectorListNextLevel = null;
- subCollectorNextLevel = null;
- }
-
- /**
- * Instantiates a new mtas data collector.
- *
- * @param collectorType the collector type
- * @param dataType the data type
- * @param statsType the stats type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param subCollectorTypes the sub collector types
- * @param subDataTypes the sub data types
- * @param subStatsTypes the sub stats types
- * @param subStatsItems the sub stats items
- * @param subSortTypes the sub sort types
- * @param subSortDirections the sub sort directions
- * @param subStart the sub start
- * @param subNumber the sub number
- * @param segmentRegistration the segment registration
- */
- protected MtasDataCollector(String collectorType, String dataType,
- String statsType, TreeSet<String> statsItems, String sortType,
- String sortDirection, Integer start, Integer number,
- String[] subCollectorTypes, String[] subDataTypes,
- String[] subStatsTypes, TreeSet<String>[] subStatsItems,
- String subSortTypes[], String[] subSortDirections, Integer[] subStart,
- Integer[] subNumber, boolean segmentRegistration) {
- // initialize
- this(collectorType, dataType, statsType, statsItems, sortType,
- sortDirection, start, number, segmentRegistration);
- // initialize subCollectors
- if (subCollectorTypes != null) {
- hasSub = true;
- this.subCollectorTypes = subCollectorTypes;
- this.subDataTypes = subDataTypes;
- this.subStatsTypes = subStatsTypes;
- this.subStatsItems = subStatsItems;
- this.subSortTypes = subSortTypes;
- this.subSortDirections = subSortDirections;
- this.subStart = subStart;
- this.subNumber = subNumber;
- if (subCollectorTypes.length > 1) {
- newSubCollectorTypes = Arrays.copyOfRange(subCollectorTypes, 1,
- subCollectorTypes.length);
- newSubDataTypes = Arrays.copyOfRange(subDataTypes, 1,
- subStatsTypes.length);
- newSubStatsTypes = Arrays.copyOfRange(subStatsTypes, 1,
- subStatsTypes.length);
- newSubStatsItems = Arrays.copyOfRange(subStatsItems, 1,
- subStatsItems.length);
- newSubSortTypes = Arrays.copyOfRange(subSortTypes, 1,
- subSortTypes.length);
- newSubSortDirections = Arrays.copyOfRange(subSortDirections, 1,
- subSortDirections.length);
- newSubStart = Arrays.copyOfRange(subStart, 1, subStart.length);
- newSubNumber = Arrays.copyOfRange(subNumber, 1, subNumber.length);
- }
- newSubCollectorListNextLevel = new MtasDataCollector[0];
- }
- }
-
- /**
- * Merge.
- *
- * @param newDataCollector the new data collector
- * @throws IOException Signals that an I/O exception has occurred.
- */
- abstract public void merge(MtasDataCollector<?, ?> newDataCollector)
- throws IOException;
-
- /**
- * Inits the new list.
- *
- * @param maxNumberOfTerms the max number of terms
- * @param segmentName the segment name
- * @param segmentNumber the segment number
- * @throws IOException Signals that an I/O exception has occurred.
- */
- protected void initNewList(int maxNumberOfTerms, String segmentName,
- int segmentNumber) throws IOException {
- initNewListBasic(maxNumberOfTerms);
- if (segmentRegistration) {
- this.segmentName = segmentName;
- this.segmentNumber = segmentNumber;
- if (!segmentKeyValueList.containsKey(segmentName)) {
- segmentKeyValueList.put(segmentName, new HashMap<String, T1>());
- segmentValueBoundary.put(segmentName, null);
- segmentValueMaxListMin.put(segmentName, null);
- }
- this.segmentValueMaxList = new ArrayList<T1>();
- }
- }
-
- /**
- * Inits the new list.
- *
- * @param maxNumberOfTerms the max number of terms
- * @throws IOException Signals that an I/O exception has occurred.
- */
- protected void initNewList(int maxNumberOfTerms) throws IOException {
- if (segmentRegistration) {
- throw new IOException("missing segment name");
- } else {
- initNewListBasic(maxNumberOfTerms);
- }
- }
-
- /**
- * Inits the new list basic.
- *
- * @param maxNumberOfTerms the max number of terms
- */
- private void initNewListBasic(int maxNumberOfTerms) {
- position = 0;
- newPosition = 0;
- newCurrentPosition = 0;
- newSize = maxNumberOfTerms + size;
- newKeyList = new String[newSize];
- newErrorNumber = new int[newSize];
- newErrorList = (HashMap<String, Integer>[]) new HashMap<?, ?>[newSize];
- if (hasSub) {
- newSubCollectorListNextLevel = new MtasDataCollector[newSize];
- }
- }
-
- /**
- * Increase new list size.
- */
- protected void increaseNewListSize() {
- String[] tmpNewKeyList = newKeyList;
- int[] tmpNewErrorNumber = newErrorNumber;
- HashMap<String, Integer>[] tmpNewErrorList = newErrorList;
- int tmpNewSize = newSize;
- newSize = 2 * newSize;
- newKeyList = new String[newSize];
- newErrorNumber = new int[newSize];
- newErrorList = (HashMap<String, Integer>[]) new HashMap<?, ?>[newSize];
- System.arraycopy(tmpNewKeyList, 0, newKeyList, 0, tmpNewSize);
- System.arraycopy(tmpNewErrorNumber, 0, newErrorNumber, 0, tmpNewSize);
- System.arraycopy(tmpNewErrorList, 0, newErrorList, 0, tmpNewSize);
- if (hasSub) {
- MtasDataCollector<?, ?>[] tmpNewSubCollectorListNextLevel = newSubCollectorListNextLevel;
- newSubCollectorListNextLevel = new MtasDataCollector[newSize];
- System.arraycopy(tmpNewSubCollectorListNextLevel, 0,
- newSubCollectorListNextLevel, 0, tmpNewSize);
- }
- }
-
- /**
- * Adds the.
- *
- * @return the mtas data collector
- * @throws IOException Signals that an I/O exception has occurred.
- */
- protected final MtasDataCollector<?, ?> add() throws IOException {
- if (!collectorType.equals(COLLECTOR_TYPE_DATA)) {
- throw new IOException("collector should be " + COLLECTOR_TYPE_DATA);
- } else {
- if (newPosition > 0) {
- newCurrentExisting = true;
- } else if (position < getSize()) {
- // copy
- newKeyList[0] = keyList[0];
- newErrorNumber[0] = errorNumber[0];
- newErrorList[0] = errorList[0];
- if (hasSub) {
- newSubCollectorNextLevel = subCollectorNextLevel;
- }
- copyToNew(0, 0);
- newPosition = 1;
- position = 1;
- newCurrentExisting = true;
- } else {
- // add key
- newKeyList[0] = COLLECTOR_TYPE_DATA;
- newErrorNumber[0] = 0;
- newErrorList[0] = new HashMap<String, Integer>();
- newPosition = 1;
- newCurrentPosition = newPosition - 1;
- newCurrentExisting = false;
- // ready, only handle sub
- if (hasSub) {
- newSubCollectorNextLevel = getCollector(subCollectorTypes[0],
- subDataTypes[0], subStatsTypes[0], subStatsItems[0],
- subSortTypes[0], subSortDirections[0], subStart[0],
- subNumber[0], newSubCollectorTypes, newSubDataTypes,
- newSubStatsTypes, newSubStatsItems, newSubSortTypes,
- newSubSortDirections, newSubStart, newSubNumber,
- segmentRegistration);
- } else {
- newSubCollectorNextLevel = null;
- }
- }
- return newSubCollectorNextLevel;
- }
- }
-
- /**
- * Adds the.
- *
- * @param key the key
- * @return the mtas data collector
- * @throws IOException Signals that an I/O exception has occurred.
- */
- protected final MtasDataCollector<?, ?> add(String key) throws IOException {
- if (collectorType.equals(COLLECTOR_TYPE_DATA)) {
- throw new IOException("collector should be " + COLLECTOR_TYPE_LIST);
- } else if (key == null) {
- throw new IOException("key shouldn't be null");
- } else {
- // check previous added
- if ((newPosition > 0)
- && newKeyList[(newPosition - 1)].compareTo(key) >= 0) {
- int i = newPosition;
- do {
- i--;
- if (newKeyList[i].equals(key)) {
- newCurrentPosition = i;
- newCurrentExisting = true;
- if (subDataTypes != null) {
- return newSubCollectorListNextLevel[newCurrentPosition];
- } else {
- return null;
- }
- }
- } while ((i > 0) && (newKeyList[i].compareTo(key) > 0));
- }
- // move position in old list
- if (position < getSize()) {
- // just add smaller or equal items
- while (keyList[position].compareTo(key) <= 0) {
- if (newPosition == newSize) {
- increaseNewListSize();
- }
- // copy
- newKeyList[newPosition] = keyList[position];
- newErrorNumber[newPosition] = errorNumber[position];
- newErrorList[newPosition] = errorList[position];
- if (hasSub) {
- newSubCollectorListNextLevel[newPosition] = subCollectorListNextLevel[position];
- }
- copyToNew(position, newPosition);
- newPosition++;
- position++;
- // check if added key from list is right key
- if (newKeyList[(newPosition - 1)].equals(key)) {
- newCurrentPosition = newPosition - 1;
- newCurrentExisting = true;
- // ready
- if (hasSub) {
- return newSubCollectorListNextLevel[newCurrentPosition];
- } else {
- return null;
- }
- // stop if position exceeds size
- } else if (position == getSize()) {
- break;
- }
- }
- }
- // check size
- if (newPosition == newSize) {
- increaseNewListSize();
- }
- // add key
- newKeyList[newPosition] = key;
- newErrorNumber[newPosition] = 0;
- newErrorList[newPosition] = new HashMap<String, Integer>();
- newPosition++;
- newCurrentPosition = newPosition - 1;
- newCurrentExisting = false;
- // ready, only handle sub
- if (hasSub) {
- newSubCollectorListNextLevel[newCurrentPosition] = getCollector(
- subCollectorTypes[0], subDataTypes[0], subStatsTypes[0],
- subStatsItems[0], subSortTypes[0], subSortDirections[0],
- subStart[0], subNumber[0], newSubCollectorTypes, newSubDataTypes,
- newSubStatsTypes, newSubStatsItems, newSubSortTypes,
- newSubSortDirections, newSubStart, newSubNumber,
- segmentRegistration);
- return newSubCollectorListNextLevel[newCurrentPosition];
- } else {
- return null;
- }
- }
- }
-
- /**
- * Copy to new.
- *
- * @param position the position
- * @param newPosition the new position
- */
- protected abstract void copyToNew(int position, int newPosition);
-
- /**
- * Copy from new.
- */
- protected abstract void copyFromNew();
-
- /**
- * Compare for computing segment.
- *
- * @param value the value
- * @param boundary the boundary
- * @return true, if successful
- */
- protected abstract boolean compareForComputingSegment(T1 value,
- T1 boundary);
-
- /**
- * Minimum for computing segment.
- *
- * @param value the value
- * @param boundary the boundary
- * @return the t1
- */
- protected abstract T1 minimumForComputingSegment(T1 value, T1 boundary);
-
- /**
- * Minimum for computing segment.
- *
- * @return the t1
- */
- protected abstract T1 minimumForComputingSegment();
-
- /**
- * Boundary for segment.
- *
- * @return the t1
- */
- protected abstract T1 boundaryForSegment();
-
- /**
- * Boundary for computing segment.
- *
- * @return the t1
- */
- protected abstract T1 boundaryForComputingSegment();
-
- /**
- * Close segment key value registration.
- */
- public void closeSegmentKeyValueRegistration() {
- if(segmentRegistration) {
- HashMap<String, T1> keyValueList = segmentKeyValueList.get(segmentName);
- T1 tmpSegmentValueBoundary = segmentValueBoundary.get(segmentName);
- for (String key : keyValueList.keySet()) {
- if (tmpSegmentValueBoundary == null || compareForComputingSegment(
- keyValueList.get(key), tmpSegmentValueBoundary)) {
- segmentKeys.add(key);
- }
- }
- }
- }
-
- /**
- * Recompute segment keys.
- */
- public void recomputeSegmentKeys() {
- segmentKeys.clear();
- segmentRecomputeKeyList = new LinkedHashMap<String, HashSet<String>>();
- // recompute boundaries
- for (String segmentName : segmentKeyValueList.keySet()) {
- this.segmentName = segmentName;
- T1 tmpSegmentValueBoundary = boundaryForSegment();
- segmentValueBoundary.put(segmentName, tmpSegmentValueBoundary);
- }
- // compute adjusted boundaries and compute keys
- for (String segmentName : segmentKeyValueList.keySet()) {
- this.segmentName = segmentName;
- HashMap<String, T1> keyValueList = segmentKeyValueList.get(segmentName);
- T1 tmpSegmentValueBoundaryForComputing = boundaryForComputingSegment();
- for (String key : keyValueList.keySet()) {
- if (tmpSegmentValueBoundaryForComputing == null
- || compareForComputingSegment(keyValueList.get(key),
- tmpSegmentValueBoundaryForComputing)) {
- if (!segmentKeys.contains(key)) {
- segmentKeys.add(key);
- }
- }
- }
- }
- HashMap<String, T1> keyValueList;
- HashSet<String> recomputeKeyList;
- for (String key : segmentKeys) {
- for (String segmentName : segmentKeyValueList.keySet()) {
- keyValueList = segmentKeyValueList.get(segmentName);
- if(!keyValueList.containsKey(key)) {
- if(!segmentRecomputeKeyList.containsKey(segmentName)) {
- recomputeKeyList = new HashSet<String>();
- segmentRecomputeKeyList.put(segmentName, recomputeKeyList);
- } else {
- recomputeKeyList = segmentRecomputeKeyList.get(segmentName);
- }
- recomputeKeyList.add(key);
- } else {
- break;
- }
- }
- }
- this.segmentName = null;
- }
-
- /**
- * Check existence necessary keys.
- *
- * @return true, if successful
- */
- public boolean checkExistenceNecessaryKeys() {
- return segmentRecomputeKeyList.size() == 0;
- }
-
- /**
- * Validate segment value.
- *
- * @param key the key
- * @param value the value
- * @param maximumNumber the maximum number
- * @param segmentNumber the segment number
- * @return true, if successful
- */
- public boolean validateSegmentValue(String key, T1 value, int maximumNumber,
- int segmentNumber) {
- if(segmentRegistration) {
- if(maximumNumber>0) {
- segmentKeyValueList.get(segmentName).put(key, value);
- T1 tmpSegmentValueMaxListMin = segmentValueMaxListMin.get(segmentName);
- T1 tmpSegmentValueBoundary = segmentValueBoundary.get(segmentName);
- if (segmentValueMaxList.size() < maximumNumber) {
- segmentValueMaxList.add(value);
- segmentValueMaxListMin.put(segmentName,
- (tmpSegmentValueMaxListMin == null) ? value
- : minimumForComputingSegment(tmpSegmentValueMaxListMin, value));
- if (segmentValueMaxList.size() == maximumNumber) {
- tmpSegmentValueMaxListMin = segmentValueMaxListMin.get(segmentName);
- segmentValueMaxListMin.put(segmentName, tmpSegmentValueMaxListMin);
- segmentValueBoundary.put(segmentName, boundaryForComputingSegment());
- }
- return true;
- } else if (compareForComputingSegment(value, tmpSegmentValueBoundary)) {
- if (compareForComputingSegment(value, tmpSegmentValueMaxListMin)) {
- segmentValueMaxList.remove(tmpSegmentValueMaxListMin);
- segmentValueMaxList.add(value);
- tmpSegmentValueMaxListMin = minimumForComputingSegment();
- segmentValueMaxListMin.put(segmentName, tmpSegmentValueMaxListMin);
- segmentValueBoundary.put(segmentName, boundaryForComputingSegment());
- }
- return true;
- } else if (segmentKeys.contains(key)) {
- return true;
- } else {
- return false;
- }
- } else {
- return false;
- }
- } else {
- return true;
- }
- }
-
- /**
- * Sets the error.
- *
- * @param newPosition the new position
- * @param errorNumberItem the error number item
- * @param errorListItem the error list item
- * @param currentExisting the current existing
- */
- protected final void setError(int newPosition, int errorNumberItem,
- HashMap<String, Integer> errorListItem, boolean currentExisting) {
- if (currentExisting) {
- newErrorNumber[newPosition] += errorNumberItem;
- HashMap<String, Integer> item = newErrorList[newPosition];
- for (Entry<String, Integer> entry : errorListItem.entrySet()) {
- if (item.containsKey(entry.getKey())) {
- item.put(entry.getKey(),
- item.get(entry.getKey()) + entry.getValue());
- } else {
- item.put(entry.getKey(), entry.getValue());
- }
- }
- } else {
- newErrorNumber[newPosition] = errorNumberItem;
- newErrorList[newPosition] = errorListItem;
- }
- }
-
- /**
- * Sorted and unique.
- *
- * @param keyList the key list
- * @param size the size
- * @return true, if successful
- */
- private boolean sortedAndUnique(String[] keyList, int size) {
- for (int i = 1; i < size; i++) {
- if (keyList[(i - 1)].compareTo(keyList[i]) >= 0) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Compute sort and unique mapping.
- *
- * @param keyList the key list
- * @param size the size
- * @return the int[][]
- */
- private int[][] computeSortAndUniqueMapping(String[] keyList, int size) {
- if (size > 0) {
- SortedMap<String, int[]> sortedMap = new TreeMap<String, int[]>();
- for (int i = 0; i < size; i++) {
- if (sortedMap.containsKey(keyList[i])) {
- int[] previousList = sortedMap.get(keyList[i]);
- int[] newList = new int[previousList.length + 1];
- System.arraycopy(previousList, 0, newList, 0, previousList.length);
- newList[previousList.length] = i;
- sortedMap.put(keyList[i], newList);
- } else {
- sortedMap.put(keyList[i], new int[] { i });
- }
- }
- Collection<int[]> values = sortedMap.values();
- int[][] result = new int[sortedMap.size()][];
- return values.toArray(result);
- } else {
- return null;
- }
- }
-
- /**
- * Remap data.
- *
- * @param mapping the mapping
- * @throws IOException Signals that an I/O exception has occurred.
- */
- protected void remapData(int[][] mapping) throws IOException {
- // remap and merge keys
- String[] newKeyList = new String[mapping.length];
- int[] newErrorNumber = new int[mapping.length];
- HashMap<String, Integer>[] newErrorList = (HashMap<String, Integer>[]) new HashMap<?, ?>[mapping.length];
- for (int i = 0; i < mapping.length; i++) {
- newKeyList[i] = keyList[mapping[i][0]];
- for (int j = 0; j < mapping[i].length; j++) {
- if (j == 0) {
- newErrorNumber[i] = errorNumber[mapping[i][j]];
- newErrorList[i] = errorList[mapping[i][j]];
- } else {
- newErrorNumber[i] += errorNumber[mapping[i][j]];
- for (Entry<String, Integer> entry : errorList[mapping[i][j]]
- .entrySet()) {
- if (newErrorList[i].containsKey(entry.getKey())) {
- newErrorList[i].put(entry.getKey(),
- newErrorList[i].get(entry.getKey()) + entry.getValue());
- } else {
- newErrorList[i].put(entry.getKey(), entry.getValue());
- }
- }
- }
- }
- }
- if (hasSub) {
- newSubCollectorListNextLevel = new MtasDataCollector<?, ?>[mapping.length];
- for (int i = 0; i < mapping.length; i++) {
- for (int j = 0; j < mapping[i].length; j++) {
- if (j == 0 || newSubCollectorListNextLevel[i] == null) {
- newSubCollectorListNextLevel[i] = subCollectorListNextLevel[mapping[i][j]];
- } else {
- newSubCollectorListNextLevel[i]
- .merge(subCollectorListNextLevel[mapping[i][j]]);
- }
- }
- }
- subCollectorListNextLevel = newSubCollectorListNextLevel;
- }
- keyList = newKeyList;
- errorNumber = newErrorNumber;
- errorList = newErrorList;
- size = keyList.length;
- position = 0;
- }
-
- /**
- * Close new list.
- *
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public void closeNewList() throws IOException {
- if (segmentRegistration) {
- this.segmentName = null;
- }
- if (newSize > 0) {
- // add remaining old
- while (position < getSize()) {
- if (newPosition == newSize) {
- increaseNewListSize();
- }
- newKeyList[newPosition] = keyList[position];
- newErrorNumber[newPosition] = errorNumber[position];
- newErrorList[newPosition] = errorList[position];
- if (hasSub) {
- newSubCollectorListNextLevel[newPosition] = subCollectorListNextLevel[position];
- }
- copyToNew(position, newPosition);
- position++;
- newPosition++;
- }
- // copy
- keyList = newKeyList;
- errorNumber = newErrorNumber;
- errorList = newErrorList;
- subCollectorListNextLevel = newSubCollectorListNextLevel;
- copyFromNew();
- size = newPosition;
- // sort and merge
- if (!sortedAndUnique(keyList, getSize())) {
- remapData(computeSortAndUniqueMapping(keyList, getSize()));
- }
- }
- position = 0;
- newSize = 0;
- newPosition = 0;
- newCurrentPosition = 0;
- }
-
- /**
- * Gets the item.
- *
- * @param i the i
- * @return the item
- */
- abstract protected T2 getItem(int i);
-
- /**
- * Checks for sub.
- *
- * @return true, if successful
- */
- protected boolean hasSub() {
- return hasSub;
- }
-
- /**
- * Error.
- *
- * @param error the error
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public abstract void error(String error) throws IOException;
-
- /**
- * Error.
- *
- * @param keys the keys
- * @param error the error
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public abstract void error(String keys[], String error) throws IOException;
-
- /**
- * Adds the.
- *
- * @param valueSum the value sum
- * @param valueN the value n
- * @return the mtas data collector
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public abstract MtasDataCollector<?, ?> add(long valueSum, long valueN)
- throws IOException;
-
- /**
- * Adds the.
- *
- * @param values the values
- * @param number the number
- * @return the mtas data collector
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public abstract MtasDataCollector<?, ?> add(long[] values, int number)
- throws IOException;
-
- /**
- * Adds the.
- *
- * @param valueSum the value sum
- * @param valueN the value n
- * @return the mtas data collector
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public abstract MtasDataCollector<?, ?> add(double valueSum, long valueN)
- throws IOException;
-
- /**
- * Adds the.
- *
- * @param values the values
- * @param number the number
- * @return the mtas data collector
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public abstract MtasDataCollector<?, ?> add(double[] values, int number)
- throws IOException;
-
- /**
- * Adds the.
- *
- * @param keys the keys
- * @param valueSum the value sum
- * @param valueN the value n
- * @return the mtas data collector[]
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public abstract MtasDataCollector<?, ?>[] add(String[] keys, long valueSum,
- long valueN) throws IOException;
-
- /**
- * Adds the.
- *
- * @param keys the keys
- * @param values the values
- * @param number the number
- * @return the mtas data collector[]
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public abstract MtasDataCollector<?, ?>[] add(String[] keys, long[] values,
- int number) throws IOException;
-
- /**
- * Adds the.
- *
- * @param keys the keys
- * @param valueSum the value sum
- * @param valueN the value n
- * @return the mtas data collector[]
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public abstract MtasDataCollector<?, ?>[] add(String[] keys,
- double valueSum, long valueN) throws IOException;
-
- /**
- * Adds the.
- *
- * @param keys the keys
- * @param values the values
- * @param number the number
- * @return the mtas data collector[]
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public abstract MtasDataCollector<?, ?>[] add(String[] keys,
- double[] values, int number) throws IOException;
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#toString()
- */
- @Override
- public String toString() {
- return this.getClass().getCanonicalName() + ": " + collectorType + " - "
- + statsType + " " + statsItems + " " + hasSub;
- }
-
- /**
- * Gets the list.
- *
- * @return the list
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public final SortedMap<String, T2> getList() throws IOException {
- final TreeMap<String, T2> basicList = getBasicList();
- SortedMap<String, T2> list = null;
- if (sortType.equals(CodecUtil.SORT_TERM)) {
- if (sortDirection.equals(CodecUtil.SORT_ASC)) {
- list = basicList;
- } else if (sortDirection.equals(CodecUtil.SORT_DESC)) {
- list = basicList.descendingMap();
- } else {
- throw new IOException("unknown sort direction " + sortDirection);
- }
- } else if (CodecUtil.STATS_TYPES.contains(sortType)) {
- // comperator
- Comparator<String> valueComparator = new Comparator<String>() {
- @Override
- public int compare(String k1, String k2) {
- int compare = basicList.get(k1).compareTo(basicList.get(k2));
- return compare == 0 ? k1.compareTo(k2) : compare;
- }
- };
- SortedMap<String, T2> sortedByValues = new TreeMap<String, T2>(
- valueComparator);
- sortedByValues.putAll(basicList);
- list = sortedByValues;
- } else {
- throw new IOException("unknown sort type " + sortType);
- }
- int start = this.start == null ? 0 : this.start;
- if (number == null || (start == 0 && number >= list.size())) {
- // ful list
- return list;
- } else if (start < list.size() && number > 0) {
- // subset
- String startKey = null, endKey = null;
- int counter = 0;
- for (String key : list.keySet()) {
- if (start == counter) {
- startKey = key;
- } else if (start + number == counter) {
- endKey = key;
- break;
- } else {
- endKey = key;
- }
- counter++;
- }
- return list.subMap(startKey, endKey);
- } else {
- // empty set
- return new TreeMap<String, T2>();
- }
- }
-
- /**
- * Gets the basic list.
- *
- * @return the basic list
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private TreeMap<String, T2> getBasicList() throws IOException {
- closeNewList();
- TreeMap<String, T2> list = new TreeMap<String, T2>();
- if (collectorType.equals(COLLECTOR_TYPE_LIST)) {
- for (int i = 0; i < getSize(); i++) {
- T2 newItem = getItem(i);
- if (list.containsKey(keyList[i])) {
- newItem.add(list.get(keyList[i]));
- }
- list.put(keyList[i], newItem);
- }
- return list;
- } else {
- throw new IOException("type " + collectorType + " not supported");
- }
- }
-
- /**
- * Gets the data.
- *
- * @return the data
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public final T2 getData() throws IOException {
- closeNewList();
- if (collectorType.equals(COLLECTOR_TYPE_DATA)) {
- if (getSize() > 0) {
- return getItem(0);
- } else {
- return null;
- }
- } else {
- throw new IOException("type " + collectorType + " not supported");
- }
- }
-
- /**
- * Gets the collector type.
- *
- * @return the collector type
- */
- public String getCollectorType() {
- return collectorType;
- }
-
- /**
- * Gets the stats type.
- *
- * @return the stats type
- */
- public String getStatsType() {
- return statsType;
- }
-
- /**
- * Gets the data type.
- *
- * @return the data type
- */
- public String getDataType() {
- return dataType;
- }
-
- /**
- * Gets the size.
- *
- * @return the size
- */
- public int getSize() {
- return size;
- }
-
- /**
- * Write object.
- *
- * @param out the out
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void writeObject(ObjectOutputStream out) throws IOException {
- segmentRegistration = false;
- out.defaultWriteObject();
- }
-
- }
-
-
-
-
-
-
-
}
diff --git a/src/mtas/codec/util/collector/MtasDataAdvanced.java b/src/mtas/codec/util/collector/MtasDataAdvanced.java
index c91b371..32e1397 100644
--- a/src/mtas/codec/util/collector/MtasDataAdvanced.java
+++ b/src/mtas/codec/util/collector/MtasDataAdvanced.java
@@ -2,20 +2,23 @@ package mtas.codec.util.collector;
import java.io.IOException;
import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Set;
import java.util.TreeSet;
+
import mtas.codec.util.CodecUtil;
import mtas.codec.util.DataCollector;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataAdvanced.
*
- * @param <T1> the generic type
- * @param <T2> the generic type
- * @param <T3> the generic type
+ * @param <T1>
+ * the generic type
+ * @param <T2>
+ * the generic type
*/
-abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends MtasDataItem<T1>>
- extends MtasDataCollector<T1, T3> implements Serializable {
+abstract class MtasDataAdvanced<T1 extends Number & Comparable<T1>, T2 extends Number & Comparable<T2>>
+ extends MtasDataCollector<T1, T2> implements Serializable {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -46,24 +49,44 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
/**
* Instantiates a new mtas data advanced.
*
- * @param collectorType the collector type
- * @param dataType the data type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param subCollectorTypes the sub collector types
- * @param subDataTypes the sub data types
- * @param subStatsTypes the sub stats types
- * @param subStatsItems the sub stats items
- * @param subSortTypes the sub sort types
- * @param subSortDirections the sub sort directions
- * @param subStart the sub start
- * @param subNumber the sub number
- * @param operations the operations
- * @param segmentRegistration the segment registration
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param collectorType
+ * the collector type
+ * @param dataType
+ * the data type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param subCollectorTypes
+ * the sub collector types
+ * @param subDataTypes
+ * the sub data types
+ * @param subStatsTypes
+ * the sub stats types
+ * @param subStatsItems
+ * the sub stats items
+ * @param subSortTypes
+ * the sub sort types
+ * @param subSortDirections
+ * the sub sort directions
+ * @param subStart
+ * the sub start
+ * @param subNumber
+ * the sub number
+ * @param operations
+ * the operations
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasDataAdvanced(String collectorType, String dataType,
TreeSet<String> statsItems, String sortType, String sortDirection,
@@ -71,12 +94,12 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
String[] subDataTypes, String[] subStatsTypes,
TreeSet<String>[] subStatsItems, String[] subSortTypes,
String[] subSortDirections, Integer[] subStart, Integer[] subNumber,
- MtasDataOperations<T1, T2> operations, boolean segmentRegistration)
- throws IOException {
+ MtasDataOperations<T1, T2> operations, String segmentRegistration,
+ String boundary) throws IOException {
super(collectorType, dataType, CodecUtil.STATS_ADVANCED, statsItems,
sortType, sortDirection, start, number, subCollectorTypes, subDataTypes,
subStatsTypes, subStatsItems, subSortTypes, subSortDirections, subStart,
- subNumber, segmentRegistration);
+ subNumber, segmentRegistration, boundary);
this.operations = operations;
}
@@ -88,7 +111,7 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
*/
@Override
public final void error(String error) throws IOException {
- add();
+ add(false);
setError(newCurrentPosition, error, newCurrentExisting);
}
@@ -103,7 +126,7 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
public final void error(String[] keys, String error) throws IOException {
if (keys != null && keys.length > 0) {
for (int i = 0; i < keys.length; i++) {
- add(keys[i]);
+ add(keys[i], false);
setError(newCurrentPosition, error, newCurrentExisting);
}
}
@@ -112,9 +135,12 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
/**
* Sets the error.
*
- * @param newPosition the new position
- * @param error the error
- * @param currentExisting the current existing
+ * @param newPosition
+ * the new position
+ * @param error
+ * the error
+ * @param currentExisting
+ * the current existing
*/
protected void setError(int newPosition, String error,
boolean currentExisting) {
@@ -141,7 +167,7 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
* @see mtas.codec.util.DataCollector.MtasDataCollector#increaseNewListSize()
*/
@Override
- protected final void increaseNewListSize() {
+ protected final void increaseNewListSize() throws IOException {
// register old situation
int tmpOldSize = newKeyList.length;
int tmpNewPosition = newPosition;
@@ -178,6 +204,56 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
/*
* (non-Javadoc)
*
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#reduceToKeys(java.util.Set)
+ */
+ @SuppressWarnings("unchecked")
+ public void reduceToKeys(Set<String> keys) {
+ if (size > 0) {
+ int sizeCopy = size;
+ String[] keyListCopy = keyList.clone();
+ int[] errorNumberCopy = errorNumber.clone();
+ HashMap<String, Integer>[] errorListCopy = errorList.clone();
+ int[] sourceNumberListCopy = sourceNumberList.clone();
+ T1[] advancedValueSumListCopy = advancedValueSumList.clone();
+ T1[] advancedValueMaxListCopy = advancedValueMaxList.clone();
+ T1[] advancedValueMinListCopy = advancedValueMinList.clone();
+ T1[] advancedValueSumOfSquaresListCopy = advancedValueSumOfSquaresList
+ .clone();
+ T2[] advancedValueSumOfLogsListCopy = advancedValueSumOfLogsList.clone();
+ long[] advancedValueNListCopy = advancedValueNList.clone();
+ keyList = new String[keys.size()];
+ errorNumber = new int[keys.size()];
+ errorList = new HashMap[keys.size()];
+ sourceNumberList = new int[keys.size()];
+ advancedValueSumList = operations.createVector1(keys.size());
+ advancedValueMaxList = operations.createVector1(keys.size());
+ advancedValueMinList = operations.createVector1(keys.size());
+ advancedValueSumOfSquaresList = operations.createVector1(keys.size());
+ advancedValueSumOfLogsList = operations.createVector2(keys.size());
+ advancedValueNList = new long[keys.size()];
+ size = 0;
+ for (int i = 0; i < sizeCopy; i++) {
+ if (keys.contains(keyListCopy[i])) {
+ keyList[size] = keyListCopy[i];
+ errorNumber[size] = errorNumberCopy[i];
+ errorList[size] = errorListCopy[i];
+ sourceNumberList[size] = sourceNumberListCopy[i];
+ advancedValueSumList[size] = advancedValueSumListCopy[i];
+ advancedValueMaxList[size] = advancedValueMaxListCopy[i];
+ advancedValueMinList[size] = advancedValueMinListCopy[i];
+ advancedValueSumOfSquaresList[size] = advancedValueSumOfSquaresListCopy[i];
+ advancedValueSumOfLogsList[size] = advancedValueSumOfLogsListCopy[i];
+ advancedValueNList[size] = advancedValueNListCopy[i];
+ size++;
+ }
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.util.DataCollector.MtasDataCollector#copyToNew(int, int)
*/
@Override
@@ -208,10 +284,14 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
/**
* Sets the value.
*
- * @param newPosition the new position
- * @param values the values
- * @param number the number
- * @param currentExisting the current existing
+ * @param newPosition
+ * the new position
+ * @param values
+ * the values
+ * @param number
+ * the number
+ * @param currentExisting
+ * the current existing
*/
protected void setValue(int newPosition, T1[] values, int number,
boolean currentExisting) {
@@ -240,14 +320,22 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
/**
* Sets the value.
*
- * @param newPosition the new position
- * @param valueSum the value sum
- * @param valueSumOfLogs the value sum of logs
- * @param valueSumOfSquares the value sum of squares
- * @param valueMin the value min
- * @param valueMax the value max
- * @param valueN the value n
- * @param currentExisting the current existing
+ * @param newPosition
+ * the new position
+ * @param valueSum
+ * the value sum
+ * @param valueSumOfLogs
+ * the value sum of logs
+ * @param valueSumOfSquares
+ * the value sum of squares
+ * @param valueMin
+ * the value min
+ * @param valueMax
+ * the value max
+ * @param valueN
+ * the value n
+ * @param currentExisting
+ * the current existing
*/
private void setValue(int newPosition, T1 valueSum, T2 valueSumOfLogs,
T1 valueSumOfSquares, T1 valueMin, T1 valueMax, long valueN,
@@ -332,8 +420,9 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
* DataCollector.MtasDataCollector)
*/
@Override
- public void merge(MtasDataCollector<?, ?> newDataCollector)
- throws IOException {
+ public void merge(MtasDataCollector<?, ?> newDataCollector,
+ HashMap<MtasDataCollector<?, ?>, MtasDataCollector<?, ?>> map,
+ boolean increaseSourceNumber) throws IOException {
closeNewList();
if (!collectorType.equals(newDataCollector.getCollectorType())
|| !dataType.equals(newDataCollector.getDataType())
@@ -341,13 +430,17 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
|| !(newDataCollector instanceof MtasDataAdvanced)) {
throw new IOException("cannot merge different dataCollectors");
} else {
- MtasDataAdvanced<T1, T2, T3> newMtasDataAdvanced = (MtasDataAdvanced<T1, T2, T3>) newDataCollector;
+ segmentRegistration = null;
+ @SuppressWarnings("unchecked")
+ MtasDataAdvanced<T1, T2> newMtasDataAdvanced = (MtasDataAdvanced<T1, T2>) newDataCollector;
newMtasDataAdvanced.closeNewList();
initNewList(newMtasDataAdvanced.getSize());
if (collectorType.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ map.put(newDataCollector, this);
for (int i = 0; i < newMtasDataAdvanced.getSize(); i++) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector[1];
- subCollectors[0] = add(newMtasDataAdvanced.keyList[i]);
+ subCollectors[0] = add(newMtasDataAdvanced.keyList[i],
+ increaseSourceNumber);
setError(newCurrentPosition, newMtasDataAdvanced.errorNumber[i],
newMtasDataAdvanced.errorList[i], newCurrentExisting);
setValue(newCurrentPosition,
@@ -358,14 +451,16 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
newMtasDataAdvanced.advancedValueMaxList[i],
newMtasDataAdvanced.advancedValueNList[i], newCurrentExisting);
if (hasSub() && newMtasDataAdvanced.hasSub()) {
- subCollectors[0]
- .merge(newMtasDataAdvanced.subCollectorListNextLevel[i]);
+ subCollectors[0].merge(
+ newMtasDataAdvanced.subCollectorListNextLevel[i], map,
+ increaseSourceNumber);
}
}
closeNewList();
} else if (collectorType.equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ map.put(newDataCollector, this);
if (newMtasDataAdvanced.getSize() > 0) {
- MtasDataCollector subCollector = add();
+ MtasDataCollector<?, ?> subCollector = add(increaseSourceNumber);
setError(newCurrentPosition, newMtasDataAdvanced.errorNumber[0],
newMtasDataAdvanced.errorList[0], newCurrentExisting);
setValue(newCurrentPosition,
@@ -376,7 +471,8 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
newMtasDataAdvanced.advancedValueMaxList[0],
newMtasDataAdvanced.advancedValueNList[0], newCurrentExisting);
if (hasSub() && newMtasDataAdvanced.hasSub()) {
- subCollector.merge(newMtasDataAdvanced.subCollectorNextLevel);
+ subCollector.merge(newMtasDataAdvanced.subCollectorNextLevel, map,
+ increaseSourceNumber);
}
}
closeNewList();
@@ -405,15 +501,16 @@ abstract class MtasDataAdvanced<T1 extends Number, T2 extends Number, T3 extends
*/
@Override
public final void initNewList(int maxNumberOfTerms, String segmentName,
- int segmentNumber) throws IOException {
- super.initNewList(maxNumberOfTerms, segmentName, segmentNumber);
+ int segmentNumber, String boundary) throws IOException {
+ super.initNewList(maxNumberOfTerms, segmentName, segmentNumber, boundary);
initNewListBasic(maxNumberOfTerms);
}
/**
* Inits the new list basic.
*
- * @param maxNumberOfTerms the max number of terms
+ * @param maxNumberOfTerms
+ * the max number of terms
*/
private void initNewListBasic(int maxNumberOfTerms) {
newAdvancedValueSumList = operations.createVector1(newSize);
diff --git a/src/mtas/codec/util/collector/MtasDataBasic.java b/src/mtas/codec/util/collector/MtasDataBasic.java
index e7d2a31..f7e104c 100644
--- a/src/mtas/codec/util/collector/MtasDataBasic.java
+++ b/src/mtas/codec/util/collector/MtasDataBasic.java
@@ -2,29 +2,37 @@ package mtas.codec.util.collector;
import java.io.IOException;
import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Set;
import java.util.TreeSet;
import mtas.codec.util.CodecUtil;
import mtas.codec.util.DataCollector;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataBasic.
*
- * @param <T1> the generic type
- * @param <T2> the generic type
- * @param <T3> the generic type
+ * @param <T1>
+ * the generic type
+ * @param <T2>
+ * the generic type
*/
-abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends MtasDataItem<T1>>
- extends MtasDataCollector<T1, T3> implements Serializable {
+abstract class MtasDataBasic<T1 extends Number & Comparable<T1>, T2 extends Number & Comparable<T2>>
+ extends MtasDataCollector<T1, T2> implements Serializable {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
+ /** The basic value sum list. */
+ protected T1[] basicValueSumList = null;
+
+ /** The basic value n list. */
+ protected long[] basicValueNList = null;
+
/** The new basic value sum list. */
- protected T1[] basicValueSumList = null, newBasicValueSumList = null;
+ protected transient T1[] newBasicValueSumList = null;
/** The new basic value n list. */
- protected long[] basicValueNList = null, newBasicValueNList = null;
+ protected transient long[] newBasicValueNList = null;
/** The operations. */
protected MtasDataOperations<T1, T2> operations;
@@ -32,24 +40,44 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
/**
* Instantiates a new mtas data basic.
*
- * @param collectorType the collector type
- * @param dataType the data type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param subCollectorTypes the sub collector types
- * @param subDataTypes the sub data types
- * @param subStatsTypes the sub stats types
- * @param subStatsItems the sub stats items
- * @param subSortTypes the sub sort types
- * @param subSortDirections the sub sort directions
- * @param subStart the sub start
- * @param subNumber the sub number
- * @param operations the operations
- * @param segmentRegistration the segment registration
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param collectorType
+ * the collector type
+ * @param dataType
+ * the data type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param subCollectorTypes
+ * the sub collector types
+ * @param subDataTypes
+ * the sub data types
+ * @param subStatsTypes
+ * the sub stats types
+ * @param subStatsItems
+ * the sub stats items
+ * @param subSortTypes
+ * the sub sort types
+ * @param subSortDirections
+ * the sub sort directions
+ * @param subStart
+ * the sub start
+ * @param subNumber
+ * the sub number
+ * @param operations
+ * the operations
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasDataBasic(String collectorType, String dataType,
TreeSet<String> statsItems, String sortType, String sortDirection,
@@ -57,12 +85,12 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
String[] subDataTypes, String[] subStatsTypes,
TreeSet<String>[] subStatsItems, String[] subSortTypes,
String[] subSortDirections, Integer[] subStart, Integer[] subNumber,
- MtasDataOperations<T1, T2> operations, boolean segmentRegistration)
- throws IOException {
+ MtasDataOperations<T1, T2> operations, String segmentRegistration,
+ String boundary) throws IOException {
super(collectorType, dataType, CodecUtil.STATS_BASIC, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
subStatsTypes, subStatsItems, subSortTypes, subSortDirections, subStart,
- subNumber, segmentRegistration);
+ subNumber, segmentRegistration, boundary);
this.operations = operations;
}
@@ -74,7 +102,7 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
*/
@Override
public final void error(String error) throws IOException {
- add();
+ add(false);
setError(newCurrentPosition, error, newCurrentExisting);
}
@@ -89,7 +117,7 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
public final void error(String[] keys, String error) throws IOException {
if (keys != null && keys.length > 0) {
for (int i = 0; i < keys.length; i++) {
- add(keys[i]);
+ add(keys[i], false);
setError(newCurrentPosition, error, newCurrentExisting);
}
}
@@ -98,9 +126,12 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
/**
* Sets the error.
*
- * @param newPosition the new position
- * @param error the error
- * @param currentExisting the current existing
+ * @param newPosition
+ * the new position
+ * @param error
+ * the error
+ * @param currentExisting
+ * the current existing
*/
protected void setError(int newPosition, String error,
boolean currentExisting) {
@@ -120,10 +151,14 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
/**
* Sets the value.
*
- * @param newPosition the new position
- * @param valueSum the value sum
- * @param valueN the value n
- * @param currentExisting the current existing
+ * @param newPosition
+ * the new position
+ * @param valueSum
+ * the value sum
+ * @param valueN
+ * the value n
+ * @param currentExisting
+ * the current existing
*/
protected void setValue(int newPosition, T1 valueSum, long valueN,
boolean currentExisting) {
@@ -142,10 +177,14 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
/**
* Sets the value.
*
- * @param newPosition the new position
- * @param values the values
- * @param number the number
- * @param currentExisting the current existing
+ * @param newPosition
+ * the new position
+ * @param values
+ * the values
+ * @param number
+ * the number
+ * @param currentExisting
+ * the current existing
*/
protected void setValue(int newPosition, T1[] values, int number,
boolean currentExisting) {
@@ -171,7 +210,7 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
* @see mtas.codec.util.DataCollector.MtasDataCollector#increaseNewListSize()
*/
@Override
- protected final void increaseNewListSize() {
+ protected final void increaseNewListSize() throws IOException {
// register old situation
int tmpOldSize = newKeyList.length;
int tmpNewPosition = newPosition;
@@ -192,6 +231,43 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
/*
* (non-Javadoc)
*
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#reduceToKeys(java.util.Set)
+ */
+ @SuppressWarnings("unchecked")
+ public void reduceToKeys(Set<String> keys) {
+ if (size > 0) {
+ int sizeCopy = size;
+ String[] keyListCopy = keyList.clone();
+ int[] errorNumberCopy = errorNumber.clone();
+ HashMap<String, Integer>[] errorListCopy = errorList.clone();
+ int[] sourceNumberListCopy = sourceNumberList.clone();
+ T1[] basicValueSumListCopy = basicValueSumList.clone();
+ long[] basicValueNListCopy = basicValueNList.clone();
+ keyList = new String[keys.size()];
+ errorNumber = new int[keys.size()];
+ errorList = new HashMap[keys.size()];
+ sourceNumberList = new int[keys.size()];
+ basicValueSumList = operations.createVector1(keys.size());
+ basicValueNList = new long[keys.size()];
+ size = 0;
+ for (int i = 0; i < sizeCopy; i++) {
+ if (keys.contains(keyListCopy[i])) {
+ keyList[size] = keyListCopy[i];
+ errorNumber[size] = errorNumberCopy[i];
+ errorList[size] = errorListCopy[i];
+ sourceNumberList[size] = sourceNumberListCopy[i];
+ basicValueSumList[size] = basicValueSumListCopy[i];
+ basicValueNList[size] = basicValueNListCopy[i];
+ size++;
+ }
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.util.DataCollector.MtasDataCollector#copyToNew(int, int)
*/
@Override
@@ -245,8 +321,9 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
* DataCollector.MtasDataCollector)
*/
@Override
- public void merge(MtasDataCollector<?, ?> newDataCollector)
- throws IOException {
+ public void merge(MtasDataCollector<?, ?> newDataCollector,
+ HashMap<MtasDataCollector<?, ?>, MtasDataCollector<?, ?>> map,
+ boolean increaseSourceNumber) throws IOException {
closeNewList();
if (!collectorType.equals(newDataCollector.getCollectorType())
|| !dataType.equals(newDataCollector.getDataType())
@@ -254,34 +331,40 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
|| !(newDataCollector instanceof MtasDataBasic)) {
throw new IOException("cannot merge different dataCollectors");
} else {
- MtasDataBasic<T1, T2, T3> newMtasDataBasic = (MtasDataBasic<T1, T2, T3>) newDataCollector;
+ segmentRegistration = null;
+ @SuppressWarnings("unchecked")
+ MtasDataBasic<T1, T2> newMtasDataBasic = (MtasDataBasic<T1, T2>) newDataCollector;
newMtasDataBasic.closeNewList();
initNewList(newMtasDataBasic.getSize());
if (collectorType.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
- String[] keys = new String[1];
+ map.put(newDataCollector, this);
for (int i = 0; i < newMtasDataBasic.getSize(); i++) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector[1];
- subCollectors[0] = add(newMtasDataBasic.keyList[i]);
+ subCollectors[0] = add(newMtasDataBasic.keyList[i],
+ increaseSourceNumber);
setError(newCurrentPosition, newMtasDataBasic.errorNumber[i],
newMtasDataBasic.errorList[i], newCurrentExisting);
setValue(newCurrentPosition, newMtasDataBasic.basicValueSumList[i],
newMtasDataBasic.basicValueNList[i], newCurrentExisting);
if (hasSub() && newMtasDataBasic.hasSub()) {
// single key implies exactly one subCollector if hasSub
- subCollectors[0]
- .merge(newMtasDataBasic.subCollectorListNextLevel[i]);
+ subCollectors[0].merge(
+ newMtasDataBasic.subCollectorListNextLevel[i], map,
+ increaseSourceNumber);
}
}
closeNewList();
} else if (collectorType.equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ map.put(newDataCollector, this);
if (newMtasDataBasic.getSize() > 0) {
- MtasDataCollector<?, ?> subCollector = add();
+ MtasDataCollector<?, ?> subCollector = add(increaseSourceNumber);
setError(newCurrentPosition, newMtasDataBasic.errorNumber[0],
newMtasDataBasic.errorList[0], newCurrentExisting);
setValue(newCurrentPosition, newMtasDataBasic.basicValueSumList[0],
newMtasDataBasic.basicValueNList[0], newCurrentExisting);
if (hasSub() && newMtasDataBasic.hasSub()) {
- subCollector.merge(newMtasDataBasic.subCollectorNextLevel);
+ subCollector.merge(newMtasDataBasic.subCollectorNextLevel, map,
+ increaseSourceNumber);
}
}
closeNewList();
@@ -310,15 +393,16 @@ abstract class MtasDataBasic<T1 extends Number, T2 extends Number, T3 extends Mt
*/
@Override
public final void initNewList(int maxNumberOfTerms, String segmentName,
- int segmentNumber) throws IOException {
- super.initNewList(maxNumberOfTerms, segmentName, segmentNumber);
+ int segmentNumber, String boundary) throws IOException {
+ super.initNewList(maxNumberOfTerms, segmentName, segmentNumber, boundary);
initNewListBasic(maxNumberOfTerms);
}
/**
* Inits the new list basic.
*
- * @param maxNumberOfTerms the max number of terms
+ * @param maxNumberOfTerms
+ * the max number of terms
*/
private void initNewListBasic(int maxNumberOfTerms) {
newBasicValueSumList = operations.createVector1(newSize);
diff --git a/src/mtas/codec/util/collector/MtasDataCollector.java b/src/mtas/codec/util/collector/MtasDataCollector.java
new file mode 100644
index 0000000..266bd24
--- /dev/null
+++ b/src/mtas/codec/util/collector/MtasDataCollector.java
@@ -0,0 +1,1537 @@
+package mtas.codec.util.collector;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.Map.Entry;
+
+import mtas.codec.util.DataCollector;
+import mtas.codec.util.collector.MtasDataItem.NumberComparator;
+
+/**
+ * The Class MtasDataCollector.
+ *
+ * @param <T1>
+ * the generic type
+ * @param <T2>
+ * the generic type
+ */
+public abstract class MtasDataCollector<T1 extends Number & Comparable<T1>, T2 extends Number & Comparable<T2>>
+ implements Serializable {
+
+ /** The Constant serialVersionUID. */
+ private static final long serialVersionUID = 1L;
+
+ /** The segment sort asc. */
+ public static String SEGMENT_SORT_ASC = "segment_asc";
+
+ /** The segment sort desc. */
+ public static String SEGMENT_SORT_DESC = "segment_desc";
+
+ /** The segment boundary asc. */
+ public static String SEGMENT_BOUNDARY_ASC = "segment_boundary_asc";
+
+ /** The segment boundary desc. */
+ public static String SEGMENT_BOUNDARY_DESC = "segment_boundary_desc";
+
+ /** The segment key. */
+ public static String SEGMENT_KEY = "key";
+
+ /** The segment new. */
+ public static String SEGMENT_NEW = "new";
+
+ /** The size. */
+ protected int size;
+
+ /** The position. */
+ protected int position;
+
+ /** The collector type. */
+ // properties collector
+ protected String collectorType;
+
+ /** The stats type. */
+ protected String statsType;
+
+ /** The data type. */
+ protected String dataType;
+
+ /** The stats items. */
+ public TreeSet<String> statsItems;
+
+ /** The sort type. */
+ protected String sortType;
+
+ /** The sort direction. */
+ protected String sortDirection;
+
+ /** The start. */
+ protected Integer start;
+
+ /** The number. */
+ protected Integer number;
+
+ /** The error number. */
+ // error
+ protected int[] errorNumber;
+
+ /** The error list. */
+ protected HashMap<String, Integer>[] errorList;
+
+ /** The key list. */
+ protected String[] keyList;
+
+ /** The source number list. */
+ protected int[] sourceNumberList;
+
+ /** The segment registration. */
+ public transient String segmentRegistration;
+
+ /** The segment key value list. */
+ protected transient LinkedHashMap<String, HashMap<String, T1>> segmentKeyValueList;
+
+ /** The segment recompute key list. */
+ public transient LinkedHashMap<String, HashSet<String>> segmentRecomputeKeyList;
+
+ /** The segment keys. */
+ public transient HashSet<String> segmentKeys;
+
+ /** The segment values boundary. */
+ protected transient LinkedHashMap<String, T1> segmentValuesBoundary;
+
+ /** The segment value boundary. */
+ protected transient T1 segmentValueBoundary;
+
+ /** The segment value top list last. */
+ protected transient LinkedHashMap<String, T1> segmentValueTopListLast;
+
+ /** The segment value top list. */
+ protected transient ArrayList<T1> segmentValueTopList;
+
+ /** The segment name. */
+ protected transient String segmentName;
+
+ /** The segment number. */
+ protected transient int segmentNumber;
+
+ /** The has sub. */
+ private boolean hasSub;
+
+ /** The sub collector types. */
+ private String[] subCollectorTypes;
+
+ /** The sub data types. */
+ private String[] subDataTypes;
+
+ /** The sub stats types. */
+ private String[] subStatsTypes;
+
+ /** The sub stats items. */
+ private TreeSet<String>[] subStatsItems;
+
+ /** The sub sort types. */
+ private String[] subSortTypes;
+
+ /** The sub sort directions. */
+ private String[] subSortDirections;
+
+ /** The sub start. */
+ private Integer[] subStart;
+
+ /** The sub number. */
+ private Integer[] subNumber;
+
+ /** The sub collector list next level. */
+ protected MtasDataCollector<?, ?>[] subCollectorListNextLevel = null;
+
+ /** The sub collector next level. */
+ protected MtasDataCollector<?, ?> subCollectorNextLevel = null;
+
+ /** The new current position. */
+ protected transient int newSize, newPosition, newCurrentPosition;
+
+ /** The new current existing. */
+ protected transient boolean newCurrentExisting;
+
+ /** The new key list. */
+ protected transient String[] newKeyList = null;
+
+ /** The new source number list. */
+ protected transient int[] newSourceNumberList = null;
+
+ /** The new error number. */
+ protected transient int[] newErrorNumber;
+
+ /** The new error list. */
+ protected transient HashMap<String, Integer>[] newErrorList;
+
+ /** The new sub collector types. */
+ private transient String[] newSubCollectorTypes;
+
+ /** The new sub data types. */
+ private transient String[] newSubDataTypes;
+
+ /** The new sub stats types. */
+ private transient String[] newSubStatsTypes;
+
+ /** The new sub stats items. */
+ private transient TreeSet<String>[] newSubStatsItems;
+
+ /** The new sub sort types. */
+ private transient String[] newSubSortTypes;
+
+ /** The new sub sort directions. */
+ private transient String[] newSubSortDirections;
+
+ /** The new sub start. */
+ private transient Integer[] newSubStart;
+
+ /** The new sub number. */
+ private transient Integer[] newSubNumber;
+
+ /** The new sub collector list next level. */
+ // subcollectors next level for adding
+ protected transient MtasDataCollector<?, ?>[] newSubCollectorListNextLevel = null;
+
+ /** The new sub collector next level. */
+ protected transient MtasDataCollector<?, ?> newSubCollectorNextLevel = null;
+
+ /** The closed. */
+ protected transient boolean closed = false;
+
+ /** The result. */
+ private transient MtasDataCollectorResult<T1, T2> result = null;
+
+ /**
+ * Instantiates a new mtas data collector.
+ *
+ * @param collectorType
+ * the collector type
+ * @param dataType
+ * the data type
+ * @param statsType
+ * the stats type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ protected MtasDataCollector(String collectorType, String dataType,
+ String statsType, TreeSet<String> statsItems, String sortType,
+ String sortDirection, Integer start, Integer number,
+ String segmentRegistration, String boundary) throws IOException {
+ // set properties
+ this.closed = false;
+ this.collectorType = collectorType; // data or list
+ this.dataType = dataType; // long or double
+ this.statsType = statsType; // basic, advanced or full
+ this.statsItems = statsItems; // sum, n, all, ...
+ this.sortType = sortType;
+ this.sortDirection = sortDirection;
+ this.start = start;
+ this.number = number;
+ this.segmentRegistration = segmentRegistration;
+ if (segmentRegistration != null) {
+ segmentKeys = new HashSet<String>();
+ segmentKeyValueList = new LinkedHashMap<String, HashMap<String, T1>>();
+ segmentValuesBoundary = new LinkedHashMap<String, T1>();
+ segmentValueTopListLast = new LinkedHashMap<String, T1>();
+ if (segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ if (boundary != null) {
+ segmentValueBoundary = stringToBoundary(boundary);
+ } else {
+ throw new IOException("did expect boundary with segmentRegistration "
+ + segmentRegistration);
+ }
+ } else if (boundary != null) {
+ throw new IOException("didn't expect boundary with segmentRegistration "
+ + segmentRegistration);
+ }
+ }
+ // initialize administration
+ keyList = new String[0];
+ sourceNumberList = new int[0];
+ errorNumber = new int[0];
+ errorList = (HashMap<String, Integer>[]) new HashMap<?, ?>[0];
+ size = 0;
+ position = 0;
+ // subCollectors properties
+ hasSub = false;
+ subCollectorTypes = null;
+ subDataTypes = null;
+ subStatsTypes = null;
+ subStatsItems = null;
+ subSortTypes = null;
+ subSortDirections = null;
+ subStart = null;
+ subNumber = null;
+ subCollectorListNextLevel = null;
+ subCollectorNextLevel = null;
+ }
+
+ /**
+ * Instantiates a new mtas data collector.
+ *
+ * @param collectorType
+ * the collector type
+ * @param dataType
+ * the data type
+ * @param statsType
+ * the stats type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param subCollectorTypes
+ * the sub collector types
+ * @param subDataTypes
+ * the sub data types
+ * @param subStatsTypes
+ * the sub stats types
+ * @param subStatsItems
+ * the sub stats items
+ * @param subSortTypes
+ * the sub sort types
+ * @param subSortDirections
+ * the sub sort directions
+ * @param subStart
+ * the sub start
+ * @param subNumber
+ * the sub number
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected MtasDataCollector(String collectorType, String dataType,
+ String statsType, TreeSet<String> statsItems, String sortType,
+ String sortDirection, Integer start, Integer number,
+ String[] subCollectorTypes, String[] subDataTypes, String[] subStatsTypes,
+ TreeSet<String>[] subStatsItems, String subSortTypes[],
+ String[] subSortDirections, Integer[] subStart, Integer[] subNumber,
+ String segmentRegistration, String boundary) throws IOException {
+ // initialize
+ this(collectorType, dataType, statsType, statsItems, sortType,
+ sortDirection, start, number, segmentRegistration, boundary);
+ // initialize subCollectors
+ if (subCollectorTypes != null) {
+ hasSub = true;
+ this.subCollectorTypes = subCollectorTypes;
+ this.subDataTypes = subDataTypes;
+ this.subStatsTypes = subStatsTypes;
+ this.subStatsItems = subStatsItems;
+ this.subSortTypes = subSortTypes;
+ this.subSortDirections = subSortDirections;
+ this.subStart = subStart;
+ this.subNumber = subNumber;
+ if (subCollectorTypes.length > 1) {
+ newSubCollectorTypes = Arrays.copyOfRange(subCollectorTypes, 1,
+ subCollectorTypes.length);
+ newSubDataTypes = Arrays.copyOfRange(subDataTypes, 1,
+ subStatsTypes.length);
+ newSubStatsTypes = Arrays.copyOfRange(subStatsTypes, 1,
+ subStatsTypes.length);
+ newSubStatsItems = Arrays.copyOfRange(subStatsItems, 1,
+ subStatsItems.length);
+ newSubSortTypes = Arrays.copyOfRange(subSortTypes, 1,
+ subSortTypes.length);
+ newSubSortDirections = Arrays.copyOfRange(subSortDirections, 1,
+ subSortDirections.length);
+ newSubStart = Arrays.copyOfRange(subStart, 1, subStart.length);
+ newSubNumber = Arrays.copyOfRange(subNumber, 1, subNumber.length);
+ }
+ newSubCollectorListNextLevel = new MtasDataCollector[0];
+ }
+ }
+
+ /**
+ * Merge.
+ *
+ * @param newDataCollector
+ * the new data collector
+ * @param map
+ * the map
+ * @param increaseSourceNumber
+ * the increase source number
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ abstract public void merge(MtasDataCollector<?, ?> newDataCollector,
+ HashMap<MtasDataCollector<?, ?>, MtasDataCollector<?, ?>> map,
+ boolean increaseSourceNumber) throws IOException;
+
+ /**
+ * Inits the new list.
+ *
+ * @param maxNumberOfTerms
+ * the max number of terms
+ * @param segmentName
+ * the segment name
+ * @param segmentNumber
+ * the segment number
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void initNewList(int maxNumberOfTerms, String segmentName,
+ int segmentNumber, String boundary) throws IOException {
+ if (closed) {
+ result = null;
+ closed = false;
+ }
+ initNewListBasic(maxNumberOfTerms);
+ if (segmentRegistration != null) {
+ this.segmentName = segmentName;
+ this.segmentNumber = segmentNumber;
+ if (!segmentKeyValueList.containsKey(segmentName)) {
+ segmentKeyValueList.put(segmentName, new HashMap<String, T1>());
+ if (segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ if (boundary != null) {
+ segmentValuesBoundary.put(segmentName,
+ stringToBoundary(boundary, segmentNumber));
+ } else {
+ throw new IOException("expected boundary");
+ }
+ } else {
+ segmentValuesBoundary.put(segmentName, null);
+ }
+ segmentValueTopListLast.put(segmentName, null);
+ }
+ this.segmentValueTopList = new ArrayList<T1>();
+ }
+ }
+
+ /**
+ * Inits the new list.
+ *
+ * @param maxNumberOfTerms
+ * the max number of terms
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void initNewList(int maxNumberOfTerms) throws IOException {
+ if (closed) {
+ result = null;
+ closed = false;
+ }
+ if (segmentRegistration != null) {
+ throw new IOException("missing segment name");
+ } else {
+ initNewListBasic(maxNumberOfTerms);
+ }
+ }
+
+ /**
+ * Inits the new list basic.
+ *
+ * @param maxNumberOfTerms
+ * the max number of terms
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ private void initNewListBasic(int maxNumberOfTerms) throws IOException {
+ if (!closed) {
+ position = 0;
+ newPosition = 0;
+ newCurrentPosition = 0;
+ newSize = maxNumberOfTerms + size;
+ newKeyList = new String[newSize];
+ newSourceNumberList = new int[newSize];
+ newErrorNumber = new int[newSize];
+ newErrorList = (HashMap<String, Integer>[]) new HashMap<?, ?>[newSize];
+ if (hasSub) {
+ newSubCollectorListNextLevel = new MtasDataCollector[newSize];
+ }
+ } else {
+ throw new IOException("already closed");
+ }
+ }
+
+ /**
+ * Increase new list size.
+ *
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ protected void increaseNewListSize() throws IOException {
+ if (!closed) {
+ String[] tmpNewKeyList = newKeyList;
+ int[] tmpNewSourceNumberList = newSourceNumberList;
+ int[] tmpNewErrorNumber = newErrorNumber;
+ HashMap<String, Integer>[] tmpNewErrorList = newErrorList;
+ int tmpNewSize = newSize;
+ newSize = 2 * newSize;
+ newKeyList = new String[newSize];
+ newSourceNumberList = new int[newSize];
+ newErrorNumber = new int[newSize];
+ newErrorList = (HashMap<String, Integer>[]) new HashMap<?, ?>[newSize];
+ System.arraycopy(tmpNewKeyList, 0, newKeyList, 0, tmpNewSize);
+ System.arraycopy(tmpNewSourceNumberList, 0, newSourceNumberList, 0,
+ tmpNewSize);
+ System.arraycopy(tmpNewErrorNumber, 0, newErrorNumber, 0, tmpNewSize);
+ System.arraycopy(tmpNewErrorList, 0, newErrorList, 0, tmpNewSize);
+ if (hasSub) {
+ MtasDataCollector<?, ?>[] tmpNewSubCollectorListNextLevel = newSubCollectorListNextLevel;
+ newSubCollectorListNextLevel = new MtasDataCollector[newSize];
+ System.arraycopy(tmpNewSubCollectorListNextLevel, 0,
+ newSubCollectorListNextLevel, 0, tmpNewSize);
+ }
+ } else {
+ throw new IOException("already closed");
+ }
+ }
+
+ /**
+ * Adds the.
+ *
+ * @param increaseSourceNumber
+ * the increase source number
+ * @return the mtas data collector
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected final MtasDataCollector<?, ?> add(boolean increaseSourceNumber)
+ throws IOException {
+ if (!closed) {
+ if (!collectorType.equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ throw new IOException(
+ "collector should be " + DataCollector.COLLECTOR_TYPE_DATA);
+ } else {
+ if (newPosition > 0) {
+ newCurrentExisting = true;
+ } else if (position < getSize()) {
+ // copy
+ newKeyList[0] = keyList[0];
+ newSourceNumberList[0] = sourceNumberList[0];
+ if (increaseSourceNumber) {
+ newSourceNumberList[0]++;
+ }
+ newErrorNumber[0] = errorNumber[0];
+ newErrorList[0] = errorList[0];
+ if (hasSub) {
+ newSubCollectorNextLevel = subCollectorNextLevel;
+ }
+ copyToNew(0, 0);
+ newPosition = 1;
+ position = 1;
+ newCurrentExisting = true;
+ } else {
+ // add key
+ newKeyList[0] = DataCollector.COLLECTOR_TYPE_DATA;
+ newSourceNumberList[0] = 1;
+ newErrorNumber[0] = 0;
+ newErrorList[0] = new HashMap<String, Integer>();
+ newPosition = 1;
+ newCurrentPosition = newPosition - 1;
+ newCurrentExisting = false;
+ // ready, only handle sub
+ if (hasSub) {
+ newSubCollectorNextLevel = DataCollector.getCollector(
+ subCollectorTypes[0], subDataTypes[0], subStatsTypes[0],
+ subStatsItems[0], subSortTypes[0], subSortDirections[0],
+ subStart[0], subNumber[0], newSubCollectorTypes,
+ newSubDataTypes, newSubStatsTypes, newSubStatsItems,
+ newSubSortTypes, newSubSortDirections, newSubStart,
+ newSubNumber, segmentRegistration, null);
+ } else {
+ newSubCollectorNextLevel = null;
+ }
+ }
+ return newSubCollectorNextLevel;
+ }
+ } else {
+ throw new IOException("already closed");
+ }
+ }
+
+ /**
+ * Adds the.
+ *
+ * @param key
+ * the key
+ * @param increaseSourceNumber
+ * the increase source number
+ * @return the mtas data collector
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected final MtasDataCollector<?, ?> add(String key,
+ boolean increaseSourceNumber) throws IOException {
+ if (!closed) {
+ if (collectorType.equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ throw new IOException(
+ "collector should be " + DataCollector.COLLECTOR_TYPE_LIST);
+ } else if (key == null) {
+ throw new IOException("key shouldn't be null");
+ } else {
+ // check previous added
+ if ((newPosition > 0)
+ && newKeyList[(newPosition - 1)].compareTo(key) >= 0) {
+ int i = newPosition;
+ do {
+ i--;
+ if (newKeyList[i].equals(key)) {
+ newCurrentPosition = i;
+ newCurrentExisting = true;
+ if (subDataTypes != null) {
+ return newSubCollectorListNextLevel[newCurrentPosition];
+ } else {
+ return null;
+ }
+ }
+ } while ((i > 0) && (newKeyList[i].compareTo(key) > 0));
+ }
+ // move position in old list
+ if (position < getSize()) {
+ // just add smaller or equal items
+ while (keyList[position].compareTo(key) <= 0) {
+ if (newPosition == newSize) {
+ increaseNewListSize();
+ }
+ // copy
+ newKeyList[newPosition] = keyList[position];
+ newSourceNumberList[newPosition] = sourceNumberList[position];
+ newErrorNumber[newPosition] = errorNumber[position];
+ newErrorList[newPosition] = errorList[position];
+ if (hasSub) {
+ newSubCollectorListNextLevel[newPosition] = subCollectorListNextLevel[position];
+ }
+ copyToNew(position, newPosition);
+ newPosition++;
+ position++;
+ // check if added key from list is right key
+ if (newKeyList[(newPosition - 1)].equals(key)) {
+ if (increaseSourceNumber) {
+ newSourceNumberList[(newPosition - 1)]++;
+ }
+ newCurrentPosition = newPosition - 1;
+ newCurrentExisting = true;
+ // ready
+ if (hasSub) {
+ return newSubCollectorListNextLevel[newCurrentPosition];
+ } else {
+ return null;
+ }
+ // stop if position exceeds size
+ } else if (position == getSize()) {
+ break;
+ }
+ }
+ }
+ // check size
+ if (newPosition == newSize) {
+ increaseNewListSize();
+ }
+ // add key
+ newKeyList[newPosition] = key;
+ newSourceNumberList[newPosition] = 1;
+ newErrorNumber[newPosition] = 0;
+ newErrorList[newPosition] = new HashMap<String, Integer>();
+ newPosition++;
+ newCurrentPosition = newPosition - 1;
+ newCurrentExisting = false;
+ // ready, only handle sub
+ if (hasSub) {
+ newSubCollectorListNextLevel[newCurrentPosition] = DataCollector
+ .getCollector(subCollectorTypes[0], subDataTypes[0],
+ subStatsTypes[0], subStatsItems[0], subSortTypes[0],
+ subSortDirections[0], subStart[0], subNumber[0],
+ newSubCollectorTypes, newSubDataTypes, newSubStatsTypes,
+ newSubStatsItems, newSubSortTypes, newSubSortDirections,
+ newSubStart, newSubNumber, segmentRegistration, null);
+ return newSubCollectorListNextLevel[newCurrentPosition];
+ } else {
+ return null;
+ }
+ }
+ } else {
+ throw new IOException("already closed");
+ }
+ }
+
+ /**
+ * Copy to new.
+ *
+ * @param position
+ * the position
+ * @param newPosition
+ * the new position
+ */
+ protected abstract void copyToNew(int position, int newPosition);
+
+ /**
+ * Copy from new.
+ */
+ protected abstract void copyFromNew();
+
+ /**
+ * Compare with boundary.
+ *
+ * @param value
+ * the value
+ * @param boundary
+ * the boundary
+ * @return true, if successful
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected abstract boolean compareWithBoundary(T1 value, T1 boundary)
+ throws IOException;
+
+ /**
+ * Last for computing segment.
+ *
+ * @param value
+ * the value
+ * @param boundary
+ * the boundary
+ * @return the t1
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected abstract T1 lastForComputingSegment(T1 value, T1 boundary)
+ throws IOException;
+
+ /**
+ * Last for computing segment.
+ *
+ * @return the t1
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected abstract T1 lastForComputingSegment() throws IOException;
+
+ /**
+ * Boundary for segment.
+ *
+ * @param segmentName
+ * the segment name
+ * @return the t1
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected abstract T1 boundaryForSegment(String segmentName)
+ throws IOException;
+
+ /**
+ * Boundary for segment computing.
+ *
+ * @param segmentName
+ * the segment name
+ * @return the t1
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected abstract T1 boundaryForSegmentComputing(String segmentName)
+ throws IOException;
+
+ /**
+ * String to boundary.
+ *
+ * @param boundary
+ * the boundary
+ * @param segmentNumber
+ * the segment number
+ * @return the t1
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected abstract T1 stringToBoundary(String boundary, Integer segmentNumber)
+ throws IOException;
+
+ /**
+ * String to boundary.
+ *
+ * @param boundary
+ * the boundary
+ * @return the t1
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected T1 stringToBoundary(String boundary) throws IOException {
+ return stringToBoundary(boundary, null);
+ }
+
+ /**
+ * Close segment key value registration.
+ *
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void closeSegmentKeyValueRegistration() throws IOException {
+ if (!closed) {
+ if (segmentRegistration != null) {
+ HashMap<String, T1> keyValueList = segmentKeyValueList.get(segmentName);
+ T1 tmpSegmentValueBoundary = segmentValuesBoundary.get(segmentName);
+ for (String key : keyValueList.keySet()) {
+ if (tmpSegmentValueBoundary == null || compareWithBoundary(
+ keyValueList.get(key), tmpSegmentValueBoundary)) {
+ segmentKeys.add(key);
+ }
+ }
+ }
+ } else {
+ throw new IOException("already closed");
+ }
+ }
+
+ /**
+ * Recompute segment keys.
+ *
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void recomputeSegmentKeys() throws IOException {
+ if (!closed && segmentRegistration != null) {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ segmentKeys.clear();
+ // recompute boundaries
+ for (String segmentName : segmentKeyValueList.keySet()) {
+ T1 tmpSegmentValueBoundary = boundaryForSegment(segmentName);
+ segmentValuesBoundary.put(segmentName, tmpSegmentValueBoundary);
+ }
+ // compute adjusted boundaries and compute keys
+ for (String segmentName : segmentKeyValueList.keySet()) {
+ this.segmentName = segmentName;
+ HashMap<String, T1> keyValueList = segmentKeyValueList
+ .get(segmentName);
+ T1 tmpSegmentValueBoundaryForComputing = boundaryForSegmentComputing(
+ segmentName);
+ for (String key : keyValueList.keySet()) {
+ if (tmpSegmentValueBoundaryForComputing == null
+ || compareWithBoundary(keyValueList.get(key),
+ tmpSegmentValueBoundaryForComputing)) {
+ if (!segmentKeys.contains(key)) {
+ segmentKeys.add(key);
+ }
+ }
+ }
+ }
+ }
+
+ HashMap<String, T1> keyValueList;
+ HashSet<String> recomputeKeyList;
+ segmentRecomputeKeyList = new LinkedHashMap<String, HashSet<String>>();
+ for (String key : segmentKeys) {
+ for (String segmentName : segmentKeyValueList.keySet()) {
+ keyValueList = segmentKeyValueList.get(segmentName);
+ if (!keyValueList.containsKey(key)) {
+ if (!segmentRecomputeKeyList.containsKey(segmentName)) {
+ recomputeKeyList = new HashSet<String>();
+ segmentRecomputeKeyList.put(segmentName, recomputeKeyList);
+ } else {
+ recomputeKeyList = segmentRecomputeKeyList.get(segmentName);
+ }
+ recomputeKeyList.add(key);
+ } else {
+ break;
+ }
+ }
+ }
+ this.segmentName = null;
+ } else {
+ throw new IOException(
+ "not for segmentRegistration " + segmentRegistration);
+ }
+ } else {
+ throw new IOException("already closed or no segmentRegistration ("
+ + segmentRegistration + ")");
+ }
+ }
+
+ /**
+ * Reduce to keys.
+ *
+ * @param keys
+ * the keys
+ */
+ public abstract void reduceToKeys(Set<String> keys);
+
+ /**
+ * Reduce to segment keys.
+ */
+ public final void reduceToSegmentKeys() {
+ if (segmentRegistration != null) {
+ reduceToKeys(segmentKeys);
+ }
+ }
+
+ /**
+ * Check existence necessary keys.
+ *
+ * @return true, if successful
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public boolean checkExistenceNecessaryKeys() throws IOException {
+ if (!closed) {
+ if (segmentRegistration != null) {
+ return segmentRecomputeKeyList.size() == 0;
+ } else {
+ return true;
+ }
+ } else {
+ throw new IOException("already closed");
+ }
+ }
+
+ /**
+ * Validate segment boundary.
+ *
+ * @param o
+ * the o
+ * @return true, if successful
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ abstract public boolean validateSegmentBoundary(Object o) throws IOException;
+
+ /**
+ * Validate with segment boundary.
+ *
+ * @param value
+ * the value
+ * @return true, if successful
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected boolean validateWithSegmentBoundary(T1 value) throws IOException {
+ if (!closed) {
+ if (segmentRegistration != null) {
+ T1 tmpSegmentValueBoundary = segmentValuesBoundary.get(segmentName);
+ if (tmpSegmentValueBoundary == null) {
+ return true;
+ } else if (compareWithBoundary(value, tmpSegmentValueBoundary)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Validate segment value.
+ *
+ * @param key
+ * the key
+ * @param value
+ * the value
+ * @param maximumNumber
+ * the maximum number
+ * @param segmentNumber
+ * the segment number
+ * @return the string
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public String validateSegmentValue(String key, T1 value, int maximumNumber,
+ int segmentNumber) throws IOException {
+ if (!closed) {
+ if (segmentRegistration != null) {
+ if (maximumNumber > 0) {
+ segmentKeyValueList.get(segmentName).put(key, value);
+ T1 tmpSegmentValueMaxListMin = segmentValueTopListLast
+ .get(segmentName);
+ T1 tmpSegmentValueBoundary = segmentValuesBoundary.get(segmentName);
+ if (segmentValueTopList.size() < maximumNumber) {
+ segmentValueTopList.add(value);
+ segmentValueTopListLast.put(segmentName,
+ (tmpSegmentValueMaxListMin == null) ? value
+ : lastForComputingSegment(tmpSegmentValueMaxListMin,
+ value));
+ if (segmentValueTopList.size() == maximumNumber) {
+ tmpSegmentValueMaxListMin = segmentValueTopListLast
+ .get(segmentName);
+ segmentValueTopListLast.put(segmentName,
+ tmpSegmentValueMaxListMin);
+ segmentValuesBoundary.put(segmentName,
+ boundaryForSegmentComputing(segmentName));
+ }
+ return segmentKeys.contains(key) ? SEGMENT_KEY : SEGMENT_NEW;
+ } else if (compareWithBoundary(value, tmpSegmentValueBoundary)) {
+ if (compareWithBoundary(value, tmpSegmentValueMaxListMin)) {
+ segmentValueTopList.add(value);
+ segmentValueTopList.remove(tmpSegmentValueMaxListMin);
+ tmpSegmentValueMaxListMin = lastForComputingSegment();
+ segmentValueTopListLast.put(segmentName,
+ tmpSegmentValueMaxListMin);
+ segmentValuesBoundary.put(segmentName,
+ boundaryForSegmentComputing(segmentName));
+ }
+ return segmentKeys.contains(key) ? SEGMENT_KEY : SEGMENT_NEW;
+ } else if (segmentKeys.contains(key)) {
+ return SEGMENT_KEY;
+ } else {
+ return null;
+ }
+ } else {
+ return null;
+ }
+ } else {
+ return null;
+ }
+ } else {
+ throw new IOException("already closed");
+ }
+ }
+
+ /**
+ * Sets the error.
+ *
+ * @param newPosition
+ * the new position
+ * @param errorNumberItem
+ * the error number item
+ * @param errorListItem
+ * the error list item
+ * @param currentExisting
+ * the current existing
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected final void setError(int newPosition, int errorNumberItem,
+ HashMap<String, Integer> errorListItem, boolean currentExisting)
+ throws IOException {
+ if (!closed) {
+ if (currentExisting) {
+ newErrorNumber[newPosition] += errorNumberItem;
+ HashMap<String, Integer> item = newErrorList[newPosition];
+ for (Entry<String, Integer> entry : errorListItem.entrySet()) {
+ if (item.containsKey(entry.getKey())) {
+ item.put(entry.getKey(),
+ item.get(entry.getKey()) + entry.getValue());
+ } else {
+ item.put(entry.getKey(), entry.getValue());
+ }
+ }
+ } else {
+ newErrorNumber[newPosition] = errorNumberItem;
+ newErrorList[newPosition] = errorListItem;
+ }
+ } else {
+ throw new IOException("already closed");
+ }
+ }
+
+ /**
+ * Sorted and unique.
+ *
+ * @param keyList
+ * the key list
+ * @param size
+ * the size
+ * @return true, if successful
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ private boolean sortedAndUnique(String[] keyList, int size)
+ throws IOException {
+ if (!closed) {
+ for (int i = 1; i < size; i++) {
+ if (keyList[(i - 1)].compareTo(keyList[i]) >= 0) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ throw new IOException("already closed");
+ }
+ }
+
+ /**
+ * Compute sort and unique mapping.
+ *
+ * @param keyList
+ * the key list
+ * @param size
+ * the size
+ * @return the int[][]
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ private int[][] computeSortAndUniqueMapping(String[] keyList, int size)
+ throws IOException {
+ if (!closed) {
+ if (size > 0) {
+ SortedMap<String, int[]> sortedMap = new TreeMap<String, int[]>();
+ for (int i = 0; i < size; i++) {
+ if (sortedMap.containsKey(keyList[i])) {
+ int[] previousList = sortedMap.get(keyList[i]);
+ int[] newList = new int[previousList.length + 1];
+ System.arraycopy(previousList, 0, newList, 0, previousList.length);
+ newList[previousList.length] = i;
+ sortedMap.put(keyList[i], newList);
+ } else {
+ sortedMap.put(keyList[i], new int[] { i });
+ }
+ }
+ Collection<int[]> values = sortedMap.values();
+ int[][] result = new int[sortedMap.size()][];
+ return values.toArray(result);
+ } else {
+ return null;
+ }
+ } else {
+ throw new IOException("already closed");
+ }
+ }
+
+ /**
+ * Remap data.
+ *
+ * @param mapping
+ * the mapping
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected void remapData(int[][] mapping) throws IOException {
+ if (!closed) {
+ // remap and merge keys
+ String[] newKeyList = new String[mapping.length];
+ // TODO: process mapping for functions
+ HashMap<MtasDataCollector<?, ?>, MtasDataCollector<?, ?>> map = new HashMap<MtasDataCollector<?, ?>, MtasDataCollector<?, ?>>();
+ int[] newSourceNumberList = new int[mapping.length];
+ int[] newErrorNumber = new int[mapping.length];
+ @SuppressWarnings("unchecked")
+ HashMap<String, Integer>[] newErrorList = (HashMap<String, Integer>[]) new HashMap<?, ?>[mapping.length];
+ for (int i = 0; i < mapping.length; i++) {
+ newKeyList[i] = keyList[mapping[i][0]];
+ newSourceNumberList[i] = sourceNumberList[mapping[i][0]];
+ for (int j = 0; j < mapping[i].length; j++) {
+ if (j == 0) {
+ newErrorNumber[i] = errorNumber[mapping[i][j]];
+ newErrorList[i] = errorList[mapping[i][j]];
+ } else {
+ newErrorNumber[i] += errorNumber[mapping[i][j]];
+ for (Entry<String, Integer> entry : errorList[mapping[i][j]]
+ .entrySet()) {
+ if (newErrorList[i].containsKey(entry.getKey())) {
+ newErrorList[i].put(entry.getKey(),
+ newErrorList[i].get(entry.getKey()) + entry.getValue());
+ } else {
+ newErrorList[i].put(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+ }
+ }
+ if (hasSub) {
+ newSubCollectorListNextLevel = new MtasDataCollector<?, ?>[mapping.length];
+ for (int i = 0; i < mapping.length; i++) {
+ for (int j = 0; j < mapping[i].length; j++) {
+ if (j == 0 || newSubCollectorListNextLevel[i] == null) {
+ newSubCollectorListNextLevel[i] = subCollectorListNextLevel[mapping[i][j]];
+ } else {
+ newSubCollectorListNextLevel[i]
+ .merge(subCollectorListNextLevel[mapping[i][j]], map, false);
+ }
+ }
+ }
+ subCollectorListNextLevel = newSubCollectorListNextLevel;
+ }
+ keyList = newKeyList;
+ sourceNumberList = newSourceNumberList;
+ errorNumber = newErrorNumber;
+ errorList = newErrorList;
+ size = keyList.length;
+ position = 0;
+ } else {
+ throw new IOException("already closed");
+ }
+ }
+
+ /**
+ * Close new list.
+ *
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void closeNewList() throws IOException {
+ if (!closed) {
+ if (segmentRegistration != null) {
+ this.segmentName = null;
+ }
+ if (newSize > 0) {
+ // add remaining old
+ while (position < getSize()) {
+ if (newPosition == newSize) {
+ increaseNewListSize();
+ }
+ newKeyList[newPosition] = keyList[position];
+ newSourceNumberList[newPosition] = sourceNumberList[position];
+ newErrorNumber[newPosition] = errorNumber[position];
+ newErrorList[newPosition] = errorList[position];
+ if (hasSub) {
+ newSubCollectorListNextLevel[newPosition] = subCollectorListNextLevel[position];
+ }
+ copyToNew(position, newPosition);
+ position++;
+ newPosition++;
+ }
+ // copy
+ keyList = newKeyList;
+ sourceNumberList = newSourceNumberList;
+ errorNumber = newErrorNumber;
+ errorList = newErrorList;
+ subCollectorListNextLevel = newSubCollectorListNextLevel;
+ copyFromNew();
+ size = newPosition;
+ // sort and merge
+ if (!sortedAndUnique(keyList, getSize())) {
+ remapData(computeSortAndUniqueMapping(keyList, getSize()));
+ }
+ }
+ position = 0;
+ newSize = 0;
+ newPosition = 0;
+ newCurrentPosition = 0;
+ }
+ }
+
+ /**
+ * Gets the item.
+ *
+ * @param i
+ * the i
+ * @return the item
+ */
+ abstract protected MtasDataItem<T1, T2> getItem(int i);
+
+ /**
+ * Checks for sub.
+ *
+ * @return true, if successful
+ */
+ protected boolean hasSub() {
+ return hasSub;
+ }
+
+ /**
+ * Error.
+ *
+ * @param error
+ * the error
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public abstract void error(String error) throws IOException;
+
+ /**
+ * Error.
+ *
+ * @param keys
+ * the keys
+ * @param error
+ * the error
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public abstract void error(String keys[], String error) throws IOException;
+
+ /**
+ * Adds the.
+ *
+ * @param valueSum
+ * the value sum
+ * @param valueN
+ * the value n
+ * @return the mtas data collector
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public abstract MtasDataCollector<?, ?> add(long valueSum, long valueN)
+ throws IOException;
+
+ /**
+ * Adds the.
+ *
+ * @param values
+ * the values
+ * @param number
+ * the number
+ * @return the mtas data collector
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public abstract MtasDataCollector<?, ?> add(long[] values, int number)
+ throws IOException;
+
+ /**
+ * Adds the.
+ *
+ * @param valueSum
+ * the value sum
+ * @param valueN
+ * the value n
+ * @return the mtas data collector
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public abstract MtasDataCollector<?, ?> add(double valueSum, long valueN)
+ throws IOException;
+
+ /**
+ * Adds the.
+ *
+ * @param values
+ * the values
+ * @param number
+ * the number
+ * @return the mtas data collector
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public abstract MtasDataCollector<?, ?> add(double[] values, int number)
+ throws IOException;
+
+ /**
+ * Adds the.
+ *
+ * @param keys
+ * the keys
+ * @param valueSum
+ * the value sum
+ * @param valueN
+ * the value n
+ * @return the mtas data collector[]
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public abstract MtasDataCollector<?, ?>[] add(String[] keys, long valueSum,
+ long valueN) throws IOException;
+
+ /**
+ * Adds the.
+ *
+ * @param keys
+ * the keys
+ * @param values
+ * the values
+ * @param number
+ * the number
+ * @return the mtas data collector[]
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public abstract MtasDataCollector<?, ?>[] add(String[] keys, long[] values,
+ int number) throws IOException;
+
+ /**
+ * Adds the.
+ *
+ * @param keys
+ * the keys
+ * @param valueSum
+ * the value sum
+ * @param valueN
+ * the value n
+ * @return the mtas data collector[]
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public abstract MtasDataCollector<?, ?>[] add(String[] keys, double valueSum,
+ long valueN) throws IOException;
+
+ /**
+ * Adds the.
+ *
+ * @param keys
+ * the keys
+ * @param values
+ * the values
+ * @param number
+ * the number
+ * @return the mtas data collector[]
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public abstract MtasDataCollector<?, ?>[] add(String[] keys, double[] values,
+ int number) throws IOException;
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ return this.getClass().getSimpleName() + "-" + this.hashCode() + ": "
+ + collectorType + " - " + statsType + " " + statsItems + " " + hasSub;
+ }
+
+ /**
+ * Gets the result.
+ *
+ * @return the result
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public MtasDataCollectorResult<T1, T2> getResult() throws IOException {
+ if (!closed) {
+ close();
+ }
+ return result;
+ }
+
+ /**
+ * Gets the key list.
+ *
+ * @return the key list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public Set<String> getKeyList() throws IOException {
+ if (!closed) {
+ close();
+ }
+ return new HashSet<String>(Arrays.asList(keyList));
+ }
+
+ /**
+ * Close.
+ *
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public void close() throws IOException {
+ if (!closed) {
+ closeNewList();
+ if (collectorType.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ // compute initial basic list
+ TreeMap<String, MtasDataItem<T1, T2>> basicList = new TreeMap<String, MtasDataItem<T1, T2>>();
+ for (int i = 0; i < getSize(); i++) {
+ MtasDataItem<T1, T2> newItem = getItem(i);
+ if (basicList.containsKey(keyList[i])) {
+ newItem.add(basicList.get(keyList[i]));
+ }
+ basicList.put(keyList[i], newItem);
+ }
+ // create result based on basic list
+ result = new MtasDataCollectorResult<T1, T2>(collectorType, sortType,
+ sortDirection, basicList, start, number);
+ // reduce
+ if (segmentRegistration != null) {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ reduceToKeys(result.getComparatorList().keySet());
+ } else if (segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ LinkedHashMap<String, NumberComparator> comparatorList = result
+ .getComparatorList();
+ HashSet<String> filteredKeySet = new HashSet<String>();
+ if (segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ for (String key : comparatorList.keySet()) {
+ if (comparatorList.get(key)
+ .compareTo(segmentValueBoundary) < 0) {
+ filteredKeySet.add(key);
+ }
+ }
+ } else if (segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ for (String key : comparatorList.keySet()) {
+ if (comparatorList.get(key)
+ .compareTo(segmentValueBoundary) > 0) {
+ filteredKeySet.add(key);
+ }
+ }
+ }
+ reduceToKeys(filteredKeySet);
+ basicList.keySet().retainAll(filteredKeySet);
+ result = new MtasDataCollectorResult<T1, T2>(collectorType,
+ sortType, sortDirection, basicList, start, number);
+ }
+ }
+ } else if (collectorType.equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ if (getSize() > 0) {
+ result = new MtasDataCollectorResult<T1, T2>(collectorType,
+ getItem(0));
+ } else {
+ result = new MtasDataCollectorResult<T1, T2>(collectorType, sortType,
+ sortDirection);
+ }
+ } else {
+ throw new IOException("type " + collectorType + " not supported");
+ }
+ closed = true;
+ }
+
+ }
+
+ /**
+ * Gets the collector type.
+ *
+ * @return the collector type
+ */
+ public String getCollectorType() {
+ return collectorType;
+ }
+
+ /**
+ * Gets the stats type.
+ *
+ * @return the stats type
+ */
+ public String getStatsType() {
+ return statsType;
+ }
+
+ /**
+ * Gets the data type.
+ *
+ * @return the data type
+ */
+ public String getDataType() {
+ return dataType;
+ }
+
+ /**
+ * Gets the size.
+ *
+ * @return the size
+ */
+ public int getSize() {
+ return size;
+ }
+
+}
\ No newline at end of file
diff --git a/src/mtas/codec/util/collector/MtasDataCollectorResult.java b/src/mtas/codec/util/collector/MtasDataCollectorResult.java
new file mode 100644
index 0000000..3e9c89a
--- /dev/null
+++ b/src/mtas/codec/util/collector/MtasDataCollectorResult.java
@@ -0,0 +1,245 @@
+package mtas.codec.util.collector;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import mtas.codec.util.CodecUtil;
+import mtas.codec.util.DataCollector;
+import mtas.codec.util.collector.MtasDataItem.NumberComparator;
+
+/**
+ * The Class MtasDataCollectorResult.
+ *
+ * @param <T1>
+ * the generic type
+ * @param <T2>
+ * the generic type
+ */
+public class MtasDataCollectorResult<T1 extends Number & Comparable<T1>, T2 extends Number & Comparable<T2>>
+ implements Serializable {
+
+ /** The Constant serialVersionUID. */
+ private static final long serialVersionUID = 1L;
+
+ /** The list. */
+ SortedMap<String, MtasDataItem<T1, T2>> list;
+
+ /** The item. */
+ MtasDataItem<T1, T2> item;
+
+ /** The collector type. */
+ String sortType, sortDirection, collectorType;
+
+ /** The last sort value. */
+ @SuppressWarnings("rawtypes")
+ NumberComparator lastSortValue;
+
+ /** The end key. */
+ String startKey, endKey;
+
+ /**
+ * Instantiates a new mtas data collector result.
+ *
+ * @param collectorType
+ * the collector type
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param basicList
+ * the basic list
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public MtasDataCollectorResult(String collectorType, String sortType,
+ String sortDirection, TreeMap<String, MtasDataItem<T1, T2>> basicList,
+ Integer start, Integer number) throws IOException {
+ this(collectorType, sortType, sortDirection);
+ if (sortType == null || sortType.equals(CodecUtil.SORT_TERM)) {
+ if (sortDirection == null || sortDirection.equals(CodecUtil.SORT_ASC)) {
+ list = basicList;
+ } else if (sortDirection.equals(CodecUtil.SORT_DESC)) {
+ list = basicList.descendingMap();
+ } else {
+ throw new IOException("unknown sort direction " + sortDirection);
+ }
+ } else if (CodecUtil.STATS_TYPES.contains(sortType)) {
+ // comperator
+ Comparator<String> valueComparator = new Comparator<String>() {
+ @Override
+ public int compare(String k1, String k2) {
+ int compare = basicList.get(k1).compareTo(basicList.get(k2));
+ return compare == 0 ? k1.compareTo(k2) : compare;
+ }
+ };
+ SortedMap<String, MtasDataItem<T1, T2>> sortedByValues = new TreeMap<String, MtasDataItem<T1, T2>>(
+ valueComparator);
+ sortedByValues.putAll(basicList);
+ list = sortedByValues;
+ } else {
+ throw new IOException("unknown sort type " + sortType);
+ }
+ int listStart = start == null ? 0 : start;
+ if (number == null || (start == 0 && number >= list.size())) {
+ // do nothing, full list is ok
+ } else if (listStart < list.size() && number > 0) {
+ // subset
+ String boundaryEndKey = null;
+ int counter = 0;
+ MtasDataItem<T1, T2> previous = null;
+ for (String key : list.keySet()) {
+ if (listStart == counter) {
+ startKey = key;
+ } else if (listStart + number <= counter) {
+ if (sortType.equals(CodecUtil.SORT_TERM)) {
+ endKey = key;
+ boundaryEndKey = key;
+ break;
+ } else if (previous != null) {
+ if (previous.compareTo(list.get(key)) != 0) {
+ break;
+ } else {
+ boundaryEndKey = key;
+ }
+ } else {
+ endKey = key;
+ boundaryEndKey = key;
+ previous = list.get(key);
+ }
+ } else {
+ endKey = key;
+ }
+ counter++;
+ }
+ list = list.subMap(startKey, boundaryEndKey);
+ } else {
+ list = new TreeMap<String, MtasDataItem<T1, T2>>();
+ }
+ if (list.size() > 0 && sortType != null) {
+ lastSortValue = list.get(list.lastKey()).getComparableValue();
+ }
+ }
+
+ /**
+ * Instantiates a new mtas data collector result.
+ *
+ * @param collectorType
+ * the collector type
+ * @param item
+ * the item
+ */
+ public MtasDataCollectorResult(String collectorType,
+ MtasDataItem<T1, T2> item) {
+ this(collectorType, null, null);
+ this.item = item;
+ }
+
+ /**
+ * Instantiates a new mtas data collector result.
+ *
+ * @param collectorType
+ * the collector type
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ */
+ public MtasDataCollectorResult(String collectorType, String sortType,
+ String sortDirection) {
+ list = null;
+ item = null;
+ lastSortValue = null;
+ this.collectorType = collectorType;
+ this.sortType = sortType;
+ this.sortDirection = sortDirection;
+ }
+
+ /**
+ * Gets the list.
+ *
+ * @return the list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public final SortedMap<String, MtasDataItem<T1, T2>> getList()
+ throws IOException {
+ return getList(true);
+ }
+
+ /**
+ * Gets the list.
+ *
+ * @param reduce
+ * the reduce
+ * @return the list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public final SortedMap<String, MtasDataItem<T1, T2>> getList(boolean reduce)
+ throws IOException {
+ if (collectorType.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ if (reduce && startKey != null && endKey != null) {
+ return list.subMap(startKey, endKey);
+ } else {
+ return list;
+ }
+ } else {
+ throw new IOException("type " + collectorType + " not supported");
+ }
+ }
+
+ /**
+ * Gets the comparator list.
+ *
+ * @return the comparator list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("rawtypes")
+ public final LinkedHashMap<String, NumberComparator> getComparatorList()
+ throws IOException {
+ if (collectorType.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ LinkedHashMap<String, NumberComparator> comparatorList = new LinkedHashMap<String, NumberComparator>();
+ for (String key : list.keySet()) {
+ comparatorList.put(key, list.get(key).getComparableValue());
+ }
+ return comparatorList;
+ } else {
+ throw new IOException("type " + collectorType + " not supported");
+ }
+ }
+
+ /**
+ * Gets the last sort value.
+ *
+ * @return the last sort value
+ */
+ @SuppressWarnings("rawtypes")
+ public final NumberComparator getLastSortValue() {
+ return lastSortValue;
+ }
+
+ /**
+ * Gets the data.
+ *
+ * @return the data
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public final MtasDataItem<T1, T2> getData() throws IOException {
+ if (collectorType.equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ return item;
+ } else {
+ throw new IOException("type " + collectorType + " not supported");
+ }
+ }
+
+}
diff --git a/src/mtas/codec/util/collector/MtasDataDoubleAdvanced.java b/src/mtas/codec/util/collector/MtasDataDoubleAdvanced.java
index 3eef26b..269120d 100644
--- a/src/mtas/codec/util/collector/MtasDataDoubleAdvanced.java
+++ b/src/mtas/codec/util/collector/MtasDataDoubleAdvanced.java
@@ -5,13 +5,11 @@ import java.util.Collections;
import java.util.TreeSet;
import org.apache.commons.lang.ArrayUtils;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataDoubleAdvanced.
*/
-public class MtasDataDoubleAdvanced
- extends MtasDataAdvanced<Double, Double, MtasDataItemDoubleAdvanced> {
+public class MtasDataDoubleAdvanced extends MtasDataAdvanced<Double, Double> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -19,22 +17,40 @@ public class MtasDataDoubleAdvanced
/**
* Instantiates a new mtas data double advanced.
*
- * @param collectorType the collector type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param subCollectorTypes the sub collector types
- * @param subDataTypes the sub data types
- * @param subStatsTypes the sub stats types
- * @param subStatsItems the sub stats items
- * @param subSortTypes the sub sort types
- * @param subSortDirections the sub sort directions
- * @param subStart the sub start
- * @param subNumber the sub number
- * @param segmentRegistration the segment registration
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param collectorType
+ * the collector type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param subCollectorTypes
+ * the sub collector types
+ * @param subDataTypes
+ * the sub data types
+ * @param subStatsTypes
+ * the sub stats types
+ * @param subStatsItems
+ * the sub stats items
+ * @param subSortTypes
+ * the sub sort types
+ * @param subSortDirections
+ * the sub sort directions
+ * @param subStart
+ * the sub start
+ * @param subNumber
+ * the sub number
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasDataDoubleAdvanced(String collectorType,
TreeSet<String> statsItems, String sortType, String sortDirection,
@@ -42,12 +58,12 @@ public class MtasDataDoubleAdvanced
String[] subDataTypes, String[] subStatsTypes,
TreeSet<String>[] subStatsItems, String[] subSortTypes,
String[] subSortDirections, Integer[] subStart, Integer[] subNumber,
- boolean segmentRegistration) throws IOException {
+ String segmentRegistration, String boundary) throws IOException {
super(collectorType, CodecUtil.DATA_TYPE_DOUBLE, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
- subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, new MtasDataDoubleOperations(),
- segmentRegistration);
+ subStatsTypes, subStatsItems, subSortTypes, subSortDirections, subStart,
+ subNumber, new MtasDataDoubleOperations(), segmentRegistration,
+ boundary);
}
/*
@@ -57,11 +73,16 @@ public class MtasDataDoubleAdvanced
*/
@Override
protected final MtasDataItemDoubleAdvanced getItem(int i) {
- return new MtasDataItemDoubleAdvanced(advancedValueSumList[i],
- advancedValueSumOfLogsList[i], advancedValueSumOfSquaresList[i],
- advancedValueMinList[i], advancedValueMaxList[i],
- advancedValueNList[i], hasSub() ? subCollectorListNextLevel[i] : null,
- statsItems, sortType, sortDirection, errorNumber[i], errorList[i]);
+ if (i >= 0 && i < size) {
+ return new MtasDataItemDoubleAdvanced(advancedValueSumList[i],
+ advancedValueSumOfLogsList[i], advancedValueSumOfSquaresList[i],
+ advancedValueMinList[i], advancedValueMaxList[i],
+ advancedValueNList[i], hasSub() ? subCollectorListNextLevel[i] : null,
+ statsItems, sortType, sortDirection, errorNumber[i], errorList[i],
+ sourceNumberList[i]);
+ } else {
+ return null;
+ }
}
/*
@@ -83,7 +104,7 @@ public class MtasDataDoubleAdvanced
@Override
public MtasDataCollector<?, ?> add(long[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
Double[] newValues = new Double[number];
for (int i = 0; i < values.length; i++)
newValues[i] = Long.valueOf(values[i]).doubleValue();
@@ -110,7 +131,7 @@ public class MtasDataDoubleAdvanced
@Override
public MtasDataCollector<?, ?> add(double[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
return dataCollector;
@@ -137,15 +158,15 @@ public class MtasDataDoubleAdvanced
* long[], int)
*/
@Override
- public MtasDataCollector<?, ?>[] add(String[] keys, long[] values,
- int number) throws IOException {
+ public MtasDataCollector<?, ?>[] add(String[] keys, long[] values, int number)
+ throws IOException {
if (keys != null && keys.length > 0) {
Double[] newValues = new Double[number];
for (int i = 0; i < values.length; i++)
newValues[i] = Long.valueOf(values[i]).doubleValue();
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, newValues, number, newCurrentExisting);
}
return subCollectors;
@@ -180,7 +201,7 @@ public class MtasDataDoubleAdvanced
if (keys != null && keys.length > 0) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
}
@@ -190,56 +211,179 @@ public class MtasDataDoubleAdvanced
}
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected boolean compareForComputingSegment(Double value,
- Double boundary) {
- return value >= boundary;
+ protected boolean compareWithBoundary(Double value, Double boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return value <= boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return value >= boundary;
+ } else {
+ throw new IOException(
+ "can't compare for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected Double minimumForComputingSegment(Double value, Double boundary) {
- return Math.min(value, boundary);
+ protected Double lastForComputingSegment(Double value, Double boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return Math.max(value, boundary);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return Math.min(value, boundary);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * )
*/
@Override
- protected Double minimumForComputingSegment() {
- return Collections.min(segmentValueMaxList);
+ protected Double lastForComputingSegment() throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return Collections.max(segmentValueTopList);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return Collections.min(segmentValueTopList);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment
+ * ()
*/
@Override
- protected Double boundaryForComputingSegment() {
- Double boundary = boundaryForSegment();
- double correctionBoundary = 0;
- for (String otherSegmentName : segmentValueMaxListMin.keySet()) {
- if (!otherSegmentName.equals(segmentName)) {
- Double otherBoundary = segmentValueBoundary.get(otherSegmentName);
- if (otherBoundary != null) {
- correctionBoundary += Math.max(0, otherBoundary - boundary);
+ protected Double boundaryForSegmentComputing(String segmentName)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Double boundary = boundaryForSegment(segmentName);
+ if (boundary == null) {
+ return null;
+ } else {
+ if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ long correctionBoundary = 0;
+ for (String otherSegmentName : segmentValueTopListLast.keySet()) {
+ if (!otherSegmentName.equals(segmentName)) {
+ Double otherBoundary = segmentValuesBoundary
+ .get(otherSegmentName);
+ if (otherBoundary != null) {
+ correctionBoundary += Math.max(0, otherBoundary - boundary);
+ }
+ }
+ }
+ return boundary + correctionBoundary;
+ } else {
+ return boundary;
}
}
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
}
- return boundary + correctionBoundary;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForSegment()
*/
@Override
- protected Double boundaryForSegment() {
- return segmentValueMaxListMin.get(segmentName) / segmentNumber;
+ protected Double boundaryForSegment(String segmentName) throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Double thisLast = segmentValueTopListLast.get(segmentName);
+ if (thisLast == null) {
+ return null;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_ASC)) {
+ Double boundary = thisLast * segmentNumber;
+ return boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Double boundary = thisLast / segmentNumber;
+ return boundary;
+ } else {
+ // should not happen
+ return null;
+ }
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
+ }
}
-}
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#stringToBoundary(java.lang.
+ * String, java.lang.Integer)
+ */
+ @Override
+ protected Double stringToBoundary(String boundary, Integer segmentNumber)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ if (segmentNumber == null) {
+ return Double.valueOf(boundary);
+ } else {
+ return Double.valueOf(boundary) / segmentNumber;
+ }
+ } else if (segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ if (segmentNumber == null) {
+ return Double.valueOf(boundary);
+ } else {
+ return Double.valueOf(boundary) / segmentNumber;
+ }
+ } else {
+ throw new IOException(
+ "not available for segmentRegistration " + segmentRegistration);
+ }
+ }
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#validateSegmentBoundary(java.
+ * lang.Object)
+ */
+ @Override
+ public boolean validateSegmentBoundary(Object o) throws IOException {
+ if (o instanceof Double) {
+ return validateWithSegmentBoundary((Double) o);
+ } else {
+ throw new IOException("incorrect type");
+ }
+ }
+
+}
diff --git a/src/mtas/codec/util/collector/MtasDataDoubleBasic.java b/src/mtas/codec/util/collector/MtasDataDoubleBasic.java
index fe604cd..3466d36 100644
--- a/src/mtas/codec/util/collector/MtasDataDoubleBasic.java
+++ b/src/mtas/codec/util/collector/MtasDataDoubleBasic.java
@@ -5,13 +5,11 @@ import java.util.Collections;
import java.util.TreeSet;
import org.apache.commons.lang.ArrayUtils;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataDoubleBasic.
*/
-public class MtasDataDoubleBasic
- extends MtasDataBasic<Double, Double, MtasDataItemDoubleBasic> {
+public class MtasDataDoubleBasic extends MtasDataBasic<Double, Double> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -19,34 +17,52 @@ public class MtasDataDoubleBasic
/**
* Instantiates a new mtas data double basic.
*
- * @param collectorType the collector type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param subCollectorTypes the sub collector types
- * @param subDataTypes the sub data types
- * @param subStatsTypes the sub stats types
- * @param subStatsItems the sub stats items
- * @param subSortTypes the sub sort types
- * @param subSortDirections the sub sort directions
- * @param subStart the sub start
- * @param subNumber the sub number
- * @param segmentRegistration the segment registration
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param collectorType
+ * the collector type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param subCollectorTypes
+ * the sub collector types
+ * @param subDataTypes
+ * the sub data types
+ * @param subStatsTypes
+ * the sub stats types
+ * @param subStatsItems
+ * the sub stats items
+ * @param subSortTypes
+ * the sub sort types
+ * @param subSortDirections
+ * the sub sort directions
+ * @param subStart
+ * the sub start
+ * @param subNumber
+ * the sub number
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasDataDoubleBasic(String collectorType, TreeSet<String> statsItems,
String sortType, String sortDirection, Integer start, Integer number,
- String[] subCollectorTypes, String[] subDataTypes,
- String[] subStatsTypes, TreeSet<String>[] subStatsItems,
- String[] subSortTypes, String[] subSortDirections, Integer[] subStart,
- Integer[] subNumber, boolean segmentRegistration) throws IOException {
+ String[] subCollectorTypes, String[] subDataTypes, String[] subStatsTypes,
+ TreeSet<String>[] subStatsItems, String[] subSortTypes,
+ String[] subSortDirections, Integer[] subStart, Integer[] subNumber,
+ String segmentRegistration, String boundary) throws IOException {
super(collectorType, CodecUtil.DATA_TYPE_DOUBLE, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
- subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, new MtasDataDoubleOperations(),
- segmentRegistration);
+ subStatsTypes, subStatsItems, subSortTypes, subSortDirections, subStart,
+ subNumber, new MtasDataDoubleOperations(), segmentRegistration,
+ boundary);
}
/*
@@ -56,9 +72,14 @@ public class MtasDataDoubleBasic
*/
@Override
protected MtasDataItemDoubleBasic getItem(int i) {
- return new MtasDataItemDoubleBasic(basicValueSumList[i],
- basicValueNList[i], hasSub() ? subCollectorListNextLevel[i] : null,
- statsItems, sortType, sortDirection, errorNumber[i], errorList[i]);
+ if (i >= 0 && i < size) {
+ return new MtasDataItemDoubleBasic(basicValueSumList[i],
+ basicValueNList[i], hasSub() ? subCollectorListNextLevel[i] : null,
+ statsItems, sortType, sortDirection, errorNumber[i], errorList[i],
+ sourceNumberList[i]);
+ } else {
+ return null;
+ }
}
/*
@@ -69,7 +90,7 @@ public class MtasDataDoubleBasic
@Override
public MtasDataCollector<?, ?> add(long valueSum, long valueN)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
setValue(newCurrentPosition, Double.valueOf(valueSum), valueN,
newCurrentExisting);
return dataCollector;
@@ -83,7 +104,7 @@ public class MtasDataDoubleBasic
@Override
public MtasDataCollector<?, ?> add(long[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
Double[] newValues = new Double[number];
for (int i = 0; i < values.length; i++)
newValues[i] = Long.valueOf(values[i]).doubleValue();
@@ -99,7 +120,7 @@ public class MtasDataDoubleBasic
@Override
public MtasDataCollector<?, ?> add(double valueSum, long valueN)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
setValue(newCurrentPosition, valueSum, valueN, newCurrentExisting);
return dataCollector;
}
@@ -112,7 +133,7 @@ public class MtasDataDoubleBasic
@Override
public MtasDataCollector<?, ?> add(double[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
return dataCollector;
@@ -131,7 +152,7 @@ public class MtasDataDoubleBasic
if (keys != null && keys.length > 0) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, Double.valueOf(valueSum), valueN,
newCurrentExisting);
}
@@ -149,15 +170,15 @@ public class MtasDataDoubleBasic
* long[], int)
*/
@Override
- public MtasDataCollector<?, ?>[] add(String[] keys, long[] values,
- int number) throws IOException {
+ public MtasDataCollector<?, ?>[] add(String[] keys, long[] values, int number)
+ throws IOException {
if (keys != null && keys.length > 0) {
Double[] newValues = new Double[number];
for (int i = 0; i < values.length; i++)
newValues[i] = Long.valueOf(values[i]).doubleValue();
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, newValues, number, newCurrentExisting);
}
return subCollectors;
@@ -179,7 +200,7 @@ public class MtasDataDoubleBasic
if (keys != null && keys.length > 0) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, valueSum, valueN, newCurrentExisting);
}
return subCollectors;
@@ -201,7 +222,7 @@ public class MtasDataDoubleBasic
if (keys != null && keys.length > 0) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
}
@@ -211,56 +232,177 @@ public class MtasDataDoubleBasic
}
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected boolean compareForComputingSegment(Double value,
- Double boundary) {
- return value >= boundary;
+ protected boolean compareWithBoundary(Double value, Double boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return value <= boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return value >= boundary;
+ } else {
+ throw new IOException(
+ "can't compare for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected Double minimumForComputingSegment(Double value, Double boundary) {
- return Math.min(value, boundary);
+ protected Double lastForComputingSegment(Double value, Double boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)) {
+ return Math.max(value, boundary);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ return Math.min(value, boundary);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * )
*/
@Override
- protected Double minimumForComputingSegment() {
- return Collections.min(segmentValueMaxList);
+ protected Double lastForComputingSegment() throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return Collections.max(segmentValueTopList);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return Collections.min(segmentValueTopList);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment
+ * ()
*/
@Override
- protected Double boundaryForComputingSegment() {
- Double boundary = boundaryForSegment();
- double correctionBoundary = 0;
- for (String otherSegmentName : segmentValueMaxListMin.keySet()) {
- if (!otherSegmentName.equals(segmentName)) {
- Double otherBoundary = segmentValueBoundary.get(otherSegmentName);
- if (otherBoundary != null) {
- correctionBoundary += Math.max(0, otherBoundary - boundary);
+ protected Double boundaryForSegmentComputing(String segmentName)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Double boundary = boundaryForSegment(segmentName);
+ if (boundary == null) {
+ return null;
+ } else {
+ if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ long correctionBoundary = 0;
+ for (String otherSegmentName : segmentValueTopListLast.keySet()) {
+ if (!otherSegmentName.equals(segmentName)) {
+ Double otherBoundary = segmentValuesBoundary
+ .get(otherSegmentName);
+ if (otherBoundary != null) {
+ correctionBoundary += Math.max(0, otherBoundary - boundary);
+ }
+ }
+ }
+ return boundary + correctionBoundary;
+ } else {
+ return boundary;
}
}
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
}
- return boundary + correctionBoundary;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForSegment()
*/
@Override
- protected Double boundaryForSegment() {
- return segmentValueMaxListMin.get(segmentName) / segmentNumber;
+ protected Double boundaryForSegment(String segmentName) throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Double thisLast = segmentValueTopListLast.get(segmentName);
+ if (thisLast == null) {
+ return null;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_ASC)) {
+ Double boundary = thisLast * segmentNumber;
+ return boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Double boundary = thisLast / segmentNumber;
+ return boundary;
+ } else {
+ // should not happen
+ return null;
+ }
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
+ }
}
-}
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#stringToBoundary(java.lang.
+ * String, java.lang.Integer)
+ */
+ @Override
+ protected Double stringToBoundary(String boundary, Integer segmentNumber)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ if (segmentNumber == null) {
+ return Double.valueOf(boundary);
+ } else {
+ return Double.valueOf(boundary) / segmentNumber;
+ }
+ } else if (segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ if (segmentNumber == null) {
+ return Double.valueOf(boundary);
+ } else {
+ return Double.valueOf(boundary) / segmentNumber;
+ }
+ } else {
+ throw new IOException(
+ "not available for segmentRegistration " + segmentRegistration);
+ }
+ }
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#validateSegmentBoundary(java.
+ * lang.Object)
+ */
+ @Override
+ public boolean validateSegmentBoundary(Object o) throws IOException {
+ if (o instanceof Double) {
+ return validateWithSegmentBoundary((Double) o);
+ } else {
+ throw new IOException("incorrect type");
+ }
+ }
+
+}
diff --git a/src/mtas/codec/util/collector/MtasDataDoubleFull.java b/src/mtas/codec/util/collector/MtasDataDoubleFull.java
index 60f3eb3..f513a82 100644
--- a/src/mtas/codec/util/collector/MtasDataDoubleFull.java
+++ b/src/mtas/codec/util/collector/MtasDataDoubleFull.java
@@ -5,13 +5,11 @@ import java.util.Collections;
import java.util.TreeSet;
import org.apache.commons.lang.ArrayUtils;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataDoubleFull.
*/
-public class MtasDataDoubleFull
- extends MtasDataFull<Double, Double, MtasDataItemDoubleFull> {
+public class MtasDataDoubleFull extends MtasDataFull<Double, Double> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -19,34 +17,52 @@ public class MtasDataDoubleFull
/**
* Instantiates a new mtas data double full.
*
- * @param collectorType the collector type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param subCollectorTypes the sub collector types
- * @param subDataTypes the sub data types
- * @param subStatsTypes the sub stats types
- * @param subStatsItems the sub stats items
- * @param subSortTypes the sub sort types
- * @param subSortDirections the sub sort directions
- * @param subStart the sub start
- * @param subNumber the sub number
- * @param segmentRegistration the segment registration
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param collectorType
+ * the collector type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param subCollectorTypes
+ * the sub collector types
+ * @param subDataTypes
+ * the sub data types
+ * @param subStatsTypes
+ * the sub stats types
+ * @param subStatsItems
+ * the sub stats items
+ * @param subSortTypes
+ * the sub sort types
+ * @param subSortDirections
+ * the sub sort directions
+ * @param subStart
+ * the sub start
+ * @param subNumber
+ * the sub number
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasDataDoubleFull(String collectorType, TreeSet<String> statsItems,
String sortType, String sortDirection, Integer start, Integer number,
- String[] subCollectorTypes, String[] subDataTypes,
- String[] subStatsTypes, TreeSet<String>[] subStatsItems,
- String[] subSortTypes, String[] subSortDirections, Integer[] subStart,
- Integer[] subNumber, boolean segmentRegistration) throws IOException {
+ String[] subCollectorTypes, String[] subDataTypes, String[] subStatsTypes,
+ TreeSet<String>[] subStatsItems, String[] subSortTypes,
+ String[] subSortDirections, Integer[] subStart, Integer[] subNumber,
+ String segmentRegistration, String boundary) throws IOException {
super(collectorType, CodecUtil.DATA_TYPE_DOUBLE, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
- subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, new MtasDataDoubleOperations(),
- segmentRegistration);
+ subStatsTypes, subStatsItems, subSortTypes, subSortDirections, subStart,
+ subNumber, new MtasDataDoubleOperations(), segmentRegistration,
+ boundary);
}
/*
@@ -56,10 +72,14 @@ public class MtasDataDoubleFull
*/
@Override
protected MtasDataItemDoubleFull getItem(int i) {
- return new MtasDataItemDoubleFull(
- ArrayUtils.toPrimitive(fullValueList[i]),
- hasSub() ? subCollectorListNextLevel[i] : null, statsItems, sortType,
- sortDirection, errorNumber[i], errorList[i]);
+ if (i >= 0 && i < size) {
+ return new MtasDataItemDoubleFull(
+ ArrayUtils.toPrimitive(fullValueList[i]),
+ hasSub() ? subCollectorListNextLevel[i] : null, statsItems, sortType,
+ sortDirection, errorNumber[i], errorList[i], sourceNumberList[i]);
+ } else {
+ return null;
+ }
}
/*
@@ -81,7 +101,7 @@ public class MtasDataDoubleFull
@Override
public MtasDataCollector<?, ?> add(long[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
Double[] newValues = new Double[number];
for (int i = 0; i < values.length; i++)
newValues[i] = Long.valueOf(values[i]).doubleValue();
@@ -108,7 +128,7 @@ public class MtasDataDoubleFull
@Override
public MtasDataCollector<?, ?> add(double[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
return dataCollector;
@@ -135,15 +155,15 @@ public class MtasDataDoubleFull
* long[], int)
*/
@Override
- public MtasDataCollector<?, ?>[] add(String[] keys, long[] values,
- int number) throws IOException {
+ public MtasDataCollector<?, ?>[] add(String[] keys, long[] values, int number)
+ throws IOException {
if (keys != null && keys.length > 0) {
Double[] newValues = new Double[number];
for (int i = 0; i < values.length; i++)
newValues[i] = Long.valueOf(values[i]).doubleValue();
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, newValues, number, newCurrentExisting);
}
return subCollectors;
@@ -178,7 +198,7 @@ public class MtasDataDoubleFull
if (keys != null && keys.length > 0) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
}
@@ -188,55 +208,179 @@ public class MtasDataDoubleFull
}
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected boolean compareForComputingSegment(Double value,
- Double boundary) {
- return value >= boundary;
+ protected boolean compareWithBoundary(Double value, Double boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return value <= boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return value >= boundary;
+ } else {
+ throw new IOException(
+ "can't compare for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected Double minimumForComputingSegment(Double value, Double boundary) {
- return Math.min(value, boundary);
+ protected Double lastForComputingSegment(Double value, Double boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return Math.max(value, boundary);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return Math.min(value, boundary);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * )
*/
@Override
- protected Double minimumForComputingSegment() {
- return Collections.min(segmentValueMaxList);
+ protected Double lastForComputingSegment() throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return Collections.max(segmentValueTopList);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return Collections.min(segmentValueTopList);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment
+ * ()
*/
@Override
- protected Double boundaryForComputingSegment() {
- Double boundary = boundaryForSegment();
- double correctionBoundary = 0;
- for (String otherSegmentName : segmentValueMaxListMin.keySet()) {
- if (!otherSegmentName.equals(segmentName)) {
- Double otherBoundary = segmentValueBoundary.get(otherSegmentName);
- if (otherBoundary != null) {
- correctionBoundary += Math.max(0, otherBoundary - boundary);
+ protected Double boundaryForSegmentComputing(String segmentName)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Double boundary = boundaryForSegment(segmentName);
+ if (boundary == null) {
+ return null;
+ } else {
+ if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ long correctionBoundary = 0;
+ for (String otherSegmentName : segmentValueTopListLast.keySet()) {
+ if (!otherSegmentName.equals(segmentName)) {
+ Double otherBoundary = segmentValuesBoundary
+ .get(otherSegmentName);
+ if (otherBoundary != null) {
+ correctionBoundary += Math.max(0, otherBoundary - boundary);
+ }
+ }
+ }
+ return boundary + correctionBoundary;
+ } else {
+ return boundary;
}
}
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
}
- return boundary + correctionBoundary;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForSegment()
*/
@Override
- protected Double boundaryForSegment() {
- return segmentValueMaxListMin.get(segmentName) / segmentNumber;
+ protected Double boundaryForSegment(String segmentName) throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Double thisLast = segmentValueTopListLast.get(segmentName);
+ if (thisLast == null) {
+ return null;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_ASC)) {
+ Double boundary = thisLast * segmentNumber;
+ return boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Double boundary = thisLast / segmentNumber;
+ return boundary;
+ } else {
+ // should not happen
+ return null;
+ }
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#stringToBoundary(java.lang.
+ * String, java.lang.Integer)
+ */
+ @Override
+ protected Double stringToBoundary(String boundary, Integer segmentNumber)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ if (segmentNumber == null) {
+ return Double.valueOf(boundary);
+ } else {
+ return Double.valueOf(boundary) / segmentNumber;
+ }
+ } else if (segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ if (segmentNumber == null) {
+ return Double.valueOf(boundary);
+ } else {
+ return Double.valueOf(boundary) / segmentNumber;
+ }
+ } else {
+ throw new IOException(
+ "not available for segmentRegistration " + segmentRegistration);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#validateSegmentBoundary(java.
+ * lang.Object)
+ */
+ @Override
+ public boolean validateSegmentBoundary(Object o) throws IOException {
+ if (o instanceof Double) {
+ return validateWithSegmentBoundary((Double) o);
+ } else {
+ throw new IOException("incorrect type");
+ }
}
}
diff --git a/src/mtas/codec/util/collector/MtasDataDoubleOperations.java b/src/mtas/codec/util/collector/MtasDataDoubleOperations.java
index 4b80257..61999c1 100644
--- a/src/mtas/codec/util/collector/MtasDataDoubleOperations.java
+++ b/src/mtas/codec/util/collector/MtasDataDoubleOperations.java
@@ -14,13 +14,16 @@ class MtasDataDoubleOperations
/*
* (non-Javadoc)
*
- * @see
- * mtas.codec.util.DataCollector.MtasDataOperations#product11(java.lang.
+ * @see mtas.codec.util.DataCollector.MtasDataOperations#product11(java.lang.
* Number, java.lang.Number)
*/
@Override
public Double product11(Double arg1, Double arg2) {
- return arg1 * arg2;
+ if (arg1 == null || arg2 == null) {
+ return Double.NaN;
+ } else {
+ return arg1 * arg2;
+ }
}
/*
@@ -32,7 +35,11 @@ class MtasDataDoubleOperations
*/
@Override
public Double add11(Double arg1, Double arg2) {
- return arg1 + arg2;
+ if (arg1 == null || arg2 == null) {
+ return Double.NaN;
+ } else {
+ return arg1 + arg2;
+ }
}
/*
@@ -44,19 +51,26 @@ class MtasDataDoubleOperations
*/
@Override
public Double add22(Double arg1, Double arg2) {
- return arg1 + arg2;
+ if (arg1 == null || arg2 == null) {
+ return Double.NaN;
+ } else {
+ return arg1 + arg2;
+ }
}
/*
* (non-Javadoc)
*
- * @see
- * mtas.codec.util.DataCollector.MtasDataOperations#subtract12(java.lang.
+ * @see mtas.codec.util.DataCollector.MtasDataOperations#subtract12(java.lang.
* Number, java.lang.Number)
*/
@Override
public Double subtract12(Double arg1, Double arg2) {
- return arg1 - arg2;
+ if (arg1 == null || arg2 == null) {
+ return Double.NaN;
+ } else {
+ return arg1 - arg2;
+ }
}
/*
@@ -67,7 +81,11 @@ class MtasDataDoubleOperations
*/
@Override
public Double divide1(Double arg1, long arg2) {
- return arg1 / arg2;
+ if (arg1 == null) {
+ return Double.NaN;
+ } else {
+ return arg1 / arg2;
+ }
}
/*
@@ -78,7 +96,11 @@ class MtasDataDoubleOperations
*/
@Override
public Double divide2(Double arg1, long arg2) {
- return arg1 / arg2;
+ if (arg1 == null) {
+ return Double.NaN;
+ } else {
+ return arg1 / arg2;
+ }
}
/*
@@ -90,7 +112,11 @@ class MtasDataDoubleOperations
*/
@Override
public Double min11(Double arg1, Double arg2) {
- return Math.min(arg1, arg2);
+ if (arg1 == null || arg2 == null) {
+ return Double.NaN;
+ } else {
+ return Math.min(arg1, arg2);
+ }
}
/*
@@ -102,7 +128,11 @@ class MtasDataDoubleOperations
*/
@Override
public Double max11(Double arg1, Double arg2) {
- return Math.max(arg1, arg2);
+ if (arg1 == null || arg2 == null) {
+ return Double.NaN;
+ } else {
+ return Math.max(arg1, arg2);
+ }
}
/*
@@ -113,7 +143,11 @@ class MtasDataDoubleOperations
*/
@Override
public Double exp2(Double arg1) {
- return Math.exp(arg1);
+ if (arg1 == null) {
+ return Double.NaN;
+ } else {
+ return Math.exp(arg1);
+ }
}
/*
@@ -124,7 +158,11 @@ class MtasDataDoubleOperations
*/
@Override
public Double sqrt2(Double arg1) {
- return Math.sqrt(arg1);
+ if (arg1 == null) {
+ return Double.NaN;
+ } else {
+ return Math.sqrt(arg1);
+ }
}
/*
@@ -135,7 +173,11 @@ class MtasDataDoubleOperations
*/
@Override
public Double log1(Double arg1) {
- return Math.log(arg1);
+ if (arg1 == null) {
+ return Double.NaN;
+ } else {
+ return Math.log(arg1);
+ }
}
/*
diff --git a/src/mtas/codec/util/collector/MtasDataFull.java b/src/mtas/codec/util/collector/MtasDataFull.java
index 239a411..7a25520 100644
--- a/src/mtas/codec/util/collector/MtasDataFull.java
+++ b/src/mtas/codec/util/collector/MtasDataFull.java
@@ -2,20 +2,23 @@ package mtas.codec.util.collector;
import java.io.IOException;
import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Set;
import java.util.TreeSet;
+
import mtas.codec.util.CodecUtil;
import mtas.codec.util.DataCollector;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataFull.
*
- * @param <T1> the generic type
- * @param <T2> the generic type
- * @param <T3> the generic type
+ * @param <T1>
+ * the generic type
+ * @param <T2>
+ * the generic type
*/
-abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends MtasDataItem<T1>>
- extends MtasDataCollector<T1, T3> implements Serializable {
+abstract class MtasDataFull<T1 extends Number & Comparable<T1>, T2 extends Number & Comparable<T2>>
+ extends MtasDataCollector<T1, T2> implements Serializable {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -29,24 +32,44 @@ abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends Mta
/**
* Instantiates a new mtas data full.
*
- * @param collectorType the collector type
- * @param dataType the data type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param subCollectorTypes the sub collector types
- * @param subDataTypes the sub data types
- * @param subStatsTypes the sub stats types
- * @param subStatsItems the sub stats items
- * @param subSortTypes the sub sort types
- * @param subSortDirections the sub sort directions
- * @param subStart the sub start
- * @param subNumber the sub number
- * @param operations the operations
- * @param segmentRegistration the segment registration
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param collectorType
+ * the collector type
+ * @param dataType
+ * the data type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param subCollectorTypes
+ * the sub collector types
+ * @param subDataTypes
+ * the sub data types
+ * @param subStatsTypes
+ * the sub stats types
+ * @param subStatsItems
+ * the sub stats items
+ * @param subSortTypes
+ * the sub sort types
+ * @param subSortDirections
+ * the sub sort directions
+ * @param subStart
+ * the sub start
+ * @param subNumber
+ * the sub number
+ * @param operations
+ * the operations
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasDataFull(String collectorType, String dataType,
TreeSet<String> statsItems, String sortType, String sortDirection,
@@ -54,12 +77,12 @@ abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends Mta
String[] subDataTypes, String[] subStatsTypes,
TreeSet<String>[] subStatsItems, String[] subSortTypes,
String[] subSortDirections, Integer[] subStart, Integer[] subNumber,
- MtasDataOperations<T1, T2> operations, boolean segmentRegistration)
- throws IOException {
+ MtasDataOperations<T1, T2> operations, String segmentRegistration,
+ String boundary) throws IOException {
super(collectorType, dataType, CodecUtil.STATS_FULL, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
- subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, segmentRegistration);
+ subStatsTypes, subStatsItems, subSortTypes, subSortDirections, subStart,
+ subNumber, segmentRegistration, boundary);
this.operations = operations;
}
@@ -71,7 +94,7 @@ abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends Mta
*/
@Override
public final void error(String error) throws IOException {
- add();
+ add(false);
setError(newCurrentPosition, error, newCurrentExisting);
}
@@ -86,7 +109,7 @@ abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends Mta
public final void error(String[] keys, String error) throws IOException {
if (keys != null && keys.length > 0) {
for (int i = 0; i < keys.length; i++) {
- add(keys[i]);
+ add(keys[i], false);
setError(newCurrentPosition, error, newCurrentExisting);
}
}
@@ -95,9 +118,12 @@ abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends Mta
/**
* Sets the error.
*
- * @param newPosition the new position
- * @param error the error
- * @param currentExisting the current existing
+ * @param newPosition
+ * the new position
+ * @param error
+ * the error
+ * @param currentExisting
+ * the current existing
*/
protected void setError(int newPosition, String error,
boolean currentExisting) {
@@ -116,11 +142,10 @@ abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends Mta
/*
* (non-Javadoc)
*
- * @see
- * mtas.codec.util.DataCollector.MtasDataCollector#increaseNewListSize()
+ * @see mtas.codec.util.DataCollector.MtasDataCollector#increaseNewListSize()
*/
@Override
- protected final void increaseNewListSize() {
+ protected final void increaseNewListSize() throws IOException {
// register old situation
int tmpOldSize = newKeyList.length;
int tmpNewPosition = newPosition;
@@ -136,6 +161,45 @@ abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends Mta
/*
* (non-Javadoc)
*
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#reduceToKeys(java.util.Set)
+ */
+ @SuppressWarnings("unchecked")
+ public void reduceToKeys(Set<String> keys) {
+ if (size > 0) {
+ int sizeCopy = size;
+ String[] keyListCopy = keyList.clone();
+ int[] errorNumberCopy = errorNumber.clone();
+ HashMap<String, Integer>[] errorListCopy = errorList.clone();
+ int[] sourceNumberListCopy = sourceNumberList.clone();
+ T1[][] fullValueListCopy = fullValueList.clone();
+ for (int i = 0; i < fullValueListCopy.length; i++) {
+ if (fullValueListCopy[i] != null) {
+ fullValueListCopy[i] = fullValueListCopy[i].clone();
+ }
+ }
+ keyList = new String[keys.size()];
+ errorNumber = new int[keys.size()];
+ errorList = new HashMap[keys.size()];
+ sourceNumberList = new int[keys.size()];
+ fullValueList = operations.createMatrix1(keys.size());
+ size = 0;
+ for (int i = 0; i < sizeCopy; i++) {
+ if (keys.contains(keyListCopy[i])) {
+ keyList[size] = keyListCopy[i];
+ errorNumber[size] = errorNumberCopy[i];
+ errorList[size] = errorListCopy[i];
+ sourceNumberList[size] = sourceNumberListCopy[i];
+ fullValueList[size] = fullValueListCopy[i];
+ size++;
+ }
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.util.DataCollector.MtasDataCollector#copyToNew(int, int)
*/
@Override
@@ -156,10 +220,14 @@ abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends Mta
/**
* Sets the value.
*
- * @param newPosition the new position
- * @param values the values
- * @param number the number
- * @param currentExisting the current existing
+ * @param newPosition
+ * the new position
+ * @param values
+ * the values
+ * @param number
+ * the number
+ * @param currentExisting
+ * the current existing
*/
protected void setValue(int newPosition, T1[] values, int number,
boolean currentExisting) {
@@ -211,13 +279,13 @@ abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends Mta
/*
* (non-Javadoc)
*
- * @see
- * mtas.codec.util.DataCollector.MtasDataCollector#merge(mtas.codec.util.
+ * @see mtas.codec.util.DataCollector.MtasDataCollector#merge(mtas.codec.util.
* DataCollector.MtasDataCollector)
*/
@Override
- public void merge(MtasDataCollector<?, ?> newDataCollector)
- throws IOException {
+ public void merge(MtasDataCollector<?, ?> newDataCollector,
+ HashMap<MtasDataCollector<?, ?>, MtasDataCollector<?, ?>> map,
+ boolean increaseSourceNumber) throws IOException {
closeNewList();
if (!collectorType.equals(newDataCollector.getCollectorType())
|| !dataType.equals(newDataCollector.getDataType())
@@ -225,35 +293,41 @@ abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends Mta
|| !(newDataCollector instanceof MtasDataFull)) {
throw new IOException("cannot merge different dataCollectors");
} else {
- MtasDataFull<T1, T2, T3> newMtasDataFull = (MtasDataFull<T1, T2, T3>) newDataCollector;
+ segmentRegistration = null;
+ @SuppressWarnings("unchecked")
+ MtasDataFull<T1, T2> newMtasDataFull = (MtasDataFull<T1, T2>) newDataCollector;
closeNewList();
initNewList(newMtasDataFull.getSize());
if (collectorType.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
- String[] keys = new String[1];
+ map.put(newDataCollector, this);
for (int i = 0; i < newMtasDataFull.getSize(); i++) {
if (newMtasDataFull.fullValueList[i].length > 0) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[1];
- subCollectors[0] = add(newMtasDataFull.keyList[i]);
+ subCollectors[0] = add(newMtasDataFull.keyList[i],
+ increaseSourceNumber);
setError(newCurrentPosition, newMtasDataFull.errorNumber[i],
newMtasDataFull.errorList[i], newCurrentExisting);
setValue(newCurrentPosition, newMtasDataFull.fullValueList[i],
newMtasDataFull.fullValueList[i].length, newCurrentExisting);
if (hasSub() && newMtasDataFull.hasSub()) {
// single key implies exactly one subCollector if hasSub
- subCollectors[0]
- .merge(newMtasDataFull.subCollectorListNextLevel[i]);
+ subCollectors[0].merge(
+ newMtasDataFull.subCollectorListNextLevel[i], map,
+ increaseSourceNumber);
}
}
}
} else if (collectorType.equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ map.put(newDataCollector, this);
if (newMtasDataFull.getSize() > 0) {
- MtasDataCollector<?, ?> subCollector = add();
+ MtasDataCollector<?, ?> subCollector = add(increaseSourceNumber);
setError(newCurrentPosition, newMtasDataFull.errorNumber[0],
newMtasDataFull.errorList[0], newCurrentExisting);
setValue(newCurrentPosition, newMtasDataFull.fullValueList[0],
newMtasDataFull.fullValueList[0].length, newCurrentExisting);
if (hasSub() && newMtasDataFull.hasSub()) {
- subCollector.merge(newMtasDataFull.subCollectorNextLevel);
+ subCollector.merge(newMtasDataFull.subCollectorNextLevel, map,
+ increaseSourceNumber);
}
}
} else {
@@ -282,20 +356,19 @@ abstract class MtasDataFull<T1 extends Number, T2 extends Number, T3 extends Mta
*/
@Override
public final void initNewList(int maxNumberOfTerms, String segmentName,
- int segmentNumber) throws IOException {
- super.initNewList(maxNumberOfTerms, segmentName, segmentNumber);
+ int segmentNumber, String boundary) throws IOException {
+ super.initNewList(maxNumberOfTerms, segmentName, segmentNumber, boundary);
initNewListBasic(maxNumberOfTerms);
}
/**
* Inits the new list basic.
*
- * @param maxNumberOfTerms the max number of terms
+ * @param maxNumberOfTerms
+ * the max number of terms
*/
private void initNewListBasic(int maxNumberOfTerms) {
newFullValueList = operations.createMatrix1(newSize);
}
}
-
-
diff --git a/src/mtas/codec/util/collector/MtasDataItem.java b/src/mtas/codec/util/collector/MtasDataItem.java
index 8f9fa97..7582324 100644
--- a/src/mtas/codec/util/collector/MtasDataItem.java
+++ b/src/mtas/codec/util/collector/MtasDataItem.java
@@ -6,15 +6,20 @@ import java.util.HashMap;
import java.util.Map;
import java.util.TreeSet;
-import mtas.codec.util.DataCollector.MtasDataCollector;
-
/**
* The Class MtasDataItem.
*
- * @param <T> the generic type
+ * @param <T1> the generic type
+ * @param <T2> the generic type
+ */
+/**
+ * @author matthijs
+ *
+ * @param <T1>
+ * @param <T2>
*/
-public abstract class MtasDataItem<T extends Number>
- implements Serializable, Comparable<MtasDataItem<T>> {
+public abstract class MtasDataItem<T1 extends Number & Comparable<T1>, T2 extends Number & Comparable<T2>>
+ implements Serializable, Comparable<MtasDataItem<T1, T2>> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -34,42 +39,68 @@ public abstract class MtasDataItem<T extends Number>
/** The error list. */
protected HashMap<String, Integer> errorList;
+ /** The comparable sort value. */
+ protected NumberComparator<?> comparableSortValue;
+
+ /** The recompute comparable sort value. */
+ protected boolean recomputeComparableSortValue;
+
+ /** The source number. */
+ protected int sourceNumber;
+
/**
* Instantiates a new mtas data item.
*
- * @param sub the sub
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param errorNumber the error number
- * @param errorList the error list
+ * @param sub
+ * the sub
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param errorNumber
+ * the error number
+ * @param errorList
+ * the error list
+ * @param sourceNumber
+ * the source number
*/
public MtasDataItem(MtasDataCollector<?, ?> sub, TreeSet<String> statsItems,
String sortType, String sortDirection, int errorNumber,
- HashMap<String, Integer> errorList) {
+ HashMap<String, Integer> errorList, int sourceNumber) {
this.sub = sub;
this.statsItems = statsItems;
this.sortType = sortType;
this.sortDirection = sortDirection;
this.errorNumber = errorNumber;
this.errorList = errorList;
+ this.sourceNumber = sourceNumber;
+ this.comparableSortValue = null;
+ this.recomputeComparableSortValue = true;
}
/**
* Adds the.
*
- * @param newItem the new item
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param newItem
+ * the new item
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- public abstract void add(MtasDataItem<T> newItem) throws IOException;
+ public abstract void add(MtasDataItem<T1, T2> newItem) throws IOException;
/**
* Rewrite.
*
+ * @param showDebugInfo
+ * the show debug info
* @return the map
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- public abstract Map<String, Object> rewrite() throws IOException;
+ public abstract Map<String, Object> rewrite(boolean showDebugInfo)
+ throws IOException;
/**
* Gets the sub.
@@ -80,4 +111,183 @@ public abstract class MtasDataItem<T extends Number>
return sub;
}
+ /**
+ * Gets the compare value type.
+ *
+ * @return the compare value type
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ protected abstract int getCompareValueType() throws IOException;
+
+ /**
+ * Compute comparable value.
+ */
+ private void computeComparableValue() {
+ recomputeComparableSortValue = false;
+ try {
+ int type = getCompareValueType();
+ switch (type) {
+ case 0:
+ comparableSortValue = getCompareValue0();
+ break;
+ case 1:
+ comparableSortValue = getCompareValue1();
+ break;
+ case 2:
+ comparableSortValue = getCompareValue2();
+ break;
+ default:
+ comparableSortValue = null;
+ break;
+ }
+ } catch (IOException e) {
+ comparableSortValue = null;
+ }
+ }
+
+ /**
+ * Gets the comparable value.
+ *
+ * @return the comparable value
+ */
+ protected final NumberComparator<?> getComparableValue() {
+ if (recomputeComparableSortValue) {
+ computeComparableValue();
+ }
+ return comparableSortValue;
+ }
+
+ /**
+ * Gets the compare value0.
+ *
+ * @return the compare value0
+ */
+ protected abstract NumberComparator<Long> getCompareValue0();
+
+ /**
+ * Gets the compare value1.
+ *
+ * @return the compare value1
+ */
+ protected abstract NumberComparator<T1> getCompareValue1();
+
+ /**
+ * Gets the compare value2.
+ *
+ * @return the compare value2
+ */
+ protected abstract NumberComparator<T2> getCompareValue2();
+
+ /**
+ * The Class NumberComparator.
+ *
+ * @param <T>
+ * the generic type
+ */
+ public class NumberComparator<T extends Number & Comparable<T>>
+ implements Comparable<T>, Serializable, Cloneable {
+
+ /** The Constant serialVersionUID. */
+ private static final long serialVersionUID = 1L;
+
+ /** The value. */
+ T value;
+
+ /**
+ * Instantiates a new number comparator.
+ *
+ * @param value
+ * the value
+ */
+ public NumberComparator(T value) {
+ this.value = value;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#clone()
+ */
+ @Override
+ public NumberComparator<T> clone() {
+ return new NumberComparator<T>(this.value);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Comparable#compareTo(java.lang.Object)
+ */
+ @Override
+ public int compareTo(T compareValue) {
+ return value.compareTo(compareValue);
+ }
+
+ /**
+ * Gets the value.
+ *
+ * @return the value
+ */
+ public T getValue() {
+ return value;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ public String toString() {
+ return value.toString();
+ }
+
+ /**
+ * Adds the.
+ *
+ * @param newValue
+ * the new value
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ public void add(T newValue) throws IOException {
+ if (value instanceof Integer && newValue instanceof Integer) {
+ value = (T) Integer.valueOf(value.intValue() + newValue.intValue());
+ } else if (value instanceof Long && newValue instanceof Long) {
+ value = (T) Long.valueOf(value.longValue() + newValue.longValue());
+ } else if (value instanceof Float && newValue instanceof Float) {
+ value = (T) Float.valueOf(value.floatValue() + newValue.floatValue());
+ } else if (value instanceof Double && newValue instanceof Double) {
+ value = (T) Double.valueOf(value.doubleValue() + newValue.longValue());
+ } else {
+ throw new IOException("incompatible NumberComparators");
+ }
+ }
+
+ /**
+ * Subtract.
+ *
+ * @param newValue
+ * the new value
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ public void subtract(T newValue) throws IOException {
+ if (value instanceof Integer && newValue instanceof Integer) {
+ value = (T) Integer.valueOf(value.intValue() - newValue.intValue());
+ } else if (value instanceof Long && newValue instanceof Long) {
+ value = (T) Long.valueOf(value.longValue() - newValue.longValue());
+ } else if (value instanceof Float && newValue instanceof Float) {
+ value = (T) Float.valueOf(value.floatValue() - newValue.floatValue());
+ } else if (value instanceof Double && newValue instanceof Double) {
+ value = (T) Double.valueOf(value.doubleValue() - newValue.longValue());
+ } else {
+ throw new IOException("incompatible NumberComparators");
+ }
+ }
+
+ }
+
}
\ No newline at end of file
diff --git a/src/mtas/codec/util/collector/MtasDataItemAdvanced.java b/src/mtas/codec/util/collector/MtasDataItemAdvanced.java
index 7b080db..54a9de5 100644
--- a/src/mtas/codec/util/collector/MtasDataItemAdvanced.java
+++ b/src/mtas/codec/util/collector/MtasDataItemAdvanced.java
@@ -7,16 +7,17 @@ import java.util.Map;
import java.util.TreeSet;
import java.util.Map.Entry;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataItemAdvanced.
*
- * @param <T1> the generic type
- * @param <T2> the generic type
+ * @param <T1>
+ * the generic type
+ * @param <T2>
+ * the generic type
*/
-abstract class MtasDataItemAdvanced<T1 extends Number, T2 extends Number>
- extends MtasDataItem<T1> implements Serializable {
+abstract class MtasDataItemAdvanced<T1 extends Number & Comparable<T1>, T2 extends Number & Comparable<T2>>
+ extends MtasDataItem<T1, T2> implements Serializable {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -45,27 +46,42 @@ abstract class MtasDataItemAdvanced<T1 extends Number, T2 extends Number>
/**
* Instantiates a new mtas data item advanced.
*
- * @param valueSum the value sum
- * @param valueSumOfLogs the value sum of logs
- * @param valueSumOfSquares the value sum of squares
- * @param valueMin the value min
- * @param valueMax the value max
- * @param valueN the value n
- * @param sub the sub
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param errorNumber the error number
- * @param errorList the error list
- * @param operations the operations
+ * @param valueSum
+ * the value sum
+ * @param valueSumOfLogs
+ * the value sum of logs
+ * @param valueSumOfSquares
+ * the value sum of squares
+ * @param valueMin
+ * the value min
+ * @param valueMax
+ * the value max
+ * @param valueN
+ * the value n
+ * @param sub
+ * the sub
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param errorNumber
+ * the error number
+ * @param errorList
+ * the error list
+ * @param operations
+ * the operations
+ * @param sourceNumber
+ * the source number
*/
public MtasDataItemAdvanced(T1 valueSum, T2 valueSumOfLogs,
- T1 valueSumOfSquares, T1 valueMin, T1 valueMax, long valueN,
- MtasDataCollector<?, ?> sub, TreeSet<String> statsItems,
- String sortType, String sortDirection, int errorNumber,
- HashMap<String, Integer> errorList,
- MtasDataOperations<T1, T2> operations) {
- super(sub, statsItems, sortType, sortDirection, errorNumber, errorList);
+ T1 valueSumOfSquares, T1 valueMin, T1 valueMax, Long valueN,
+ MtasDataCollector<?, ?> sub, TreeSet<String> statsItems, String sortType,
+ String sortDirection, int errorNumber, HashMap<String, Integer> errorList,
+ MtasDataOperations<T1, T2> operations, int sourceNumber) {
+ super(sub, statsItems, sortType, sortDirection, errorNumber, errorList,
+ sourceNumber);
this.valueSum = valueSum;
this.valueSumOfLogs = valueSumOfLogs;
this.valueSumOfSquares = valueSumOfSquares;
@@ -82,7 +98,7 @@ abstract class MtasDataItemAdvanced<T1 extends Number, T2 extends Number>
* DataCollector.MtasDataItem)
*/
@Override
- public void add(MtasDataItem<T1> newItem) throws IOException {
+ public void add(MtasDataItem<T1, T2> newItem) throws IOException {
if (newItem instanceof MtasDataItemAdvanced) {
MtasDataItemAdvanced<T1, T2> newTypedItem = (MtasDataItemAdvanced<T1, T2>) newItem;
valueSum = operations.add11(valueSum, newTypedItem.valueSum);
@@ -93,6 +109,7 @@ abstract class MtasDataItemAdvanced<T1 extends Number, T2 extends Number>
valueMin = operations.min11(valueMin, newTypedItem.valueMin);
valueMax = operations.max11(valueMax, newTypedItem.valueMax);
valueN += newTypedItem.valueN;
+ recomputeComparableSortValue = true;
} else {
throw new IOException("can only add MtasDataItemAdvanced");
}
@@ -104,7 +121,7 @@ abstract class MtasDataItemAdvanced<T1 extends Number, T2 extends Number>
* @see mtas.codec.util.DataCollector.MtasDataItem#rewrite()
*/
@Override
- public Map<String, Object> rewrite() throws IOException {
+ public Map<String, Object> rewrite(boolean showDebugInfo) throws IOException {
Map<String, Object> response = new HashMap<String, Object>();
for (String statsItem : statsItems) {
if (statsItem.equals(CodecUtil.STATS_TYPE_SUM)) {
@@ -143,14 +160,18 @@ abstract class MtasDataItemAdvanced<T1 extends Number, T2 extends Number>
response.put("errorNumber", errorNumber);
response.put("errorList", errorResponse);
}
- // response.put("stats", "advanced");
+ if (showDebugInfo) {
+ response.put("sourceNumber", sourceNumber);
+ response.put("stats", "advanced");
+ }
return response;
}
/**
* Gets the value.
*
- * @param statsType the stats type
+ * @param statsType
+ * the stats type
* @return the value
*/
protected T2 getValue(String statsType) {
@@ -189,6 +210,56 @@ abstract class MtasDataItemAdvanced<T1 extends Number, T2 extends Number>
}
}
-}
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValueType()
+ */
+ @Override
+ public int getCompareValueType() throws IOException {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_N:
+ return 0;
+ case CodecUtil.STATS_TYPE_SUM:
+ return 1;
+ case CodecUtil.STATS_TYPE_MAX:
+ return 1;
+ case CodecUtil.STATS_TYPE_MIN:
+ return 1;
+ case CodecUtil.STATS_TYPE_SUMSQ:
+ return 1;
+ case CodecUtil.STATS_TYPE_SUMOFLOGS:
+ return 2;
+ case CodecUtil.STATS_TYPE_MEAN:
+ return 2;
+ case CodecUtil.STATS_TYPE_GEOMETRICMEAN:
+ return 2;
+ case CodecUtil.STATS_TYPE_STANDARDDEVIATION:
+ return 2;
+ case CodecUtil.STATS_TYPE_VARIANCE:
+ return 2;
+ case CodecUtil.STATS_TYPE_POPULATIONVARIANCE:
+ return 2;
+ case CodecUtil.STATS_TYPE_QUADRATICMEAN:
+ return 2;
+ default:
+ throw new IOException("sortType " + sortType + " not supported");
+ }
+ }
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue0()
+ */
+ @Override
+ public final NumberComparator<Long> getCompareValue0() {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_N:
+ return new NumberComparator<Long>(valueN);
+ default:
+ return null;
+ }
+ }
+}
diff --git a/src/mtas/codec/util/collector/MtasDataItemBasic.java b/src/mtas/codec/util/collector/MtasDataItemBasic.java
index d58d4ab..9befd69 100644
--- a/src/mtas/codec/util/collector/MtasDataItemBasic.java
+++ b/src/mtas/codec/util/collector/MtasDataItemBasic.java
@@ -7,16 +7,17 @@ import java.util.Map;
import java.util.TreeSet;
import java.util.Map.Entry;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataItemBasic.
*
- * @param <T1> the generic type
- * @param <T2> the generic type
+ * @param <T1>
+ * the generic type
+ * @param <T2>
+ * the generic type
*/
-abstract class MtasDataItemBasic<T1 extends Number, T2 extends Number>
- extends MtasDataItem<T1> implements Serializable {
+abstract class MtasDataItemBasic<T1 extends Number & Comparable<T1>, T2 extends Number & Comparable<T2>>
+ extends MtasDataItem<T1, T2> implements Serializable {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -33,22 +34,33 @@ abstract class MtasDataItemBasic<T1 extends Number, T2 extends Number>
/**
* Instantiates a new mtas data item basic.
*
- * @param valueSum the value sum
- * @param valueN the value n
- * @param sub the sub
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param errorNumber the error number
- * @param errorList the error list
- * @param operations the operations
+ * @param valueSum
+ * the value sum
+ * @param valueN
+ * the value n
+ * @param sub
+ * the sub
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param errorNumber
+ * the error number
+ * @param errorList
+ * the error list
+ * @param operations
+ * the operations
+ * @param sourceNumber
+ * the source number
*/
public MtasDataItemBasic(T1 valueSum, long valueN,
- MtasDataCollector<?, ?> sub, TreeSet<String> statsItems,
- String sortType, String sortDirection, int errorNumber,
- HashMap<String, Integer> errorList,
- MtasDataOperations<T1, T2> operations) {
- super(sub, statsItems, sortType, sortDirection, errorNumber, errorList);
+ MtasDataCollector<?, ?> sub, TreeSet<String> statsItems, String sortType,
+ String sortDirection, int errorNumber, HashMap<String, Integer> errorList,
+ MtasDataOperations<T1, T2> operations, int sourceNumber) {
+ super(sub, statsItems, sortType, sortDirection, errorNumber, errorList,
+ sourceNumber);
this.valueSum = valueSum;
this.valueN = valueN;
this.operations = operations;
@@ -61,11 +73,12 @@ abstract class MtasDataItemBasic<T1 extends Number, T2 extends Number>
* DataCollector.MtasDataItem)
*/
@Override
- public void add(MtasDataItem<T1> newItem) throws IOException {
+ public void add(MtasDataItem<T1, T2> newItem) throws IOException {
if (newItem instanceof MtasDataItemBasic) {
MtasDataItemBasic<T1, T2> newTypedItem = (MtasDataItemBasic<T1, T2>) newItem;
this.valueSum = operations.add11(this.valueSum, newTypedItem.valueSum);
this.valueN += newTypedItem.valueN;
+ recomputeComparableSortValue = true;
} else {
throw new IOException("can only add MtasDataItemBasic");
}
@@ -77,7 +90,7 @@ abstract class MtasDataItemBasic<T1 extends Number, T2 extends Number>
* @see mtas.codec.util.DataCollector.MtasDataItem#rewrite()
*/
@Override
- public Map<String, Object> rewrite() throws IOException {
+ public Map<String, Object> rewrite(boolean showDebugInfo) throws IOException {
Map<String, Object> response = new HashMap<String, Object>();
for (String statsItem : statsItems) {
if (statsItem.equals(CodecUtil.STATS_TYPE_SUM)) {
@@ -98,14 +111,18 @@ abstract class MtasDataItemBasic<T1 extends Number, T2 extends Number>
response.put("errorNumber", errorNumber);
response.put("errorList", errorResponse);
}
- // response.put("stats", "basic");
+ if (showDebugInfo) {
+ response.put("sourceNumber", sourceNumber);
+ response.put("stats", "basic");
+ }
return response;
}
/**
* Gets the value.
*
- * @param statsType the stats type
+ * @param statsType
+ * the stats type
* @return the value
*/
protected T2 getValue(String statsType) {
@@ -116,5 +133,36 @@ abstract class MtasDataItemBasic<T1 extends Number, T2 extends Number>
}
}
-}
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValueType()
+ */
+ public final int getCompareValueType() throws IOException {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_N:
+ return 0;
+ case CodecUtil.STATS_TYPE_SUM:
+ return 1;
+ case CodecUtil.STATS_TYPE_MEAN:
+ return 2;
+ default:
+ throw new IOException("sortType " + sortType + " not supported");
+ }
+ }
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue0()
+ */
+ public final NumberComparator<Long> getCompareValue0() {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_N:
+ return new NumberComparator<Long>(valueN);
+ default:
+ return null;
+ }
+ }
+
+}
diff --git a/src/mtas/codec/util/collector/MtasDataItemDoubleAdvanced.java b/src/mtas/codec/util/collector/MtasDataItemDoubleAdvanced.java
index 2fd5c2f..07ce22f 100644
--- a/src/mtas/codec/util/collector/MtasDataItemDoubleAdvanced.java
+++ b/src/mtas/codec/util/collector/MtasDataItemDoubleAdvanced.java
@@ -3,7 +3,6 @@ package mtas.codec.util.collector;
import java.util.HashMap;
import java.util.TreeSet;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataItemDoubleAdvanced.
@@ -17,27 +16,41 @@ public class MtasDataItemDoubleAdvanced
/**
* Instantiates a new mtas data item double advanced.
*
- * @param valueSum the value sum
- * @param valueSumOfLogs the value sum of logs
- * @param valueSumOfSquares the value sum of squares
- * @param valueMin the value min
- * @param valueMax the value max
- * @param valueN the value n
- * @param sub the sub
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param errorNumber the error number
- * @param errorList the error list
+ * @param valueSum
+ * the value sum
+ * @param valueSumOfLogs
+ * the value sum of logs
+ * @param valueSumOfSquares
+ * the value sum of squares
+ * @param valueMin
+ * the value min
+ * @param valueMax
+ * the value max
+ * @param valueN
+ * the value n
+ * @param sub
+ * the sub
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param errorNumber
+ * the error number
+ * @param errorList
+ * the error list
+ * @param sourceNumber
+ * the source number
*/
- public MtasDataItemDoubleAdvanced(double valueSum, double valueSumOfLogs,
- double valueSumOfSquares, double valueMin, double valueMax, long valueN,
- MtasDataCollector<?, ?> sub, TreeSet<String> statsItems,
- String sortType, String sortDirection, int errorNumber,
- HashMap<String, Integer> errorList) {
+ public MtasDataItemDoubleAdvanced(Double valueSum, Double valueSumOfLogs,
+ Double valueSumOfSquares, Double valueMin, Double valueMax, long valueN,
+ MtasDataCollector<?, ?> sub, TreeSet<String> statsItems, String sortType,
+ String sortDirection, int errorNumber, HashMap<String, Integer> errorList,
+ int sourceNumber) {
super(valueSum, valueSumOfLogs, valueSumOfSquares, valueMin, valueMax,
valueN, sub, statsItems, sortType, sortDirection, errorNumber,
- errorList, new MtasDataDoubleOperations());
+ errorList, new MtasDataDoubleOperations(), sourceNumber);
}
/*
@@ -45,38 +58,74 @@ public class MtasDataItemDoubleAdvanced
*
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/
+ @SuppressWarnings({ "rawtypes", "unchecked" })
@Override
- public int compareTo(MtasDataItem<Double> o) {
+ public int compareTo(MtasDataItem<Double, Double> o) {
int compare = 0;
if (o instanceof MtasDataItemDoubleAdvanced) {
MtasDataItemDoubleAdvanced to = (MtasDataItemDoubleAdvanced) o;
- if (sortType.equals(CodecUtil.STATS_TYPE_N)) {
- compare = valueN.compareTo(to.valueN);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUM)) {
- compare = valueSum.compareTo(to.valueSum);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MAX)) {
- compare = valueMax.compareTo(to.valueMax);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MIN)) {
- compare = valueMin.compareTo(to.valueMin);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUMSQ)) {
- compare = valueSumOfSquares.compareTo(to.valueSumOfSquares);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUMOFLOGS)) {
- compare = valueSumOfLogs.compareTo(to.valueSumOfLogs);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MEAN)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_GEOMETRICMEAN)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_STANDARDDEVIATION)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_VARIANCE)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_POPULATIONVARIANCE)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_QUADRATICMEAN)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- }
+ NumberComparator c1 = getComparableValue();
+ NumberComparator c2 = to.getComparableValue();
+ compare = (c1 != null && c2 != null) ? c1.compareTo(c2.getValue()) : 0;
}
return sortDirection.equals(CodecUtil.SORT_DESC) ? -1 * compare : compare;
}
-}
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue1()
+ */
+ @Override
+ public NumberComparator<Double> getCompareValue1() {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_SUM:
+ return new NumberComparator<Double>(valueSum);
+ case CodecUtil.STATS_TYPE_MAX:
+ return new NumberComparator<Double>(valueMax);
+ case CodecUtil.STATS_TYPE_MIN:
+ return new NumberComparator<Double>(valueMin);
+ case CodecUtil.STATS_TYPE_SUMSQ:
+ return new NumberComparator<Double>(valueSumOfSquares);
+ default:
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue2()
+ */
+ public NumberComparator<Double> getCompareValue2() {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_SUMOFLOGS:
+ return new NumberComparator<Double>(valueSumOfLogs);
+ case CodecUtil.STATS_TYPE_MEAN:
+ return new NumberComparator<Double>(getValue(sortType));
+ case CodecUtil.STATS_TYPE_GEOMETRICMEAN:
+ return new NumberComparator<Double>(getValue(sortType));
+ case CodecUtil.STATS_TYPE_STANDARDDEVIATION:
+ return new NumberComparator<Double>(getValue(sortType));
+ case CodecUtil.STATS_TYPE_VARIANCE:
+ return new NumberComparator<Double>(getValue(sortType));
+ case CodecUtil.STATS_TYPE_POPULATIONVARIANCE:
+ return new NumberComparator<Double>(getValue(sortType));
+ case CodecUtil.STATS_TYPE_QUADRATICMEAN:
+ return new NumberComparator<Double>(getValue(sortType));
+ default:
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ public String toString() {
+ return this.getClass().getSimpleName() + "[" + valueSum + "," + valueN
+ + "]";
+ }
+
+}
diff --git a/src/mtas/codec/util/collector/MtasDataItemDoubleBasic.java b/src/mtas/codec/util/collector/MtasDataItemDoubleBasic.java
index 0235702..cfee3e2 100644
--- a/src/mtas/codec/util/collector/MtasDataItemDoubleBasic.java
+++ b/src/mtas/codec/util/collector/MtasDataItemDoubleBasic.java
@@ -3,13 +3,11 @@ package mtas.codec.util.collector;
import java.util.HashMap;
import java.util.TreeSet;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataItemDoubleBasic.
*/
-public class MtasDataItemDoubleBasic
- extends MtasDataItemBasic<Double, Double> {
+public class MtasDataItemDoubleBasic extends MtasDataItemBasic<Double, Double> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -17,21 +15,31 @@ public class MtasDataItemDoubleBasic
/**
* Instantiates a new mtas data item double basic.
*
- * @param valueSum the value sum
- * @param valueN the value n
- * @param sub the sub
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param errorNumber the error number
- * @param errorList the error list
+ * @param valueSum
+ * the value sum
+ * @param valueN
+ * the value n
+ * @param sub
+ * the sub
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param errorNumber
+ * the error number
+ * @param errorList
+ * the error list
+ * @param sourceNumber
+ * the source number
*/
- public MtasDataItemDoubleBasic(double valueSum, long valueN,
- MtasDataCollector<?, ?> sub, TreeSet<String> statsItems,
- String sortType, String sortDirection, int errorNumber,
- HashMap<String, Integer> errorList) {
+ public MtasDataItemDoubleBasic(Double valueSum, long valueN,
+ MtasDataCollector<?, ?> sub, TreeSet<String> statsItems, String sortType,
+ String sortDirection, int errorNumber, HashMap<String, Integer> errorList,
+ int sourceNumber) {
super(valueSum, valueN, sub, statsItems, sortType, sortDirection,
- errorNumber, errorList, new MtasDataDoubleOperations());
+ errorNumber, errorList, new MtasDataDoubleOperations(), sourceNumber);
}
/*
@@ -39,21 +47,56 @@ public class MtasDataItemDoubleBasic
*
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/
+ @SuppressWarnings({ "rawtypes", "unchecked" })
@Override
- public int compareTo(MtasDataItem<Double> o) {
+ public int compareTo(MtasDataItem<Double, Double> o) {
int compare = 0;
if (o instanceof MtasDataItemDoubleBasic) {
MtasDataItemDoubleBasic to = (MtasDataItemDoubleBasic) o;
- if (sortType.equals(CodecUtil.STATS_TYPE_N)) {
- compare = valueN.compareTo(to.valueN);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUM)) {
- compare = valueSum.compareTo(to.valueSum);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MEAN)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- }
+ NumberComparator c1 = getComparableValue();
+ NumberComparator c2 = to.getComparableValue();
+ compare = (c1 != null && c2 != null) ? c1.compareTo(c2.getValue()) : 0;
}
return sortDirection.equals(CodecUtil.SORT_DESC) ? -1 * compare : compare;
}
-}
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue()
+ */
+ @Override
+ public NumberComparator<Double> getCompareValue1() {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_SUM:
+ return new NumberComparator<Double>(valueSum);
+ default:
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue2()
+ */
+ public NumberComparator<Double> getCompareValue2() {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_MEAN:
+ return new NumberComparator<Double>(getValue(sortType));
+ default:
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ public String toString() {
+ return this.getClass().getSimpleName() + "[" + valueSum + "," + valueN
+ + "]";
+ }
+}
diff --git a/src/mtas/codec/util/collector/MtasDataItemDoubleFull.java b/src/mtas/codec/util/collector/MtasDataItemDoubleFull.java
index 9789b17..a8aa855 100644
--- a/src/mtas/codec/util/collector/MtasDataItemDoubleFull.java
+++ b/src/mtas/codec/util/collector/MtasDataItemDoubleFull.java
@@ -7,13 +7,11 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.ArrayUtils;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataItemDoubleFull.
*/
-public class MtasDataItemDoubleFull
- extends MtasDataItemFull<Double, Double> {
+public class MtasDataItemDoubleFull extends MtasDataItemFull<Double, Double> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -24,26 +22,35 @@ public class MtasDataItemDoubleFull
/**
* Instantiates a new mtas data item double full.
*
- * @param value the value
- * @param sub the sub
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param errorNumber the error number
- * @param errorList the error list
+ * @param value
+ * the value
+ * @param sub
+ * the sub
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param errorNumber
+ * the error number
+ * @param errorList
+ * the error list
+ * @param sourceNumber
+ * the source number
*/
public MtasDataItemDoubleFull(double[] value, MtasDataCollector<?, ?> sub,
TreeSet<String> statsItems, String sortType, String sortDirection,
- int errorNumber, HashMap<String, Integer> errorList) {
- super(ArrayUtils.toObject(value), sub, statsItems, sortType,
- sortDirection, errorNumber, errorList,
- new MtasDataDoubleOperations());
+ int errorNumber, HashMap<String, Integer> errorList, int sourceNumber) {
+ super(ArrayUtils.toObject(value), sub, statsItems, sortType, sortDirection,
+ errorNumber, errorList, new MtasDataDoubleOperations(), sourceNumber);
}
/**
* Gets the number of decimals.
*
- * @param ds the ds
+ * @param ds
+ * the ds
* @return the number of decimals
*/
private int getNumberOfDecimals(String ds) {
@@ -114,8 +121,7 @@ public class MtasDataItemDoubleFull
if (step == null) {
step = (tmpEnd - tmpStart) / number;
}
- number = Double.valueOf(Math.ceil((tmpEnd - tmpStart) / step))
- .intValue();
+ number = Double.valueOf(Math.ceil((tmpEnd - tmpStart) / step)).intValue();
start = tmpStart;
end = start + (number * step);
} else if (start == null) {
@@ -137,8 +143,7 @@ public class MtasDataItemDoubleFull
number = Double.valueOf(Math.ceil((end - start) / step)).intValue();
}
// round step to agreeable format and recompute number
- int tmpD = Double
- .valueOf(Math.max(0, 1 + Math.ceil(-1 * Math.log10(step))))
+ int tmpD = Double.valueOf(Math.max(0, 1 + Math.ceil(-1 * Math.log10(step))))
.intValue();
d = (d == null) ? tmpD : Math.max(d, tmpD);
double tmp = Math.pow(10.0, d);
@@ -169,59 +174,87 @@ public class MtasDataItemDoubleFull
return result;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/
- public int compareTo(MtasDataItem<Double> o) {
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public int compareTo(MtasDataItem<Double, Double> o) {
int compare = 0;
if (o instanceof MtasDataItemDoubleFull) {
MtasDataItemDoubleFull to = (MtasDataItemDoubleFull) o;
- createStats();
- to.createStats();
- if (sortType.equals(CodecUtil.STATS_TYPE_N)) {
- compare = Long.valueOf(stats.getN()).compareTo(to.stats.getN());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUM)) {
- compare = Double.valueOf(stats.getSum()).compareTo(to.stats.getSum());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MAX)) {
- compare = Double.valueOf(stats.getMax()).compareTo(to.stats.getMax());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MIN)) {
- compare = Double.valueOf(stats.getMin()).compareTo(to.stats.getMin());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUMSQ)) {
- compare = Double.valueOf(stats.getSumsq())
- .compareTo(to.stats.getSumsq());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUMOFLOGS)) {
- compare = Double
- .valueOf(stats.getN() * Math.log(stats.getGeometricMean()))
- .compareTo(to.stats.getN() * Math.log(to.stats.getGeometricMean()));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MEAN)) {
- compare = Double.valueOf(stats.getMean()).compareTo(to.stats.getMean());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_GEOMETRICMEAN)) {
- compare = Double.valueOf(stats.getGeometricMean())
- .compareTo(to.stats.getGeometricMean());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_STANDARDDEVIATION)) {
- compare = Double.valueOf(stats.getStandardDeviation())
- .compareTo(to.stats.getStandardDeviation());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_VARIANCE)) {
- compare = Double.valueOf(stats.getVariance())
- .compareTo(to.stats.getVariance());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_POPULATIONVARIANCE)) {
- compare = Double.valueOf(stats.getPopulationVariance())
- .compareTo(to.stats.getPopulationVariance());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_QUADRATICMEAN)) {
- compare = Double.valueOf(stats.getQuadraticMean())
- .compareTo(to.stats.getQuadraticMean());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_KURTOSIS)) {
- compare = Double.valueOf(stats.getKurtosis())
- .compareTo(to.stats.getKurtosis());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MEDIAN)) {
- compare = Double.valueOf(stats.getPercentile(50))
- .compareTo(to.stats.getPercentile(50));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SKEWNESS)) {
- compare = Double.valueOf(stats.getSkewness())
- .compareTo(to.stats.getSkewness());
- }
+ NumberComparator c1 = getComparableValue();
+ NumberComparator c2 = to.getComparableValue();
+ compare = (c1 != null && c2 != null) ? c1.compareTo(c2.getValue()) : 0;
}
return sortDirection.equals(CodecUtil.SORT_DESC) ? -1 * compare : compare;
}
-
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue1()
+ */
+ @Override
+ public NumberComparator<Double> getCompareValue1() {
+ createStats();
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_SUM:
+ return new NumberComparator<Double>(stats.getSum());
+ case CodecUtil.STATS_TYPE_MAX:
+ return new NumberComparator<Double>(stats.getMax());
+ case CodecUtil.STATS_TYPE_MIN:
+ return new NumberComparator<Double>(stats.getMin());
+ case CodecUtil.STATS_TYPE_SUMSQ:
+ return new NumberComparator<Double>(stats.getSumsq());
+ default:
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue2()
+ */
+ @Override
+ public NumberComparator<Double> getCompareValue2() {
+ createStats();
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_SUMOFLOGS:
+ return new NumberComparator<Double>(
+ stats.getN() * Math.log(stats.getGeometricMean()));
+ case CodecUtil.STATS_TYPE_MEAN:
+ return new NumberComparator<Double>(stats.getMean());
+ case CodecUtil.STATS_TYPE_GEOMETRICMEAN:
+ return new NumberComparator<Double>(stats.getGeometricMean());
+ case CodecUtil.STATS_TYPE_STANDARDDEVIATION:
+ return new NumberComparator<Double>(stats.getStandardDeviation());
+ case CodecUtil.STATS_TYPE_VARIANCE:
+ return new NumberComparator<Double>(stats.getVariance());
+ case CodecUtil.STATS_TYPE_POPULATIONVARIANCE:
+ return new NumberComparator<Double>(stats.getPopulationVariance());
+ case CodecUtil.STATS_TYPE_QUADRATICMEAN:
+ return new NumberComparator<Double>(stats.getQuadraticMean());
+ case CodecUtil.STATS_TYPE_KURTOSIS:
+ return new NumberComparator<Double>(stats.getKurtosis());
+ case CodecUtil.STATS_TYPE_MEDIAN:
+ return new NumberComparator<Double>(stats.getPercentile(50));
+ case CodecUtil.STATS_TYPE_SKEWNESS:
+ return new NumberComparator<Double>(stats.getSkewness());
+ default:
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ public String toString() {
+ return this.getClass().getSimpleName() + "[" + fullValues.length + "]";
+ }
+
}
diff --git a/src/mtas/codec/util/collector/MtasDataItemFull.java b/src/mtas/codec/util/collector/MtasDataItemFull.java
index 6420144..cf062c1 100644
--- a/src/mtas/codec/util/collector/MtasDataItemFull.java
+++ b/src/mtas/codec/util/collector/MtasDataItemFull.java
@@ -10,16 +10,17 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataItemFull.
*
- * @param <T1> the generic type
- * @param <T2> the generic type
+ * @param <T1>
+ * the generic type
+ * @param <T2>
+ * the generic type
*/
-abstract class MtasDataItemFull<T1 extends Number, T2 extends Number>
- extends MtasDataItem<T1> implements Serializable {
+abstract class MtasDataItemFull<T1 extends Number & Comparable<T1>, T2 extends Number & Comparable<T2>>
+ extends MtasDataItem<T1, T2> implements Serializable {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -40,20 +41,31 @@ abstract class MtasDataItemFull<T1 extends Number, T2 extends Number>
/**
* Instantiates a new mtas data item full.
*
- * @param value the value
- * @param sub the sub
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param errorNumber the error number
- * @param errorList the error list
- * @param operations the operations
+ * @param value
+ * the value
+ * @param sub
+ * the sub
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param errorNumber
+ * the error number
+ * @param errorList
+ * the error list
+ * @param operations
+ * the operations
+ * @param sourceNumber
+ * the source number
*/
public MtasDataItemFull(T1[] value, MtasDataCollector<?, ?> sub,
TreeSet<String> statsItems, String sortType, String sortDirection,
int errorNumber, HashMap<String, Integer> errorList,
- MtasDataOperations<T1, T2> operations) {
- super(sub, statsItems, sortType, sortDirection, errorNumber, errorList);
+ MtasDataOperations<T1, T2> operations, int sourceNumber) {
+ super(sub, statsItems, sortType, sortDirection, errorNumber, errorList,
+ sourceNumber);
this.fullValues = value;
this.operations = operations;
}
@@ -65,15 +77,16 @@ abstract class MtasDataItemFull<T1 extends Number, T2 extends Number>
* DataCollector.MtasDataItem)
*/
@Override
- public void add(MtasDataItem<T1> newItem) throws IOException {
+ public void add(MtasDataItem<T1, T2> newItem) throws IOException {
if (newItem instanceof MtasDataItemFull) {
MtasDataItemFull<T1, T2> newTypedItem = (MtasDataItemFull<T1, T2>) newItem;
T1[] tmpValue = operations
.createVector1(fullValues.length + newTypedItem.fullValues.length);
System.arraycopy(fullValues, 0, tmpValue, 0, fullValues.length);
- System.arraycopy(newTypedItem.fullValues, 0, tmpValue,
- fullValues.length, newTypedItem.fullValues.length);
+ System.arraycopy(newTypedItem.fullValues, 0, tmpValue, fullValues.length,
+ newTypedItem.fullValues.length);
fullValues = tmpValue;
+ recomputeComparableSortValue = true;
} else {
throw new IOException("can only add MtasDataItemFull");
}
@@ -94,11 +107,11 @@ abstract class MtasDataItemFull<T1 extends Number, T2 extends Number>
/**
* Gets the distribution.
*
- * @param arguments the arguments
+ * @param arguments
+ * the arguments
* @return the distribution
*/
- abstract protected HashMap<String, Object> getDistribution(
- String arguments);
+ abstract protected HashMap<String, Object> getDistribution(String arguments);
/*
* (non-Javadoc)
@@ -106,7 +119,7 @@ abstract class MtasDataItemFull<T1 extends Number, T2 extends Number>
* @see mtas.codec.util.DataCollector.MtasDataItem#rewrite()
*/
@Override
- public Map<String, Object> rewrite() throws IOException {
+ public Map<String, Object> rewrite(boolean showDebugInfo) throws IOException {
createStats();
Map<String, Object> response = new HashMap<String, Object>();
for (String statsItem : statsItems) {
@@ -147,11 +160,7 @@ abstract class MtasDataItemFull<T1 extends Number, T2 extends Number>
String function = m.group(2).trim();
if (function.equals(CodecUtil.STATS_FUNCTION_DISTRIBUTION)) {
response.put(statsItem, getDistribution(m.group(4)));
- } else {
- response.put(statsItem, "test");
}
- } else {
- response.put(statsItem, "niet");
}
}
}
@@ -163,12 +172,69 @@ abstract class MtasDataItemFull<T1 extends Number, T2 extends Number>
response.put("errorNumber", errorNumber);
response.put("errorList", errorResponse);
}
- // response.put("stats", "full");
+ if (showDebugInfo) {
+ response.put("sourceNumber", sourceNumber);
+ response.put("stats", "full");
+ }
return response;
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValueType()
+ */
+ @Override
+ public final int getCompareValueType() throws IOException {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_N:
+ return 0;
+ case CodecUtil.STATS_TYPE_SUM:
+ return 1;
+ case CodecUtil.STATS_TYPE_MAX:
+ return 1;
+ case CodecUtil.STATS_TYPE_MIN:
+ return 1;
+ case CodecUtil.STATS_TYPE_SUMSQ:
+ return 1;
+ case CodecUtil.STATS_TYPE_SUMOFLOGS:
+ return 2;
+ case CodecUtil.STATS_TYPE_MEAN:
+ return 2;
+ case CodecUtil.STATS_TYPE_GEOMETRICMEAN:
+ return 2;
+ case CodecUtil.STATS_TYPE_STANDARDDEVIATION:
+ return 2;
+ case CodecUtil.STATS_TYPE_VARIANCE:
+ return 2;
+ case CodecUtil.STATS_TYPE_POPULATIONVARIANCE:
+ return 2;
+ case CodecUtil.STATS_TYPE_QUADRATICMEAN:
+ return 2;
+ case CodecUtil.STATS_TYPE_KURTOSIS:
+ return 2;
+ case CodecUtil.STATS_TYPE_MEDIAN:
+ return 2;
+ case CodecUtil.STATS_TYPE_SKEWNESS:
+ return 2;
+ default:
+ throw new IOException("sortType " + sortType + " not supported");
+ }
+ }
-
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue0()
+ */
+ public final NumberComparator<Long> getCompareValue0() {
+ createStats();
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_N:
+ return new NumberComparator<Long>(stats.getN());
+ default:
+ return null;
+ }
+ }
}
-
diff --git a/src/mtas/codec/util/collector/MtasDataItemLongAdvanced.java b/src/mtas/codec/util/collector/MtasDataItemLongAdvanced.java
index 356ef5e..456b6f1 100644
--- a/src/mtas/codec/util/collector/MtasDataItemLongAdvanced.java
+++ b/src/mtas/codec/util/collector/MtasDataItemLongAdvanced.java
@@ -3,13 +3,11 @@ package mtas.codec.util.collector;
import java.util.HashMap;
import java.util.TreeSet;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataItemLongAdvanced.
*/
-class MtasDataItemLongAdvanced
- extends MtasDataItemAdvanced<Long, Double> {
+class MtasDataItemLongAdvanced extends MtasDataItemAdvanced<Long, Double> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -17,27 +15,41 @@ class MtasDataItemLongAdvanced
/**
* Instantiates a new mtas data item long advanced.
*
- * @param valueSum the value sum
- * @param valueSumOfLogs the value sum of logs
- * @param valueSumOfSquares the value sum of squares
- * @param valueMin the value min
- * @param valueMax the value max
- * @param valueN the value n
- * @param sub the sub
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param errorNumber the error number
- * @param errorList the error list
+ * @param valueSum
+ * the value sum
+ * @param valueSumOfLogs
+ * the value sum of logs
+ * @param valueSumOfSquares
+ * the value sum of squares
+ * @param valueMin
+ * the value min
+ * @param valueMax
+ * the value max
+ * @param valueN
+ * the value n
+ * @param sub
+ * the sub
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param errorNumber
+ * the error number
+ * @param errorList
+ * the error list
+ * @param sourceNumber
+ * the source number
*/
- public MtasDataItemLongAdvanced(long valueSum, double valueSumOfLogs,
- long valueSumOfSquares, long valueMin, long valueMax, long valueN,
- MtasDataCollector<?, ?> sub, TreeSet<String> statsItems,
- String sortType, String sortDirection, int errorNumber,
- HashMap<String, Integer> errorList) {
+ public MtasDataItemLongAdvanced(Long valueSum, Double valueSumOfLogs,
+ Long valueSumOfSquares, Long valueMin, Long valueMax, long valueN,
+ MtasDataCollector<?, ?> sub, TreeSet<String> statsItems, String sortType,
+ String sortDirection, int errorNumber, HashMap<String, Integer> errorList,
+ int sourceNumber) {
super(valueSum, valueSumOfLogs, valueSumOfSquares, valueMin, valueMax,
valueN, sub, statsItems, sortType, sortDirection, errorNumber,
- errorList, new MtasDataLongOperations());
+ errorList, new MtasDataLongOperations(), sourceNumber);
}
/*
@@ -45,38 +57,74 @@ class MtasDataItemLongAdvanced
*
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/
+ @SuppressWarnings({ "rawtypes", "unchecked" })
@Override
- public int compareTo(MtasDataItem<Long> o) {
+ public int compareTo(MtasDataItem<Long, Double> o) {
int compare = 0;
if (o instanceof MtasDataItemLongAdvanced) {
MtasDataItemLongAdvanced to = (MtasDataItemLongAdvanced) o;
- if (sortType.equals(CodecUtil.STATS_TYPE_N)) {
- compare = valueN.compareTo(to.valueN);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUM)) {
- compare = valueSum.compareTo(to.valueSum);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MAX)) {
- compare = valueMax.compareTo(to.valueMax);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MIN)) {
- compare = valueMin.compareTo(to.valueMin);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUMSQ)) {
- compare = valueSumOfSquares.compareTo(to.valueSumOfSquares);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUMOFLOGS)) {
- compare = valueSumOfLogs.compareTo(to.valueSumOfLogs);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MEAN)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_GEOMETRICMEAN)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_STANDARDDEVIATION)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_VARIANCE)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_POPULATIONVARIANCE)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_QUADRATICMEAN)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- }
+ NumberComparator c1 = getComparableValue();
+ NumberComparator c2 = to.getComparableValue();
+ compare = (c1 != null && c2 != null) ? c1.compareTo(c2.getValue()) : 0;
}
return sortDirection.equals(CodecUtil.SORT_DESC) ? -1 * compare : compare;
}
-}
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue1()
+ */
+ @Override
+ public NumberComparator<Long> getCompareValue1() {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_SUM:
+ return new NumberComparator<Long>(valueSum);
+ case CodecUtil.STATS_TYPE_MAX:
+ return new NumberComparator<Long>(valueMax);
+ case CodecUtil.STATS_TYPE_MIN:
+ return new NumberComparator<Long>(valueMin);
+ case CodecUtil.STATS_TYPE_SUMSQ:
+ return new NumberComparator<Long>(valueSumOfSquares);
+ default:
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue2()
+ */
+ public NumberComparator<Double> getCompareValue2() {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_SUMOFLOGS:
+ return new NumberComparator<Double>(valueSumOfLogs);
+ case CodecUtil.STATS_TYPE_MEAN:
+ return new NumberComparator<Double>(getValue(sortType));
+ case CodecUtil.STATS_TYPE_GEOMETRICMEAN:
+ return new NumberComparator<Double>(getValue(sortType));
+ case CodecUtil.STATS_TYPE_STANDARDDEVIATION:
+ return new NumberComparator<Double>(getValue(sortType));
+ case CodecUtil.STATS_TYPE_VARIANCE:
+ return new NumberComparator<Double>(getValue(sortType));
+ case CodecUtil.STATS_TYPE_POPULATIONVARIANCE:
+ return new NumberComparator<Double>(getValue(sortType));
+ case CodecUtil.STATS_TYPE_QUADRATICMEAN:
+ return new NumberComparator<Double>(getValue(sortType));
+ default:
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ public String toString() {
+ return this.getClass().getSimpleName() + "[" + valueSum + "," + valueN
+ + "]";
+ }
+
+}
diff --git a/src/mtas/codec/util/collector/MtasDataItemLongBasic.java b/src/mtas/codec/util/collector/MtasDataItemLongBasic.java
index f1b5051..69e5a42 100644
--- a/src/mtas/codec/util/collector/MtasDataItemLongBasic.java
+++ b/src/mtas/codec/util/collector/MtasDataItemLongBasic.java
@@ -3,13 +3,11 @@ package mtas.codec.util.collector;
import java.util.HashMap;
import java.util.TreeSet;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataItemLongBasic.
*/
-class MtasDataItemLongBasic
- extends MtasDataItemBasic<Long, Double> {
+class MtasDataItemLongBasic extends MtasDataItemBasic<Long, Double> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -17,21 +15,31 @@ class MtasDataItemLongBasic
/**
* Instantiates a new mtas data item long basic.
*
- * @param valueSum the value sum
- * @param valueN the value n
- * @param sub the sub
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param errorNumber the error number
- * @param errorList the error list
+ * @param valueSum
+ * the value sum
+ * @param valueN
+ * the value n
+ * @param sub
+ * the sub
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param errorNumber
+ * the error number
+ * @param errorList
+ * the error list
+ * @param sourceNumber
+ * the source number
*/
- public MtasDataItemLongBasic(long valueSum, long valueN,
- MtasDataCollector<?, ?> sub, TreeSet<String> statsItems,
- String sortType, String sortDirection, int errorNumber,
- HashMap<String, Integer> errorList) {
+ public MtasDataItemLongBasic(Long valueSum, long valueN,
+ MtasDataCollector<?, ?> sub, TreeSet<String> statsItems, String sortType,
+ String sortDirection, int errorNumber, HashMap<String, Integer> errorList,
+ int sourceNumber) {
super(valueSum, valueN, sub, statsItems, sortType, sortDirection,
- errorNumber, errorList, new MtasDataLongOperations());
+ errorNumber, errorList, new MtasDataLongOperations(), sourceNumber);
}
/*
@@ -39,21 +47,59 @@ class MtasDataItemLongBasic
*
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/
+ @SuppressWarnings({ "rawtypes", "unchecked" })
@Override
- public int compareTo(MtasDataItem<Long> o) {
+ public int compareTo(MtasDataItem<Long, Double> o) {
int compare = 0;
if (o instanceof MtasDataItemLongBasic) {
MtasDataItemLongBasic to = (MtasDataItemLongBasic) o;
- if (sortType.equals(CodecUtil.STATS_TYPE_N)) {
- compare = valueN.compareTo(to.valueN);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUM)) {
- compare = valueSum.compareTo(to.valueSum);
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MEAN)) {
- compare = getValue(sortType).compareTo(to.getValue(sortType));
- }
+ NumberComparator c1 = getComparableValue();
+ NumberComparator c2 = to.getComparableValue();
+ compare = (c1 != null && c2 != null) ? c1.compareTo(c2.getValue()) : 0;
}
return sortDirection.equals(CodecUtil.SORT_DESC) ? -1 * compare : compare;
}
-}
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue()
+ */
+ @Override
+ public NumberComparator<Long> getCompareValue1() {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_N:
+ return new NumberComparator<Long>(valueN);
+ case CodecUtil.STATS_TYPE_SUM:
+ return new NumberComparator<Long>(valueSum);
+ default:
+ return null;
+ }
+ }
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue2()
+ */
+ @Override
+ public NumberComparator<Double> getCompareValue2() {
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_MEAN:
+ return new NumberComparator<Double>(getValue(sortType));
+ default:
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ public String toString() {
+ return this.getClass().getSimpleName() + "[" + valueSum + "," + valueN
+ + "]";
+ }
+
+}
diff --git a/src/mtas/codec/util/collector/MtasDataItemLongFull.java b/src/mtas/codec/util/collector/MtasDataItemLongFull.java
index 9aae494..f3b7345 100644
--- a/src/mtas/codec/util/collector/MtasDataItemLongFull.java
+++ b/src/mtas/codec/util/collector/MtasDataItemLongFull.java
@@ -7,7 +7,6 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.ArrayUtils;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataItemLongFull.
@@ -23,19 +22,28 @@ class MtasDataItemLongFull extends MtasDataItemFull<Long, Double> {
/**
* Instantiates a new mtas data item long full.
*
- * @param value the value
- * @param sub the sub
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param errorNumber the error number
- * @param errorList the error list
+ * @param value
+ * the value
+ * @param sub
+ * the sub
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param errorNumber
+ * the error number
+ * @param errorList
+ * the error list
+ * @param sourceNumber
+ * the source number
*/
public MtasDataItemLongFull(long[] value, MtasDataCollector<?, ?> sub,
TreeSet<String> statsItems, String sortType, String sortDirection,
- int errorNumber, HashMap<String, Integer> errorList) {
+ int errorNumber, HashMap<String, Integer> errorList, int sourceNumber) {
super(ArrayUtils.toObject(value), sub, statsItems, sortType, sortDirection,
- errorNumber, errorList, new MtasDataLongOperations());
+ errorNumber, errorList, new MtasDataLongOperations(), sourceNumber);
}
/*
@@ -137,59 +145,87 @@ class MtasDataItemLongFull extends MtasDataItemFull<Long, Double> {
return result;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/
- public int compareTo(MtasDataItem<Long> o) {
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public int compareTo(MtasDataItem<Long, Double> o) {
int compare = 0;
if (o instanceof MtasDataItemLongFull) {
MtasDataItemLongFull to = (MtasDataItemLongFull) o;
- createStats();
- to.createStats();
- if (sortType.equals(CodecUtil.STATS_TYPE_N)) {
- compare = Long.valueOf(stats.getN()).compareTo(to.stats.getN());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUM)) {
- compare = Double.valueOf(stats.getSum()).compareTo(to.stats.getSum());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MAX)) {
- compare = Double.valueOf(stats.getMax()).compareTo(to.stats.getMax());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MIN)) {
- compare = Double.valueOf(stats.getMin()).compareTo(to.stats.getMin());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUMSQ)) {
- compare = Double.valueOf(stats.getSumsq())
- .compareTo(to.stats.getSumsq());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SUMOFLOGS)) {
- compare = Double
- .valueOf(stats.getN() * Math.log(stats.getGeometricMean()))
- .compareTo(to.stats.getN() * Math.log(to.stats.getGeometricMean()));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MEAN)) {
- compare = Double.valueOf(stats.getMean()).compareTo(to.stats.getMean());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_GEOMETRICMEAN)) {
- compare = Double.valueOf(stats.getGeometricMean())
- .compareTo(to.stats.getGeometricMean());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_STANDARDDEVIATION)) {
- compare = Double.valueOf(stats.getStandardDeviation())
- .compareTo(to.stats.getStandardDeviation());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_VARIANCE)) {
- compare = Double.valueOf(stats.getVariance())
- .compareTo(to.stats.getVariance());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_POPULATIONVARIANCE)) {
- compare = Double.valueOf(stats.getPopulationVariance())
- .compareTo(to.stats.getPopulationVariance());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_QUADRATICMEAN)) {
- compare = Double.valueOf(stats.getQuadraticMean())
- .compareTo(to.stats.getQuadraticMean());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_KURTOSIS)) {
- compare = Double.valueOf(stats.getKurtosis())
- .compareTo(to.stats.getKurtosis());
- } else if (sortType.equals(CodecUtil.STATS_TYPE_MEDIAN)) {
- compare = Double.valueOf(stats.getPercentile(50))
- .compareTo(to.stats.getPercentile(50));
- } else if (sortType.equals(CodecUtil.STATS_TYPE_SKEWNESS)) {
- compare = Double.valueOf(stats.getSkewness())
- .compareTo(to.stats.getSkewness());
- }
+ NumberComparator c1 = getComparableValue();
+ NumberComparator c2 = to.getComparableValue();
+ compare = (c1 != null && c2 != null) ? c1.compareTo(c2.getValue()) : 0;
}
return sortDirection.equals(CodecUtil.SORT_DESC) ? -1 * compare : compare;
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue1()
+ */
+ @Override
+ public NumberComparator<Long> getCompareValue1() {
+ createStats();
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_SUM:
+ return new NumberComparator<Long>(Math.round(stats.getSum()));
+ case CodecUtil.STATS_TYPE_MAX:
+ return new NumberComparator<Long>(Math.round(stats.getMax()));
+ case CodecUtil.STATS_TYPE_MIN:
+ return new NumberComparator<Long>(Math.round(stats.getMin()));
+ case CodecUtil.STATS_TYPE_SUMSQ:
+ return new NumberComparator<Long>(Math.round(stats.getSumsq()));
+ default:
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.codec.util.collector.MtasDataItem#getCompareValue2()
+ */
+ @Override
+ public NumberComparator<Double> getCompareValue2() {
+ createStats();
+ switch (sortType) {
+ case CodecUtil.STATS_TYPE_SUMOFLOGS:
+ return new NumberComparator<Double>(
+ stats.getN() * Math.log(stats.getGeometricMean()));
+ case CodecUtil.STATS_TYPE_MEAN:
+ return new NumberComparator<Double>(stats.getMean());
+ case CodecUtil.STATS_TYPE_GEOMETRICMEAN:
+ return new NumberComparator<Double>(stats.getGeometricMean());
+ case CodecUtil.STATS_TYPE_STANDARDDEVIATION:
+ return new NumberComparator<Double>(stats.getStandardDeviation());
+ case CodecUtil.STATS_TYPE_VARIANCE:
+ return new NumberComparator<Double>(stats.getVariance());
+ case CodecUtil.STATS_TYPE_POPULATIONVARIANCE:
+ return new NumberComparator<Double>(stats.getPopulationVariance());
+ case CodecUtil.STATS_TYPE_QUADRATICMEAN:
+ return new NumberComparator<Double>(stats.getQuadraticMean());
+ case CodecUtil.STATS_TYPE_KURTOSIS:
+ return new NumberComparator<Double>(stats.getKurtosis());
+ case CodecUtil.STATS_TYPE_MEDIAN:
+ return new NumberComparator<Double>(stats.getPercentile(50));
+ case CodecUtil.STATS_TYPE_SKEWNESS:
+ return new NumberComparator<Double>(stats.getSkewness());
+ default:
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ public String toString() {
+ return this.getClass().getSimpleName() + "[" + fullValues.length + "]";
+ }
+
}
diff --git a/src/mtas/codec/util/collector/MtasDataLongAdvanced.java b/src/mtas/codec/util/collector/MtasDataLongAdvanced.java
index 77a8716..a66cbef 100644
--- a/src/mtas/codec/util/collector/MtasDataLongAdvanced.java
+++ b/src/mtas/codec/util/collector/MtasDataLongAdvanced.java
@@ -5,13 +5,11 @@ import java.util.Collections;
import java.util.TreeSet;
import org.apache.commons.lang.ArrayUtils;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataLongAdvanced.
*/
-public class MtasDataLongAdvanced
- extends MtasDataAdvanced<Long, Double, MtasDataItemLongAdvanced> {
+public class MtasDataLongAdvanced extends MtasDataAdvanced<Long, Double> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -19,35 +17,51 @@ public class MtasDataLongAdvanced
/**
* Instantiates a new mtas data long advanced.
*
- * @param collectorType the collector type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param subCollectorTypes the sub collector types
- * @param subDataTypes the sub data types
- * @param subStatsTypes the sub stats types
- * @param subStatsItems the sub stats items
- * @param subSortTypes the sub sort types
- * @param subSortDirections the sub sort directions
- * @param subStart the sub start
- * @param subNumber the sub number
- * @param segmentRegistration the segment registration
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public MtasDataLongAdvanced(String collectorType,
- TreeSet<String> statsItems, String sortType, String sortDirection,
- Integer start, Integer number, String[] subCollectorTypes,
- String[] subDataTypes, String[] subStatsTypes,
+ * @param collectorType
+ * the collector type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param subCollectorTypes
+ * the sub collector types
+ * @param subDataTypes
+ * the sub data types
+ * @param subStatsTypes
+ * the sub stats types
+ * @param subStatsItems
+ * the sub stats items
+ * @param subSortTypes
+ * the sub sort types
+ * @param subSortDirections
+ * the sub sort directions
+ * @param subStart
+ * the sub start
+ * @param subNumber
+ * the sub number
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public MtasDataLongAdvanced(String collectorType, TreeSet<String> statsItems,
+ String sortType, String sortDirection, Integer start, Integer number,
+ String[] subCollectorTypes, String[] subDataTypes, String[] subStatsTypes,
TreeSet<String>[] subStatsItems, String[] subSortTypes,
String[] subSortDirections, Integer[] subStart, Integer[] subNumber,
- boolean segmentRegistration) throws IOException {
+ String segmentRegistration, String boundary) throws IOException {
super(collectorType, CodecUtil.DATA_TYPE_LONG, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
- subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, new MtasDataLongOperations(),
- segmentRegistration);
+ subStatsTypes, subStatsItems, subSortTypes, subSortDirections, subStart,
+ subNumber, new MtasDataLongOperations(), segmentRegistration, boundary);
}
/*
@@ -57,11 +71,16 @@ public class MtasDataLongAdvanced
*/
@Override
protected final MtasDataItemLongAdvanced getItem(int i) {
- return new MtasDataItemLongAdvanced(advancedValueSumList[i],
- advancedValueSumOfLogsList[i], advancedValueSumOfSquaresList[i],
- advancedValueMinList[i], advancedValueMaxList[i],
- advancedValueNList[i], hasSub() ? subCollectorListNextLevel[i] : null,
- statsItems, sortType, sortDirection, errorNumber[i], errorList[i]);
+ if (i >= 0 && i < size) {
+ return new MtasDataItemLongAdvanced(advancedValueSumList[i],
+ advancedValueSumOfLogsList[i], advancedValueSumOfSquaresList[i],
+ advancedValueMinList[i], advancedValueMaxList[i],
+ advancedValueNList[i], hasSub() ? subCollectorListNextLevel[i] : null,
+ statsItems, sortType, sortDirection, errorNumber[i], errorList[i],
+ sourceNumberList[i]);
+ } else {
+ return null;
+ }
}
/*
@@ -83,7 +102,7 @@ public class MtasDataLongAdvanced
@Override
public MtasDataCollector<?, ?> add(long[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
return dataCollector;
@@ -108,7 +127,7 @@ public class MtasDataLongAdvanced
@Override
public MtasDataCollector<?, ?> add(double[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
Long[] newValues = new Long[number];
for (int i = 0; i < values.length; i++)
newValues[i] = Double.valueOf(values[i]).longValue();
@@ -137,12 +156,12 @@ public class MtasDataLongAdvanced
* long[], int)
*/
@Override
- public MtasDataCollector<?, ?>[] add(String[] keys, long[] values,
- int number) throws IOException {
+ public MtasDataCollector<?, ?>[] add(String[] keys, long[] values, int number)
+ throws IOException {
if (keys != null && keys.length > 0) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
}
@@ -181,7 +200,7 @@ public class MtasDataLongAdvanced
newValues[i] = Double.valueOf(values[i]).longValue();
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, newValues, number, newCurrentExisting);
}
return subCollectors;
@@ -190,56 +209,178 @@ public class MtasDataLongAdvanced
}
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected boolean compareForComputingSegment(Long value, Long boundary) {
- return value >= boundary;
+ protected boolean compareWithBoundary(Long value, Long boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return value <= boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return value >= boundary;
+ } else {
+ throw new IOException(
+ "can't compare for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected Long minimumForComputingSegment(Long value, Long boundary) {
- return Math.min(value, boundary);
+ protected Long lastForComputingSegment(Long value, Long boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return Math.max(value, boundary);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return Math.min(value, boundary);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * )
*/
@Override
- protected Long minimumForComputingSegment() {
- return Collections.min(segmentValueMaxList);
+ protected Long lastForComputingSegment() throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return Collections.max(segmentValueTopList);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return Collections.min(segmentValueTopList);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment
+ * ()
*/
@Override
- protected Long boundaryForComputingSegment() {
- Long boundary = boundaryForSegment();
- long correctionBoundary = 0;
- for (String otherSegmentName : segmentValueMaxListMin.keySet()) {
- if (!otherSegmentName.equals(segmentName)) {
- Long otherBoundary = segmentValueBoundary.get(otherSegmentName);
- if (otherBoundary != null) {
- correctionBoundary += Math.max(0, otherBoundary - boundary);
+ protected Long boundaryForSegmentComputing(String segmentName)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Long boundary = boundaryForSegment(segmentName);
+ if (boundary == null) {
+ return null;
+ } else {
+ if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ long correctionBoundary = 0;
+ for (String otherSegmentName : segmentValueTopListLast.keySet()) {
+ if (!otherSegmentName.equals(segmentName)) {
+ Long otherBoundary = segmentValuesBoundary.get(otherSegmentName);
+ if (otherBoundary != null) {
+ correctionBoundary += Math.max(0, otherBoundary - boundary);
+ }
+ }
+ }
+ return boundary + correctionBoundary;
+ } else {
+ return boundary;
}
}
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
}
- return boundary + correctionBoundary;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForSegment()
*/
@Override
- protected Long boundaryForSegment() {
- long thisMin = segmentValueMaxListMin.get(segmentName);
- Long boundary = Math.floorDiv(thisMin, segmentNumber);
- return boundary;
+ protected Long boundaryForSegment(String segmentName) throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Long thisLast = segmentValueTopListLast.get(segmentName);
+ if (thisLast == null) {
+ return null;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_ASC)) {
+ Long boundary = thisLast * segmentNumber;
+ return boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Long boundary = Math.floorDiv(thisLast, segmentNumber);
+ return boundary;
+ } else {
+ // should not happen
+ return null;
+ }
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#stringToBoundary(java.lang.
+ * String, java.lang.Integer)
+ */
+ @Override
+ protected Long stringToBoundary(String boundary, Integer segmentNumber)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ if (segmentNumber == null) {
+ return Long.valueOf(boundary);
+ } else {
+ return Math.floorDiv(Long.valueOf(boundary), segmentNumber);
+ }
+ } else if (segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ if (segmentNumber == null) {
+ return Long.valueOf(boundary);
+ } else {
+ return Math.floorDiv(Long.valueOf(boundary), segmentNumber);
+ }
+ } else {
+ throw new IOException(
+ "not available for segmentRegistration " + segmentRegistration);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#validateSegmentBoundary(java.
+ * lang.Object)
+ */
+ @Override
+ public boolean validateSegmentBoundary(Object o) throws IOException {
+ if (o instanceof Long) {
+ return validateWithSegmentBoundary((Long) o);
+ } else {
+ throw new IOException("incorrect type");
+ }
}
}
diff --git a/src/mtas/codec/util/collector/MtasDataLongBasic.java b/src/mtas/codec/util/collector/MtasDataLongBasic.java
index 23cfd5e..b56fcac 100644
--- a/src/mtas/codec/util/collector/MtasDataLongBasic.java
+++ b/src/mtas/codec/util/collector/MtasDataLongBasic.java
@@ -5,13 +5,11 @@ import java.util.Collections;
import java.util.TreeSet;
import org.apache.commons.lang.ArrayUtils;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataLongBasic.
*/
-public class MtasDataLongBasic
- extends MtasDataBasic<Long, Double, MtasDataItemLongBasic> {
+public class MtasDataLongBasic extends MtasDataBasic<Long, Double> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -19,34 +17,51 @@ public class MtasDataLongBasic
/**
* Instantiates a new mtas data long basic.
*
- * @param collectorType the collector type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param subCollectorTypes the sub collector types
- * @param subDataTypes the sub data types
- * @param subStatsTypes the sub stats types
- * @param subStatsItems the sub stats items
- * @param subSortTypes the sub sort types
- * @param subSortDirections the sub sort directions
- * @param subStart the sub start
- * @param subNumber the sub number
- * @param segmentRegistration the segment registration
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param collectorType
+ * the collector type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param subCollectorTypes
+ * the sub collector types
+ * @param subDataTypes
+ * the sub data types
+ * @param subStatsTypes
+ * the sub stats types
+ * @param subStatsItems
+ * the sub stats items
+ * @param subSortTypes
+ * the sub sort types
+ * @param subSortDirections
+ * the sub sort directions
+ * @param subStart
+ * the sub start
+ * @param subNumber
+ * the sub number
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasDataLongBasic(String collectorType, TreeSet<String> statsItems,
String sortType, String sortDirection, Integer start, Integer number,
- String[] subCollectorTypes, String[] subDataTypes,
- String[] subStatsTypes, TreeSet<String>[] subStatsItems,
- String[] subSortTypes, String[] subSortDirections, Integer[] subStart,
- Integer[] subNumber, boolean segmentRegistration) throws IOException {
+ String[] subCollectorTypes, String[] subDataTypes, String[] subStatsTypes,
+ TreeSet<String>[] subStatsItems, String[] subSortTypes,
+ String[] subSortDirections, Integer[] subStart, Integer[] subNumber,
+ String segmentRegistration, String boundary) throws IOException {
super(collectorType, CodecUtil.DATA_TYPE_LONG, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
- subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, new MtasDataLongOperations(),
- segmentRegistration);
+ subStatsTypes, subStatsItems, subSortTypes, subSortDirections, subStart,
+ subNumber, new MtasDataLongOperations(), segmentRegistration, boundary);
}
/*
@@ -56,9 +71,13 @@ public class MtasDataLongBasic
*/
@Override
protected MtasDataItemLongBasic getItem(int i) {
- return new MtasDataItemLongBasic(basicValueSumList[i], basicValueNList[i],
- hasSub() ? subCollectorListNextLevel[i] : null, statsItems, sortType,
- sortDirection, errorNumber[i], errorList[i]);
+ if (i >= 0 && i < size) {
+ return new MtasDataItemLongBasic(basicValueSumList[i], basicValueNList[i],
+ hasSub() ? subCollectorListNextLevel[i] : null, statsItems, sortType,
+ sortDirection, errorNumber[i], errorList[i], sourceNumberList[i]);
+ } else {
+ return null;
+ }
}
/*
@@ -69,7 +88,7 @@ public class MtasDataLongBasic
@Override
public MtasDataCollector<?, ?> add(long valueSum, long valueN)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
setValue(newCurrentPosition, valueSum, valueN, newCurrentExisting);
return dataCollector;
}
@@ -82,7 +101,7 @@ public class MtasDataLongBasic
@Override
public MtasDataCollector<?, ?> add(long[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
return dataCollector;
@@ -96,7 +115,7 @@ public class MtasDataLongBasic
@Override
public MtasDataCollector<?, ?> add(double valueSum, long valueN)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
setValue(newCurrentPosition, Double.valueOf(valueSum).longValue(), valueN,
newCurrentExisting);
return dataCollector;
@@ -110,7 +129,7 @@ public class MtasDataLongBasic
@Override
public MtasDataCollector<?, ?> add(double[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
Long[] newValues = new Long[number];
for (int i = 0; i < values.length; i++)
newValues[i] = Double.valueOf(values[i]).longValue();
@@ -131,7 +150,7 @@ public class MtasDataLongBasic
if (keys != null && keys.length > 0) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, valueSum, valueN, newCurrentExisting);
}
return subCollectors;
@@ -148,12 +167,12 @@ public class MtasDataLongBasic
* long[], int)
*/
@Override
- public MtasDataCollector<?, ?>[] add(String[] keys, long[] values,
- int number) throws IOException {
+ public MtasDataCollector<?, ?>[] add(String[] keys, long[] values, int number)
+ throws IOException {
if (keys != null && keys.length > 0) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
}
@@ -176,7 +195,7 @@ public class MtasDataLongBasic
if (keys != null && keys.length > 0) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, Double.valueOf(valueSum).longValue(),
valueN, newCurrentExisting);
}
@@ -202,7 +221,7 @@ public class MtasDataLongBasic
newValues[i] = Double.valueOf(values[i]).longValue();
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, newValues, number, newCurrentExisting);
}
return subCollectors;
@@ -211,57 +230,178 @@ public class MtasDataLongBasic
}
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected boolean compareForComputingSegment(Long value, Long boundary) {
- return value >= boundary;
+ protected boolean compareWithBoundary(Long value, Long boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return value <= boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return value >= boundary;
+ } else {
+ throw new IOException(
+ "can't compare for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected Long minimumForComputingSegment(Long value, Long boundary) {
- return Math.min(value, boundary);
+ protected Long lastForComputingSegment(Long value, Long boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return Math.max(value, boundary);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return Math.min(value, boundary);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * )
*/
@Override
- protected Long minimumForComputingSegment() {
- return Collections.min(segmentValueMaxList);
+ protected Long lastForComputingSegment() throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return Collections.max(segmentValueTopList);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return Collections.min(segmentValueTopList);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment
+ * ()
*/
@Override
- protected Long boundaryForComputingSegment() {
- Long boundary = boundaryForSegment();
- long correctionBoundary = 0;
- for (String otherSegmentName : segmentValueMaxListMin.keySet()) {
- if (!otherSegmentName.equals(segmentName)) {
- Long otherBoundary = segmentValueBoundary.get(otherSegmentName);
- if (otherBoundary != null) {
- correctionBoundary += Math.max(0, otherBoundary - boundary);
+ protected Long boundaryForSegmentComputing(String segmentName)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Long boundary = boundaryForSegment(segmentName);
+ if (boundary == null) {
+ return null;
+ } else {
+ if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ long correctionBoundary = 0;
+ for (String otherSegmentName : segmentValueTopListLast.keySet()) {
+ if (!otherSegmentName.equals(segmentName)) {
+ Long otherBoundary = segmentValuesBoundary.get(otherSegmentName);
+ if (otherBoundary != null) {
+ correctionBoundary += Math.max(0, otherBoundary - boundary);
+ }
+ }
+ }
+ return boundary + correctionBoundary;
+ } else {
+ return boundary;
}
}
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
}
- return boundary + correctionBoundary;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForSegment()
*/
@Override
- protected Long boundaryForSegment() {
- long thisMin = segmentValueMaxListMin.get(segmentName);
- Long boundary = Math.floorDiv(thisMin, segmentNumber);
- return boundary;
+ protected Long boundaryForSegment(String segmentName) throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Long thisLast = segmentValueTopListLast.get(segmentName);
+ if (thisLast == null) {
+ return null;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_ASC)) {
+ Long boundary = thisLast * segmentNumber;
+ return boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Long boundary = Math.floorDiv(thisLast, segmentNumber);
+ return boundary;
+ } else {
+ // should not happen
+ return null;
+ }
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
+ }
}
-}
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#stringToBoundary(java.lang.
+ * String, java.lang.Integer)
+ */
+ @Override
+ protected Long stringToBoundary(String boundary, Integer segmentNumber)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ if (segmentNumber == null) {
+ return Long.valueOf(boundary);
+ } else {
+ return Math.floorDiv(Long.valueOf(boundary), segmentNumber);
+ }
+ } else if (segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ if (segmentNumber == null) {
+ return Long.valueOf(boundary);
+ } else {
+ return Math.floorDiv(Long.valueOf(boundary), segmentNumber);
+ }
+ } else {
+ throw new IOException(
+ "not available for segmentRegistration " + segmentRegistration);
+ }
+ }
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#validateSegmentBoundary(java.
+ * lang.Object)
+ */
+ @Override
+ public boolean validateSegmentBoundary(Object o) throws IOException {
+ if (o instanceof Long) {
+ return validateWithSegmentBoundary((Long) o);
+ } else {
+ throw new IOException("incorrect type ");
+ }
+ }
+
+}
diff --git a/src/mtas/codec/util/collector/MtasDataLongFull.java b/src/mtas/codec/util/collector/MtasDataLongFull.java
index d7f62f3..0c752c8 100644
--- a/src/mtas/codec/util/collector/MtasDataLongFull.java
+++ b/src/mtas/codec/util/collector/MtasDataLongFull.java
@@ -5,13 +5,11 @@ import java.util.Collections;
import java.util.TreeSet;
import org.apache.commons.lang.ArrayUtils;
import mtas.codec.util.CodecUtil;
-import mtas.codec.util.DataCollector.MtasDataCollector;
/**
* The Class MtasDataLongFull.
*/
-public class MtasDataLongFull
- extends MtasDataFull<Long, Double, MtasDataItemLongFull> {
+public class MtasDataLongFull extends MtasDataFull<Long, Double> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -19,34 +17,51 @@ public class MtasDataLongFull
/**
* Instantiates a new mtas data long full.
*
- * @param collectorType the collector type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- * @param subCollectorTypes the sub collector types
- * @param subDataTypes the sub data types
- * @param subStatsTypes the sub stats types
- * @param subStatsItems the sub stats items
- * @param subSortTypes the sub sort types
- * @param subSortDirections the sub sort directions
- * @param subStart the sub start
- * @param subNumber the sub number
- * @param segmentRegistration the segment registration
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param collectorType
+ * the collector type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param subCollectorTypes
+ * the sub collector types
+ * @param subDataTypes
+ * the sub data types
+ * @param subStatsTypes
+ * the sub stats types
+ * @param subStatsItems
+ * the sub stats items
+ * @param subSortTypes
+ * the sub sort types
+ * @param subSortDirections
+ * the sub sort directions
+ * @param subStart
+ * the sub start
+ * @param subNumber
+ * the sub number
+ * @param segmentRegistration
+ * the segment registration
+ * @param boundary
+ * the boundary
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public MtasDataLongFull(String collectorType, TreeSet<String> statsItems,
String sortType, String sortDirection, Integer start, Integer number,
- String[] subCollectorTypes, String[] subDataTypes,
- String[] subStatsTypes, TreeSet<String>[] subStatsItems,
- String[] subSortTypes, String[] subSortDirections, Integer[] subStart,
- Integer[] subNumber, boolean segmentRegistration) throws IOException {
+ String[] subCollectorTypes, String[] subDataTypes, String[] subStatsTypes,
+ TreeSet<String>[] subStatsItems, String[] subSortTypes,
+ String[] subSortDirections, Integer[] subStart, Integer[] subNumber,
+ String segmentRegistration, String boundary) throws IOException {
super(collectorType, CodecUtil.DATA_TYPE_LONG, statsItems, sortType,
sortDirection, start, number, subCollectorTypes, subDataTypes,
- subStatsTypes, subStatsItems, subSortTypes, subSortDirections,
- subStart, subNumber, new MtasDataLongOperations(),
- segmentRegistration);
+ subStatsTypes, subStatsItems, subSortTypes, subSortDirections, subStart,
+ subNumber, new MtasDataLongOperations(), segmentRegistration, boundary);
}
/*
@@ -56,9 +71,13 @@ public class MtasDataLongFull
*/
@Override
protected MtasDataItemLongFull getItem(int i) {
- return new MtasDataItemLongFull(ArrayUtils.toPrimitive(fullValueList[i]),
- hasSub() ? subCollectorListNextLevel[i] : null, statsItems, sortType,
- sortDirection, errorNumber[i], errorList[i]);
+ if (i >= 0 && i < size) {
+ return new MtasDataItemLongFull(ArrayUtils.toPrimitive(fullValueList[i]),
+ hasSub() ? subCollectorListNextLevel[i] : null, statsItems, sortType,
+ sortDirection, errorNumber[i], errorList[i], sourceNumberList[i]);
+ } else {
+ return null;
+ }
}
/*
@@ -80,7 +99,7 @@ public class MtasDataLongFull
@Override
public MtasDataCollector<?, ?> add(long[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
return dataCollector;
@@ -105,7 +124,7 @@ public class MtasDataLongFull
@Override
public MtasDataCollector<?, ?> add(double[] values, int number)
throws IOException {
- MtasDataCollector<?, ?> dataCollector = add();
+ MtasDataCollector<?, ?> dataCollector = add(false);
Long[] newValues = new Long[number];
for (int i = 0; i < values.length; i++)
newValues[i] = Double.valueOf(values[i]).longValue();
@@ -134,12 +153,12 @@ public class MtasDataLongFull
* long[], int)
*/
@Override
- public MtasDataCollector<?, ?>[] add(String[] keys, long[] values,
- int number) throws IOException {
+ public MtasDataCollector<?, ?>[] add(String[] keys, long[] values, int number)
+ throws IOException {
if (keys != null && keys.length > 0) {
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, ArrayUtils.toObject(values), number,
newCurrentExisting);
}
@@ -178,7 +197,7 @@ public class MtasDataLongFull
newValues[i] = Double.valueOf(values[i]).longValue();
MtasDataCollector<?, ?>[] subCollectors = new MtasDataCollector<?, ?>[keys.length];
for (int i = 0; i < keys.length; i++) {
- subCollectors[i] = add(keys[i]);
+ subCollectors[i] = add(keys[i], false);
setValue(newCurrentPosition, newValues, number, newCurrentExisting);
}
return subCollectors;
@@ -187,56 +206,178 @@ public class MtasDataLongFull
}
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#compareForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected boolean compareForComputingSegment(Long value, Long boundary) {
- return value >= boundary;
+ protected boolean compareWithBoundary(Long value, Long boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return value <= boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return value >= boundary;
+ } else {
+ throw new IOException(
+ "can't compare for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(java.lang.Number, java.lang.Number)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * java.lang.Number, java.lang.Number)
*/
@Override
- protected Long minimumForComputingSegment(Long value, Long boundary) {
- return Math.min(value, boundary);
+ protected Long lastForComputingSegment(Long value, Long boundary)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return Math.max(value, boundary);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return Math.min(value, boundary);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#minimumForComputingSegment(
+ * )
*/
@Override
- protected Long minimumForComputingSegment() {
- return Collections.min(segmentValueMaxList);
+ protected Long lastForComputingSegment() throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ return Collections.max(segmentValueTopList);
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)
+ || segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ return Collections.min(segmentValueTopList);
+ } else {
+ throw new IOException(
+ "can't compute last for segmentRegistration " + segmentRegistration);
+ }
}
- /* (non-Javadoc)
- * @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.DataCollector.MtasDataCollector#boundaryForComputingSegment
+ * ()
*/
@Override
- protected Long boundaryForComputingSegment() {
- Long boundary = boundaryForSegment();
- long correctionBoundary = 0;
- for (String otherSegmentName : segmentValueMaxListMin.keySet()) {
- if (!otherSegmentName.equals(segmentName)) {
- Long otherBoundary = segmentValueBoundary.get(otherSegmentName);
- if (otherBoundary != null) {
- correctionBoundary += Math.max(0, otherBoundary - boundary);
+ protected Long boundaryForSegmentComputing(String segmentName)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Long boundary = boundaryForSegment(segmentName);
+ if (boundary == null) {
+ return null;
+ } else {
+ if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ long correctionBoundary = 0;
+ for (String otherSegmentName : segmentValueTopListLast.keySet()) {
+ if (!otherSegmentName.equals(segmentName)) {
+ Long otherBoundary = segmentValuesBoundary.get(otherSegmentName);
+ if (otherBoundary != null) {
+ correctionBoundary += Math.max(0, otherBoundary - boundary);
+ }
+ }
+ }
+ return boundary + correctionBoundary;
+ } else {
+ return boundary;
}
}
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
}
- return boundary + correctionBoundary;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.codec.util.DataCollector.MtasDataCollector#boundaryForSegment()
*/
@Override
- protected Long boundaryForSegment() {
- long thisMin = segmentValueMaxListMin.get(segmentName);
- Long boundary = Math.floorDiv(thisMin, segmentNumber);
- return boundary;
+ protected Long boundaryForSegment(String segmentName) throws IOException {
+ if (segmentRegistration.equals(SEGMENT_SORT_ASC)
+ || segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Long thisLast = segmentValueTopListLast.get(segmentName);
+ if (thisLast == null) {
+ return null;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_ASC)) {
+ Long boundary = thisLast * segmentNumber;
+ return boundary;
+ } else if (segmentRegistration.equals(SEGMENT_SORT_DESC)) {
+ Long boundary = Math.floorDiv(thisLast, segmentNumber);
+ return boundary;
+ } else {
+ // should not happen
+ return null;
+ }
+ } else {
+ throw new IOException("can't compute boundary for segmentRegistration "
+ + segmentRegistration);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#stringToBoundary(java.lang.
+ * String, java.lang.Integer)
+ */
+ @Override
+ protected Long stringToBoundary(String boundary, Integer segmentNumber)
+ throws IOException {
+ if (segmentRegistration.equals(SEGMENT_BOUNDARY_ASC)) {
+ if (segmentNumber == null) {
+ return Long.valueOf(boundary);
+ } else {
+ return Math.floorDiv(Long.valueOf(boundary), segmentNumber);
+ }
+ } else if (segmentRegistration.equals(SEGMENT_BOUNDARY_DESC)) {
+ if (segmentNumber == null) {
+ return Long.valueOf(boundary);
+ } else {
+ return Math.floorDiv(Long.valueOf(boundary), segmentNumber);
+ }
+ } else {
+ throw new IOException(
+ "not available for segmentRegistration " + segmentRegistration);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.codec.util.collector.MtasDataCollector#validateSegmentBoundary(java.
+ * lang.Object)
+ */
+ @Override
+ public boolean validateSegmentBoundary(Object o) throws IOException {
+ if (o instanceof Long) {
+ return validateWithSegmentBoundary((Long) o);
+ } else {
+ throw new IOException("incorrect type");
+ }
}
}
diff --git a/src/mtas/codec/util/collector/MtasDataLongOperations.java b/src/mtas/codec/util/collector/MtasDataLongOperations.java
index 8b9802b..bd88bd2 100644
--- a/src/mtas/codec/util/collector/MtasDataLongOperations.java
+++ b/src/mtas/codec/util/collector/MtasDataLongOperations.java
@@ -2,7 +2,6 @@ package mtas.codec.util.collector;
import java.io.Serializable;
-
/**
* The Class MtasDataLongOperations.
*/
@@ -15,13 +14,16 @@ class MtasDataLongOperations
/*
* (non-Javadoc)
*
- * @see
- * mtas.codec.util.DataCollector.MtasDataOperations#product11(java.lang.
+ * @see mtas.codec.util.DataCollector.MtasDataOperations#product11(java.lang.
* Number, java.lang.Number)
*/
@Override
public Long product11(Long arg1, Long arg2) {
- return arg1 * arg2;
+ if (arg1 == null || arg2 == null) {
+ return null;
+ } else {
+ return arg1 * arg2;
+ }
}
/*
@@ -33,7 +35,11 @@ class MtasDataLongOperations
*/
@Override
public Long add11(Long arg1, Long arg2) {
- return arg1 + arg2;
+ if (arg1 == null || arg2 == null) {
+ return null;
+ } else {
+ return arg1 + arg2;
+ }
}
/*
@@ -45,19 +51,26 @@ class MtasDataLongOperations
*/
@Override
public Double add22(Double arg1, Double arg2) {
- return arg1 + arg2;
+ if (arg1 == null || arg2 == null) {
+ return Double.NaN;
+ } else {
+ return arg1 + arg2;
+ }
}
/*
* (non-Javadoc)
*
- * @see
- * mtas.codec.util.DataCollector.MtasDataOperations#subtract12(java.lang.
+ * @see mtas.codec.util.DataCollector.MtasDataOperations#subtract12(java.lang.
* Number, java.lang.Number)
*/
@Override
public Double subtract12(Long arg1, Double arg2) {
- return arg1.doubleValue() - arg2;
+ if (arg1 == null || arg2 == null) {
+ return Double.NaN;
+ } else {
+ return arg1.doubleValue() - arg2;
+ }
}
/*
@@ -68,7 +81,11 @@ class MtasDataLongOperations
*/
@Override
public Double divide1(Long arg1, long arg2) {
- return arg1 / (double) arg2;
+ if (arg1 == null) {
+ return Double.NaN;
+ } else {
+ return arg1 / (double) arg2;
+ }
}
/*
@@ -79,7 +96,11 @@ class MtasDataLongOperations
*/
@Override
public Double divide2(Double arg1, long arg2) {
- return arg1 / arg2;
+ if (arg1 == null) {
+ return Double.NaN;
+ } else {
+ return arg1 / arg2;
+ }
}
/*
@@ -91,7 +112,11 @@ class MtasDataLongOperations
*/
@Override
public Long min11(Long arg1, Long arg2) {
- return Math.min(arg1, arg2);
+ if (arg1 == null || arg2 == null) {
+ return null;
+ } else {
+ return Math.min(arg1, arg2);
+ }
}
/*
@@ -103,7 +128,11 @@ class MtasDataLongOperations
*/
@Override
public Long max11(Long arg1, Long arg2) {
- return Math.max(arg1, arg2);
+ if (arg1 == null || arg2 == null) {
+ return null;
+ } else {
+ return Math.max(arg1, arg2);
+ }
}
/*
@@ -114,7 +143,11 @@ class MtasDataLongOperations
*/
@Override
public Double exp2(Double arg1) {
- return Math.exp(arg1);
+ if (arg1 == null) {
+ return Double.NaN;
+ } else {
+ return Math.exp(arg1);
+ }
}
/*
@@ -125,7 +158,11 @@ class MtasDataLongOperations
*/
@Override
public Double sqrt2(Double arg1) {
- return Math.sqrt(arg1);
+ if (arg1 == null) {
+ return Double.NaN;
+ } else {
+ return Math.sqrt(arg1);
+ }
}
/*
@@ -136,7 +173,11 @@ class MtasDataLongOperations
*/
@Override
public Double log1(Long arg1) {
- return Math.log(arg1);
+ if (arg1 == null) {
+ return Double.NaN;
+ } else {
+ return Math.log(arg1);
+ }
}
/*
diff --git a/src/mtas/codec/util/collector/MtasDataOperations.java b/src/mtas/codec/util/collector/MtasDataOperations.java
index 87d8a7d..502e628 100644
--- a/src/mtas/codec/util/collector/MtasDataOperations.java
+++ b/src/mtas/codec/util/collector/MtasDataOperations.java
@@ -3,16 +3,20 @@ package mtas.codec.util.collector;
/**
* The Interface MtasDataOperations.
*
- * @param <T1> the generic type
- * @param <T2> the generic type
+ * @param <T1>
+ * the generic type
+ * @param <T2>
+ * the generic type
*/
abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Product11.
*
- * @param arg1 the arg1
- * @param arg2 the arg2
+ * @param arg1
+ * the arg1
+ * @param arg2
+ * the arg2
* @return the t1
*/
public T1 product11(T1 arg1, T1 arg2);
@@ -20,8 +24,10 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Add11.
*
- * @param arg1 the arg1
- * @param arg2 the arg2
+ * @param arg1
+ * the arg1
+ * @param arg2
+ * the arg2
* @return the t1
*/
public T1 add11(T1 arg1, T1 arg2);
@@ -29,8 +35,10 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Add22.
*
- * @param arg1 the arg1
- * @param arg2 the arg2
+ * @param arg1
+ * the arg1
+ * @param arg2
+ * the arg2
* @return the t2
*/
public T2 add22(T2 arg1, T2 arg2);
@@ -38,8 +46,10 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Subtract12.
*
- * @param arg1 the arg1
- * @param arg2 the arg2
+ * @param arg1
+ * the arg1
+ * @param arg2
+ * the arg2
* @return the t2
*/
public T2 subtract12(T1 arg1, T2 arg2);
@@ -47,8 +57,10 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Divide1.
*
- * @param arg1 the arg1
- * @param arg2 the arg2
+ * @param arg1
+ * the arg1
+ * @param arg2
+ * the arg2
* @return the t2
*/
public T2 divide1(T1 arg1, long arg2);
@@ -56,8 +68,10 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Divide2.
*
- * @param arg1 the arg1
- * @param arg2 the arg2
+ * @param arg1
+ * the arg1
+ * @param arg2
+ * the arg2
* @return the t2
*/
public T2 divide2(T2 arg1, long arg2);
@@ -65,7 +79,8 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Exp2.
*
- * @param arg1 the arg1
+ * @param arg1
+ * the arg1
* @return the t2
*/
public T2 exp2(T2 arg1);
@@ -73,7 +88,8 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Sqrt2.
*
- * @param arg1 the arg1
+ * @param arg1
+ * the arg1
* @return the t2
*/
public T2 sqrt2(T2 arg1);
@@ -81,7 +97,8 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Log1.
*
- * @param arg1 the arg1
+ * @param arg1
+ * the arg1
* @return the t2
*/
public T2 log1(T1 arg1);
@@ -89,8 +106,10 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Min11.
*
- * @param arg1 the arg1
- * @param arg2 the arg2
+ * @param arg1
+ * the arg1
+ * @param arg2
+ * the arg2
* @return the t1
*/
public T1 min11(T1 arg1, T1 arg2);
@@ -98,8 +117,10 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Max11.
*
- * @param arg1 the arg1
- * @param arg2 the arg2
+ * @param arg1
+ * the arg1
+ * @param arg2
+ * the arg2
* @return the t1
*/
public T1 max11(T1 arg1, T1 arg2);
@@ -107,7 +128,8 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Creates the vector1.
*
- * @param length the length
+ * @param length
+ * the length
* @return the t1[]
*/
public T1[] createVector1(int length);
@@ -115,7 +137,8 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Creates the vector2.
*
- * @param length the length
+ * @param length
+ * the length
* @return the t2[]
*/
public T2[] createVector2(int length);
@@ -123,7 +146,8 @@ abstract interface MtasDataOperations<T1 extends Number, T2 extends Number> {
/**
* Creates the matrix1.
*
- * @param length the length
+ * @param length
+ * the length
* @return the t1[][]
*/
public T1[][] createMatrix1(int length);
diff --git a/src/mtas/parser/cql/MtasCQLParser.java b/src/mtas/parser/cql/MtasCQLParser.java
new file mode 100644
index 0000000..9eaefa4
--- /dev/null
+++ b/src/mtas/parser/cql/MtasCQLParser.java
@@ -0,0 +1,2257 @@
+/* Generated By:JavaCC: Do not edit this line. MtasCQLParser.java */
+package mtas.parser.cql;
+import mtas.analysis.token.MtasToken;
+import mtas.parser.cql.util.MtasCQLParserGroupCondition;
+import mtas.parser.cql.util.MtasCQLParserGroupFullCondition;
+import mtas.parser.cql.util.MtasCQLParserWordCondition;
+import mtas.parser.cql.util.MtasCQLParserWordPositionQuery;
+import mtas.parser.cql.util.MtasCQLParserWordFullCondition;
+import mtas.parser.cql.util.MtasCQLParserBasicSentenceCondition;
+import mtas.parser.cql.util.MtasCQLParserSentenceCondition;
+import mtas.parser.cql.util.MtasCQLParserSentencePartCondition;
+import mtas.parser.cql.util.MtasCQLParserGroupQuery;
+import mtas.parser.cql.util.MtasCQLParserWordQuery;
+import mtas.parser.cql.util.MtasCQLParserDefaultPrefixCondition;
+import org.apache.lucene.search.spans.SpanContainingQuery;
+import org.apache.lucene.search.spans.SpanNotQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanWithinQuery;
+import org.apache.lucene.index.Term;
+import mtas.search.spans.MtasSpanSequenceItem;
+import mtas.search.spans.MtasSpanSequenceQuery;
+import java.util.ArrayList;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class MtasCQLParser implements MtasCQLParserConstants {
+ public SpanQuery parse(String field, String defaultPrefix) throws ParseException
+ {
+ return cql(field, defaultPrefix);
+ }
+
+ private String unquoteString(String unfiltered)
+ {
+ if (unfiltered.startsWith("\u005c"") && unfiltered.endsWith("\u005c""))
+ {
+ unfiltered = unfiltered.substring(1, unfiltered.length());
+ unfiltered = unfiltered.substring(0, unfiltered.length() - 1);
+ }
+ return unfiltered;
+ }
+
+ final private SpanQuery cql(String field, String defaultPrefix) throws ParseException, ParseException {
+ SpanQuery q;
+ ArrayList < MtasSpanSequenceItem > itemList = new ArrayList < MtasSpanSequenceItem > ();
+ q = cqlBlock(field, defaultPrefix);
+ itemList.add(new MtasSpanSequenceItem(q, false));
+ label_1:
+ while (true) {
+ if (jj_2_1(1000)) {
+ ;
+ } else {
+ break label_1;
+ }
+ q = cqlBlock(field, defaultPrefix);
+ itemList.add(new MtasSpanSequenceItem(q, false));
+ }
+ jj_consume_token(0);
+ if (itemList.size() > 1)
+ {
+ {if (true) return new MtasSpanSequenceQuery(itemList);}
+ }
+ else
+ {
+ {if (true) return itemList.get(0).getQuery();}
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final private SpanQuery cqlBlock(String field, String defaultPrefix) throws ParseException, ParseException {
+ MtasCQLParserSentenceCondition s1 = null, s2 = null;
+ SpanQuery q1 = null, q2 = null;
+ Token end = null;
+ String operator = null;
+ String OPERATOR_CONTAINING = "containing";
+ String OPERATOR_NOT_CONTAINING = "not_containing";
+ String OPERATOR_WITHIN = "within";
+ String OPERATOR_NOT_WITHIN = "not_within";
+ if (jj_2_2(1000)) {
+ s1 = sentence(field, defaultPrefix);
+ } else if (jj_2_3(1000)) {
+ jj_consume_token(BRACKET_START);
+ q1 = cqlBlock(field, defaultPrefix);
+ jj_consume_token(BRACKET_END);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ if (jj_2_10(1000)) {
+ if (jj_2_4(1000)) {
+ jj_consume_token(CONTAINING);
+ operator = OPERATOR_CONTAINING;
+ } else if (jj_2_5(1000)) {
+ jj_consume_token(NOT_CONTAINING);
+ operator = OPERATOR_NOT_CONTAINING;
+ } else if (jj_2_6(1000)) {
+ jj_consume_token(WITHIN);
+ operator = OPERATOR_WITHIN;
+ } else if (jj_2_7(1000)) {
+ jj_consume_token(NOT_WITHIN);
+ operator = OPERATOR_NOT_WITHIN;
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ if (jj_2_8(1000)) {
+ s2 = sentence(field, defaultPrefix);
+ } else if (jj_2_9(1000)) {
+ jj_consume_token(BRACKET_START);
+ q2 = cqlBlock(field, defaultPrefix);
+ jj_consume_token(BRACKET_END);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ } else {
+ ;
+ }
+ if (s1 != null)
+ {
+ q1 = s1.getQuery();
+ }
+ if (operator != null)
+ {
+ if (s2 != null)
+ {
+ q2 = s2.getQuery();
+ }
+ if (operator.equals(OPERATOR_CONTAINING))
+ {
+ {if (true) return new SpanContainingQuery(q1, q2);}
+ }
+ else if (operator.equals(OPERATOR_NOT_CONTAINING))
+ {
+ {if (true) return new SpanNotQuery(q1, new SpanContainingQuery(q1, q2));}
+ }
+ else if (operator.equals(OPERATOR_WITHIN))
+ {
+ {if (true) return new SpanWithinQuery(q2, q1);}
+ }
+ else if (operator.equals(OPERATOR_NOT_WITHIN))
+ {
+ {if (true) return new SpanNotQuery(q1, new SpanWithinQuery(q2, q1));}
+ }
+ else
+ {
+ {if (true) throw new ParseException("unexpected operator " + operator);}
+ }
+ }
+ else
+ {
+ {if (true) return q1;}
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final private MtasCQLParserSentenceCondition sentence(String field, String defaultPrefix) throws ParseException, ParseException {
+ MtasCQLParserSentenceCondition sentenceCondition;
+ MtasCQLParserSentencePartCondition condition;
+ Token questionMark = null;
+ Token minValue = null;
+ Token maxValue = null;
+ int minimumOccurence = 1;
+ int maximumOccurence = 1;
+ condition = sentencePart(field, defaultPrefix);
+ sentenceCondition = condition.createFullSentence();
+ {if (true) return sentenceCondition;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final private MtasCQLParserSentencePartCondition sentencePart(String field, String defaultPrefix) throws ParseException, ParseException {
+ Token operator;
+ MtasCQLParserSentencePartCondition condition, sentencePart;
+ MtasCQLParserBasicSentenceCondition basicSentence;
+ Token questionMark = null;
+ Token minValue = null;
+ Token maxValue = null;
+ int minimumOccurence = 1;
+ int maximumOccurence = 1;
+ if (jj_2_15(1000)) {
+ basicSentence = basicSentence(field, defaultPrefix);
+ condition = new MtasCQLParserSentencePartCondition(basicSentence);
+ } else if (jj_2_16(1000)) {
+ jj_consume_token(BRACKET_START);
+ sentencePart = sentencePart(field, defaultPrefix);
+ jj_consume_token(BRACKET_END);
+ if (jj_2_14(1000)) {
+ questionMark = null;
+ if (jj_2_11(1000)) {
+ jj_consume_token(CURLY_BRACKET_START);
+ minValue = jj_consume_token(NUMBER);
+ jj_consume_token(KOMMA);
+ maxValue = jj_consume_token(NUMBER);
+ jj_consume_token(CURLY_BRACKET_END);
+ } else if (jj_2_12(1000)) {
+ jj_consume_token(CURLY_BRACKET_START);
+ minValue = jj_consume_token(NUMBER);
+ jj_consume_token(CURLY_BRACKET_END);
+ } else if (jj_2_13(1000)) {
+ questionMark = jj_consume_token(QUESTION_MARK);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ } else {
+ ;
+ }
+ condition = new MtasCQLParserSentencePartCondition(sentencePart.createFullSentence());
+ if (questionMark != null)
+ {
+ minimumOccurence = 0;
+ maximumOccurence = 1;
+ }
+ else if (minValue != null)
+ {
+ minimumOccurence = Integer.parseInt(minValue.image);
+ if (maxValue != null)
+ {
+ maximumOccurence = Integer.parseInt(maxValue.image);
+ }
+ else
+ {
+ maximumOccurence = minimumOccurence;
+ }
+ }
+ condition.setFirstOccurence(minimumOccurence, maximumOccurence);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ if (jj_2_18(1000)) {
+ operator = null;
+ if (jj_2_17(1000)) {
+ operator = jj_consume_token(OR);
+ } else {
+ ;
+ }
+ sentencePart = sentencePart(field, defaultPrefix);
+ if (operator == null)
+ {
+ condition.setOr(false);
+ }
+ else
+ {
+ condition.setOr(true);
+ }
+ condition.setSecondPart(sentencePart);
+ } else {
+ ;
+ }
+ {if (true) return condition;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final private MtasCQLParserBasicSentenceCondition basicSentence(String field, String defaultPrefix) throws ParseException, ParseException {
+ MtasCQLParserWordFullCondition subWordCondition;
+ MtasCQLParserGroupFullCondition subGroupCondition;
+ MtasCQLParserBasicSentenceCondition condition = new MtasCQLParserBasicSentenceCondition();
+ if (jj_2_19(1000)) {
+ subWordCondition = word(field, defaultPrefix);
+ condition.addWord(subWordCondition);
+ } else if (jj_2_20(1000)) {
+ subGroupCondition = group(field);
+ condition.addGroup(subGroupCondition);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ label_2:
+ while (true) {
+ if (jj_2_21(1000)) {
+ ;
+ } else {
+ break label_2;
+ }
+ if (jj_2_22(1000)) {
+ subWordCondition = word(field, defaultPrefix);
+ condition.addWord(subWordCondition);
+ } else if (jj_2_23(1000)) {
+ subGroupCondition = group(field);
+ condition.addGroup(subGroupCondition);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ }
+ {if (true) return condition;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final private MtasCQLParserGroupFullCondition group(String field) throws ParseException, ParseException {
+ Token questionMark = null;
+ MtasCQLParserGroupFullCondition groupCondition;
+ MtasCQLParserGroupCondition condition;
+ Boolean startGroup, endGroup;
+ Token minValue = null;
+ Token maxValue = null;
+ Token slash = null;
+ int minimumOccurence = 1;
+ int maximumOccurence = 1;
+ jj_consume_token(GROUP_START);
+ if (jj_2_25(1000)) {
+ condition = groupCondition(field);
+ if (jj_2_24(1000)) {
+ slash = jj_consume_token(SLASH);
+ } else {
+ ;
+ }
+ if (slash == null)
+ {
+ startGroup = true;
+ }
+ else
+ {
+ startGroup = false;
+ }
+ endGroup = false;
+ } else if (jj_2_26(1000)) {
+ jj_consume_token(SLASH);
+ condition = groupCondition(field);
+ startGroup = true;
+ endGroup = false;
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ jj_consume_token(GROUP_END);
+ if (jj_2_30(1000)) {
+ questionMark = null;
+ if (jj_2_27(1000)) {
+ jj_consume_token(CURLY_BRACKET_START);
+ minValue = jj_consume_token(NUMBER);
+ jj_consume_token(KOMMA);
+ maxValue = jj_consume_token(NUMBER);
+ jj_consume_token(CURLY_BRACKET_END);
+ } else if (jj_2_28(1000)) {
+ jj_consume_token(CURLY_BRACKET_START);
+ minValue = jj_consume_token(NUMBER);
+ jj_consume_token(CURLY_BRACKET_END);
+ } else if (jj_2_29(1000)) {
+ questionMark = jj_consume_token(QUESTION_MARK);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ } else {
+ ;
+ }
+ if (startGroup)
+ {
+ groupCondition = new MtasCQLParserGroupFullCondition(condition, MtasCQLParserGroupFullCondition.GROUP_START);
+ }
+ else if (endGroup)
+ {
+ groupCondition = new MtasCQLParserGroupFullCondition(condition, MtasCQLParserGroupFullCondition.GROUP_END);
+ }
+ else
+ {
+ groupCondition = new MtasCQLParserGroupFullCondition(condition, MtasCQLParserGroupFullCondition.GROUP_FULL);
+ }
+ if (questionMark != null)
+ {
+ minimumOccurence = 0;
+ maximumOccurence = 1;
+ }
+ else if (minValue != null)
+ {
+ minimumOccurence = Integer.parseInt(minValue.image);
+ if (maxValue != null)
+ {
+ maximumOccurence = Integer.parseInt(maxValue.image);
+ }
+ else
+ {
+ maximumOccurence = minimumOccurence;
+ }
+ }
+ groupCondition.setOccurence(minimumOccurence, maximumOccurence);
+ {if (true) return groupCondition;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final private MtasCQLParserGroupCondition groupCondition(String field) throws ParseException, ParseException {
+ Token prefix;
+ Token value = null;
+ if (jj_2_31(1000)) {
+ prefix = jj_consume_token(UNQUOTED_VALUE);
+ jj_consume_token(TOKEN_EQUALS);
+ value = jj_consume_token(QUOTED_VALUE);
+ } else if (jj_2_32(1000)) {
+ prefix = jj_consume_token(UNQUOTED_VALUE);
+ value = null;
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ if (value != null)
+ {
+ SpanQuery q = new MtasCQLParserGroupQuery(field, prefix.image, unquoteString(value.image));
+ MtasCQLParserGroupCondition condition = new MtasCQLParserGroupCondition(field, q);
+ {if (true) return condition;}
+ }
+ else
+ {
+ SpanQuery q = new MtasCQLParserGroupQuery(field, prefix.image, null);
+ MtasCQLParserGroupCondition condition = new MtasCQLParserGroupCondition(field, q);
+ {if (true) return condition;}
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final private MtasCQLParserWordFullCondition word(String field, String defaultPrefix) throws ParseException, ParseException {
+ Token questionMark = null;
+ Token value;
+ MtasCQLParserWordFullCondition wordCondition;
+ MtasCQLParserWordCondition condition, subCondition;
+ Token minValue = null;
+ Token maxValue = null;
+ int minimumOccurence = 1;
+ int maximumOccurence = 1;
+ if (jj_2_39(1000)) {
+ value = jj_consume_token(QUOTED_VALUE);
+ condition = new MtasCQLParserDefaultPrefixCondition(field, defaultPrefix, unquoteString(value.image));
+ } else if (jj_2_40(1000)) {
+ jj_consume_token(WORD_START);
+ if (jj_2_37(1000)) {
+ subCondition = wordCondition(field);
+ if (jj_2_35(1000)) {
+ jj_consume_token(AND);
+ condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_AND);
+ condition.addCondition(subCondition);
+ subCondition = wordCondition(field);
+ condition.addCondition(subCondition);
+ label_3:
+ while (true) {
+ if (jj_2_33(1000)) {
+ ;
+ } else {
+ break label_3;
+ }
+ jj_consume_token(AND);
+ subCondition = wordCondition(field);
+ condition.addCondition(subCondition);
+ }
+ } else if (jj_2_36(1000)) {
+ jj_consume_token(OR);
+ condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_OR);
+ condition.addCondition(subCondition);
+ subCondition = wordCondition(field);
+ condition.addCondition(subCondition);
+ label_4:
+ while (true) {
+ if (jj_2_34(1000)) {
+ ;
+ } else {
+ break label_4;
+ }
+ jj_consume_token(OR);
+ subCondition = wordCondition(field);
+ condition.addCondition(subCondition);
+ }
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ } else if (jj_2_38(1000)) {
+ condition = wordCondition(field);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ jj_consume_token(WORD_END);
+ } else if (jj_2_41(1000)) {
+ condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_AND);
+ jj_consume_token(WORD_START);
+ jj_consume_token(WORD_END);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ if (jj_2_45(1000)) {
+ questionMark = null;
+ if (jj_2_42(1000)) {
+ jj_consume_token(CURLY_BRACKET_START);
+ minValue = jj_consume_token(NUMBER);
+ jj_consume_token(KOMMA);
+ maxValue = jj_consume_token(NUMBER);
+ jj_consume_token(CURLY_BRACKET_END);
+ } else if (jj_2_43(1000)) {
+ jj_consume_token(CURLY_BRACKET_START);
+ minValue = jj_consume_token(NUMBER);
+ jj_consume_token(CURLY_BRACKET_END);
+ } else if (jj_2_44(1000)) {
+ questionMark = jj_consume_token(QUESTION_MARK);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ } else {
+ ;
+ }
+ condition.simplify();
+ wordCondition = new MtasCQLParserWordFullCondition(condition);
+ if (questionMark != null)
+ {
+ minimumOccurence = 0;
+ maximumOccurence = 1;
+ }
+ else if (minValue != null)
+ {
+ minimumOccurence = Integer.parseInt(minValue.image);
+ if (maxValue != null)
+ {
+ maximumOccurence = Integer.parseInt(maxValue.image);
+ }
+ else
+ {
+ maximumOccurence = minimumOccurence;
+ }
+ }
+ wordCondition.setOccurence(minimumOccurence, maximumOccurence);
+ //System.out.println(wordCondition + "\n");
+ {if (true) return wordCondition;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final private MtasCQLParserWordCondition wordCondition(String field) throws ParseException, ParseException {
+ Token negation = null;
+ MtasCQLParserWordCondition condition, subCondition;
+ if (jj_2_66(1000)) {
+ if (jj_2_46(1000)) {
+ negation = jj_consume_token(NEGATION);
+ } else {
+ ;
+ }
+ jj_consume_token(BRACKET_START);
+ if (jj_2_63(1000)) {
+ condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_AND);
+ if (jj_2_47(1000)) {
+ subCondition = wordAtomCondition(field);
+ } else if (jj_2_48(1000)) {
+ subCondition = wordCondition(field);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ condition.addCondition(subCondition);
+ jj_consume_token(AND);
+ if (jj_2_49(1000)) {
+ subCondition = wordAtomCondition(field);
+ } else if (jj_2_50(1000)) {
+ subCondition = wordCondition(field);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ condition.addCondition(subCondition);
+ label_5:
+ while (true) {
+ if (jj_2_51(1000)) {
+ ;
+ } else {
+ break label_5;
+ }
+ jj_consume_token(AND);
+ if (jj_2_52(1000)) {
+ subCondition = wordAtomCondition(field);
+ } else if (jj_2_53(1000)) {
+ subCondition = wordCondition(field);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ condition.addCondition(subCondition);
+ }
+ } else if (jj_2_64(1000)) {
+ condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_OR);
+ if (jj_2_54(1000)) {
+ subCondition = wordAtomCondition(field);
+ } else if (jj_2_55(1000)) {
+ subCondition = wordCondition(field);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ condition.addCondition(subCondition);
+ jj_consume_token(OR);
+ if (jj_2_56(1000)) {
+ subCondition = wordAtomCondition(field);
+ } else if (jj_2_57(1000)) {
+ subCondition = wordCondition(field);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ condition.addCondition(subCondition);
+ label_6:
+ while (true) {
+ if (jj_2_58(1000)) {
+ ;
+ } else {
+ break label_6;
+ }
+ jj_consume_token(OR);
+ if (jj_2_59(1000)) {
+ subCondition = wordAtomCondition(field);
+ } else if (jj_2_60(1000)) {
+ subCondition = wordCondition(field);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ condition.addCondition(subCondition);
+ }
+ } else if (jj_2_65(1000)) {
+ if (jj_2_61(1000)) {
+ condition = wordAtomCondition(field);
+ } else if (jj_2_62(1000)) {
+ condition = wordCondition(field);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ jj_consume_token(BRACKET_END);
+ if (negation != null)
+ {
+ condition.swapNot();
+ }
+ //System.out.println("=== wordCondition ===\n" + condition + "\n");
+ {if (true) return condition;}
+ } else if (jj_2_67(1000)) {
+ //plain atom is a valid condition
+ subCondition = wordAtomCondition(field);
+ //System.out.println("=== wordCondition ===\n" + subCondition + "\n");
+ {if (true) return subCondition;}
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ //System.out.println("=== wordCondition ===\n" + condition + "\n");
+ {if (true) return null;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final private MtasCQLParserWordCondition wordAtomCondition(String field) throws ParseException, ParseException {
+ Token negation = null;
+ Token nequals = null;
+ Token prefix;
+ Token value;
+ if (jj_2_68(1000)) {
+ negation = jj_consume_token(NEGATION);
+ } else {
+ ;
+ }
+ if (jj_2_73(1000)) {
+ jj_consume_token(OCTOTHORPE);
+ if (jj_2_69(1000)) {
+ value = jj_consume_token(NUMBER);
+ } else if (jj_2_70(1000)) {
+ value = jj_consume_token(UNQUOTED_VALUE);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ Integer startPosition = null;
+ Integer endPosition = null;
+ Pattern range = Pattern.compile("([0-9]+)(\u005c\u005c-([0-9]+))?");
+ Matcher m = range.matcher(value.image);
+ if (m.find())
+ {
+ startPosition = m.group(1) != null ? Integer.parseInt(m.group(1)) : null;
+ endPosition = m.group(3) != null ? Integer.parseInt(m.group(3)) : null;
+ }
+ else
+ {
+ {if (true) throw new ParseException("invalid range '" + value.image + "'");}
+ }
+ MtasCQLParserWordCondition condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_AND);
+ if (startPosition != null && endPosition != null)
+ {
+ SpanQuery q = new MtasCQLParserWordPositionQuery(field, startPosition, endPosition);
+ condition.addPositiveQuery(q);
+ }
+ else if (startPosition != null)
+ {
+ SpanQuery q = new MtasCQLParserWordPositionQuery(field, startPosition, startPosition);
+ condition.addPositiveQuery(q);
+ }
+ else
+ {
+ //should not happen
+ {if (true) throw new ParseException("invalid range");}
+ }
+ {if (true) return condition;}
+ } else if (jj_2_74(1000)) {
+ prefix = jj_consume_token(UNQUOTED_VALUE);
+ if (jj_2_71(1000)) {
+ nequals = jj_consume_token(TOKEN_NOTEQUALS);
+ } else if (jj_2_72(1000)) {
+ jj_consume_token(TOKEN_EQUALS);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ value = jj_consume_token(QUOTED_VALUE);
+ if (nequals != null)
+ {
+ //use RegexpQuery combined with SpanMultiTermQueryWrapper
+ {if (true) throw new ParseException("TODO: not implemented '" + prefix.image + nequals.image + value.image + "'");}
+ }
+ else
+ {
+ MtasCQLParserWordCondition condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_AND);
+ Term term = new Term(field, prefix.image + MtasToken.DELIMITER + unquoteString(value.image));
+ SpanQuery q = new MtasCQLParserWordQuery(field, prefix.image, unquoteString(value.image), MtasCQLParserWordQuery.MTAS_CQL_REGEXP_QUERY);
+ if (negation != null)
+ {
+ condition.swapNot();
+ }
+ condition.addPositiveQuery(q);
+ //System.out.println("=== wordAtomCondition ===\n" + condition + "\n");
+ {if (true) return condition;}
+ }
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ private boolean jj_2_1(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_1(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(0, xla); }
+ }
+
+ private boolean jj_2_2(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_2(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(1, xla); }
+ }
+
+ private boolean jj_2_3(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_3(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(2, xla); }
+ }
+
+ private boolean jj_2_4(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_4(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(3, xla); }
+ }
+
+ private boolean jj_2_5(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_5(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(4, xla); }
+ }
+
+ private boolean jj_2_6(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_6(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(5, xla); }
+ }
+
+ private boolean jj_2_7(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_7(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(6, xla); }
+ }
+
+ private boolean jj_2_8(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_8(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(7, xla); }
+ }
+
+ private boolean jj_2_9(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_9(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(8, xla); }
+ }
+
+ private boolean jj_2_10(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_10(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(9, xla); }
+ }
+
+ private boolean jj_2_11(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_11(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(10, xla); }
+ }
+
+ private boolean jj_2_12(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_12(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(11, xla); }
+ }
+
+ private boolean jj_2_13(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_13(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(12, xla); }
+ }
+
+ private boolean jj_2_14(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_14(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(13, xla); }
+ }
+
+ private boolean jj_2_15(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_15(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(14, xla); }
+ }
+
+ private boolean jj_2_16(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_16(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(15, xla); }
+ }
+
+ private boolean jj_2_17(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_17(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(16, xla); }
+ }
+
+ private boolean jj_2_18(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_18(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(17, xla); }
+ }
+
+ private boolean jj_2_19(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_19(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(18, xla); }
+ }
+
+ private boolean jj_2_20(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_20(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(19, xla); }
+ }
+
+ private boolean jj_2_21(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_21(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(20, xla); }
+ }
+
+ private boolean jj_2_22(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_22(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(21, xla); }
+ }
+
+ private boolean jj_2_23(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_23(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(22, xla); }
+ }
+
+ private boolean jj_2_24(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_24(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(23, xla); }
+ }
+
+ private boolean jj_2_25(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_25(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(24, xla); }
+ }
+
+ private boolean jj_2_26(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_26(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(25, xla); }
+ }
+
+ private boolean jj_2_27(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_27(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(26, xla); }
+ }
+
+ private boolean jj_2_28(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_28(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(27, xla); }
+ }
+
+ private boolean jj_2_29(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_29(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(28, xla); }
+ }
+
+ private boolean jj_2_30(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_30(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(29, xla); }
+ }
+
+ private boolean jj_2_31(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_31(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(30, xla); }
+ }
+
+ private boolean jj_2_32(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_32(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(31, xla); }
+ }
+
+ private boolean jj_2_33(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_33(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(32, xla); }
+ }
+
+ private boolean jj_2_34(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_34(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(33, xla); }
+ }
+
+ private boolean jj_2_35(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_35(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(34, xla); }
+ }
+
+ private boolean jj_2_36(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_36(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(35, xla); }
+ }
+
+ private boolean jj_2_37(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_37(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(36, xla); }
+ }
+
+ private boolean jj_2_38(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_38(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(37, xla); }
+ }
+
+ private boolean jj_2_39(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_39(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(38, xla); }
+ }
+
+ private boolean jj_2_40(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_40(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(39, xla); }
+ }
+
+ private boolean jj_2_41(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_41(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(40, xla); }
+ }
+
+ private boolean jj_2_42(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_42(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(41, xla); }
+ }
+
+ private boolean jj_2_43(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_43(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(42, xla); }
+ }
+
+ private boolean jj_2_44(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_44(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(43, xla); }
+ }
+
+ private boolean jj_2_45(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_45(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(44, xla); }
+ }
+
+ private boolean jj_2_46(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_46(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(45, xla); }
+ }
+
+ private boolean jj_2_47(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_47(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(46, xla); }
+ }
+
+ private boolean jj_2_48(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_48(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(47, xla); }
+ }
+
+ private boolean jj_2_49(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_49(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(48, xla); }
+ }
+
+ private boolean jj_2_50(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_50(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(49, xla); }
+ }
+
+ private boolean jj_2_51(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_51(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(50, xla); }
+ }
+
+ private boolean jj_2_52(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_52(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(51, xla); }
+ }
+
+ private boolean jj_2_53(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_53(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(52, xla); }
+ }
+
+ private boolean jj_2_54(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_54(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(53, xla); }
+ }
+
+ private boolean jj_2_55(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_55(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(54, xla); }
+ }
+
+ private boolean jj_2_56(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_56(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(55, xla); }
+ }
+
+ private boolean jj_2_57(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_57(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(56, xla); }
+ }
+
+ private boolean jj_2_58(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_58(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(57, xla); }
+ }
+
+ private boolean jj_2_59(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_59(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(58, xla); }
+ }
+
+ private boolean jj_2_60(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_60(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(59, xla); }
+ }
+
+ private boolean jj_2_61(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_61(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(60, xla); }
+ }
+
+ private boolean jj_2_62(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_62(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(61, xla); }
+ }
+
+ private boolean jj_2_63(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_63(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(62, xla); }
+ }
+
+ private boolean jj_2_64(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_64(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(63, xla); }
+ }
+
+ private boolean jj_2_65(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_65(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(64, xla); }
+ }
+
+ private boolean jj_2_66(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_66(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(65, xla); }
+ }
+
+ private boolean jj_2_67(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_67(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(66, xla); }
+ }
+
+ private boolean jj_2_68(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_68(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(67, xla); }
+ }
+
+ private boolean jj_2_69(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_69(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(68, xla); }
+ }
+
+ private boolean jj_2_70(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_70(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(69, xla); }
+ }
+
+ private boolean jj_2_71(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_71(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(70, xla); }
+ }
+
+ private boolean jj_2_72(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_72(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(71, xla); }
+ }
+
+ private boolean jj_2_73(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_73(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(72, xla); }
+ }
+
+ private boolean jj_2_74(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_74(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(73, xla); }
+ }
+
+ private boolean jj_3R_11() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_39()) {
+ jj_scanpos = xsp;
+ if (jj_3_40()) {
+ jj_scanpos = xsp;
+ if (jj_3_41()) return true;
+ }
+ }
+ xsp = jj_scanpos;
+ if (jj_3_45()) jj_scanpos = xsp;
+ return false;
+ }
+
+ private boolean jj_3_68() {
+ if (jj_scan_token(NEGATION)) return true;
+ return false;
+ }
+
+ private boolean jj_3R_15() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_68()) jj_scanpos = xsp;
+ xsp = jj_scanpos;
+ if (jj_3_73()) {
+ jj_scanpos = xsp;
+ if (jj_3_74()) return true;
+ }
+ return false;
+ }
+
+ private boolean jj_3_13() {
+ if (jj_scan_token(QUESTION_MARK)) return true;
+ return false;
+ }
+
+ private boolean jj_3_12() {
+ if (jj_scan_token(CURLY_BRACKET_START)) return true;
+ if (jj_scan_token(NUMBER)) return true;
+ if (jj_scan_token(CURLY_BRACKET_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_11() {
+ if (jj_scan_token(CURLY_BRACKET_START)) return true;
+ if (jj_scan_token(NUMBER)) return true;
+ if (jj_scan_token(KOMMA)) return true;
+ if (jj_scan_token(NUMBER)) return true;
+ if (jj_scan_token(CURLY_BRACKET_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_14() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_11()) {
+ jj_scanpos = xsp;
+ if (jj_3_12()) {
+ jj_scanpos = xsp;
+ if (jj_3_13()) return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3_67() {
+ if (jj_3R_15()) return true;
+ return false;
+ }
+
+ private boolean jj_3_16() {
+ if (jj_scan_token(BRACKET_START)) return true;
+ if (jj_3R_10()) return true;
+ if (jj_scan_token(BRACKET_END)) return true;
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_14()) jj_scanpos = xsp;
+ return false;
+ }
+
+ private boolean jj_3_15() {
+ if (jj_3R_9()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_10() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_15()) {
+ jj_scanpos = xsp;
+ if (jj_3_16()) return true;
+ }
+ xsp = jj_scanpos;
+ if (jj_3_18()) jj_scanpos = xsp;
+ return false;
+ }
+
+ private boolean jj_3_62() {
+ if (jj_3R_14()) return true;
+ return false;
+ }
+
+ private boolean jj_3_61() {
+ if (jj_3R_15()) return true;
+ return false;
+ }
+
+ private boolean jj_3_32() {
+ if (jj_scan_token(UNQUOTED_VALUE)) return true;
+ return false;
+ }
+
+ private boolean jj_3_31() {
+ if (jj_scan_token(UNQUOTED_VALUE)) return true;
+ if (jj_scan_token(TOKEN_EQUALS)) return true;
+ if (jj_scan_token(QUOTED_VALUE)) return true;
+ return false;
+ }
+
+ private boolean jj_3_65() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_61()) {
+ jj_scanpos = xsp;
+ if (jj_3_62()) return true;
+ }
+ return false;
+ }
+
+ private boolean jj_3R_13() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_31()) {
+ jj_scanpos = xsp;
+ if (jj_3_32()) return true;
+ }
+ return false;
+ }
+
+ private boolean jj_3_60() {
+ if (jj_3R_14()) return true;
+ return false;
+ }
+
+ private boolean jj_3_59() {
+ if (jj_3R_15()) return true;
+ return false;
+ }
+
+ private boolean jj_3_58() {
+ if (jj_scan_token(OR)) return true;
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_59()) {
+ jj_scanpos = xsp;
+ if (jj_3_60()) return true;
+ }
+ return false;
+ }
+
+ private boolean jj_3_57() {
+ if (jj_3R_14()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_8() {
+ if (jj_3R_10()) return true;
+ return false;
+ }
+
+ private boolean jj_3_56() {
+ if (jj_3R_15()) return true;
+ return false;
+ }
+
+ private boolean jj_3_55() {
+ if (jj_3R_14()) return true;
+ return false;
+ }
+
+ private boolean jj_3_54() {
+ if (jj_3R_15()) return true;
+ return false;
+ }
+
+ private boolean jj_3_64() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_54()) {
+ jj_scanpos = xsp;
+ if (jj_3_55()) return true;
+ }
+ if (jj_scan_token(OR)) return true;
+ xsp = jj_scanpos;
+ if (jj_3_56()) {
+ jj_scanpos = xsp;
+ if (jj_3_57()) return true;
+ }
+ while (true) {
+ xsp = jj_scanpos;
+ if (jj_3_58()) { jj_scanpos = xsp; break; }
+ }
+ return false;
+ }
+
+ private boolean jj_3_53() {
+ if (jj_3R_14()) return true;
+ return false;
+ }
+
+ private boolean jj_3_52() {
+ if (jj_3R_15()) return true;
+ return false;
+ }
+
+ private boolean jj_3_24() {
+ if (jj_scan_token(SLASH)) return true;
+ return false;
+ }
+
+ private boolean jj_3_51() {
+ if (jj_scan_token(AND)) return true;
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_52()) {
+ jj_scanpos = xsp;
+ if (jj_3_53()) return true;
+ }
+ return false;
+ }
+
+ private boolean jj_3_29() {
+ if (jj_scan_token(QUESTION_MARK)) return true;
+ return false;
+ }
+
+ private boolean jj_3_28() {
+ if (jj_scan_token(CURLY_BRACKET_START)) return true;
+ if (jj_scan_token(NUMBER)) return true;
+ if (jj_scan_token(CURLY_BRACKET_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_50() {
+ if (jj_3R_14()) return true;
+ return false;
+ }
+
+ private boolean jj_3_27() {
+ if (jj_scan_token(CURLY_BRACKET_START)) return true;
+ if (jj_scan_token(NUMBER)) return true;
+ if (jj_scan_token(KOMMA)) return true;
+ if (jj_scan_token(NUMBER)) return true;
+ if (jj_scan_token(CURLY_BRACKET_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_49() {
+ if (jj_3R_15()) return true;
+ return false;
+ }
+
+ private boolean jj_3_30() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_27()) {
+ jj_scanpos = xsp;
+ if (jj_3_28()) {
+ jj_scanpos = xsp;
+ if (jj_3_29()) return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3_48() {
+ if (jj_3R_14()) return true;
+ return false;
+ }
+
+ private boolean jj_3_47() {
+ if (jj_3R_15()) return true;
+ return false;
+ }
+
+ private boolean jj_3_26() {
+ if (jj_scan_token(SLASH)) return true;
+ if (jj_3R_13()) return true;
+ return false;
+ }
+
+ private boolean jj_3_63() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_47()) {
+ jj_scanpos = xsp;
+ if (jj_3_48()) return true;
+ }
+ if (jj_scan_token(AND)) return true;
+ xsp = jj_scanpos;
+ if (jj_3_49()) {
+ jj_scanpos = xsp;
+ if (jj_3_50()) return true;
+ }
+ while (true) {
+ xsp = jj_scanpos;
+ if (jj_3_51()) { jj_scanpos = xsp; break; }
+ }
+ return false;
+ }
+
+ private boolean jj_3_46() {
+ if (jj_scan_token(NEGATION)) return true;
+ return false;
+ }
+
+ private boolean jj_3_66() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_46()) jj_scanpos = xsp;
+ if (jj_scan_token(BRACKET_START)) return true;
+ xsp = jj_scanpos;
+ if (jj_3_63()) {
+ jj_scanpos = xsp;
+ if (jj_3_64()) {
+ jj_scanpos = xsp;
+ if (jj_3_65()) return true;
+ }
+ }
+ if (jj_scan_token(BRACKET_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_9() {
+ if (jj_scan_token(BRACKET_START)) return true;
+ if (jj_3R_7()) return true;
+ if (jj_scan_token(BRACKET_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_8() {
+ if (jj_3R_8()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_14() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_66()) {
+ jj_scanpos = xsp;
+ if (jj_3_67()) return true;
+ }
+ return false;
+ }
+
+ private boolean jj_3_25() {
+ if (jj_3R_13()) return true;
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_24()) jj_scanpos = xsp;
+ return false;
+ }
+
+ private boolean jj_3_7() {
+ if (jj_scan_token(NOT_WITHIN)) return true;
+ return false;
+ }
+
+ private boolean jj_3R_12() {
+ if (jj_scan_token(GROUP_START)) return true;
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_25()) {
+ jj_scanpos = xsp;
+ if (jj_3_26()) return true;
+ }
+ if (jj_scan_token(GROUP_END)) return true;
+ xsp = jj_scanpos;
+ if (jj_3_30()) jj_scanpos = xsp;
+ return false;
+ }
+
+ private boolean jj_3_6() {
+ if (jj_scan_token(WITHIN)) return true;
+ return false;
+ }
+
+ private boolean jj_3_5() {
+ if (jj_scan_token(NOT_CONTAINING)) return true;
+ return false;
+ }
+
+ private boolean jj_3_4() {
+ if (jj_scan_token(CONTAINING)) return true;
+ return false;
+ }
+
+ private boolean jj_3_10() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_4()) {
+ jj_scanpos = xsp;
+ if (jj_3_5()) {
+ jj_scanpos = xsp;
+ if (jj_3_6()) {
+ jj_scanpos = xsp;
+ if (jj_3_7()) return true;
+ }
+ }
+ }
+ xsp = jj_scanpos;
+ if (jj_3_8()) {
+ jj_scanpos = xsp;
+ if (jj_3_9()) return true;
+ }
+ return false;
+ }
+
+ private boolean jj_3_3() {
+ if (jj_scan_token(BRACKET_START)) return true;
+ if (jj_3R_7()) return true;
+ if (jj_scan_token(BRACKET_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_2() {
+ if (jj_3R_8()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_7() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_2()) {
+ jj_scanpos = xsp;
+ if (jj_3_3()) return true;
+ }
+ xsp = jj_scanpos;
+ if (jj_3_10()) jj_scanpos = xsp;
+ return false;
+ }
+
+ private boolean jj_3_23() {
+ if (jj_3R_12()) return true;
+ return false;
+ }
+
+ private boolean jj_3_44() {
+ if (jj_scan_token(QUESTION_MARK)) return true;
+ return false;
+ }
+
+ private boolean jj_3_22() {
+ if (jj_3R_11()) return true;
+ return false;
+ }
+
+ private boolean jj_3_43() {
+ if (jj_scan_token(CURLY_BRACKET_START)) return true;
+ if (jj_scan_token(NUMBER)) return true;
+ if (jj_scan_token(CURLY_BRACKET_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_42() {
+ if (jj_scan_token(CURLY_BRACKET_START)) return true;
+ if (jj_scan_token(NUMBER)) return true;
+ if (jj_scan_token(KOMMA)) return true;
+ if (jj_scan_token(NUMBER)) return true;
+ if (jj_scan_token(CURLY_BRACKET_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_21() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_22()) {
+ jj_scanpos = xsp;
+ if (jj_3_23()) return true;
+ }
+ return false;
+ }
+
+ private boolean jj_3_45() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_42()) {
+ jj_scanpos = xsp;
+ if (jj_3_43()) {
+ jj_scanpos = xsp;
+ if (jj_3_44()) return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3_72() {
+ if (jj_scan_token(TOKEN_EQUALS)) return true;
+ return false;
+ }
+
+ private boolean jj_3_20() {
+ if (jj_3R_12()) return true;
+ return false;
+ }
+
+ private boolean jj_3_71() {
+ if (jj_scan_token(TOKEN_NOTEQUALS)) return true;
+ return false;
+ }
+
+ private boolean jj_3_19() {
+ if (jj_3R_11()) return true;
+ return false;
+ }
+
+ private boolean jj_3_41() {
+ if (jj_scan_token(WORD_START)) return true;
+ if (jj_scan_token(WORD_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_38() {
+ if (jj_3R_14()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_9() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_19()) {
+ jj_scanpos = xsp;
+ if (jj_3_20()) return true;
+ }
+ while (true) {
+ xsp = jj_scanpos;
+ if (jj_3_21()) { jj_scanpos = xsp; break; }
+ }
+ return false;
+ }
+
+ private boolean jj_3_74() {
+ if (jj_scan_token(UNQUOTED_VALUE)) return true;
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_71()) {
+ jj_scanpos = xsp;
+ if (jj_3_72()) return true;
+ }
+ if (jj_scan_token(QUOTED_VALUE)) return true;
+ return false;
+ }
+
+ private boolean jj_3_34() {
+ if (jj_scan_token(OR)) return true;
+ if (jj_3R_14()) return true;
+ return false;
+ }
+
+ private boolean jj_3_1() {
+ if (jj_3R_7()) return true;
+ return false;
+ }
+
+ private boolean jj_3_36() {
+ if (jj_scan_token(OR)) return true;
+ if (jj_3R_14()) return true;
+ Token xsp;
+ while (true) {
+ xsp = jj_scanpos;
+ if (jj_3_34()) { jj_scanpos = xsp; break; }
+ }
+ return false;
+ }
+
+ private boolean jj_3_33() {
+ if (jj_scan_token(AND)) return true;
+ if (jj_3R_14()) return true;
+ return false;
+ }
+
+ private boolean jj_3_17() {
+ if (jj_scan_token(OR)) return true;
+ return false;
+ }
+
+ private boolean jj_3_35() {
+ if (jj_scan_token(AND)) return true;
+ if (jj_3R_14()) return true;
+ Token xsp;
+ while (true) {
+ xsp = jj_scanpos;
+ if (jj_3_33()) { jj_scanpos = xsp; break; }
+ }
+ return false;
+ }
+
+ private boolean jj_3_18() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_17()) jj_scanpos = xsp;
+ if (jj_3R_10()) return true;
+ return false;
+ }
+
+ private boolean jj_3_37() {
+ if (jj_3R_14()) return true;
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_35()) {
+ jj_scanpos = xsp;
+ if (jj_3_36()) return true;
+ }
+ return false;
+ }
+
+ private boolean jj_3_70() {
+ if (jj_scan_token(UNQUOTED_VALUE)) return true;
+ return false;
+ }
+
+ private boolean jj_3_69() {
+ if (jj_scan_token(NUMBER)) return true;
+ return false;
+ }
+
+ private boolean jj_3_40() {
+ if (jj_scan_token(WORD_START)) return true;
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_37()) {
+ jj_scanpos = xsp;
+ if (jj_3_38()) return true;
+ }
+ if (jj_scan_token(WORD_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_39() {
+ if (jj_scan_token(QUOTED_VALUE)) return true;
+ return false;
+ }
+
+ private boolean jj_3_73() {
+ if (jj_scan_token(OCTOTHORPE)) return true;
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_69()) {
+ jj_scanpos = xsp;
+ if (jj_3_70()) return true;
+ }
+ return false;
+ }
+
+ /** Generated Token Manager. */
+ public MtasCQLParserTokenManager token_source;
+ SimpleCharStream jj_input_stream;
+ /** Current token. */
+ public Token token;
+ /** Next token. */
+ public Token jj_nt;
+ private int jj_ntk;
+ private Token jj_scanpos, jj_lastpos;
+ private int jj_la;
+ private int jj_gen;
+ final private int[] jj_la1 = new int[0];
+ static private int[] jj_la1_0;
+ static {
+ jj_la1_init_0();
+ }
+ private static void jj_la1_init_0() {
+ jj_la1_0 = new int[] {};
+ }
+ final private JJCalls[] jj_2_rtns = new JJCalls[74];
+ private boolean jj_rescan = false;
+ private int jj_gc = 0;
+
+ /** Constructor with InputStream. */
+ public MtasCQLParser(java.io.InputStream stream) {
+ this(stream, null);
+ }
+ /** Constructor with InputStream and supplied encoding */
+ public MtasCQLParser(java.io.InputStream stream, String encoding) {
+ try { jj_input_stream = new SimpleCharStream(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
+ token_source = new MtasCQLParserTokenManager(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream stream) {
+ ReInit(stream, null);
+ }
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream stream, String encoding) {
+ try { jj_input_stream.ReInit(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
+ token_source.ReInit(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Constructor. */
+ public MtasCQLParser(java.io.Reader stream) {
+ jj_input_stream = new SimpleCharStream(stream, 1, 1);
+ token_source = new MtasCQLParserTokenManager(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.Reader stream) {
+ jj_input_stream.ReInit(stream, 1, 1);
+ token_source.ReInit(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Constructor with generated Token Manager. */
+ public MtasCQLParser(MtasCQLParserTokenManager tm) {
+ token_source = tm;
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Reinitialise. */
+ public void ReInit(MtasCQLParserTokenManager tm) {
+ token_source = tm;
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ private Token jj_consume_token(int kind) throws ParseException {
+ Token oldToken;
+ if ((oldToken = token).next != null) token = token.next;
+ else token = token.next = token_source.getNextToken();
+ jj_ntk = -1;
+ if (token.kind == kind) {
+ jj_gen++;
+ if (++jj_gc > 100) {
+ jj_gc = 0;
+ for (int i = 0; i < jj_2_rtns.length; i++) {
+ JJCalls c = jj_2_rtns[i];
+ while (c != null) {
+ if (c.gen < jj_gen) c.first = null;
+ c = c.next;
+ }
+ }
+ }
+ return token;
+ }
+ token = oldToken;
+ jj_kind = kind;
+ throw generateParseException();
+ }
+
+ static private final class LookaheadSuccess extends java.lang.Error { }
+ final private LookaheadSuccess jj_ls = new LookaheadSuccess();
+ private boolean jj_scan_token(int kind) {
+ if (jj_scanpos == jj_lastpos) {
+ jj_la--;
+ if (jj_scanpos.next == null) {
+ jj_lastpos = jj_scanpos = jj_scanpos.next = token_source.getNextToken();
+ } else {
+ jj_lastpos = jj_scanpos = jj_scanpos.next;
+ }
+ } else {
+ jj_scanpos = jj_scanpos.next;
+ }
+ if (jj_rescan) {
+ int i = 0; Token tok = token;
+ while (tok != null && tok != jj_scanpos) { i++; tok = tok.next; }
+ if (tok != null) jj_add_error_token(kind, i);
+ }
+ if (jj_scanpos.kind != kind) return true;
+ if (jj_la == 0 && jj_scanpos == jj_lastpos) throw jj_ls;
+ return false;
+ }
+
+
+/** Get the next Token. */
+ final public Token getNextToken() {
+ if (token.next != null) token = token.next;
+ else token = token.next = token_source.getNextToken();
+ jj_ntk = -1;
+ jj_gen++;
+ return token;
+ }
+
+/** Get the specific Token. */
+ final public Token getToken(int index) {
+ Token t = token;
+ for (int i = 0; i < index; i++) {
+ if (t.next != null) t = t.next;
+ else t = t.next = token_source.getNextToken();
+ }
+ return t;
+ }
+
+ private int jj_ntk() {
+ if ((jj_nt=token.next) == null)
+ return (jj_ntk = (token.next=token_source.getNextToken()).kind);
+ else
+ return (jj_ntk = jj_nt.kind);
+ }
+
+ private java.util.List<int[]> jj_expentries = new java.util.ArrayList<int[]>();
+ private int[] jj_expentry;
+ private int jj_kind = -1;
+ private int[] jj_lasttokens = new int[100];
+ private int jj_endpos;
+
+ private void jj_add_error_token(int kind, int pos) {
+ if (pos >= 100) return;
+ if (pos == jj_endpos + 1) {
+ jj_lasttokens[jj_endpos++] = kind;
+ } else if (jj_endpos != 0) {
+ jj_expentry = new int[jj_endpos];
+ for (int i = 0; i < jj_endpos; i++) {
+ jj_expentry[i] = jj_lasttokens[i];
+ }
+ jj_entries_loop: for (java.util.Iterator<?> it = jj_expentries.iterator(); it.hasNext();) {
+ int[] oldentry = (int[])(it.next());
+ if (oldentry.length == jj_expentry.length) {
+ for (int i = 0; i < jj_expentry.length; i++) {
+ if (oldentry[i] != jj_expentry[i]) {
+ continue jj_entries_loop;
+ }
+ }
+ jj_expentries.add(jj_expentry);
+ break jj_entries_loop;
+ }
+ }
+ if (pos != 0) jj_lasttokens[(jj_endpos = pos) - 1] = kind;
+ }
+ }
+
+ /** Generate ParseException. */
+ public ParseException generateParseException() {
+ jj_expentries.clear();
+ boolean[] la1tokens = new boolean[31];
+ if (jj_kind >= 0) {
+ la1tokens[jj_kind] = true;
+ jj_kind = -1;
+ }
+ for (int i = 0; i < 0; i++) {
+ if (jj_la1[i] == jj_gen) {
+ for (int j = 0; j < 32; j++) {
+ if ((jj_la1_0[i] & (1<<j)) != 0) {
+ la1tokens[j] = true;
+ }
+ }
+ }
+ }
+ for (int i = 0; i < 31; i++) {
+ if (la1tokens[i]) {
+ jj_expentry = new int[1];
+ jj_expentry[0] = i;
+ jj_expentries.add(jj_expentry);
+ }
+ }
+ jj_endpos = 0;
+ jj_rescan_token();
+ jj_add_error_token(0, 0);
+ int[][] exptokseq = new int[jj_expentries.size()][];
+ for (int i = 0; i < jj_expentries.size(); i++) {
+ exptokseq[i] = jj_expentries.get(i);
+ }
+ return new ParseException(token, exptokseq, tokenImage);
+ }
+
+ /** Enable tracing. */
+ final public void enable_tracing() {
+ }
+
+ /** Disable tracing. */
+ final public void disable_tracing() {
+ }
+
+ private void jj_rescan_token() {
+ jj_rescan = true;
+ for (int i = 0; i < 74; i++) {
+ try {
+ JJCalls p = jj_2_rtns[i];
+ do {
+ if (p.gen > jj_gen) {
+ jj_la = p.arg; jj_lastpos = jj_scanpos = p.first;
+ switch (i) {
+ case 0: jj_3_1(); break;
+ case 1: jj_3_2(); break;
+ case 2: jj_3_3(); break;
+ case 3: jj_3_4(); break;
+ case 4: jj_3_5(); break;
+ case 5: jj_3_6(); break;
+ case 6: jj_3_7(); break;
+ case 7: jj_3_8(); break;
+ case 8: jj_3_9(); break;
+ case 9: jj_3_10(); break;
+ case 10: jj_3_11(); break;
+ case 11: jj_3_12(); break;
+ case 12: jj_3_13(); break;
+ case 13: jj_3_14(); break;
+ case 14: jj_3_15(); break;
+ case 15: jj_3_16(); break;
+ case 16: jj_3_17(); break;
+ case 17: jj_3_18(); break;
+ case 18: jj_3_19(); break;
+ case 19: jj_3_20(); break;
+ case 20: jj_3_21(); break;
+ case 21: jj_3_22(); break;
+ case 22: jj_3_23(); break;
+ case 23: jj_3_24(); break;
+ case 24: jj_3_25(); break;
+ case 25: jj_3_26(); break;
+ case 26: jj_3_27(); break;
+ case 27: jj_3_28(); break;
+ case 28: jj_3_29(); break;
+ case 29: jj_3_30(); break;
+ case 30: jj_3_31(); break;
+ case 31: jj_3_32(); break;
+ case 32: jj_3_33(); break;
+ case 33: jj_3_34(); break;
+ case 34: jj_3_35(); break;
+ case 35: jj_3_36(); break;
+ case 36: jj_3_37(); break;
+ case 37: jj_3_38(); break;
+ case 38: jj_3_39(); break;
+ case 39: jj_3_40(); break;
+ case 40: jj_3_41(); break;
+ case 41: jj_3_42(); break;
+ case 42: jj_3_43(); break;
+ case 43: jj_3_44(); break;
+ case 44: jj_3_45(); break;
+ case 45: jj_3_46(); break;
+ case 46: jj_3_47(); break;
+ case 47: jj_3_48(); break;
+ case 48: jj_3_49(); break;
+ case 49: jj_3_50(); break;
+ case 50: jj_3_51(); break;
+ case 51: jj_3_52(); break;
+ case 52: jj_3_53(); break;
+ case 53: jj_3_54(); break;
+ case 54: jj_3_55(); break;
+ case 55: jj_3_56(); break;
+ case 56: jj_3_57(); break;
+ case 57: jj_3_58(); break;
+ case 58: jj_3_59(); break;
+ case 59: jj_3_60(); break;
+ case 60: jj_3_61(); break;
+ case 61: jj_3_62(); break;
+ case 62: jj_3_63(); break;
+ case 63: jj_3_64(); break;
+ case 64: jj_3_65(); break;
+ case 65: jj_3_66(); break;
+ case 66: jj_3_67(); break;
+ case 67: jj_3_68(); break;
+ case 68: jj_3_69(); break;
+ case 69: jj_3_70(); break;
+ case 70: jj_3_71(); break;
+ case 71: jj_3_72(); break;
+ case 72: jj_3_73(); break;
+ case 73: jj_3_74(); break;
+ }
+ }
+ p = p.next;
+ } while (p != null);
+ } catch(LookaheadSuccess ls) { }
+ }
+ jj_rescan = false;
+ }
+
+ private void jj_save(int index, int xla) {
+ JJCalls p = jj_2_rtns[index];
+ while (p.gen > jj_gen) {
+ if (p.next == null) { p = p.next = new JJCalls(); break; }
+ p = p.next;
+ }
+ p.gen = jj_gen + xla - jj_la; p.first = token; p.arg = xla;
+ }
+
+ static final class JJCalls {
+ int gen;
+ Token first;
+ int arg;
+ JJCalls next;
+ }
+
+}
diff --git a/src/mtas/parser/cql/MtasCQLParser.jj b/src/mtas/parser/cql/MtasCQLParser.jj
index 9dc5ea7..41199f2 100644
--- a/src/mtas/parser/cql/MtasCQLParser.jj
+++ b/src/mtas/parser/cql/MtasCQLParser.jj
@@ -8,8 +8,8 @@ options
DEBUG_TOKEN_MANAGER = false;
LOOKAHEAD= 1000;
-}
-
+}
+
PARSER_BEGIN(MtasCQLParser)
package mtas.parser.cql;
import mtas.analysis.token.MtasToken;
@@ -23,6 +23,7 @@ import mtas.parser.cql.util.MtasCQLParserSentenceCondition;
import mtas.parser.cql.util.MtasCQLParserSentencePartCondition;
import mtas.parser.cql.util.MtasCQLParserGroupQuery;
import mtas.parser.cql.util.MtasCQLParserWordQuery;
+import mtas.parser.cql.util.MtasCQLParserDefaultPrefixCondition;
import org.apache.lucene.search.spans.SpanContainingQuery;
import org.apache.lucene.search.spans.SpanNotQuery;
import org.apache.lucene.search.spans.SpanQuery;
@@ -36,9 +37,9 @@ import java.util.regex.Pattern;
public class MtasCQLParser
{
- public SpanQuery parse(String field) throws ParseException
+ public SpanQuery parse(String field, String defaultPrefix) throws ParseException
{
- return cql(field);
+ return cql(field, defaultPrefix);
}
private String unquoteString(String unfiltered)
@@ -47,7 +48,7 @@ public class MtasCQLParser
{
unfiltered = unfiltered.substring(1, unfiltered.length());
unfiltered = unfiltered.substring(0, unfiltered.length() - 1);
- //unfiltered = unfiltered.replace("\\\"", "\"");
}
+ }
return unfiltered;
}
}
@@ -202,18 +203,18 @@ TOKEN :
>
}
-private SpanQuery cql(String field) throws ParseException :
+private SpanQuery cql(String field, String defaultPrefix) throws ParseException :
{
SpanQuery q;
ArrayList < MtasSpanSequenceItem > itemList = new ArrayList < MtasSpanSequenceItem > ();
}
{
- q = cqlBlock(field)
+ q = cqlBlock(field, defaultPrefix)
{
itemList.add(new MtasSpanSequenceItem(q, false));
}
(
- q = cqlBlock(field)
+ q = cqlBlock(field, defaultPrefix)
{
itemList.add(new MtasSpanSequenceItem(q, false));
}
@@ -231,7 +232,7 @@ private SpanQuery cql(String field) throws ParseException :
}
}
-private SpanQuery cqlBlock(String field) throws ParseException :
+private SpanQuery cqlBlock(String field, String defaultPrefix) throws ParseException :
{
MtasCQLParserSentenceCondition s1 = null, s2 = null;
SpanQuery q1 = null, q2 = null;
@@ -244,9 +245,9 @@ private SpanQuery cqlBlock(String field) throws ParseException :
}
{
(
- s1 = sentence(field)
+ s1 = sentence(field, defaultPrefix)
|
- < BRACKET_START > q1 = cqlBlock(field) < BRACKET_END >
+ < BRACKET_START > q1 = cqlBlock(field, defaultPrefix) < BRACKET_END >
)
[
(
@@ -268,8 +269,8 @@ private SpanQuery cqlBlock(String field) throws ParseException :
}
)
(
- s2 = sentence(field)
- | < BRACKET_START > q2 = cqlBlock(field) < BRACKET_END >
+ s2 = sentence(field, defaultPrefix)
+ | < BRACKET_START > q2 = cqlBlock(field, defaultPrefix) < BRACKET_END >
)
]
{
@@ -311,7 +312,7 @@ private SpanQuery cqlBlock(String field) throws ParseException :
}
}
-private MtasCQLParserSentenceCondition sentence(String field) throws ParseException :
+private MtasCQLParserSentenceCondition sentence(String field, String defaultPrefix) throws ParseException :
{
MtasCQLParserSentenceCondition sentenceCondition;
MtasCQLParserSentencePartCondition condition;
@@ -322,32 +323,14 @@ private MtasCQLParserSentenceCondition sentence(String field) throws ParseExcept
int maximumOccurence = 1;
}
{
- condition = sentencePart(field)
+ condition = sentencePart(field, defaultPrefix)
{
sentenceCondition = condition.createFullSentence();
- if (questionMark != null)
- {
- minimumOccurence = 0;
- maximumOccurence = 1;
- }
- else if (minValue != null)
- {
- minimumOccurence = Integer.parseInt(minValue.image);
- if (maxValue != null)
- {
- maximumOccurence = Integer.parseInt(maxValue.image);
- }
- else
- {
- maximumOccurence = minimumOccurence;
- }
- }
- sentenceCondition.setOccurence(minimumOccurence, maximumOccurence);
return sentenceCondition;
}
}
-private MtasCQLParserSentencePartCondition sentencePart(String field) throws ParseException :
+private MtasCQLParserSentencePartCondition sentencePart(String field, String defaultPrefix) throws ParseException :
{
Token operator;
MtasCQLParserSentencePartCondition condition, sentencePart;
@@ -360,13 +343,13 @@ private MtasCQLParserSentencePartCondition sentencePart(String field) throws Par
}
{
(
- basicSentence = basicSentence(field)
+ basicSentence = basicSentence(field, defaultPrefix)
{
condition = new MtasCQLParserSentencePartCondition(basicSentence);
}
|
(
- < BRACKET_START > sentencePart = sentencePart(field) < BRACKET_END >
+ < BRACKET_START > sentencePart = sentencePart(field, defaultPrefix) < BRACKET_END >
[
{
questionMark = null;
@@ -389,11 +372,11 @@ private MtasCQLParserSentencePartCondition sentencePart(String field) throws Par
minimumOccurence = Integer.parseInt(minValue.image);
if (maxValue != null)
{
- maximumOccurence = Integer.parseInt(maxValue.image);
+ maximumOccurence = Integer.parseInt(maxValue.image);
}
else
{
- maximumOccurence = minimumOccurence;
+ maximumOccurence = minimumOccurence;
}
}
condition.setFirstOccurence(minimumOccurence, maximumOccurence);
@@ -406,7 +389,7 @@ private MtasCQLParserSentencePartCondition sentencePart(String field) throws Par
}
[ operator = < OR > ]
(
- sentencePart = sentencePart(field)
+ sentencePart = sentencePart(field, defaultPrefix)
{
if (operator == null)
{
@@ -425,7 +408,7 @@ private MtasCQLParserSentencePartCondition sentencePart(String field) throws Par
}
}
-private MtasCQLParserBasicSentenceCondition basicSentence(String field) throws ParseException :
+private MtasCQLParserBasicSentenceCondition basicSentence(String field, String defaultPrefix) throws ParseException :
{
MtasCQLParserWordFullCondition subWordCondition;
MtasCQLParserGroupFullCondition subGroupCondition;
@@ -433,7 +416,7 @@ private MtasCQLParserBasicSentenceCondition basicSentence(String field) throws P
}
{
(
- subWordCondition = word(field)
+ subWordCondition = word(field, defaultPrefix)
{
condition.addWord(subWordCondition);
}
@@ -444,7 +427,7 @@ private MtasCQLParserBasicSentenceCondition basicSentence(String field) throws P
)
(
(
- subWordCondition = word(field)
+ subWordCondition = word(field, defaultPrefix)
{
condition.addWord(subWordCondition);
}
@@ -567,9 +550,10 @@ private MtasCQLParserGroupCondition groupCondition(String field) throws ParseExc
}
}
-private MtasCQLParserWordFullCondition word(String field) throws ParseException :
+private MtasCQLParserWordFullCondition word(String field, String defaultPrefix) throws ParseException :
{
Token questionMark = null;
+ Token value;
MtasCQLParserWordFullCondition wordCondition;
MtasCQLParserWordCondition condition, subCondition;
Token minValue = null;
@@ -579,6 +563,11 @@ private MtasCQLParserWordFullCondition word(String field) throws ParseException
}
{
(
+ value = < QUOTED_VALUE >
+ {
+ condition = new MtasCQLParserDefaultPrefixCondition(field, defaultPrefix, unquoteString(value.image));
+ }
+ |
< WORD_START >
(
subCondition = wordCondition(field)
@@ -723,14 +712,14 @@ private MtasCQLParserWordCondition wordCondition(String field) throws ParseExcep
{
condition.addCondition(subCondition);
}
- )*
+ )*
|
(
- condition = wordAtomCondition(field)
- | condition = wordCondition(field)
+ condition = wordAtomCondition(field)
+ | condition = wordCondition(field)
)
)
- < BRACKET_END >
+ < BRACKET_END >
{
if (negation != null)
{
diff --git a/src/mtas/parser/cql/MtasCQLParserConstants.java b/src/mtas/parser/cql/MtasCQLParserConstants.java
new file mode 100644
index 0000000..e52ad32
--- /dev/null
+++ b/src/mtas/parser/cql/MtasCQLParserConstants.java
@@ -0,0 +1,104 @@
+/* Generated By:JavaCC: Do not edit this line. MtasCQLParserConstants.java */
+package mtas.parser.cql;
+
+
+/**
+ * Token literal values and constants.
+ * Generated by org.javacc.parser.OtherFilesGen#start()
+ */
+public interface MtasCQLParserConstants {
+
+ /** End of File. */
+ int EOF = 0;
+ /** RegularExpression Id. */
+ int CONTAINING = 5;
+ /** RegularExpression Id. */
+ int NOT_CONTAINING = 6;
+ /** RegularExpression Id. */
+ int WITHIN = 7;
+ /** RegularExpression Id. */
+ int NOT_WITHIN = 8;
+ /** RegularExpression Id. */
+ int GROUP_START = 9;
+ /** RegularExpression Id. */
+ int GROUP_END = 10;
+ /** RegularExpression Id. */
+ int WORD_START = 11;
+ /** RegularExpression Id. */
+ int WORD_END = 12;
+ /** RegularExpression Id. */
+ int BRACKET_START = 13;
+ /** RegularExpression Id. */
+ int BRACKET_END = 14;
+ /** RegularExpression Id. */
+ int CURLY_BRACKET_START = 15;
+ /** RegularExpression Id. */
+ int CURLY_BRACKET_END = 16;
+ /** RegularExpression Id. */
+ int SLASH = 17;
+ /** RegularExpression Id. */
+ int QUESTION_MARK = 18;
+ /** RegularExpression Id. */
+ int NEGATION = 19;
+ /** RegularExpression Id. */
+ int AND = 20;
+ /** RegularExpression Id. */
+ int OR = 21;
+ /** RegularExpression Id. */
+ int KOMMA = 22;
+ /** RegularExpression Id. */
+ int TOKEN_NOTEQUALS = 23;
+ /** RegularExpression Id. */
+ int TOKEN_EQUALS = 24;
+ /** RegularExpression Id. */
+ int NUMBER = 25;
+ /** RegularExpression Id. */
+ int QUOTED_VALUE = 26;
+ /** RegularExpression Id. */
+ int UNQUOTED_VALUE = 27;
+ /** RegularExpression Id. */
+ int OCTOTHORPE = 28;
+ /** RegularExpression Id. */
+ int DIGIT = 29;
+ /** RegularExpression Id. */
+ int ALLOWED_UNQUOTED_CHARACTER = 30;
+
+ /** Lexical state. */
+ int DEFAULT = 0;
+
+ /** Literal token values. */
+ String[] tokenImage = {
+ "<EOF>",
+ "\" \"",
+ "\"\\r\"",
+ "\"\\t\"",
+ "\"\\n\"",
+ "\"containing\"",
+ "\"!containing\"",
+ "\"within\"",
+ "\"!within\"",
+ "\"<\"",
+ "\">\"",
+ "\"[\"",
+ "\"]\"",
+ "\"(\"",
+ "\")\"",
+ "\"{\"",
+ "\"}\"",
+ "\"/\"",
+ "\"?\"",
+ "\"!\"",
+ "\"&\"",
+ "\"|\"",
+ "\",\"",
+ "\"!=\"",
+ "\"=\"",
+ "<NUMBER>",
+ "<QUOTED_VALUE>",
+ "<UNQUOTED_VALUE>",
+ "<OCTOTHORPE>",
+ "<DIGIT>",
+ "<ALLOWED_UNQUOTED_CHARACTER>",
+ };
+
+}
diff --git a/src/mtas/parser/cql/MtasCQLParserTokenManager.java b/src/mtas/parser/cql/MtasCQLParserTokenManager.java
new file mode 100644
index 0000000..a62f7dd
--- /dev/null
+++ b/src/mtas/parser/cql/MtasCQLParserTokenManager.java
@@ -0,0 +1,750 @@
+/* Generated By:JavaCC: Do not edit this line. MtasCQLParserTokenManager.java */
+package mtas.parser.cql;
+import mtas.analysis.token.MtasToken;
+import mtas.parser.cql.util.MtasCQLParserGroupCondition;
+import mtas.parser.cql.util.MtasCQLParserGroupFullCondition;
+import mtas.parser.cql.util.MtasCQLParserWordCondition;
+import mtas.parser.cql.util.MtasCQLParserWordPositionQuery;
+import mtas.parser.cql.util.MtasCQLParserWordFullCondition;
+import mtas.parser.cql.util.MtasCQLParserBasicSentenceCondition;
+import mtas.parser.cql.util.MtasCQLParserSentenceCondition;
+import mtas.parser.cql.util.MtasCQLParserSentencePartCondition;
+import mtas.parser.cql.util.MtasCQLParserGroupQuery;
+import mtas.parser.cql.util.MtasCQLParserWordQuery;
+import mtas.parser.cql.util.MtasCQLParserDefaultPrefixCondition;
+import org.apache.lucene.search.spans.SpanContainingQuery;
+import org.apache.lucene.search.spans.SpanNotQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanWithinQuery;
+import org.apache.lucene.index.Term;
+import mtas.search.spans.MtasSpanSequenceItem;
+import mtas.search.spans.MtasSpanSequenceQuery;
+import java.util.ArrayList;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** Token Manager. */
+public class MtasCQLParserTokenManager implements MtasCQLParserConstants
+{
+
+ /** Debug output. */
+ public java.io.PrintStream debugStream = System.out;
+ /** Set debug output. */
+ public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
+private final int jjStopStringLiteralDfa_0(int pos, long active0)
+{
+ switch (pos)
+ {
+ case 0:
+ if ((active0 & 0xa0L) != 0L)
+ {
+ jjmatchedKind = 27;
+ return 6;
+ }
+ return -1;
+ case 1:
+ if ((active0 & 0xa0L) != 0L)
+ {
+ jjmatchedKind = 27;
+ jjmatchedPos = 1;
+ return 6;
+ }
+ return -1;
+ case 2:
+ if ((active0 & 0xa0L) != 0L)
+ {
+ jjmatchedKind = 27;
+ jjmatchedPos = 2;
+ return 6;
+ }
+ return -1;
+ case 3:
+ if ((active0 & 0xa0L) != 0L)
+ {
+ jjmatchedKind = 27;
+ jjmatchedPos = 3;
+ return 6;
+ }
+ return -1;
+ case 4:
+ if ((active0 & 0xa0L) != 0L)
+ {
+ jjmatchedKind = 27;
+ jjmatchedPos = 4;
+ return 6;
+ }
+ return -1;
+ case 5:
+ if ((active0 & 0x80L) != 0L)
+ return 6;
+ if ((active0 & 0x20L) != 0L)
+ {
+ jjmatchedKind = 27;
+ jjmatchedPos = 5;
+ return 6;
+ }
+ return -1;
+ case 6:
+ if ((active0 & 0x20L) != 0L)
+ {
+ jjmatchedKind = 27;
+ jjmatchedPos = 6;
+ return 6;
+ }
+ return -1;
+ case 7:
+ if ((active0 & 0x20L) != 0L)
+ {
+ jjmatchedKind = 27;
+ jjmatchedPos = 7;
+ return 6;
+ }
+ return -1;
+ case 8:
+ if ((active0 & 0x20L) != 0L)
+ {
+ jjmatchedKind = 27;
+ jjmatchedPos = 8;
+ return 6;
+ }
+ return -1;
+ case 9:
+ if ((active0 & 0x20L) != 0L)
+ return 6;
+ return -1;
+ default :
+ return -1;
+ }
+}
+private final int jjStartNfa_0(int pos, long active0)
+{
+ return jjMoveNfa_0(jjStopStringLiteralDfa_0(pos, active0), pos + 1);
+}
+private int jjStopAtPos(int pos, int kind)
+{
+ jjmatchedKind = kind;
+ jjmatchedPos = pos;
+ return pos + 1;
+}
+private int jjMoveStringLiteralDfa0_0()
+{
+ switch(curChar)
+ {
+ case 33:
+ jjmatchedKind = 19;
+ return jjMoveStringLiteralDfa1_0(0x800140L);
+ case 38:
+ return jjStopAtPos(0, 20);
+ case 40:
+ return jjStopAtPos(0, 13);
+ case 41:
+ return jjStopAtPos(0, 14);
+ case 44:
+ return jjStopAtPos(0, 22);
+ case 47:
+ return jjStopAtPos(0, 17);
+ case 60:
+ return jjStopAtPos(0, 9);
+ case 61:
+ return jjStopAtPos(0, 24);
+ case 62:
+ return jjStopAtPos(0, 10);
+ case 63:
+ return jjStopAtPos(0, 18);
+ case 91:
+ return jjStopAtPos(0, 11);
+ case 93:
+ return jjStopAtPos(0, 12);
+ case 99:
+ return jjMoveStringLiteralDfa1_0(0x20L);
+ case 119:
+ return jjMoveStringLiteralDfa1_0(0x80L);
+ case 123:
+ return jjStopAtPos(0, 15);
+ case 124:
+ return jjStopAtPos(0, 21);
+ case 125:
+ return jjStopAtPos(0, 16);
+ default :
+ return jjMoveNfa_0(1, 0);
+ }
+}
+private int jjMoveStringLiteralDfa1_0(long active0)
+{
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(0, active0);
+ return 1;
+ }
+ switch(curChar)
+ {
+ case 61:
+ if ((active0 & 0x800000L) != 0L)
+ return jjStopAtPos(1, 23);
+ break;
+ case 99:
+ return jjMoveStringLiteralDfa2_0(active0, 0x40L);
+ case 105:
+ return jjMoveStringLiteralDfa2_0(active0, 0x80L);
+ case 111:
+ return jjMoveStringLiteralDfa2_0(active0, 0x20L);
+ case 119:
+ return jjMoveStringLiteralDfa2_0(active0, 0x100L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(0, active0);
+}
+private int jjMoveStringLiteralDfa2_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(0, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(1, active0);
+ return 2;
+ }
+ switch(curChar)
+ {
+ case 105:
+ return jjMoveStringLiteralDfa3_0(active0, 0x100L);
+ case 110:
+ return jjMoveStringLiteralDfa3_0(active0, 0x20L);
+ case 111:
+ return jjMoveStringLiteralDfa3_0(active0, 0x40L);
+ case 116:
+ return jjMoveStringLiteralDfa3_0(active0, 0x80L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(1, active0);
+}
+private int jjMoveStringLiteralDfa3_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(1, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(2, active0);
+ return 3;
+ }
+ switch(curChar)
+ {
+ case 104:
+ return jjMoveStringLiteralDfa4_0(active0, 0x80L);
+ case 110:
+ return jjMoveStringLiteralDfa4_0(active0, 0x40L);
+ case 116:
+ return jjMoveStringLiteralDfa4_0(active0, 0x120L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(2, active0);
+}
+private int jjMoveStringLiteralDfa4_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(2, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(3, active0);
+ return 4;
+ }
+ switch(curChar)
+ {
+ case 97:
+ return jjMoveStringLiteralDfa5_0(active0, 0x20L);
+ case 104:
+ return jjMoveStringLiteralDfa5_0(active0, 0x100L);
+ case 105:
+ return jjMoveStringLiteralDfa5_0(active0, 0x80L);
+ case 116:
+ return jjMoveStringLiteralDfa5_0(active0, 0x40L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(3, active0);
+}
+private int jjMoveStringLiteralDfa5_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(3, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(4, active0);
+ return 5;
+ }
+ switch(curChar)
+ {
+ case 97:
+ return jjMoveStringLiteralDfa6_0(active0, 0x40L);
+ case 105:
+ return jjMoveStringLiteralDfa6_0(active0, 0x120L);
+ case 110:
+ if ((active0 & 0x80L) != 0L)
+ return jjStartNfaWithStates_0(5, 7, 6);
+ break;
+ default :
+ break;
+ }
+ return jjStartNfa_0(4, active0);
+}
+private int jjMoveStringLiteralDfa6_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(4, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(5, active0);
+ return 6;
+ }
+ switch(curChar)
+ {
+ case 105:
+ return jjMoveStringLiteralDfa7_0(active0, 0x40L);
+ case 110:
+ if ((active0 & 0x100L) != 0L)
+ return jjStopAtPos(6, 8);
+ return jjMoveStringLiteralDfa7_0(active0, 0x20L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(5, active0);
+}
+private int jjMoveStringLiteralDfa7_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(5, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(6, active0);
+ return 7;
+ }
+ switch(curChar)
+ {
+ case 105:
+ return jjMoveStringLiteralDfa8_0(active0, 0x20L);
+ case 110:
+ return jjMoveStringLiteralDfa8_0(active0, 0x40L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(6, active0);
+}
+private int jjMoveStringLiteralDfa8_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(6, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(7, active0);
+ return 8;
+ }
+ switch(curChar)
+ {
+ case 105:
+ return jjMoveStringLiteralDfa9_0(active0, 0x40L);
+ case 110:
+ return jjMoveStringLiteralDfa9_0(active0, 0x20L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(7, active0);
+}
+private int jjMoveStringLiteralDfa9_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(7, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(8, active0);
+ return 9;
+ }
+ switch(curChar)
+ {
+ case 103:
+ if ((active0 & 0x20L) != 0L)
+ return jjStartNfaWithStates_0(9, 5, 6);
+ break;
+ case 110:
+ return jjMoveStringLiteralDfa10_0(active0, 0x40L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(8, active0);
+}
+private int jjMoveStringLiteralDfa10_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(8, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(9, active0);
+ return 10;
+ }
+ switch(curChar)
+ {
+ case 103:
+ if ((active0 & 0x40L) != 0L)
+ return jjStopAtPos(10, 6);
+ break;
+ default :
+ break;
+ }
+ return jjStartNfa_0(9, active0);
+}
+private int jjStartNfaWithStates_0(int pos, int kind, int state)
+{
+ jjmatchedKind = kind;
+ jjmatchedPos = pos;
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) { return pos + 1; }
+ return jjMoveNfa_0(state, pos + 1);
+}
+static final long[] jjbitVec0 = {
+ 0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
+};
+static final long[] jjbitVec2 = {
+ 0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
+};
+private int jjMoveNfa_0(int startState, int curPos)
+{
+ int startsAt = 0;
+ jjnewStateCnt = 8;
+ int i = 1;
+ jjstateSet[0] = startState;
+ int kind = 0x7fffffff;
+ for (;;)
+ {
+ if (++jjround == 0x7fffffff)
+ ReInitRounds();
+ if (curChar < 64)
+ {
+ long l = 1L << curChar;
+ do
+ {
+ switch(jjstateSet[--i])
+ {
+ case 1:
+ if ((0x3ff600000000000L & l) != 0L)
+ {
+ if (kind > 27)
+ kind = 27;
+ jjCheckNAdd(6);
+ }
+ else if (curChar == 35)
+ {
+ if (kind > 28)
+ kind = 28;
+ }
+ else if (curChar == 34)
+ jjCheckNAddStates(0, 2);
+ if ((0x3ff000000000000L & l) != 0L)
+ {
+ if (kind > 25)
+ kind = 25;
+ jjCheckNAdd(0);
+ }
+ break;
+ case 0:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 25)
+ kind = 25;
+ jjCheckNAdd(0);
+ break;
+ case 3:
+ jjCheckNAddStates(0, 2);
+ break;
+ case 4:
+ if ((0xfffffffbffffffffL & l) != 0L)
+ jjCheckNAddStates(0, 2);
+ break;
+ case 5:
+ if (curChar == 34 && kind > 26)
+ kind = 26;
+ break;
+ case 6:
+ if ((0x3ff600000000000L & l) == 0L)
+ break;
+ if (kind > 27)
+ kind = 27;
+ jjCheckNAdd(6);
+ break;
+ case 7:
+ if (curChar == 35 && kind > 28)
+ kind = 28;
+ break;
+ default : break;
+ }
+ } while(i != startsAt);
+ }
+ else if (curChar < 128)
+ {
+ long l = 1L << (curChar & 077);
+ do
+ {
+ switch(jjstateSet[--i])
+ {
+ case 1:
+ case 6:
+ if ((0x7fffffe87fffffeL & l) == 0L)
+ break;
+ if (kind > 27)
+ kind = 27;
+ jjCheckNAdd(6);
+ break;
+ case 2:
+ if (curChar == 92)
+ jjstateSet[jjnewStateCnt++] = 3;
+ break;
+ case 3:
+ jjCheckNAddStates(0, 2);
+ break;
+ case 4:
+ if ((0xffffffffefffffffL & l) != 0L)
+ jjCheckNAddStates(0, 2);
+ break;
+ default : break;
+ }
+ } while(i != startsAt);
+ }
+ else
+ {
+ int hiByte = (int)(curChar >> 8);
+ int i1 = hiByte >> 6;
+ long l1 = 1L << (hiByte & 077);
+ int i2 = (curChar & 0xff) >> 6;
+ long l2 = 1L << (curChar & 077);
+ do
+ {
+ switch(jjstateSet[--i])
+ {
+ case 3:
+ case 4:
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+ jjCheckNAddStates(0, 2);
+ break;
+ default : break;
+ }
+ } while(i != startsAt);
+ }
+ if (kind != 0x7fffffff)
+ {
+ jjmatchedKind = kind;
+ jjmatchedPos = curPos;
+ kind = 0x7fffffff;
+ }
+ ++curPos;
+ if ((i = jjnewStateCnt) == (startsAt = 8 - (jjnewStateCnt = startsAt)))
+ return curPos;
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) { return curPos; }
+ }
+}
+static final int[] jjnextStates = {
+ 2, 4, 5,
+};
+private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
+{
+ switch(hiByte)
+ {
+ case 0:
+ return ((jjbitVec2[i2] & l2) != 0L);
+ default :
+ if ((jjbitVec0[i1] & l1) != 0L)
+ return true;
+ return false;
+ }
+}
+
+/** Token literal values. */
+public static final String[] jjstrLiteralImages = {
+"", null, null, null, null, "\143\157\156\164\141\151\156\151\156\147",
+"\41\143\157\156\164\141\151\156\151\156\147", "\167\151\164\150\151\156", "\41\167\151\164\150\151\156", "\74", "\76",
+"\133", "\135", "\50", "\51", "\173", "\175", "\57", "\77", "\41", "\46", "\174",
+"\54", "\41\75", "\75", null, null, null, null, null, null, };
+
+/** Lexer state names. */
+public static final String[] lexStateNames = {
+ "DEFAULT",
+};
+static final long[] jjtoToken = {
+ 0x1fffffe1L,
+};
+static final long[] jjtoSkip = {
+ 0x1eL,
+};
+protected SimpleCharStream input_stream;
+private final int[] jjrounds = new int[8];
+private final int[] jjstateSet = new int[16];
+protected char curChar;
+/** Constructor. */
+public MtasCQLParserTokenManager(SimpleCharStream stream){
+ if (SimpleCharStream.staticFlag)
+ throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.");
+ input_stream = stream;
+}
+
+/** Constructor. */
+public MtasCQLParserTokenManager(SimpleCharStream stream, int lexState){
+ this(stream);
+ SwitchTo(lexState);
+}
+
+/** Reinitialise parser. */
+public void ReInit(SimpleCharStream stream)
+{
+ jjmatchedPos = jjnewStateCnt = 0;
+ curLexState = defaultLexState;
+ input_stream = stream;
+ ReInitRounds();
+}
+private void ReInitRounds()
+{
+ int i;
+ jjround = 0x80000001;
+ for (i = 8; i-- > 0;)
+ jjrounds[i] = 0x80000000;
+}
+
+/** Reinitialise parser. */
+public void ReInit(SimpleCharStream stream, int lexState)
+{
+ ReInit(stream);
+ SwitchTo(lexState);
+}
+
+/** Switch to specified lex state. */
+public void SwitchTo(int lexState)
+{
+ if (lexState >= 1 || lexState < 0)
+ throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE);
+ else
+ curLexState = lexState;
+}
+
+protected Token jjFillToken()
+{
+ final Token t;
+ final String curTokenImage;
+ final int beginLine;
+ final int endLine;
+ final int beginColumn;
+ final int endColumn;
+ String im = jjstrLiteralImages[jjmatchedKind];
+ curTokenImage = (im == null) ? input_stream.GetImage() : im;
+ beginLine = input_stream.getBeginLine();
+ beginColumn = input_stream.getBeginColumn();
+ endLine = input_stream.getEndLine();
+ endColumn = input_stream.getEndColumn();
+ t = Token.newToken(jjmatchedKind, curTokenImage);
+
+ t.beginLine = beginLine;
+ t.endLine = endLine;
+ t.beginColumn = beginColumn;
+ t.endColumn = endColumn;
+
+ return t;
+}
+
+int curLexState = 0;
+int defaultLexState = 0;
+int jjnewStateCnt;
+int jjround;
+int jjmatchedPos;
+int jjmatchedKind;
+
+/** Get the next Token. */
+public Token getNextToken()
+{
+ Token matchedToken;
+ int curPos = 0;
+
+ EOFLoop :
+ for (;;)
+ {
+ try
+ {
+ curChar = input_stream.BeginToken();
+ }
+ catch(java.io.IOException e)
+ {
+ jjmatchedKind = 0;
+ matchedToken = jjFillToken();
+ return matchedToken;
+ }
+
+ try { input_stream.backup(0);
+ while (curChar <= 32 && (0x100002600L & (1L << curChar)) != 0L)
+ curChar = input_stream.BeginToken();
+ }
+ catch (java.io.IOException e1) { continue EOFLoop; }
+ jjmatchedKind = 0x7fffffff;
+ jjmatchedPos = 0;
+ curPos = jjMoveStringLiteralDfa0_0();
+ if (jjmatchedKind != 0x7fffffff)
+ {
+ if (jjmatchedPos + 1 < curPos)
+ input_stream.backup(curPos - jjmatchedPos - 1);
+ if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L)
+ {
+ matchedToken = jjFillToken();
+ return matchedToken;
+ }
+ else
+ {
+ continue EOFLoop;
+ }
+ }
+ int error_line = input_stream.getEndLine();
+ int error_column = input_stream.getEndColumn();
+ String error_after = null;
+ boolean EOFSeen = false;
+ try { input_stream.readChar(); input_stream.backup(1); }
+ catch (java.io.IOException e1) {
+ EOFSeen = true;
+ error_after = curPos <= 1 ? "" : input_stream.GetImage();
+ if (curChar == '\n' || curChar == '\r') {
+ error_line++;
+ error_column = 0;
+ }
+ else
+ error_column++;
+ }
+ if (!EOFSeen) {
+ input_stream.backup(1);
+ error_after = curPos <= 1 ? "" : input_stream.GetImage();
+ }
+ throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
+ }
+}
+
+private void jjCheckNAdd(int state)
+{
+ if (jjrounds[state] != jjround)
+ {
+ jjstateSet[jjnewStateCnt++] = state;
+ jjrounds[state] = jjround;
+ }
+}
+private void jjAddStates(int start, int end)
+{
+ do {
+ jjstateSet[jjnewStateCnt++] = jjnextStates[start];
+ } while (start++ != end);
+}
+private void jjCheckNAddTwoStates(int state1, int state2)
+{
+ jjCheckNAdd(state1);
+ jjCheckNAdd(state2);
+}
+
+private void jjCheckNAddStates(int start, int end)
+{
+ do {
+ jjCheckNAdd(jjnextStates[start]);
+ } while (start++ != end);
+}
+
+}
diff --git a/src/mtas/parser/cql/ParseException.java b/src/mtas/parser/cql/ParseException.java
new file mode 100644
index 0000000..f8c628c
--- /dev/null
+++ b/src/mtas/parser/cql/ParseException.java
@@ -0,0 +1,187 @@
+/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 5.0 */
+/* JavaCCOptions:KEEP_LINE_COL=null */
+package mtas.parser.cql;
+
+/**
+ * This exception is thrown when parse errors are encountered.
+ * You can explicitly create objects of this exception type by
+ * calling the method generateParseException in the generated
+ * parser.
+ *
+ * You can modify this class to customize your error reporting
+ * mechanisms so long as you retain the public fields.
+ */
+public class ParseException extends Exception {
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * This constructor is used by the method "generateParseException"
+ * in the generated parser. Calling this constructor generates
+ * a new object of this type with the fields "currentToken",
+ * "expectedTokenSequences", and "tokenImage" set.
+ */
+ public ParseException(Token currentTokenVal,
+ int[][] expectedTokenSequencesVal,
+ String[] tokenImageVal
+ )
+ {
+ super(initialise(currentTokenVal, expectedTokenSequencesVal, tokenImageVal));
+ currentToken = currentTokenVal;
+ expectedTokenSequences = expectedTokenSequencesVal;
+ tokenImage = tokenImageVal;
+ }
+
+ /**
+ * The following constructors are for use by you for whatever
+ * purpose you can think of. Constructing the exception in this
+ * manner makes the exception behave in the normal way - i.e., as
+ * documented in the class "Throwable". The fields "errorToken",
+ * "expectedTokenSequences", and "tokenImage" do not contain
+ * relevant information. The JavaCC generated code does not use
+ * these constructors.
+ */
+
+ public ParseException() {
+ super();
+ }
+
+ /** Constructor with message. */
+ public ParseException(String message) {
+ super(message);
+ }
+
+
+ /**
+ * This is the last token that has been consumed successfully. If
+ * this object has been created due to a parse error, the token
+ * followng this token will (therefore) be the first error token.
+ */
+ public Token currentToken;
+
+ /**
+ * Each entry in this array is an array of integers. Each array
+ * of integers represents a sequence of tokens (by their ordinal
+ * values) that is expected at this point of the parse.
+ */
+ public int[][] expectedTokenSequences;
+
+ /**
+ * This is a reference to the "tokenImage" array of the generated
+ * parser within which the parse error occurred. This array is
+ * defined in the generated ...Constants interface.
+ */
+ public String[] tokenImage;
+
+ /**
+ * It uses "currentToken" and "expectedTokenSequences" to generate a parse
+ * error message and returns it. If this object has been created
+ * due to a parse error, and you do not catch it (it gets thrown
+ * from the parser) the correct error message
+ * gets displayed.
+ */
+ private static String initialise(Token currentToken,
+ int[][] expectedTokenSequences,
+ String[] tokenImage) {
+ String eol = System.getProperty("line.separator", "\n");
+ StringBuffer expected = new StringBuffer();
+ int maxSize = 0;
+ for (int i = 0; i < expectedTokenSequences.length; i++) {
+ if (maxSize < expectedTokenSequences[i].length) {
+ maxSize = expectedTokenSequences[i].length;
+ }
+ for (int j = 0; j < expectedTokenSequences[i].length; j++) {
+ expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' ');
+ }
+ if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
+ expected.append("...");
+ }
+ expected.append(eol).append(" ");
+ }
+ String retval = "Encountered \"";
+ Token tok = currentToken.next;
+ for (int i = 0; i < maxSize; i++) {
+ if (i != 0) retval += " ";
+ if (tok.kind == 0) {
+ retval += tokenImage[0];
+ break;
+ }
+ retval += " " + tokenImage[tok.kind];
+ retval += " \"";
+ retval += add_escapes(tok.image);
+ retval += " \"";
+ tok = tok.next;
+ }
+ retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn;
+ retval += "." + eol;
+ if (expectedTokenSequences.length == 1) {
+ retval += "Was expecting:" + eol + " ";
+ } else {
+ retval += "Was expecting one of:" + eol + " ";
+ }
+ retval += expected.toString();
+ return retval;
+ }
+
+ /**
+ * The end of line string for this machine.
+ */
+ protected String eol = System.getProperty("line.separator", "\n");
+
+ /**
+ * Used to convert raw characters to their escaped version
+ * when these raw version cannot be used as part of an ASCII
+ * string literal.
+ */
+ static String add_escapes(String str) {
+ StringBuffer retval = new StringBuffer();
+ char ch;
+ for (int i = 0; i < str.length(); i++) {
+ switch (str.charAt(i))
+ {
+ case 0 :
+ continue;
+ case '\b':
+ retval.append("\\b");
+ continue;
+ case '\t':
+ retval.append("\\t");
+ continue;
+ case '\n':
+ retval.append("\\n");
+ continue;
+ case '\f':
+ retval.append("\\f");
+ continue;
+ case '\r':
+ retval.append("\\r");
+ continue;
+ case '\"':
+ retval.append("\\\"");
+ continue;
+ case '\'':
+ retval.append("\\\'");
+ continue;
+ case '\\':
+ retval.append("\\\\");
+ continue;
+ default:
+ if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+ } else {
+ retval.append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.toString();
+ }
+
+}
+/* JavaCC - OriginalChecksum=a5744d0f598c98bc282befd42ee797ff (do not edit this line) */
diff --git a/src/mtas/parser/cql/SimpleCharStream.java b/src/mtas/parser/cql/SimpleCharStream.java
new file mode 100644
index 0000000..8f5ab97
--- /dev/null
+++ b/src/mtas/parser/cql/SimpleCharStream.java
@@ -0,0 +1,471 @@
+/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 5.0 */
+/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
+package mtas.parser.cql;
+
+/**
+ * An implementation of interface CharStream, where the stream is assumed to
+ * contain only ASCII characters (without unicode processing).
+ */
+
+public class SimpleCharStream
+{
+/** Whether parser is static. */
+ public static final boolean staticFlag = false;
+ int bufsize;
+ int available;
+ int tokenBegin;
+/** Position in buffer. */
+ public int bufpos = -1;
+ protected int bufline[];
+ protected int bufcolumn[];
+
+ protected int column = 0;
+ protected int line = 1;
+
+ protected boolean prevCharIsCR = false;
+ protected boolean prevCharIsLF = false;
+
+ protected java.io.Reader inputStream;
+
+ protected char[] buffer;
+ protected int maxNextCharInd = 0;
+ protected int inBuf = 0;
+ protected int tabSize = 8;
+
+ protected void setTabSize(int i) { tabSize = i; }
+ protected int getTabSize(int i) { return tabSize; }
+
+
+ protected void ExpandBuff(boolean wrapAround)
+ {
+ char[] newbuffer = new char[bufsize + 2048];
+ int newbufline[] = new int[bufsize + 2048];
+ int newbufcolumn[] = new int[bufsize + 2048];
+
+ try
+ {
+ if (wrapAround)
+ {
+ System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+ System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos);
+ buffer = newbuffer;
+
+ System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+ System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos);
+ bufline = newbufline;
+
+ System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+ System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos);
+ bufcolumn = newbufcolumn;
+
+ maxNextCharInd = (bufpos += (bufsize - tokenBegin));
+ }
+ else
+ {
+ System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+ buffer = newbuffer;
+
+ System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+ bufline = newbufline;
+
+ System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+ bufcolumn = newbufcolumn;
+
+ maxNextCharInd = (bufpos -= tokenBegin);
+ }
+ }
+ catch (Throwable t)
+ {
+ throw new Error(t.getMessage());
+ }
+
+
+ bufsize += 2048;
+ available = bufsize;
+ tokenBegin = 0;
+ }
+
+ protected void FillBuff() throws java.io.IOException
+ {
+ if (maxNextCharInd == available)
+ {
+ if (available == bufsize)
+ {
+ if (tokenBegin > 2048)
+ {
+ bufpos = maxNextCharInd = 0;
+ available = tokenBegin;
+ }
+ else if (tokenBegin < 0)
+ bufpos = maxNextCharInd = 0;
+ else
+ ExpandBuff(false);
+ }
+ else if (available > tokenBegin)
+ available = bufsize;
+ else if ((tokenBegin - available) < 2048)
+ ExpandBuff(true);
+ else
+ available = tokenBegin;
+ }
+
+ int i;
+ try {
+ if ((i = inputStream.read(buffer, maxNextCharInd, available - maxNextCharInd)) == -1)
+ {
+ inputStream.close();
+ throw new java.io.IOException();
+ }
+ else
+ maxNextCharInd += i;
+ return;
+ }
+ catch(java.io.IOException e) {
+ --bufpos;
+ backup(0);
+ if (tokenBegin == -1)
+ tokenBegin = bufpos;
+ throw e;
+ }
+ }
+
+/** Start. */
+ public char BeginToken() throws java.io.IOException
+ {
+ tokenBegin = -1;
+ char c = readChar();
+ tokenBegin = bufpos;
+
+ return c;
+ }
+
+ protected void UpdateLineColumn(char c)
+ {
+ column++;
+
+ if (prevCharIsLF)
+ {
+ prevCharIsLF = false;
+ line += (column = 1);
+ }
+ else if (prevCharIsCR)
+ {
+ prevCharIsCR = false;
+ if (c == '\n')
+ {
+ prevCharIsLF = true;
+ }
+ else
+ line += (column = 1);
+ }
+
+ switch (c)
+ {
+ case '\r' :
+ prevCharIsCR = true;
+ break;
+ case '\n' :
+ prevCharIsLF = true;
+ break;
+ case '\t' :
+ column--;
+ column += (tabSize - (column % tabSize));
+ break;
+ default :
+ break;
+ }
+
+ bufline[bufpos] = line;
+ bufcolumn[bufpos] = column;
+ }
+
+/** Read a character. */
+ public char readChar() throws java.io.IOException
+ {
+ if (inBuf > 0)
+ {
+ --inBuf;
+
+ if (++bufpos == bufsize)
+ bufpos = 0;
+
+ return buffer[bufpos];
+ }
+
+ if (++bufpos >= maxNextCharInd)
+ FillBuff();
+
+ char c = buffer[bufpos];
+
+ UpdateLineColumn(c);
+ return c;
+ }
+
+ @Deprecated
+ /**
+ * @deprecated
+ * @see #getEndColumn
+ */
+
+ public int getColumn() {
+ return bufcolumn[bufpos];
+ }
+
+ @Deprecated
+ /**
+ * @deprecated
+ * @see #getEndLine
+ */
+
+ public int getLine() {
+ return bufline[bufpos];
+ }
+
+ /** Get token end column number. */
+ public int getEndColumn() {
+ return bufcolumn[bufpos];
+ }
+
+ /** Get token end line number. */
+ public int getEndLine() {
+ return bufline[bufpos];
+ }
+
+ /** Get token beginning column number. */
+ public int getBeginColumn() {
+ return bufcolumn[tokenBegin];
+ }
+
+ /** Get token beginning line number. */
+ public int getBeginLine() {
+ return bufline[tokenBegin];
+ }
+
+/** Backup a number of characters. */
+ public void backup(int amount) {
+
+ inBuf += amount;
+ if ((bufpos -= amount) < 0)
+ bufpos += bufsize;
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.Reader dstream, int startline,
+ int startcolumn, int buffersize)
+ {
+ inputStream = dstream;
+ line = startline;
+ column = startcolumn - 1;
+
+ available = bufsize = buffersize;
+ buffer = new char[buffersize];
+ bufline = new int[buffersize];
+ bufcolumn = new int[buffersize];
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.Reader dstream, int startline,
+ int startcolumn)
+ {
+ this(dstream, startline, startcolumn, 4096);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.Reader dstream)
+ {
+ this(dstream, 1, 1, 4096);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.Reader dstream, int startline,
+ int startcolumn, int buffersize)
+ {
+ inputStream = dstream;
+ line = startline;
+ column = startcolumn - 1;
+
+ if (buffer == null || buffersize != buffer.length)
+ {
+ available = bufsize = buffersize;
+ buffer = new char[buffersize];
+ bufline = new int[buffersize];
+ bufcolumn = new int[buffersize];
+ }
+ prevCharIsLF = prevCharIsCR = false;
+ tokenBegin = inBuf = maxNextCharInd = 0;
+ bufpos = -1;
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.Reader dstream, int startline,
+ int startcolumn)
+ {
+ ReInit(dstream, startline, startcolumn, 4096);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.Reader dstream)
+ {
+ ReInit(dstream, 1, 1, 4096);
+ }
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
+ {
+ this(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream, int startline,
+ int startcolumn, int buffersize)
+ {
+ this(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn) throws java.io.UnsupportedEncodingException
+ {
+ this(dstream, encoding, startline, startcolumn, 4096);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream, int startline,
+ int startcolumn)
+ {
+ this(dstream, startline, startcolumn, 4096);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
+ {
+ this(dstream, encoding, 1, 1, 4096);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream)
+ {
+ this(dstream, 1, 1, 4096);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
+ {
+ ReInit(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, int startline,
+ int startcolumn, int buffersize)
+ {
+ ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
+ {
+ ReInit(dstream, encoding, 1, 1, 4096);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream)
+ {
+ ReInit(dstream, 1, 1, 4096);
+ }
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn) throws java.io.UnsupportedEncodingException
+ {
+ ReInit(dstream, encoding, startline, startcolumn, 4096);
+ }
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, int startline,
+ int startcolumn)
+ {
+ ReInit(dstream, startline, startcolumn, 4096);
+ }
+ /** Get token literal value. */
+ public String GetImage()
+ {
+ if (bufpos >= tokenBegin)
+ return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
+ else
+ return new String(buffer, tokenBegin, bufsize - tokenBegin) +
+ new String(buffer, 0, bufpos + 1);
+ }
+
+ /** Get the suffix. */
+ public char[] GetSuffix(int len)
+ {
+ char[] ret = new char[len];
+
+ if ((bufpos + 1) >= len)
+ System.arraycopy(buffer, bufpos - len + 1, ret, 0, len);
+ else
+ {
+ System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0,
+ len - bufpos - 1);
+ System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1);
+ }
+
+ return ret;
+ }
+
+ /** Reset buffer when finished. */
+ public void Done()
+ {
+ buffer = null;
+ bufline = null;
+ bufcolumn = null;
+ }
+
+ /**
+ * Method to adjust line and column numbers for the start of a token.
+ */
+ public void adjustBeginLineColumn(int newLine, int newCol)
+ {
+ int start = tokenBegin;
+ int len;
+
+ if (bufpos >= tokenBegin)
+ {
+ len = bufpos - tokenBegin + inBuf + 1;
+ }
+ else
+ {
+ len = bufsize - tokenBegin + bufpos + 1 + inBuf;
+ }
+
+ int i = 0, j = 0, k = 0;
+ int nextColDiff = 0, columnDiff = 0;
+
+ while (i < len && bufline[j = start % bufsize] == bufline[k = ++start % bufsize])
+ {
+ bufline[j] = newLine;
+ nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j];
+ bufcolumn[j] = newCol + columnDiff;
+ columnDiff = nextColDiff;
+ i++;
+ }
+
+ if (i < len)
+ {
+ bufline[j] = newLine++;
+ bufcolumn[j] = newCol + columnDiff;
+
+ while (i++ < len)
+ {
+ if (bufline[j = start % bufsize] != bufline[++start % bufsize])
+ bufline[j] = newLine++;
+ else
+ bufline[j] = newLine;
+ }
+ }
+
+ line = bufline[j];
+ column = bufcolumn[j];
+ }
+
+}
+/* JavaCC - OriginalChecksum=684019f26036db3f6d2af88727527a69 (do not edit this line) */
diff --git a/src/mtas/parser/cql/Token.java b/src/mtas/parser/cql/Token.java
new file mode 100644
index 0000000..44b780e
--- /dev/null
+++ b/src/mtas/parser/cql/Token.java
@@ -0,0 +1,131 @@
+/* Generated By:JavaCC: Do not edit this line. Token.java Version 5.0 */
+/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
+package mtas.parser.cql;
+
+/**
+ * Describes the input token stream.
+ */
+
+public class Token implements java.io.Serializable {
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * An integer that describes the kind of this token. This numbering
+ * system is determined by JavaCCParser, and a table of these numbers is
+ * stored in the file ...Constants.java.
+ */
+ public int kind;
+
+ /** The line number of the first character of this Token. */
+ public int beginLine;
+ /** The column number of the first character of this Token. */
+ public int beginColumn;
+ /** The line number of the last character of this Token. */
+ public int endLine;
+ /** The column number of the last character of this Token. */
+ public int endColumn;
+
+ /**
+ * The string image of the token.
+ */
+ public String image;
+
+ /**
+ * A reference to the next regular (non-special) token from the input
+ * stream. If this is the last token from the input stream, or if the
+ * token manager has not read tokens beyond this one, this field is
+ * set to null. This is true only if this token is also a regular
+ * token. Otherwise, see below for a description of the contents of
+ * this field.
+ */
+ public Token next;
+
+ /**
+ * This field is used to access special tokens that occur prior to this
+ * token, but after the immediately preceding regular (non-special) token.
+ * If there are no such special tokens, this field is set to null.
+ * When there are more than one such special token, this field refers
+ * to the last of these special tokens, which in turn refers to the next
+ * previous special token through its specialToken field, and so on
+ * until the first special token (whose specialToken field is null).
+ * The next fields of special tokens refer to other special tokens that
+ * immediately follow it (without an intervening regular token). If there
+ * is no such token, this field is null.
+ */
+ public Token specialToken;
+
+ /**
+ * An optional attribute value of the Token.
+ * Tokens which are not used as syntactic sugar will often contain
+ * meaningful values that will be used later on by the compiler or
+ * interpreter. This attribute value is often different from the image.
+ * Any subclass of Token that actually wants to return a non-null value can
+ * override this method as appropriate.
+ */
+ public Object getValue() {
+ return null;
+ }
+
+ /**
+ * No-argument constructor
+ */
+ public Token() {}
+
+ /**
+ * Constructs a new token for the specified Image.
+ */
+ public Token(int kind)
+ {
+ this(kind, null);
+ }
+
+ /**
+ * Constructs a new token for the specified Image and Kind.
+ */
+ public Token(int kind, String image)
+ {
+ this.kind = kind;
+ this.image = image;
+ }
+
+ /**
+ * Returns the image.
+ */
+ public String toString()
+ {
+ return image;
+ }
+
+ /**
+ * Returns a new Token object, by default. However, if you want, you
+ * can create and return subclass objects based on the value of ofKind.
+ * Simply add the cases to the switch for all those special cases.
+ * For example, if you have a subclass of Token called IDToken that
+ * you want to create if ofKind is ID, simply add something like :
+ *
+ * case MyParserConstants.ID : return new IDToken(ofKind, image);
+ *
+ * to the following switch statement. Then you can cast matchedToken
+ * variable to the appropriate type and use sit in your lexical actions.
+ */
+ public static Token newToken(int ofKind, String image)
+ {
+ switch(ofKind)
+ {
+ default : return new Token(ofKind, image);
+ }
+ }
+
+ public static Token newToken(int ofKind)
+ {
+ return newToken(ofKind, null);
+ }
+
+}
+/* JavaCC - OriginalChecksum=e03311f40a067ab1c4ea0bcdd2b5d3be (do not edit this line) */
diff --git a/src/mtas/parser/cql/TokenMgrError.java b/src/mtas/parser/cql/TokenMgrError.java
new file mode 100644
index 0000000..83a720f
--- /dev/null
+++ b/src/mtas/parser/cql/TokenMgrError.java
@@ -0,0 +1,147 @@
+/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 5.0 */
+/* JavaCCOptions: */
+package mtas.parser.cql;
+
+/** Token Manager Error. */
+public class TokenMgrError extends Error
+{
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /*
+ * Ordinals for various reasons why an Error of this type can be thrown.
+ */
+
+ /**
+ * Lexical error occurred.
+ */
+ static final int LEXICAL_ERROR = 0;
+
+ /**
+ * An attempt was made to create a second instance of a static token manager.
+ */
+ static final int STATIC_LEXER_ERROR = 1;
+
+ /**
+ * Tried to change to an invalid lexical state.
+ */
+ static final int INVALID_LEXICAL_STATE = 2;
+
+ /**
+ * Detected (and bailed out of) an infinite loop in the token manager.
+ */
+ static final int LOOP_DETECTED = 3;
+
+ /**
+ * Indicates the reason why the exception is thrown. It will have
+ * one of the above 4 values.
+ */
+ int errorCode;
+
+ /**
+ * Replaces unprintable characters by their escaped (or unicode escaped)
+ * equivalents in the given string
+ */
+ protected static final String addEscapes(String str) {
+ StringBuffer retval = new StringBuffer();
+ char ch;
+ for (int i = 0; i < str.length(); i++) {
+ switch (str.charAt(i))
+ {
+ case 0 :
+ continue;
+ case '\b':
+ retval.append("\\b");
+ continue;
+ case '\t':
+ retval.append("\\t");
+ continue;
+ case '\n':
+ retval.append("\\n");
+ continue;
+ case '\f':
+ retval.append("\\f");
+ continue;
+ case '\r':
+ retval.append("\\r");
+ continue;
+ case '\"':
+ retval.append("\\\"");
+ continue;
+ case '\'':
+ retval.append("\\\'");
+ continue;
+ case '\\':
+ retval.append("\\\\");
+ continue;
+ default:
+ if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+ } else {
+ retval.append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.toString();
+ }
+
+ /**
+ * Returns a detailed message for the Error when it is thrown by the
+ * token manager to indicate a lexical error.
+ * Parameters :
+ * EOFSeen : indicates if EOF caused the lexical error
+ * curLexState : lexical state in which this error occurred
+ * errorLine : line number when the error occurred
+ * errorColumn : column number when the error occurred
+ * errorAfter : prefix that was seen before this error occurred
+ * curchar : the offending character
+ * Note: You can customize the lexical error message by modifying this method.
+ */
+ protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) {
+ return("Lexical error at line " +
+ errorLine + ", column " +
+ errorColumn + ". Encountered: " +
+ (EOFSeen ? "<EOF> " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") +
+ "after : \"" + addEscapes(errorAfter) + "\"");
+ }
+
+ /**
+ * You can also modify the body of this method to customize your error messages.
+ * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
+ * of end-users concern, so you can return something like :
+ *
+ * "Internal Error : Please file a bug report .... "
+ *
+ * from this method for such cases in the release version of your parser.
+ */
+ public String getMessage() {
+ return super.getMessage();
+ }
+
+ /*
+ * Constructors of various flavors follow.
+ */
+
+ /** No arg constructor. */
+ public TokenMgrError() {
+ }
+
+ /** Constructor with message and reason. */
+ public TokenMgrError(String message, int reason) {
+ super(message);
+ errorCode = reason;
+ }
+
+ /** Full Constructor. */
+ public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) {
+ this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
+ }
+}
+/* JavaCC - OriginalChecksum=e6fb0e781afa8bd7df734ee8575150b6 (do not edit this line) */
diff --git a/src/mtas/parser/cql/util/MtasCQLParserBasicSentenceCondition.java b/src/mtas/parser/cql/util/MtasCQLParserBasicSentenceCondition.java
index 9a540b9..eb4371d 100644
--- a/src/mtas/parser/cql/util/MtasCQLParserBasicSentenceCondition.java
+++ b/src/mtas/parser/cql/util/MtasCQLParserBasicSentenceCondition.java
@@ -17,10 +17,10 @@ public class MtasCQLParserBasicSentenceCondition {
/** The part list. */
private List<MtasCQLParserBasicSentencePartCondition> partList;
-
+
/** The maximum occurence. */
private int minimumOccurence, maximumOccurence;
-
+
/** The optional parts. */
private boolean simplified, optional, optionalParts;
@@ -38,25 +38,31 @@ public class MtasCQLParserBasicSentenceCondition {
/**
* Adds the word.
*
- * @param w the w
- * @throws ParseException the parse exception
+ * @param w
+ * the w
+ * @throws ParseException
+ * the parse exception
*/
public void addWord(MtasCQLParserWordFullCondition w) throws ParseException {
- assert w.getCondition().not() == false : "condition word should be positive in sentence definition";
+ assert w.getCondition()
+ .not() == false : "condition word should be positive in sentence definition";
if (!simplified) {
partList.add(w);
} else {
throw new ParseException("already simplified");
}
}
-
+
/**
* Adds the group.
*
- * @param w the w
- * @throws ParseException the parse exception
+ * @param w
+ * the w
+ * @throws ParseException
+ * the parse exception
*/
- public void addGroup(MtasCQLParserGroupFullCondition w) throws ParseException {
+ public void addGroup(MtasCQLParserGroupFullCondition w)
+ throws ParseException {
if (!simplified) {
partList.add(w);
} else {
@@ -67,13 +73,16 @@ public class MtasCQLParserBasicSentenceCondition {
/**
* Adds the basic sentence.
*
- * @param s the s
- * @throws ParseException the parse exception
+ * @param s
+ * the s
+ * @throws ParseException
+ * the parse exception
*/
public void addBasicSentence(MtasCQLParserBasicSentenceCondition s)
throws ParseException {
if (!simplified) {
- List<MtasCQLParserBasicSentencePartCondition> newWordList = s.getPartList();
+ List<MtasCQLParserBasicSentencePartCondition> newWordList = s
+ .getPartList();
partList.addAll(newWordList);
} else {
throw new ParseException("already simplified");
@@ -101,9 +110,12 @@ public class MtasCQLParserBasicSentenceCondition {
/**
* Sets the occurence.
*
- * @param min the min
- * @param max the max
- * @throws ParseException the parse exception
+ * @param min
+ * the min
+ * @param max
+ * the max
+ * @throws ParseException
+ * the parse exception
*/
public void setOccurence(int min, int max) throws ParseException {
if (!simplified) {
@@ -128,15 +140,16 @@ public class MtasCQLParserBasicSentenceCondition {
public boolean isOptional() {
return optional;
}
-
+
/**
* Checks for optional parts.
*
* @return true, if successful
- * @throws ParseException the parse exception
+ * @throws ParseException
+ * the parse exception
*/
public boolean hasOptionalParts() throws ParseException {
- if(simplified) {
+ if (simplified) {
return optionalParts;
} else {
throw new ParseException("can't be called when not simplified");
@@ -146,8 +159,10 @@ public class MtasCQLParserBasicSentenceCondition {
/**
* Sets the optional.
*
- * @param status the new optional
- * @throws ParseException the parse exception
+ * @param status
+ * the new optional
+ * @throws ParseException
+ * the parse exception
*/
public void setOptional(boolean status) throws ParseException {
optional = status;
@@ -156,7 +171,8 @@ public class MtasCQLParserBasicSentenceCondition {
/**
* Simplify.
*
- * @throws ParseException the parse exception
+ * @throws ParseException
+ * the parse exception
*/
public void simplify() throws ParseException {
if (!simplified) {
@@ -166,11 +182,11 @@ public class MtasCQLParserBasicSentenceCondition {
MtasCQLParserBasicSentencePartCondition lastPart = null;
newPartList = new ArrayList<MtasCQLParserBasicSentencePartCondition>();
// try and merge equal word conditions
- for (MtasCQLParserBasicSentencePartCondition part: partList) {
+ for (MtasCQLParserBasicSentencePartCondition part : partList) {
if ((lastPart == null) || !lastPart.equals(part)) {
lastPart = part;
newPartList.add(part);
- if(!part.isOptional()) {
+ if (!part.isOptional()) {
optionalParts = false;
}
} else {
@@ -193,7 +209,7 @@ public class MtasCQLParserBasicSentenceCondition {
} else {
lastPart = part;
newPartList.add(part);
- if(!part.isOptional()) {
+ if (!part.isOptional()) {
optionalParts = false;
}
}
@@ -209,7 +225,7 @@ public class MtasCQLParserBasicSentenceCondition {
} else {
lastPart = part;
newPartList.add(part);
- if(!part.isOptional()) {
+ if (!part.isOptional()) {
optionalParts = false;
}
}
@@ -225,7 +241,7 @@ public class MtasCQLParserBasicSentenceCondition {
} else {
lastPart = part;
newPartList.add(part);
- if(!part.isOptional()) {
+ if (!part.isOptional()) {
optionalParts = false;
}
}
@@ -233,7 +249,7 @@ public class MtasCQLParserBasicSentenceCondition {
}
}
partList = newPartList;
- if(optionalParts) {
+ if (optionalParts) {
optional = true;
}
}
@@ -252,7 +268,8 @@ public class MtasCQLParserBasicSentenceCondition {
* Gets the query.
*
* @return the query
- * @throws ParseException the parse exception
+ * @throws ParseException
+ * the parse exception
*/
public SpanQuery getQuery() throws ParseException {
simplify();
@@ -291,7 +308,9 @@ public class MtasCQLParserBasicSentenceCondition {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#toString()
*/
@Override
diff --git a/src/mtas/parser/cql/util/MtasCQLParserBasicSentencePartCondition.java b/src/mtas/parser/cql/util/MtasCQLParserBasicSentencePartCondition.java
index 327f930..4ac268d 100644
--- a/src/mtas/parser/cql/util/MtasCQLParserBasicSentencePartCondition.java
+++ b/src/mtas/parser/cql/util/MtasCQLParserBasicSentencePartCondition.java
@@ -8,18 +8,19 @@ import mtas.parser.cql.ParseException;
* The Class MtasCQLParserBasicSentencePartCondition.
*/
public abstract class MtasCQLParserBasicSentencePartCondition {
-
+
/** The maximum occurence. */
protected int minimumOccurence, maximumOccurence;
-
+
/** The not. */
protected boolean optional, not;
-
+
/**
* Gets the query.
*
* @return the query
- * @throws ParseException the parse exception
+ * @throws ParseException
+ * the parse exception
*/
public abstract SpanQuery getQuery() throws ParseException;
@@ -44,21 +45,24 @@ public abstract class MtasCQLParserBasicSentencePartCondition {
/**
* Sets the occurence.
*
- * @param min the min
- * @param max the max
- * @throws ParseException the parse exception
+ * @param min
+ * the min
+ * @param max
+ * the max
+ * @throws ParseException
+ * the parse exception
*/
public void setOccurence(int min, int max) throws ParseException {
if ((min < 0) || (min > max) || (max < 1)) {
throw new ParseException("Illegal number {" + min + "," + max + "}");
}
- if(min==0) {
+ if (min == 0) {
optional = true;
}
- minimumOccurence = Math.max(1,min);
- maximumOccurence = max;
+ minimumOccurence = Math.max(1, min);
+ maximumOccurence = max;
}
-
+
/**
* Checks if is optional.
*
@@ -67,17 +71,20 @@ public abstract class MtasCQLParserBasicSentencePartCondition {
public boolean isOptional() {
return optional;
}
-
+
/**
* Sets the optional.
*
- * @param status the new optional
+ * @param status
+ * the new optional
*/
public void setOptional(boolean status) {
optional = status;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#toString()
*/
@Override
@@ -88,14 +95,16 @@ public abstract class MtasCQLParserBasicSentencePartCondition {
/**
* To string.
*
- * @param firstIndent the first indent
- * @param indent the indent
+ * @param firstIndent
+ * the first indent
+ * @param indent
+ * the indent
* @return the string
*/
public String toString(String firstIndent, String indent) {
String text = "";
text += firstIndent + "PART";
- if(optional) {
+ if (optional) {
text += " OPTIONAL";
}
if ((minimumOccurence > 1) || (minimumOccurence != maximumOccurence)) {
@@ -106,12 +115,13 @@ public abstract class MtasCQLParserBasicSentencePartCondition {
}
}
try {
- text += "\n" + indent + "- Query: " + getQuery().toString(getQuery().getField());
+ text += "\n" + indent + "- Query: "
+ + getQuery().toString(getQuery().getField());
} catch (ParseException e) {
text += "\n" + indent + "- Query: " + e.getMessage();
}
text += "\n";
return text;
}
-
+
}
diff --git a/src/mtas/parser/cql/util/MtasCQLParserDefaultPrefixCondition.java b/src/mtas/parser/cql/util/MtasCQLParserDefaultPrefixCondition.java
new file mode 100644
index 0000000..58a28c3
--- /dev/null
+++ b/src/mtas/parser/cql/util/MtasCQLParserDefaultPrefixCondition.java
@@ -0,0 +1,33 @@
+package mtas.parser.cql.util;
+
+import mtas.parser.cql.ParseException;
+
+/**
+ * The Class MtasCQLParserDefaultPrefixCondition.
+ */
+public class MtasCQLParserDefaultPrefixCondition
+ extends MtasCQLParserWordCondition {
+
+ /**
+ * Instantiates a new mtas cql parser default prefix condition.
+ *
+ * @param field
+ * the field
+ * @param prefix
+ * the prefix
+ * @param value
+ * the value
+ * @throws ParseException
+ * the parse exception
+ */
+ public MtasCQLParserDefaultPrefixCondition(String field, String prefix,
+ String value) throws ParseException {
+ super(field, TYPE_AND);
+ if (prefix == null) {
+ throw new ParseException("no default prefix defined");
+ } else {
+ addPositiveQuery(new MtasCQLParserWordQuery(field, prefix, value));
+ }
+ }
+
+}
diff --git a/src/mtas/parser/cql/util/MtasCQLParserGroupCondition.java b/src/mtas/parser/cql/util/MtasCQLParserGroupCondition.java
index 1452b19..4ee27fa 100644
--- a/src/mtas/parser/cql/util/MtasCQLParserGroupCondition.java
+++ b/src/mtas/parser/cql/util/MtasCQLParserGroupCondition.java
@@ -9,15 +9,17 @@ public class MtasCQLParserGroupCondition {
/** The condition. */
private SpanQuery condition;
-
+
/** The field. */
private String field;
/**
* Instantiates a new mtas cql parser group condition.
*
- * @param field the field
- * @param condition the condition
+ * @param field
+ * the field
+ * @param condition
+ * the condition
*/
public MtasCQLParserGroupCondition(String field, SpanQuery condition) {
this.field = field;
@@ -39,10 +41,12 @@ public class MtasCQLParserGroupCondition {
* @return the query
*/
public SpanQuery getQuery() {
- return condition;
+ return condition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
@@ -57,7 +61,7 @@ public class MtasCQLParserGroupCondition {
} else {
if (!this.condition.equals(condition)) {
return false;
- }
+ }
return true;
}
} else {
diff --git a/src/mtas/parser/cql/util/MtasCQLParserGroupFullCondition.java b/src/mtas/parser/cql/util/MtasCQLParserGroupFullCondition.java
index e50a76a..76cd637 100644
--- a/src/mtas/parser/cql/util/MtasCQLParserGroupFullCondition.java
+++ b/src/mtas/parser/cql/util/MtasCQLParserGroupFullCondition.java
@@ -9,42 +9,46 @@ import org.apache.lucene.search.spans.SpanQuery;
/**
* The Class MtasCQLParserGroupFullCondition.
*/
-public class MtasCQLParserGroupFullCondition extends MtasCQLParserBasicSentencePartCondition {
+public class MtasCQLParserGroupFullCondition
+ extends MtasCQLParserBasicSentencePartCondition {
/** The group full. */
public static String GROUP_FULL = "full";
-
+
/** The group start. */
public static String GROUP_START = "start";
-
+
/** The group end. */
public static String GROUP_END = "end";
-
+
/** The group condition. */
private MtasCQLParserGroupCondition groupCondition;
-
+
/** The type. */
private String type;
-
+
/**
* Instantiates a new mtas cql parser group full condition.
*
- * @param condition the condition
- * @param type the type
+ * @param condition
+ * the condition
+ * @param type
+ * the type
*/
- public MtasCQLParserGroupFullCondition(MtasCQLParserGroupCondition condition, String type) {
+ public MtasCQLParserGroupFullCondition(MtasCQLParserGroupCondition condition,
+ String type) {
minimumOccurence = 1;
- maximumOccurence = 1;
+ maximumOccurence = 1;
optional = false;
not = false;
groupCondition = condition;
- if(type.equals(GROUP_START)) {
+ if (type.equals(GROUP_START)) {
this.type = GROUP_START;
- } else if(type.equals(GROUP_END)) {
+ } else if (type.equals(GROUP_END)) {
this.type = GROUP_END;
} else {
this.type = GROUP_FULL;
- }
+ }
}
/**
@@ -56,81 +60,101 @@ public class MtasCQLParserGroupFullCondition extends MtasCQLParserBasicSentenceP
return groupCondition;
}
- /* (non-Javadoc)
- * @see mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#getMinimumOccurence()
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#
+ * getMinimumOccurence()
*/
@Override
public int getMinimumOccurence() {
return minimumOccurence;
}
- /* (non-Javadoc)
- * @see mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#getMaximumOccurence()
+ /*
+ * (non-Javadoc)
+ *
+ * @see mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#
+ * getMaximumOccurence()
*/
@Override
public int getMaximumOccurence() {
return maximumOccurence;
}
- /* (non-Javadoc)
- * @see mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#setOccurence(int, int)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#setOccurence(
+ * int, int)
*/
@Override
public void setOccurence(int min, int max) throws ParseException {
if ((min < 0) || (min > max) || (max < 1)) {
throw new ParseException("Illegal number {" + min + "," + max + "}");
}
- if(min==0) {
+ if (min == 0) {
optional = true;
}
- minimumOccurence = Math.max(1,min);
- maximumOccurence = max;
+ minimumOccurence = Math.max(1, min);
+ maximumOccurence = max;
}
-
- /* (non-Javadoc)
- * @see mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#isOptional()
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#isOptional()
*/
@Override
public boolean isOptional() {
return optional;
}
-
- /* (non-Javadoc)
- * @see mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#setOptional(boolean)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#setOptional(
+ * boolean)
*/
@Override
public void setOptional(boolean status) {
optional = status;
}
- /* (non-Javadoc)
- * @see mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#getQuery()
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#getQuery()
*/
@Override
public SpanQuery getQuery() throws ParseException {
- if(type.equals(MtasCQLParserGroupFullCondition.GROUP_START)) {
+ if (type.equals(MtasCQLParserGroupFullCondition.GROUP_START)) {
return new MtasSpanStartQuery(groupCondition.getQuery());
- } else if(type.equals(MtasCQLParserGroupFullCondition.GROUP_END)) {
+ } else if (type.equals(MtasCQLParserGroupFullCondition.GROUP_END)) {
return new MtasSpanEndQuery(groupCondition.getQuery());
} else {
return groupCondition.getQuery();
}
}
-
-
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
public boolean equals(Object object) {
- if(object==null)
+ if (object == null)
return false;
- if(object instanceof MtasCQLParserGroupFullCondition) {
- MtasCQLParserGroupFullCondition word = (MtasCQLParserGroupFullCondition) object;
- if(!groupCondition.equals(word.groupCondition))
- return false;
- return true;
+ if (object instanceof MtasCQLParserGroupFullCondition) {
+ MtasCQLParserGroupFullCondition word = (MtasCQLParserGroupFullCondition) object;
+ if (!groupCondition.equals(word.groupCondition))
+ return false;
+ return true;
} else {
return false;
}
diff --git a/src/mtas/parser/cql/util/MtasCQLParserGroupQuery.java b/src/mtas/parser/cql/util/MtasCQLParserGroupQuery.java
index 0cd379e..a53ccf3 100644
--- a/src/mtas/parser/cql/util/MtasCQLParserGroupQuery.java
+++ b/src/mtas/parser/cql/util/MtasCQLParserGroupQuery.java
@@ -15,88 +15,165 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanWeight;
+/**
+ * The Class MtasCQLParserGroupQuery.
+ */
public class MtasCQLParserGroupQuery extends SpanQuery {
+ /** The query. */
SpanQuery query;
+
+ /** The term. */
Term term;
-
+
+ /** The query name. */
private static String QUERY_NAME = "mtasCQLParserGroupQuery";
-
+
+ /** The Constant MTAS_CQL_TERM_QUERY. */
public static final String MTAS_CQL_TERM_QUERY = "term";
+
+ /** The Constant MTAS_CQL_REGEXP_QUERY. */
public static final String MTAS_CQL_REGEXP_QUERY = "regexp";
+
+ /** The Constant MTAS_CQL_WILDCARD_QUERY. */
public static final String MTAS_CQL_WILDCARD_QUERY = "wildcard";
-
- public MtasCQLParserGroupQuery(String field, String prefix) {
- term = new Term(field,prefix+MtasToken.DELIMITER);
- query = new MtasSpanPrefixQuery(term, false);
+
+ /**
+ * Instantiates a new mtas cql parser group query.
+ *
+ * @param field
+ * the field
+ * @param prefix
+ * the prefix
+ */
+ public MtasCQLParserGroupQuery(String field, String prefix) {
+ term = new Term(field, prefix + MtasToken.DELIMITER);
+ query = new MtasSpanPrefixQuery(term, false);
}
-
+
+ /**
+ * Instantiates a new mtas cql parser group query.
+ *
+ * @param field
+ * the field
+ * @param prefix
+ * the prefix
+ * @param value
+ * the value
+ */
public MtasCQLParserGroupQuery(String field, String prefix, String value) {
- this(field, prefix, value, MTAS_CQL_REGEXP_QUERY);
+ this(field, prefix, value, MTAS_CQL_REGEXP_QUERY);
}
-
- public MtasCQLParserGroupQuery(String field, String prefix, String value, String type) {
- if(value==null || value.trim().equals("")) {
- term = new Term(field,prefix+MtasToken.DELIMITER);
- query = new MtasSpanPrefixQuery(term, false);
- } else {
- if(type==null) {
- term = new Term(field,prefix+MtasToken.DELIMITER+value);
+
+ /**
+ * Instantiates a new mtas cql parser group query.
+ *
+ * @param field
+ * the field
+ * @param prefix
+ * the prefix
+ * @param value
+ * the value
+ * @param type
+ * the type
+ */
+ public MtasCQLParserGroupQuery(String field, String prefix, String value,
+ String type) {
+ if (value == null || value.trim().equals("")) {
+ term = new Term(field, prefix + MtasToken.DELIMITER);
+ query = new MtasSpanPrefixQuery(term, false);
+ } else {
+ if (type == null) {
+ term = new Term(field, prefix + MtasToken.DELIMITER + value);
query = new MtasSpanTermQuery(term, false);
- } else if(type.equals(MTAS_CQL_REGEXP_QUERY)) {
- term = new Term(field,prefix+MtasToken.DELIMITER+value+"\u0000*");
+ } else if (type.equals(MTAS_CQL_REGEXP_QUERY)) {
+ term = new Term(field,
+ prefix + MtasToken.DELIMITER + value + "\u0000*");
query = new MtasSpanRegexpQuery(term, false);
- } else if(type.equals(MTAS_CQL_WILDCARD_QUERY)) {
- term = new Term(field,prefix+MtasToken.DELIMITER+value);
+ } else if (type.equals(MTAS_CQL_WILDCARD_QUERY)) {
+ term = new Term(field, prefix + MtasToken.DELIMITER + value);
query = new MtasSpanWildcardQuery(term, false);
- } else if(type.equals(MTAS_CQL_TERM_QUERY)) {
- term = new Term(field,prefix+MtasToken.DELIMITER+value);
+ } else if (type.equals(MTAS_CQL_TERM_QUERY)) {
+ term = new Term(field, prefix + MtasToken.DELIMITER + value);
query = new MtasSpanTermQuery(term, false);
} else {
- term = new Term(field,prefix+MtasToken.DELIMITER+value);
+ term = new Term(field, prefix + MtasToken.DELIMITER + value);
query = new MtasSpanTermQuery(term, false);
- }
- }
+ }
+ }
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.search.spans.SpanQuery#getField()
+ */
@Override
public String getField() {
return term.field();
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
+ */
@Override
public Query rewrite(IndexReader reader) throws IOException {
return query.rewrite(reader);
}
-
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
+ */
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
throws IOException {
return query.createWeight(searcher, needsScores);
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.search.Query#toString(java.lang.String)
+ */
@Override
public String toString(String field) {
return query.toString(term.field());
}
-
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.search.Query#equals(java.lang.Object)
+ */
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
- if (obj== null)
+ if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final MtasCQLParserGroupQuery that = (MtasCQLParserGroupQuery) obj;
- return query.equals(that.query);
+ return query.equals(that.query);
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.search.Query#hashCode()
+ */
@Override
public int hashCode() {
int h = QUERY_NAME.hashCode();
h = (h * 7) ^ query.hashCode();
return h;
}
-
+
}
diff --git a/src/mtas/parser/cql/util/MtasCQLParserSentenceCondition.java b/src/mtas/parser/cql/util/MtasCQLParserSentenceCondition.java
index 48c9a85..503bfee 100644
--- a/src/mtas/parser/cql/util/MtasCQLParserSentenceCondition.java
+++ b/src/mtas/parser/cql/util/MtasCQLParserSentenceCondition.java
@@ -20,27 +20,29 @@ public class MtasCQLParserSentenceCondition {
/** The sequence list. */
// child list: sequence
private List<List<MtasCQLParserSentenceCondition>> sequenceList;
-
+
/** The sentence sequence. */
private List<MtasCQLParserSentenceCondition> sentenceSequence = null;
-
+
/** The sentence current. */
private MtasCQLParserSentenceCondition sentenceCurrent = null;
-
+
/** The basic sentence. */
private MtasCQLParserBasicSentenceCondition basicSentence = null;
-
+
/** The maximum occurence. */
private int minimumOccurence, maximumOccurence;
-
+
/** The optional parts. */
private boolean basic, simplified, optional, optionalParts;
/**
* Instantiates a new mtas cql parser sentence condition.
*
- * @param s the s
- * @throws ParseException the parse exception
+ * @param s
+ * the s
+ * @throws ParseException
+ * the parse exception
*/
public MtasCQLParserSentenceCondition(MtasCQLParserBasicSentenceCondition s)
throws ParseException {
@@ -56,8 +58,10 @@ public class MtasCQLParserSentenceCondition {
/**
* Instantiates a new mtas cql parser sentence condition.
*
- * @param sp the sp
- * @throws ParseException the parse exception
+ * @param sp
+ * the sp
+ * @throws ParseException
+ * the parse exception
*/
public MtasCQLParserSentenceCondition(MtasCQLParserSentenceCondition sp)
throws ParseException {
@@ -73,8 +77,10 @@ public class MtasCQLParserSentenceCondition {
/**
* Adds the basic sentence to end latest sequence.
*
- * @param s the s
- * @throws ParseException the parse exception
+ * @param s
+ * the s
+ * @throws ParseException
+ * the parse exception
*/
public void addBasicSentenceToEndLatestSequence(
MtasCQLParserBasicSentenceCondition s) throws ParseException {
@@ -97,8 +103,10 @@ public class MtasCQLParserSentenceCondition {
/**
* Adds the basic sentence as option.
*
- * @param s the s
- * @throws ParseException the parse exception
+ * @param s
+ * the s
+ * @throws ParseException
+ * the parse exception
*/
public void addBasicSentenceAsOption(MtasCQLParserBasicSentenceCondition s)
throws ParseException {
@@ -135,8 +143,10 @@ public class MtasCQLParserSentenceCondition {
/**
* Adds the sentence to start first sequence.
*
- * @param s the s
- * @throws ParseException the parse exception
+ * @param s
+ * the s
+ * @throws ParseException
+ * the parse exception
*/
public void addSentenceToStartFirstSequence(MtasCQLParserSentenceCondition s)
throws ParseException {
@@ -179,8 +189,10 @@ public class MtasCQLParserSentenceCondition {
/**
* Adds the sentence to end latest sequence.
*
- * @param s the s
- * @throws ParseException the parse exception
+ * @param s
+ * the s
+ * @throws ParseException
+ * the parse exception
*/
public void addSentenceToEndLatestSequence(MtasCQLParserSentenceCondition s)
throws ParseException {
@@ -208,10 +220,10 @@ public class MtasCQLParserSentenceCondition {
}
} else {
sentenceCurrent = s;
- if(sentenceSequence==null) {
- sentenceSequence = new ArrayList<MtasCQLParserSentenceCondition>();
+ if (sentenceSequence == null) {
+ sentenceSequence = new ArrayList<MtasCQLParserSentenceCondition>();
sequenceList.add(sentenceSequence);
- }
+ }
sentenceSequence.add(sentenceCurrent);
}
} else {
@@ -222,8 +234,10 @@ public class MtasCQLParserSentenceCondition {
/**
* Adds the sentence as first option.
*
- * @param s the s
- * @throws ParseException the parse exception
+ * @param s
+ * the s
+ * @throws ParseException
+ * the parse exception
*/
public void addSentenceAsFirstOption(MtasCQLParserSentenceCondition s)
throws ParseException {
@@ -268,8 +282,10 @@ public class MtasCQLParserSentenceCondition {
/**
* Adds the sentence as option.
*
- * @param s the s
- * @throws ParseException the parse exception
+ * @param s
+ * the s
+ * @throws ParseException
+ * the parse exception
*/
public void addSentenceAsOption(MtasCQLParserSentenceCondition s)
throws ParseException {
@@ -329,7 +345,8 @@ public class MtasCQLParserSentenceCondition {
/**
* Simplify.
*
- * @throws ParseException the parse exception
+ * @throws ParseException
+ * the parse exception
*/
public void simplify() throws ParseException {
if (!simplified) {
@@ -338,13 +355,13 @@ public class MtasCQLParserSentenceCondition {
simplifySequence(sequence);
}
}
- //flatten
- if(sequenceList.size()>1) {
+ // flatten
+ if (sequenceList.size() > 1) {
List<List<MtasCQLParserSentenceCondition>> newSequenceList = new ArrayList<List<MtasCQLParserSentenceCondition>>();
for (List<MtasCQLParserSentenceCondition> sequence : sequenceList) {
- if(sequence.size()==1) {
+ if (sequence.size() == 1) {
MtasCQLParserSentenceCondition subSentence = sequence.get(0);
- if(subSentence.isBasic()) {
+ if (subSentence.isBasic()) {
newSequenceList.add(sequence);
} else {
newSequenceList.addAll(subSentence.getsequenceList());
@@ -360,8 +377,10 @@ public class MtasCQLParserSentenceCondition {
/**
* Simplify sequence.
*
- * @param sequence the sequence
- * @throws ParseException the parse exception
+ * @param sequence
+ * the sequence
+ * @throws ParseException
+ * the parse exception
*/
private void simplifySequence(List<MtasCQLParserSentenceCondition> sequence)
throws ParseException {
@@ -383,11 +402,12 @@ public class MtasCQLParserSentenceCondition {
// ..-.....-.....+........+.....+.....-.....-
// ..+.....-.....+........+.....+.....+.....-
// ..+.....+.....+........+.....+.....+.....+
- if ((((lastSentence.getMaximumOccurence() == 1) && (sentence
- .getMaximumOccurence() == 1))
+ if ((((lastSentence.getMaximumOccurence() == 1)
+ && (sentence.getMaximumOccurence() == 1))
|| lastSentence.getQuery().equals(sentence.getQuery())
- && ((!lastSentence.isOptional() && !sentence.isOptional()))
- || lastSentence.hasOptionalParts() || sentence.hasOptionalParts())) {
+ && ((!lastSentence.isOptional() && !sentence.isOptional()))
+ || lastSentence.hasOptionalParts()
+ || sentence.hasOptionalParts())) {
// create new basic sentence
MtasCQLParserBasicSentenceCondition newBasicSentence = new MtasCQLParserBasicSentenceCondition();
newBasicSentence.addBasicSentence(lastSentence.basicSentence);
@@ -405,8 +425,8 @@ public class MtasCQLParserSentenceCondition {
} else if (lastSentence.isBasic()) {
if (sentence.isSingle()
&& (!sentence.isOptional() || sentence.hasOptionalParts())
- && ((sentence.getMaximumOccurence() == 1) && (lastSentence
- .getMaximumOccurence() == 1))) {
+ && ((sentence.getMaximumOccurence() == 1)
+ && (lastSentence.getMaximumOccurence() == 1))) {
for (MtasCQLParserSentenceCondition subSentence : sentence
.getsequenceList().get(0)) {
newSequence.add(lastSentence);
@@ -418,9 +438,11 @@ public class MtasCQLParserSentenceCondition {
}
} else if (sentence.isBasic()) {
if (lastSentence.isSingle()
- && (!lastSentence.isOptional() || lastSentence.hasOptionalParts())&& ((sentence.getMaximumOccurence() == 1) && (lastSentence
- .getMaximumOccurence() == 1))) {
- lastSentence.addBasicSentenceToEndLatestSequence(sentence.getBasicSentence());
+ && (!lastSentence.isOptional() || lastSentence.hasOptionalParts())
+ && ((sentence.getMaximumOccurence() == 1)
+ && (lastSentence.getMaximumOccurence() == 1))) {
+ lastSentence
+ .addBasicSentenceToEndLatestSequence(sentence.getBasicSentence());
} else {
newSequence.add(lastSentence);
lastSentence = sentence;
@@ -476,9 +498,12 @@ public class MtasCQLParserSentenceCondition {
/**
* Sets the occurence.
*
- * @param min the min
- * @param max the max
- * @throws ParseException the parse exception
+ * @param min
+ * the min
+ * @param max
+ * the max
+ * @throws ParseException
+ * the parse exception
*/
public void setOccurence(int min, int max) throws ParseException {
if ((min < 0) || (min > max) || (max < 1)) {
@@ -504,7 +529,8 @@ public class MtasCQLParserSentenceCondition {
* Checks for optional parts.
*
* @return true, if successful
- * @throws ParseException the parse exception
+ * @throws ParseException
+ * the parse exception
*/
public boolean hasOptionalParts() throws ParseException {
if (simplified) {
@@ -517,7 +543,8 @@ public class MtasCQLParserSentenceCondition {
/**
* Sets the optional.
*
- * @param status the new optional
+ * @param status
+ * the new optional
*/
public void setOptional(boolean status) {
optional = status;
@@ -526,27 +553,31 @@ public class MtasCQLParserSentenceCondition {
/**
* Creates the query.
*
- * @param sentenceSequence the sentence sequence
+ * @param sentenceSequence
+ * the sentence sequence
* @return the span query
- * @throws ParseException the parse exception
+ * @throws ParseException
+ * the parse exception
*/
private SpanQuery createQuery(
List<MtasCQLParserSentenceCondition> sentenceSequence)
throws ParseException {
if (sentenceSequence.size() == 1) {
- if(maximumOccurence>1) {
- return new MtasSpanRecurrenceQuery(sentenceSequence.get(0).getQuery(), minimumOccurence, maximumOccurence);
+ if (maximumOccurence > 1) {
+ return new MtasSpanRecurrenceQuery(sentenceSequence.get(0).getQuery(),
+ minimumOccurence, maximumOccurence);
} else {
return sentenceSequence.get(0).getQuery();
}
} else {
List<MtasSpanSequenceItem> clauses = new ArrayList<MtasSpanSequenceItem>();
for (MtasCQLParserSentenceCondition sentence : sentenceSequence) {
- clauses.add(new MtasSpanSequenceItem(sentence.getQuery(),
- sentence.optional));
+ clauses.add(
+ new MtasSpanSequenceItem(sentence.getQuery(), sentence.optional));
}
- if(maximumOccurence>1) {
- return new MtasSpanRecurrenceQuery(new MtasSpanSequenceQuery(clauses), minimumOccurence, maximumOccurence);
+ if (maximumOccurence > 1) {
+ return new MtasSpanRecurrenceQuery(new MtasSpanSequenceQuery(clauses),
+ minimumOccurence, maximumOccurence);
} else {
return new MtasSpanSequenceQuery(clauses);
}
@@ -557,7 +588,8 @@ public class MtasCQLParserSentenceCondition {
* Gets the query.
*
* @return the query
- * @throws ParseException the parse exception
+ * @throws ParseException
+ * the parse exception
*/
public SpanQuery getQuery() throws ParseException {
simplify();
@@ -570,13 +602,15 @@ public class MtasCQLParserSentenceCondition {
clauses.add(new MtasSpanSequenceItem(basicSentence.getQuery(),
basicSentence.isOptional()));
query = new MtasSpanSequenceQuery(clauses);
- if(basicSentence.getMaximumOccurence()>1) {
- query = new MtasSpanRecurrenceQuery(query, basicSentence.getMinimumOccurence(), basicSentence.getMaximumOccurence());
- }
+ if (maximumOccurence > 1) {
+ query = new MtasSpanRecurrenceQuery(query, minimumOccurence,
+ maximumOccurence);
+ }
} else {
query = basicSentence.getQuery();
- if(basicSentence.getMaximumOccurence()>1) {
- query = new MtasSpanRecurrenceQuery(query, basicSentence.getMinimumOccurence(), basicSentence.getMaximumOccurence());
+ if (maximumOccurence > 1) {
+ query = new MtasSpanRecurrenceQuery(query, minimumOccurence,
+ maximumOccurence);
}
}
return query;
@@ -589,11 +623,14 @@ public class MtasCQLParserSentenceCondition {
for (List<MtasCQLParserSentenceCondition> sentenceSequence : sequenceList) {
clauses.add(createQuery(sentenceSequence));
}
- return new MtasSpanOrQuery(clauses.toArray(new SpanQuery[clauses.size()]));
+ return new MtasSpanOrQuery(
+ clauses.toArray(new SpanQuery[clauses.size()]));
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#toString()
*/
@Override
@@ -604,8 +641,10 @@ public class MtasCQLParserSentenceCondition {
/**
* To string.
*
- * @param firstIndent the first indent
- * @param indent the indent
+ * @param firstIndent
+ * the first indent
+ * @param indent
+ * the indent
* @return the string
*/
public String toString(String firstIndent, String indent) {
@@ -623,7 +662,8 @@ public class MtasCQLParserSentenceCondition {
text += firstIndent + "SENTENCE" + (optional ? " OPTIONAL" : "") + "\n";
if (simplified) {
try {
- text += indent + "- Query: " + getQuery().toString(getQuery().getField());
+ text += indent + "- Query: "
+ + getQuery().toString(getQuery().getField());
} catch (ParseException e) {
text += indent + "- Query: " + e.getMessage();
}
diff --git a/src/mtas/parser/cql/util/MtasCQLParserSentencePartCondition.java b/src/mtas/parser/cql/util/MtasCQLParserSentencePartCondition.java
index 5b8b212..72a1b6f 100644
--- a/src/mtas/parser/cql/util/MtasCQLParserSentencePartCondition.java
+++ b/src/mtas/parser/cql/util/MtasCQLParserSentencePartCondition.java
@@ -1,9 +1,7 @@
package mtas.parser.cql.util;
-
import mtas.parser.cql.ParseException;
-
/**
* The Class MtasCQLParserSentencePartCondition.
*/
@@ -11,41 +9,44 @@ public class MtasCQLParserSentencePartCondition {
/** The first sentence. */
private MtasCQLParserSentenceCondition firstSentence = null;
-
+
/** The first basic sentence. */
private MtasCQLParserBasicSentenceCondition firstBasicSentence = null;
-
+
/** The first maximum occurence. */
private int firstMinimumOccurence, firstMaximumOccurence;
-
+
/** The first optional. */
private boolean firstOptional;
-
+
/** The second sentence part. */
MtasCQLParserSentencePartCondition secondSentencePart = null;
-
+
/** The or operator. */
private boolean orOperator = false;
-
+
/** The full condition. */
private MtasCQLParserSentenceCondition fullCondition = null;
-
+
/**
* Instantiates a new mtas cql parser sentence part condition.
*
- * @param bs the bs
+ * @param bs
+ * the bs
*/
- public MtasCQLParserSentencePartCondition(MtasCQLParserBasicSentenceCondition bs) {
+ public MtasCQLParserSentencePartCondition(
+ MtasCQLParserBasicSentenceCondition bs) {
firstMinimumOccurence = 1;
firstMaximumOccurence = 1;
firstOptional = false;
firstBasicSentence = bs;
}
-
+
/**
* Instantiates a new mtas cql parser sentence part condition.
*
- * @param s the s
+ * @param s
+ * the s
*/
public MtasCQLParserSentencePartCondition(MtasCQLParserSentenceCondition s) {
firstMinimumOccurence = 1;
@@ -53,7 +54,7 @@ public class MtasCQLParserSentencePartCondition {
firstOptional = false;
firstSentence = s;
}
-
+
/**
* Gets the first minimum occurence.
*
@@ -75,25 +76,28 @@ public class MtasCQLParserSentencePartCondition {
/**
* Sets the first occurence.
*
- * @param min the min
- * @param max the max
- * @throws ParseException the parse exception
+ * @param min
+ * the min
+ * @param max
+ * the max
+ * @throws ParseException
+ * the parse exception
*/
public void setFirstOccurence(int min, int max) throws ParseException {
- if(fullCondition==null) {
- if ((min < 0) || (min > max) || (max < 1)) {
- throw new ParseException("Illegal number {" + min + "," + max + "}");
+ if (fullCondition == null) {
+ if ((min < 0) || (min > max) || (max < 1)) {
+ throw new ParseException("Illegal number {" + min + "," + max + "}");
}
- if(min==0) {
+ if (min == 0) {
firstOptional = true;
}
- firstMinimumOccurence = Math.max(1,min);
- firstMaximumOccurence = max;
+ firstMinimumOccurence = Math.max(1, min);
+ firstMaximumOccurence = max;
} else {
throw new ParseException("fullCondition already generated");
}
}
-
+
/**
* Checks if is first optional.
*
@@ -102,81 +106,99 @@ public class MtasCQLParserSentencePartCondition {
public boolean isFirstOptional() {
return firstOptional;
}
-
+
/**
* Sets the first optional.
*
- * @param status the new first optional
- * @throws ParseException the parse exception
+ * @param status
+ * the new first optional
+ * @throws ParseException
+ * the parse exception
*/
public void setFirstOptional(boolean status) throws ParseException {
- if(fullCondition==null) {
+ if (fullCondition == null) {
firstOptional = status;
} else {
throw new ParseException("fullCondition already generated");
}
}
-
+
/**
* Sets the or.
*
- * @param status the new or
- * @throws ParseException the parse exception
+ * @param status
+ * the new or
+ * @throws ParseException
+ * the parse exception
*/
public void setOr(boolean status) throws ParseException {
- if(fullCondition==null) {
+ if (fullCondition == null) {
orOperator = status;
} else {
throw new ParseException("fullCondition already generated");
- }
+ }
}
-
+
/**
* Sets the second part.
*
- * @param sp the new second part
- * @throws ParseException the parse exception
+ * @param sp
+ * the new second part
+ * @throws ParseException
+ * the parse exception
*/
- public void setSecondPart(MtasCQLParserSentencePartCondition sp) throws ParseException {
- if(fullCondition==null) {
- secondSentencePart = sp;
+ public void setSecondPart(MtasCQLParserSentencePartCondition sp)
+ throws ParseException {
+ if (fullCondition == null) {
+ secondSentencePart = sp;
} else {
throw new ParseException("fullCondition already generated");
- }
+ }
}
-
+
/**
* Creates the full sentence.
*
* @return the mtas cql parser sentence condition
- * @throws ParseException the parse exception
+ * @throws ParseException
+ * the parse exception
*/
- public MtasCQLParserSentenceCondition createFullSentence() throws ParseException {
- if(fullCondition==null) {
- if(secondSentencePart == null) {
- if(firstBasicSentence!=null) {
- fullCondition = new MtasCQLParserSentenceCondition(firstBasicSentence);
- fullCondition.setOccurence(firstMinimumOccurence, firstMaximumOccurence);
+ public MtasCQLParserSentenceCondition createFullSentence()
+ throws ParseException {
+ if (fullCondition == null) {
+ if (secondSentencePart == null) {
+ if (firstBasicSentence != null) {
+ fullCondition = new MtasCQLParserSentenceCondition(
+ firstBasicSentence);
+ fullCondition.setOccurence(firstMinimumOccurence,
+ firstMaximumOccurence);
return fullCondition;
} else {
fullCondition = firstSentence;
- fullCondition.setOccurence(firstMinimumOccurence, firstMaximumOccurence);
+ fullCondition.setOccurence(firstMinimumOccurence,
+ firstMaximumOccurence);
return fullCondition;
}
} else {
- if(!orOperator) {
- if(firstBasicSentence!=null) {
- firstBasicSentence.setOccurence(firstMinimumOccurence, firstMaximumOccurence);
- fullCondition = new MtasCQLParserSentenceCondition(firstBasicSentence);
+ if (!orOperator) {
+ if (firstBasicSentence != null) {
+ firstBasicSentence.setOccurence(firstMinimumOccurence,
+ firstMaximumOccurence);
+ fullCondition = new MtasCQLParserSentenceCondition(
+ firstBasicSentence);
} else {
- firstSentence.setOccurence(firstMinimumOccurence, firstMaximumOccurence);
+ firstSentence.setOccurence(firstMinimumOccurence,
+ firstMaximumOccurence);
fullCondition = new MtasCQLParserSentenceCondition(firstSentence);
}
- fullCondition.addSentenceToEndLatestSequence(secondSentencePart.createFullSentence());
+ fullCondition.addSentenceToEndLatestSequence(
+ secondSentencePart.createFullSentence());
} else {
- MtasCQLParserSentenceCondition sentence = secondSentencePart.createFullSentence();
- if(firstBasicSentence!=null) {
- sentence.addSentenceAsFirstOption(new MtasCQLParserSentenceCondition(firstBasicSentence));
+ MtasCQLParserSentenceCondition sentence = secondSentencePart
+ .createFullSentence();
+ if (firstBasicSentence != null) {
+ sentence.addSentenceAsFirstOption(
+ new MtasCQLParserSentenceCondition(firstBasicSentence));
} else {
sentence.addSentenceAsFirstOption(firstSentence);
}
@@ -188,5 +210,5 @@ public class MtasCQLParserSentencePartCondition {
return fullCondition;
}
}
-
+
}
diff --git a/src/mtas/parser/cql/util/MtasCQLParserWordCondition.java b/src/mtas/parser/cql/util/MtasCQLParserWordCondition.java
index 1a81426..9bd0da1 100644
--- a/src/mtas/parser/cql/util/MtasCQLParserWordCondition.java
+++ b/src/mtas/parser/cql/util/MtasCQLParserWordCondition.java
@@ -16,33 +16,35 @@ public class MtasCQLParserWordCondition {
/** The type and. */
public static String TYPE_AND = "and";
-
+
/** The type or. */
public static String TYPE_OR = "or";
/** The negative query list. */
private List<SpanQuery> positiveQueryList, negativeQueryList;
-
+
/** The condition list. */
private List<MtasCQLParserWordCondition> conditionList;
/** The simplified. */
private boolean simplified;
-
+
/** The not. */
private boolean not;
-
+
/** The type. */
private String type;
-
+
/** The field. */
private String field;
/**
* Instantiates a new mtas cql parser word condition.
*
- * @param field the field
- * @param type the type
+ * @param field
+ * the field
+ * @param type
+ * the type
*/
public MtasCQLParserWordCondition(String field, String type) {
this.field = field;
@@ -92,7 +94,8 @@ public class MtasCQLParserWordCondition {
/**
* Adds the positive query.
*
- * @param q the q
+ * @param q
+ * the q
*/
public void addPositiveQuery(SpanQuery q) {
positiveQueryList.add(q);
@@ -101,7 +104,8 @@ public class MtasCQLParserWordCondition {
/**
* Adds the negative query.
*
- * @param q the q
+ * @param q
+ * the q
*/
public void addNegativeQuery(SpanQuery q) {
negativeQueryList.add(q);
@@ -119,7 +123,8 @@ public class MtasCQLParserWordCondition {
/**
* Gets the positive query.
*
- * @param index the index
+ * @param index
+ * the index
* @return the positive query
*/
public SpanQuery getPositiveQuery(int index) {
@@ -142,7 +147,8 @@ public class MtasCQLParserWordCondition {
/**
* Gets the negative query.
*
- * @param index the index
+ * @param index
+ * the index
* @return the negative query
*/
public SpanQuery getNegativeQuery(int index) {
@@ -156,7 +162,8 @@ public class MtasCQLParserWordCondition {
/**
* Adds the condition.
*
- * @param c the c
+ * @param c
+ * the c
*/
public void addCondition(MtasCQLParserWordCondition c) {
conditionList.add(c);
@@ -275,9 +282,8 @@ public class MtasCQLParserWordCondition {
} else if (c.isSimplePositive()) {
// A | B | ( C & D )
if (c.type().equals(TYPE_AND)) {
- SpanQuery q = new MtasSpanAndQuery(
- c.positiveQueryList.toArray(new SpanQuery[c.positiveQueryList
- .size()]));
+ SpanQuery q = new MtasSpanAndQuery(c.positiveQueryList
+ .toArray(new SpanQuery[c.positiveQueryList.size()]));
if (c.not()) {
negativeQueryList.add(q);
} else {
@@ -285,9 +291,8 @@ public class MtasCQLParserWordCondition {
}
// A & B & ( C | D )
} else {
- SpanQuery q = new MtasSpanOrQuery(
- c.positiveQueryList.toArray(new SpanQuery[c.positiveQueryList
- .size()]));
+ SpanQuery q = new MtasSpanOrQuery(c.positiveQueryList
+ .toArray(new SpanQuery[c.positiveQueryList.size()]));
if (c.not()) {
negativeQueryList.add(q);
} else {
@@ -297,9 +302,8 @@ public class MtasCQLParserWordCondition {
} else if (c.isSimpleNegative()) {
// A | B | ( !C | !D )
if (c.type().equals(TYPE_OR)) {
- SpanQuery q = new MtasSpanAndQuery(
- c.negativeQueryList.toArray(new SpanQuery[c.negativeQueryList
- .size()]));
+ SpanQuery q = new MtasSpanAndQuery(c.negativeQueryList
+ .toArray(new SpanQuery[c.negativeQueryList.size()]));
if (c.not()) {
positiveQueryList.add(q);
} else {
@@ -307,9 +311,8 @@ public class MtasCQLParserWordCondition {
}
// A | B | ( !C & !D )
} else {
- SpanQuery q = new MtasSpanOrQuery(
- c.negativeQueryList.toArray(new SpanQuery[c.negativeQueryList
- .size()]));
+ SpanQuery q = new MtasSpanOrQuery(c.negativeQueryList
+ .toArray(new SpanQuery[c.negativeQueryList.size()]));
if (c.not()) {
positiveQueryList.add(q);
} else {
@@ -325,12 +328,10 @@ public class MtasCQLParserWordCondition {
}
// A | B | ( C & !D )
if (c.type().equals(TYPE_AND)) {
- SpanQuery positiveQuery = new MtasSpanAndQuery(
- c.positiveQueryList.toArray(new SpanQuery[c.positiveQueryList
- .size()]));
- SpanQuery negativeQuery = new MtasSpanAndQuery(
- c.negativeQueryList.toArray(new SpanQuery[c.negativeQueryList
- .size()]));
+ SpanQuery positiveQuery = new MtasSpanAndQuery(c.positiveQueryList
+ .toArray(new SpanQuery[c.positiveQueryList.size()]));
+ SpanQuery negativeQuery = new MtasSpanAndQuery(c.negativeQueryList
+ .toArray(new SpanQuery[c.negativeQueryList.size()]));
SpanQuery q = new SpanNotQuery(positiveQuery, negativeQuery);
if (c.not()) {
negativeQueryList.add(q);
@@ -339,12 +340,10 @@ public class MtasCQLParserWordCondition {
}
// A & B & ( C | !D )
} else {
- SpanQuery positiveQuery = new MtasSpanOrQuery(
- c.positiveQueryList.toArray(new SpanQuery[c.positiveQueryList
- .size()]));
- SpanQuery negativeQuery = new MtasSpanOrQuery(
- c.negativeQueryList.toArray(new SpanQuery[c.negativeQueryList
- .size()]));
+ SpanQuery positiveQuery = new MtasSpanOrQuery(c.positiveQueryList
+ .toArray(new SpanQuery[c.positiveQueryList.size()]));
+ SpanQuery negativeQuery = new MtasSpanOrQuery(c.negativeQueryList
+ .toArray(new SpanQuery[c.negativeQueryList.size()]));
SpanQuery q = new SpanNotQuery(positiveQuery, negativeQuery);
if (c.not()) {
negativeQueryList.add(q);
@@ -363,7 +362,9 @@ public class MtasCQLParserWordCondition {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#toString()
*/
@Override
@@ -374,8 +375,10 @@ public class MtasCQLParserWordCondition {
/**
* To string.
*
- * @param firstIndent the first indent
- * @param indent the indent
+ * @param firstIndent
+ * the first indent
+ * @param indent
+ * the indent
* @return the string
*/
public String toString(String firstIndent, String indent) {
@@ -407,7 +410,9 @@ public class MtasCQLParserWordCondition {
return text;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
@@ -420,8 +425,8 @@ public class MtasCQLParserWordCondition {
if (!field.equals(condition.field) || not ^ condition.not
|| !type.equals(condition.type) || isSingle() ^ condition.isSingle()
|| isSimplePositive() ^ condition.isSimplePositive()
- || isSimpleNegative() ^ condition.isSimpleNegative() || isEmpty()
- ^ condition.isEmpty()) {
+ || isSimpleNegative() ^ condition.isSimpleNegative()
+ || isEmpty() ^ condition.isEmpty()) {
return false;
} else if (isEmpty()) {
return true;
@@ -429,27 +434,31 @@ public class MtasCQLParserWordCondition {
if (!positiveQueryList.equals(condition.positiveQueryList)) {
return false;
} else {
- for(int i=0; i<positiveQueryList.size(); i++) {
- if(positiveQueryList.get(i) instanceof MtasCQLParserWordQuery) {
- if(!(condition.positiveQueryList.get(i) instanceof MtasCQLParserWordQuery)) {
+ for (int i = 0; i < positiveQueryList.size(); i++) {
+ if (positiveQueryList.get(i) instanceof MtasCQLParserWordQuery) {
+ if (!(condition.positiveQueryList
+ .get(i) instanceof MtasCQLParserWordQuery)) {
return false;
- } else if(!((MtasCQLParserWordQuery) positiveQueryList.get(i)).equals(condition.positiveQueryList.get(i))) {
+ } else if (!((MtasCQLParserWordQuery) positiveQueryList.get(i))
+ .equals(condition.positiveQueryList.get(i))) {
return false;
- }
- }
+ }
+ }
}
}
if (!negativeQueryList.equals(condition.negativeQueryList)) {
return false;
} else {
- for(int i=0; i<negativeQueryList.size(); i++) {
- if(negativeQueryList.get(i) instanceof MtasCQLParserWordQuery) {
- if(!(condition.negativeQueryList.get(i) instanceof MtasCQLParserWordQuery)) {
+ for (int i = 0; i < negativeQueryList.size(); i++) {
+ if (negativeQueryList.get(i) instanceof MtasCQLParserWordQuery) {
+ if (!(condition.negativeQueryList
+ .get(i) instanceof MtasCQLParserWordQuery)) {
return false;
- } else if(!((MtasCQLParserWordQuery) negativeQueryList.get(i)).equals(condition.negativeQueryList.get(i))) {
+ } else if (!((MtasCQLParserWordQuery) negativeQueryList.get(i))
+ .equals(condition.negativeQueryList.get(i))) {
return false;
- }
- }
+ }
+ }
}
}
return true;
diff --git a/src/mtas/parser/cql/util/MtasCQLParserWordFullCondition.java b/src/mtas/parser/cql/util/MtasCQLParserWordFullCondition.java
index 6f01ed0..3ae86ac 100644
--- a/src/mtas/parser/cql/util/MtasCQLParserWordFullCondition.java
+++ b/src/mtas/parser/cql/util/MtasCQLParserWordFullCondition.java
@@ -11,19 +11,21 @@ import org.apache.lucene.search.spans.SpanQuery;
/**
* The Class MtasCQLParserWordFullCondition.
*/
-public class MtasCQLParserWordFullCondition extends MtasCQLParserBasicSentencePartCondition {
+public class MtasCQLParserWordFullCondition
+ extends MtasCQLParserBasicSentencePartCondition {
/** The word condition. */
- private MtasCQLParserWordCondition wordCondition;
+ private MtasCQLParserWordCondition wordCondition;
/**
* Instantiates a new mtas cql parser word full condition.
*
- * @param condition the condition
+ * @param condition
+ * the condition
*/
public MtasCQLParserWordFullCondition(MtasCQLParserWordCondition condition) {
minimumOccurence = 1;
- maximumOccurence = 1;
+ maximumOccurence = 1;
optional = false;
condition.simplify();
if (condition.not()) {
@@ -44,8 +46,6 @@ public class MtasCQLParserWordFullCondition extends MtasCQLParserBasicSentencePa
return wordCondition;
}
-
-
/**
* Checks if is empty.
*
@@ -54,9 +54,12 @@ public class MtasCQLParserWordFullCondition extends MtasCQLParserBasicSentencePa
public boolean isEmpty() {
return wordCondition.isEmpty();
}
-
- /* (non-Javadoc)
- * @see mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#getQuery()
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.parser.cql.util.MtasCQLParserBasicSentencePartCondition#getQuery()
*/
@Override
public SpanQuery getQuery() throws ParseException {
@@ -70,12 +73,12 @@ public class MtasCQLParserWordFullCondition extends MtasCQLParserBasicSentencePa
q = wordCondition.getPositiveQuery(0);
} else {
if (wordCondition.type().equals(MtasCQLParserWordCondition.TYPE_AND)) {
- q = new MtasSpanAndQuery(wordCondition.getPositiveQuery().toArray(
- new SpanQuery[wordCondition.getPositiveQuery().size()]));
- } else if (wordCondition.type().equals(
- MtasCQLParserWordCondition.TYPE_OR)) {
- q = new MtasSpanOrQuery(wordCondition.getPositiveQuery().toArray(
- new SpanQuery[wordCondition.getPositiveQuery().size()]));
+ q = new MtasSpanAndQuery(wordCondition.getPositiveQuery()
+ .toArray(new SpanQuery[wordCondition.getPositiveQuery().size()]));
+ } else if (wordCondition.type()
+ .equals(MtasCQLParserWordCondition.TYPE_OR)) {
+ q = new MtasSpanOrQuery(wordCondition.getPositiveQuery()
+ .toArray(new SpanQuery[wordCondition.getPositiveQuery().size()]));
} else {
throw new ParseException("unknown type " + wordCondition.type());
}
@@ -96,8 +99,8 @@ public class MtasCQLParserWordFullCondition extends MtasCQLParserBasicSentencePa
if (wordCondition.getNegativeQuery().size() == 1) {
qNegative = wordCondition.getNegativeQuery(0);
} else {
- qNegative = new MtasSpanOrQuery(wordCondition.getNegativeQuery().toArray(
- new SpanQuery[wordCondition.getNegativeQuery().size()]));
+ qNegative = new MtasSpanOrQuery(wordCondition.getNegativeQuery()
+ .toArray(new SpanQuery[wordCondition.getNegativeQuery().size()]));
}
q = new SpanNotQuery(qPositive, qNegative);
} else if (wordCondition.type()
@@ -106,8 +109,8 @@ public class MtasCQLParserWordFullCondition extends MtasCQLParserBasicSentencePa
if (wordCondition.getPositiveQuery().size() == 1) {
qPositive = wordCondition.getPositiveQuery(0);
} else {
- qPositive = new MtasSpanOrQuery(wordCondition.getPositiveQuery().toArray(
- new SpanQuery[wordCondition.getPositiveQuery().size()]));
+ qPositive = new MtasSpanOrQuery(wordCondition.getPositiveQuery()
+ .toArray(new SpanQuery[wordCondition.getPositiveQuery().size()]));
}
if (wordCondition.getNegativeQuery().size() == 1) {
qNegative = wordCondition.getNegativeQuery(0);
@@ -120,27 +123,29 @@ public class MtasCQLParserWordFullCondition extends MtasCQLParserBasicSentencePa
throw new ParseException("unknown type " + wordCondition.type());
}
}
- if(not) {
+ if (not) {
SpanQuery qPositive = new MtasSpanMatchAllQuery(wordCondition.field());
q = new SpanNotQuery(qPositive, q);
}
return q;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
public boolean equals(Object object) {
- if(object==null)
+ if (object == null)
return false;
- if(object instanceof MtasCQLParserWordFullCondition) {
- MtasCQLParserWordFullCondition word = (MtasCQLParserWordFullCondition) object;
- if(!wordCondition.equals(word.wordCondition))
+ if (object instanceof MtasCQLParserWordFullCondition) {
+ MtasCQLParserWordFullCondition word = (MtasCQLParserWordFullCondition) object;
+ if (!wordCondition.equals(word.wordCondition))
return false;
- if(not!=word.not)
+ if (not != word.not)
return false;
- return true;
+ return true;
} else {
return false;
}
diff --git a/src/mtas/parser/cql/util/MtasCQLParserWordPositionQuery.java b/src/mtas/parser/cql/util/MtasCQLParserWordPositionQuery.java
index 77f4c2d..3ccd35f 100644
--- a/src/mtas/parser/cql/util/MtasCQLParserWordPositionQuery.java
+++ b/src/mtas/parser/cql/util/MtasCQLParserWordPositionQuery.java
@@ -17,37 +17,44 @@ public class MtasCQLParserWordPositionQuery extends SpanQuery {
/** The query. */
SpanQuery query;
-
+
/** The term. */
Term term;
-
+
/** The query name. */
private static String QUERY_NAME = "mtasCQLParserWordPositionQuery";
-
+
/**
* Instantiates a new mtas cql parser word position query.
*
- * @param field the field
- * @param position the position
+ * @param field
+ * the field
+ * @param position
+ * the position
*/
public MtasCQLParserWordPositionQuery(String field, int position) {
term = new Term(field);
- query = new MtasSpanPositionQuery(field, position);
+ query = new MtasSpanPositionQuery(field, position);
}
-
+
/**
* Instantiates a new mtas cql parser word position query.
*
- * @param field the field
- * @param start the start
- * @param end the end
+ * @param field
+ * the field
+ * @param start
+ * the start
+ * @param end
+ * the end
*/
public MtasCQLParserWordPositionQuery(String field, int start, int end) {
term = new Term(field);
query = new MtasSpanPositionQuery(field, start, end);
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
@@ -55,16 +62,23 @@ public class MtasCQLParserWordPositionQuery extends SpanQuery {
return term.field();
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
*/
@Override
public Query rewrite(IndexReader reader) throws IOException {
return query.rewrite(reader);
}
-
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
@@ -72,30 +86,36 @@ public class MtasCQLParserWordPositionQuery extends SpanQuery {
return query.createWeight(searcher, needsScores);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#toString(java.lang.String)
*/
@Override
public String toString(String field) {
return query.toString(term.field());
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
- if (obj== null)
+ if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final MtasCQLParserWordPositionQuery that = (MtasCQLParserWordPositionQuery) obj;
- return query.equals(that.query);
+ return query.equals(that.query);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
@@ -104,5 +124,5 @@ public class MtasCQLParserWordPositionQuery extends SpanQuery {
h = (h * 7) ^ query.hashCode();
return h;
}
-
+
}
diff --git a/src/mtas/parser/cql/util/MtasCQLParserWordQuery.java b/src/mtas/parser/cql/util/MtasCQLParserWordQuery.java
index 72ac05a..ba5f7b6 100644
--- a/src/mtas/parser/cql/util/MtasCQLParserWordQuery.java
+++ b/src/mtas/parser/cql/util/MtasCQLParserWordQuery.java
@@ -22,69 +22,81 @@ public class MtasCQLParserWordQuery extends SpanQuery {
/** The query. */
SpanQuery query;
-
+
/** The term. */
Term term;
-
+
/** The query name. */
private static String QUERY_NAME = "mtasCQLParserWordQuery";
-
+
/** The Constant MTAS_CQL_TERM_QUERY. */
public static final String MTAS_CQL_TERM_QUERY = "term";
-
+
/** The Constant MTAS_CQL_REGEXP_QUERY. */
public static final String MTAS_CQL_REGEXP_QUERY = "regexp";
-
+
/** The Constant MTAS_CQL_WILDCARD_QUERY. */
public static final String MTAS_CQL_WILDCARD_QUERY = "wildcard";
-
+
/**
* Instantiates a new mtas cql parser word query.
*
- * @param field the field
- * @param prefix the prefix
+ * @param field
+ * the field
+ * @param prefix
+ * the prefix
*/
public MtasCQLParserWordQuery(String field, String prefix) {
- term = new Term(field,prefix+MtasToken.DELIMITER);
+ term = new Term(field, prefix + MtasToken.DELIMITER);
query = new MtasSpanPrefixQuery(term, true);
}
-
+
/**
* Instantiates a new mtas cql parser word query.
*
- * @param field the field
- * @param prefix the prefix
- * @param value the value
+ * @param field
+ * the field
+ * @param prefix
+ * the prefix
+ * @param value
+ * the value
*/
- public MtasCQLParserWordQuery(String field, String prefix, String value ) {
+ public MtasCQLParserWordQuery(String field, String prefix, String value) {
this(field, prefix, value, MTAS_CQL_REGEXP_QUERY);
}
-
+
/**
* Instantiates a new mtas cql parser word query.
*
- * @param field the field
- * @param prefix the prefix
- * @param value the value
- * @param type the type
+ * @param field
+ * the field
+ * @param prefix
+ * the prefix
+ * @param value
+ * the value
+ * @param type
+ * the type
*/
- public MtasCQLParserWordQuery(String field, String prefix, String value, String type) {
- if(type.equals(MTAS_CQL_REGEXP_QUERY)) {
- term = new Term(field,prefix+MtasToken.DELIMITER+value+"\u0000*");
+ public MtasCQLParserWordQuery(String field, String prefix, String value,
+ String type) {
+ if (type.equals(MTAS_CQL_REGEXP_QUERY)) {
+ term = new Term(field, prefix + MtasToken.DELIMITER + value + "\u0000*");
query = new MtasSpanRegexpQuery(term, true);
- } else if(type.equals(MTAS_CQL_WILDCARD_QUERY)) {
- term = new Term(field,prefix+MtasToken.DELIMITER+value);
+ } else if (type.equals(MTAS_CQL_WILDCARD_QUERY)) {
+ term = new Term(field, prefix + MtasToken.DELIMITER + value);
query = new MtasSpanWildcardQuery(term, true);
- } else if(type.equals(MTAS_CQL_TERM_QUERY)) {
- term = new Term(field,prefix+MtasToken.DELIMITER+value);
+ } else if (type.equals(MTAS_CQL_TERM_QUERY)) {
+ term = new Term(field, prefix + MtasToken.DELIMITER + value);
query = new MtasSpanTermQuery(term, true);
} else {
- term = new Term(field,prefix+MtasToken.DELIMITER+value);
+ term = new Term(field, prefix + MtasToken.DELIMITER + value);
query = new MtasSpanTermQuery(term, true);
- }
+ }
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
@@ -92,16 +104,23 @@ public class MtasCQLParserWordQuery extends SpanQuery {
return term.field();
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
*/
@Override
public Query rewrite(IndexReader reader) throws IOException {
return query.rewrite(reader);
}
-
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
@@ -109,30 +128,36 @@ public class MtasCQLParserWordQuery extends SpanQuery {
return query.createWeight(searcher, needsScores);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#toString(java.lang.String)
*/
@Override
public String toString(String field) {
return query.toString(term.field());
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
- if (obj== null)
+ if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final MtasCQLParserWordQuery that = (MtasCQLParserWordQuery) obj;
- return query.equals(that.query);
+ return query.equals(that.query);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
@@ -141,5 +166,5 @@ public class MtasCQLParserWordQuery extends SpanQuery {
h = (h * 7) ^ query.hashCode();
return h;
}
-
+
}
diff --git a/src/mtas/parser/function/MtasFunctionParser.java b/src/mtas/parser/function/MtasFunctionParser.java
new file mode 100644
index 0000000..4e0783c
--- /dev/null
+++ b/src/mtas/parser/function/MtasFunctionParser.java
@@ -0,0 +1,648 @@
+/* Generated By:JavaCC: Do not edit this line. MtasFunctionParser.java */
+package mtas.parser.function;
+import mtas.codec.util.CodecUtil;
+import mtas.parser.function.util.MtasFunctionParserFunction;
+import mtas.parser.function.util.MtasFunctionParserFunctionBasic;
+import mtas.parser.function.util.MtasFunctionParserItem;
+import java.io.IOException;
+
+public class MtasFunctionParser implements MtasFunctionParserConstants {
+ public MtasFunctionParserFunction parse() throws ParseException
+ {
+ MtasFunctionParserFunction pf = parserFunction();
+ pf.close();
+ return pf;
+ }
+
+ final private MtasFunctionParserFunction parserFunction() throws ParseException, ParseException {
+ MtasFunctionParserFunction pf;
+ pf = parserFunctionBasic();
+ jj_consume_token(0);
+ {if (true) return pf;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final private MtasFunctionParserFunction parserFunctionBasic() throws ParseException, ParseException {
+ Token v, operator;
+ long vLong;
+ double vDouble;
+ MtasFunctionParserItem item;
+ MtasFunctionParserFunctionBasic pf;
+ item = parserFunctionItem();
+ pf = new MtasFunctionParserFunctionBasic(item);
+ label_1:
+ while (true) {
+ if (jj_2_1(100)) {
+ ;
+ } else {
+ break label_1;
+ }
+ if (jj_2_2(100)) {
+ operator = jj_consume_token(BASIC_OPERATOR_ADD);
+ item = parserFunctionItem();
+ pf.add(item);
+ } else if (jj_2_3(100)) {
+ operator = jj_consume_token(BASIC_OPERATOR_SUBTRACT);
+ item = parserFunctionItem();
+ pf.subtract(item);
+ } else if (jj_2_4(100)) {
+ operator = jj_consume_token(BASIC_OPERATOR_MULTIPLY);
+ item = parserFunctionItem();
+ pf.multiply(item);
+ } else if (jj_2_5(100)) {
+ operator = jj_consume_token(BASIC_OPERATOR_DIVIDE);
+ item = parserFunctionItem();
+ pf.divide(item);
+ } else if (jj_2_6(100)) {
+ operator = jj_consume_token(BASIC_OPERATOR_POWER);
+ item = parserFunctionItem();
+ pf.power(item);
+ } else if (jj_2_7(100)) {
+ v = jj_consume_token(LONG);
+ vLong = Long.parseLong(v.image);
+ if (vLong > 0)
+ {
+ {if (true) throw new ParseException("only negative");}
+ }
+ else
+ {
+ item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_CONSTANT_LONG, - 1 * vLong);
+ pf.subtract(item);
+ }
+ } else if (jj_2_8(100)) {
+ v = jj_consume_token(DOUBLE);
+ vDouble = Double.parseDouble(v.image);
+ if (vDouble > 0)
+ {
+ {if (true) throw new ParseException("only negative");}
+ }
+ else
+ {
+ item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_CONSTANT_DOUBLE, - 1 * vDouble);
+ pf.subtract(item);
+ }
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ }
+ {if (true) return pf;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final private MtasFunctionParserItem parserFunctionItem() throws ParseException, ParseException {
+ Token v;
+ long vLong;
+ int vInt;
+ double vDouble;
+ String type;
+ MtasFunctionParserItem item;
+ MtasFunctionParserFunction pf;
+ if (jj_2_9(100)) {
+ v = jj_consume_token(VARIABLE);
+ vInt = Integer.parseInt(v.image.substring(2));
+ item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_ARGUMENT, vInt);
+ {if (true) return item;}
+ } else if (jj_2_10(100)) {
+ jj_consume_token(N);
+ item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_N);
+ {if (true) return item;}
+ } else if (jj_2_11(100)) {
+ v = jj_consume_token(LONG);
+ vLong = Long.parseLong(v.image);
+ item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_CONSTANT_LONG, vLong);
+ {if (true) return item;}
+ } else if (jj_2_12(100)) {
+ v = jj_consume_token(DOUBLE);
+ vDouble = Double.parseDouble(v.image);
+ item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_CONSTANT_DOUBLE, vDouble);
+ {if (true) return item;}
+ } else if (jj_2_13(100)) {
+ jj_consume_token(BRACKET_START);
+ pf = parserFunctionBasic();
+ jj_consume_token(BRACKET_END);
+ type = pf.getType();
+ if (type.equals(CodecUtil.DATA_TYPE_LONG))
+ {
+ item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_PARSER_LONG, pf);
+ {if (true) return item;}
+ }
+ else if (type.equals(CodecUtil.DATA_TYPE_DOUBLE))
+ {
+ item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_PARSER_DOUBLE, pf);
+ {if (true) return item;}
+ }
+ else
+ {
+ {if (true) throw new ParseException("unknown dataType");}
+ }
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ {if (true) throw new ParseException("unknown type");}
+ throw new Error("Missing return statement in function");
+ }
+
+ private boolean jj_2_1(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_1(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(0, xla); }
+ }
+
+ private boolean jj_2_2(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_2(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(1, xla); }
+ }
+
+ private boolean jj_2_3(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_3(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(2, xla); }
+ }
+
+ private boolean jj_2_4(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_4(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(3, xla); }
+ }
+
+ private boolean jj_2_5(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_5(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(4, xla); }
+ }
+
+ private boolean jj_2_6(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_6(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(5, xla); }
+ }
+
+ private boolean jj_2_7(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_7(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(6, xla); }
+ }
+
+ private boolean jj_2_8(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_8(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(7, xla); }
+ }
+
+ private boolean jj_2_9(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_9(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(8, xla); }
+ }
+
+ private boolean jj_2_10(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_10(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(9, xla); }
+ }
+
+ private boolean jj_2_11(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_11(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(10, xla); }
+ }
+
+ private boolean jj_2_12(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_12(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(11, xla); }
+ }
+
+ private boolean jj_2_13(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_13(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(12, xla); }
+ }
+
+ private boolean jj_3R_2() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_9()) {
+ jj_scanpos = xsp;
+ if (jj_3_10()) {
+ jj_scanpos = xsp;
+ if (jj_3_11()) {
+ jj_scanpos = xsp;
+ if (jj_3_12()) {
+ jj_scanpos = xsp;
+ if (jj_3_13()) return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3_12() {
+ if (jj_scan_token(DOUBLE)) return true;
+ return false;
+ }
+
+ private boolean jj_3_6() {
+ if (jj_scan_token(BASIC_OPERATOR_POWER)) return true;
+ if (jj_3R_2()) return true;
+ return false;
+ }
+
+ private boolean jj_3_1() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_2()) {
+ jj_scanpos = xsp;
+ if (jj_3_3()) {
+ jj_scanpos = xsp;
+ if (jj_3_4()) {
+ jj_scanpos = xsp;
+ if (jj_3_5()) {
+ jj_scanpos = xsp;
+ if (jj_3_6()) {
+ jj_scanpos = xsp;
+ if (jj_3_7()) {
+ jj_scanpos = xsp;
+ if (jj_3_8()) return true;
+ }
+ }
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3_2() {
+ if (jj_scan_token(BASIC_OPERATOR_ADD)) return true;
+ if (jj_3R_2()) return true;
+ return false;
+ }
+
+ private boolean jj_3_8() {
+ if (jj_scan_token(DOUBLE)) return true;
+ return false;
+ }
+
+ private boolean jj_3_11() {
+ if (jj_scan_token(LONG)) return true;
+ return false;
+ }
+
+ private boolean jj_3_5() {
+ if (jj_scan_token(BASIC_OPERATOR_DIVIDE)) return true;
+ if (jj_3R_2()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_3() {
+ if (jj_3R_2()) return true;
+ Token xsp;
+ while (true) {
+ xsp = jj_scanpos;
+ if (jj_3_1()) { jj_scanpos = xsp; break; }
+ }
+ return false;
+ }
+
+ private boolean jj_3_10() {
+ if (jj_scan_token(N)) return true;
+ return false;
+ }
+
+ private boolean jj_3_4() {
+ if (jj_scan_token(BASIC_OPERATOR_MULTIPLY)) return true;
+ if (jj_3R_2()) return true;
+ return false;
+ }
+
+ private boolean jj_3_13() {
+ if (jj_scan_token(BRACKET_START)) return true;
+ if (jj_3R_3()) return true;
+ if (jj_scan_token(BRACKET_END)) return true;
+ return false;
+ }
+
+ private boolean jj_3_7() {
+ if (jj_scan_token(LONG)) return true;
+ return false;
+ }
+
+ private boolean jj_3_9() {
+ if (jj_scan_token(VARIABLE)) return true;
+ return false;
+ }
+
+ private boolean jj_3_3() {
+ if (jj_scan_token(BASIC_OPERATOR_SUBTRACT)) return true;
+ if (jj_3R_2()) return true;
+ return false;
+ }
+
+ /** Generated Token Manager. */
+ public MtasFunctionParserTokenManager token_source;
+ SimpleCharStream jj_input_stream;
+ /** Current token. */
+ public Token token;
+ /** Next token. */
+ public Token jj_nt;
+ private int jj_ntk;
+ private Token jj_scanpos, jj_lastpos;
+ private int jj_la;
+ private int jj_gen;
+ final private int[] jj_la1 = new int[0];
+ static private int[] jj_la1_0;
+ static {
+ jj_la1_init_0();
+ }
+ private static void jj_la1_init_0() {
+ jj_la1_0 = new int[] {};
+ }
+ final private JJCalls[] jj_2_rtns = new JJCalls[13];
+ private boolean jj_rescan = false;
+ private int jj_gc = 0;
+
+ /** Constructor with InputStream. */
+ public MtasFunctionParser(java.io.InputStream stream) {
+ this(stream, null);
+ }
+ /** Constructor with InputStream and supplied encoding */
+ public MtasFunctionParser(java.io.InputStream stream, String encoding) {
+ try { jj_input_stream = new SimpleCharStream(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
+ token_source = new MtasFunctionParserTokenManager(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream stream) {
+ ReInit(stream, null);
+ }
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream stream, String encoding) {
+ try { jj_input_stream.ReInit(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
+ token_source.ReInit(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Constructor. */
+ public MtasFunctionParser(java.io.Reader stream) {
+ jj_input_stream = new SimpleCharStream(stream, 1, 1);
+ token_source = new MtasFunctionParserTokenManager(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.Reader stream) {
+ jj_input_stream.ReInit(stream, 1, 1);
+ token_source.ReInit(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Constructor with generated Token Manager. */
+ public MtasFunctionParser(MtasFunctionParserTokenManager tm) {
+ token_source = tm;
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Reinitialise. */
+ public void ReInit(MtasFunctionParserTokenManager tm) {
+ token_source = tm;
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 0; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ private Token jj_consume_token(int kind) throws ParseException {
+ Token oldToken;
+ if ((oldToken = token).next != null) token = token.next;
+ else token = token.next = token_source.getNextToken();
+ jj_ntk = -1;
+ if (token.kind == kind) {
+ jj_gen++;
+ if (++jj_gc > 100) {
+ jj_gc = 0;
+ for (int i = 0; i < jj_2_rtns.length; i++) {
+ JJCalls c = jj_2_rtns[i];
+ while (c != null) {
+ if (c.gen < jj_gen) c.first = null;
+ c = c.next;
+ }
+ }
+ }
+ return token;
+ }
+ token = oldToken;
+ jj_kind = kind;
+ throw generateParseException();
+ }
+
+ static private final class LookaheadSuccess extends java.lang.Error { }
+ final private LookaheadSuccess jj_ls = new LookaheadSuccess();
+ private boolean jj_scan_token(int kind) {
+ if (jj_scanpos == jj_lastpos) {
+ jj_la--;
+ if (jj_scanpos.next == null) {
+ jj_lastpos = jj_scanpos = jj_scanpos.next = token_source.getNextToken();
+ } else {
+ jj_lastpos = jj_scanpos = jj_scanpos.next;
+ }
+ } else {
+ jj_scanpos = jj_scanpos.next;
+ }
+ if (jj_rescan) {
+ int i = 0; Token tok = token;
+ while (tok != null && tok != jj_scanpos) { i++; tok = tok.next; }
+ if (tok != null) jj_add_error_token(kind, i);
+ }
+ if (jj_scanpos.kind != kind) return true;
+ if (jj_la == 0 && jj_scanpos == jj_lastpos) throw jj_ls;
+ return false;
+ }
+
+
+/** Get the next Token. */
+ final public Token getNextToken() {
+ if (token.next != null) token = token.next;
+ else token = token.next = token_source.getNextToken();
+ jj_ntk = -1;
+ jj_gen++;
+ return token;
+ }
+
+/** Get the specific Token. */
+ final public Token getToken(int index) {
+ Token t = token;
+ for (int i = 0; i < index; i++) {
+ if (t.next != null) t = t.next;
+ else t = t.next = token_source.getNextToken();
+ }
+ return t;
+ }
+
+ private int jj_ntk() {
+ if ((jj_nt=token.next) == null)
+ return (jj_ntk = (token.next=token_source.getNextToken()).kind);
+ else
+ return (jj_ntk = jj_nt.kind);
+ }
+
+ private java.util.List<int[]> jj_expentries = new java.util.ArrayList<int[]>();
+ private int[] jj_expentry;
+ private int jj_kind = -1;
+ private int[] jj_lasttokens = new int[100];
+ private int jj_endpos;
+
+ private void jj_add_error_token(int kind, int pos) {
+ if (pos >= 100) return;
+ if (pos == jj_endpos + 1) {
+ jj_lasttokens[jj_endpos++] = kind;
+ } else if (jj_endpos != 0) {
+ jj_expentry = new int[jj_endpos];
+ for (int i = 0; i < jj_endpos; i++) {
+ jj_expentry[i] = jj_lasttokens[i];
+ }
+ jj_entries_loop: for (java.util.Iterator<?> it = jj_expentries.iterator(); it.hasNext();) {
+ int[] oldentry = (int[])(it.next());
+ if (oldentry.length == jj_expentry.length) {
+ for (int i = 0; i < jj_expentry.length; i++) {
+ if (oldentry[i] != jj_expentry[i]) {
+ continue jj_entries_loop;
+ }
+ }
+ jj_expentries.add(jj_expentry);
+ break jj_entries_loop;
+ }
+ }
+ if (pos != 0) jj_lasttokens[(jj_endpos = pos) - 1] = kind;
+ }
+ }
+
+ /** Generate ParseException. */
+ public ParseException generateParseException() {
+ jj_expentries.clear();
+ boolean[] la1tokens = new boolean[16];
+ if (jj_kind >= 0) {
+ la1tokens[jj_kind] = true;
+ jj_kind = -1;
+ }
+ for (int i = 0; i < 0; i++) {
+ if (jj_la1[i] == jj_gen) {
+ for (int j = 0; j < 32; j++) {
+ if ((jj_la1_0[i] & (1<<j)) != 0) {
+ la1tokens[j] = true;
+ }
+ }
+ }
+ }
+ for (int i = 0; i < 16; i++) {
+ if (la1tokens[i]) {
+ jj_expentry = new int[1];
+ jj_expentry[0] = i;
+ jj_expentries.add(jj_expentry);
+ }
+ }
+ jj_endpos = 0;
+ jj_rescan_token();
+ jj_add_error_token(0, 0);
+ int[][] exptokseq = new int[jj_expentries.size()][];
+ for (int i = 0; i < jj_expentries.size(); i++) {
+ exptokseq[i] = jj_expentries.get(i);
+ }
+ return new ParseException(token, exptokseq, tokenImage);
+ }
+
+ /** Enable tracing. */
+ final public void enable_tracing() {
+ }
+
+ /** Disable tracing. */
+ final public void disable_tracing() {
+ }
+
+ private void jj_rescan_token() {
+ jj_rescan = true;
+ for (int i = 0; i < 13; i++) {
+ try {
+ JJCalls p = jj_2_rtns[i];
+ do {
+ if (p.gen > jj_gen) {
+ jj_la = p.arg; jj_lastpos = jj_scanpos = p.first;
+ switch (i) {
+ case 0: jj_3_1(); break;
+ case 1: jj_3_2(); break;
+ case 2: jj_3_3(); break;
+ case 3: jj_3_4(); break;
+ case 4: jj_3_5(); break;
+ case 5: jj_3_6(); break;
+ case 6: jj_3_7(); break;
+ case 7: jj_3_8(); break;
+ case 8: jj_3_9(); break;
+ case 9: jj_3_10(); break;
+ case 10: jj_3_11(); break;
+ case 11: jj_3_12(); break;
+ case 12: jj_3_13(); break;
+ }
+ }
+ p = p.next;
+ } while (p != null);
+ } catch(LookaheadSuccess ls) { }
+ }
+ jj_rescan = false;
+ }
+
+ private void jj_save(int index, int xla) {
+ JJCalls p = jj_2_rtns[index];
+ while (p.gen > jj_gen) {
+ if (p.next == null) { p = p.next = new JJCalls(); break; }
+ p = p.next;
+ }
+ p.gen = jj_gen + xla - jj_la; p.first = token; p.arg = xla;
+ }
+
+ static final class JJCalls {
+ int gen;
+ Token first;
+ int arg;
+ JJCalls next;
+ }
+
+}
diff --git a/src/mtas/parser/function/MtasFunctionParserConstants.java b/src/mtas/parser/function/MtasFunctionParserConstants.java
new file mode 100644
index 0000000..8733413
--- /dev/null
+++ b/src/mtas/parser/function/MtasFunctionParserConstants.java
@@ -0,0 +1,59 @@
+/* Generated By:JavaCC: Do not edit this line. MtasFunctionParserConstants.java */
+package mtas.parser.function;
+
+
+/**
+ * Token literal values and constants.
+ * Generated by org.javacc.parser.OtherFilesGen#start()
+ */
+public interface MtasFunctionParserConstants {
+
+ /** End of File. */
+ int EOF = 0;
+ /** RegularExpression Id. */
+ int VARIABLE = 5;
+ /** RegularExpression Id. */
+ int LONG = 6;
+ /** RegularExpression Id. */
+ int DOUBLE = 7;
+ /** RegularExpression Id. */
+ int N = 8;
+ /** RegularExpression Id. */
+ int BASIC_OPERATOR_ADD = 9;
+ /** RegularExpression Id. */
+ int BASIC_OPERATOR_SUBTRACT = 10;
+ /** RegularExpression Id. */
+ int BASIC_OPERATOR_MULTIPLY = 11;
+ /** RegularExpression Id. */
+ int BASIC_OPERATOR_DIVIDE = 12;
+ /** RegularExpression Id. */
+ int BASIC_OPERATOR_POWER = 13;
+ /** RegularExpression Id. */
+ int BRACKET_START = 14;
+ /** RegularExpression Id. */
+ int BRACKET_END = 15;
+
+ /** Lexical state. */
+ int DEFAULT = 0;
+
+ /** Literal token values. */
+ String[] tokenImage = {
+ "<EOF>",
+ "\" \"",
+ "\"\\r\"",
+ "\"\\t\"",
+ "\"\\n\"",
+ "<VARIABLE>",
+ "<LONG>",
+ "<DOUBLE>",
+ "\"$n\"",
+ "\"+\"",
+ "\"-\"",
+ "\"*\"",
+ "\"/\"",
+ "\"^\"",
+ "\"(\"",
+ "\")\"",
+ };
+
+}
diff --git a/src/mtas/parser/function/MtasFunctionParserTokenManager.java b/src/mtas/parser/function/MtasFunctionParserTokenManager.java
new file mode 100644
index 0000000..31c63f1
--- /dev/null
+++ b/src/mtas/parser/function/MtasFunctionParserTokenManager.java
@@ -0,0 +1,441 @@
+/* Generated By:JavaCC: Do not edit this line. MtasFunctionParserTokenManager.java */
+package mtas.parser.function;
+import mtas.codec.util.CodecUtil;
+import mtas.parser.function.util.MtasFunctionParserFunction;
+import mtas.parser.function.util.MtasFunctionParserFunctionBasic;
+import mtas.parser.function.util.MtasFunctionParserItem;
+import java.io.IOException;
+
+/** Token Manager. */
+public class MtasFunctionParserTokenManager implements MtasFunctionParserConstants
+{
+
+ /** Debug output. */
+ public java.io.PrintStream debugStream = System.out;
+ /** Set debug output. */
+ public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
+private final int jjStopStringLiteralDfa_0(int pos, long active0)
+{
+ switch (pos)
+ {
+ case 0:
+ if ((active0 & 0x100L) != 0L)
+ return 0;
+ if ((active0 & 0x400L) != 0L)
+ return 4;
+ return -1;
+ default :
+ return -1;
+ }
+}
+private final int jjStartNfa_0(int pos, long active0)
+{
+ return jjMoveNfa_0(jjStopStringLiteralDfa_0(pos, active0), pos + 1);
+}
+private int jjStopAtPos(int pos, int kind)
+{
+ jjmatchedKind = kind;
+ jjmatchedPos = pos;
+ return pos + 1;
+}
+private int jjMoveStringLiteralDfa0_0()
+{
+ switch(curChar)
+ {
+ case 36:
+ return jjMoveStringLiteralDfa1_0(0x100L);
+ case 40:
+ return jjStopAtPos(0, 14);
+ case 41:
+ return jjStopAtPos(0, 15);
+ case 42:
+ return jjStopAtPos(0, 11);
+ case 43:
+ return jjStopAtPos(0, 9);
+ case 45:
+ return jjStartNfaWithStates_0(0, 10, 4);
+ case 47:
+ return jjStopAtPos(0, 12);
+ case 94:
+ return jjStopAtPos(0, 13);
+ default :
+ return jjMoveNfa_0(2, 0);
+ }
+}
+private int jjMoveStringLiteralDfa1_0(long active0)
+{
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(0, active0);
+ return 1;
+ }
+ switch(curChar)
+ {
+ case 110:
+ if ((active0 & 0x100L) != 0L)
+ return jjStopAtPos(1, 8);
+ break;
+ default :
+ break;
+ }
+ return jjStartNfa_0(0, active0);
+}
+private int jjStartNfaWithStates_0(int pos, int kind, int state)
+{
+ jjmatchedKind = kind;
+ jjmatchedPos = pos;
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) { return pos + 1; }
+ return jjMoveNfa_0(state, pos + 1);
+}
+private int jjMoveNfa_0(int startState, int curPos)
+{
+ int startsAt = 0;
+ jjnewStateCnt = 14;
+ int i = 1;
+ jjstateSet[0] = startState;
+ int kind = 0x7fffffff;
+ for (;;)
+ {
+ if (++jjround == 0x7fffffff)
+ ReInitRounds();
+ if (curChar < 64)
+ {
+ long l = 1L << curChar;
+ do
+ {
+ switch(jjstateSet[--i])
+ {
+ case 4:
+ if ((0x3fe000000000000L & l) != 0L)
+ jjCheckNAddTwoStates(8, 9);
+ else if (curChar == 48)
+ jjCheckNAdd(9);
+ if ((0x3fe000000000000L & l) != 0L)
+ {
+ if (kind > 6)
+ kind = 6;
+ jjCheckNAdd(5);
+ }
+ else if (curChar == 48)
+ {
+ if (kind > 6)
+ kind = 6;
+ }
+ break;
+ case 2:
+ if ((0x3fe000000000000L & l) != 0L)
+ {
+ if (kind > 6)
+ kind = 6;
+ jjCheckNAddStates(0, 2);
+ }
+ else if (curChar == 48)
+ {
+ if (kind > 6)
+ kind = 6;
+ jjCheckNAdd(9);
+ }
+ else if (curChar == 45)
+ jjAddStates(3, 6);
+ else if (curChar == 36)
+ jjstateSet[jjnewStateCnt++] = 0;
+ break;
+ case 1:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 5)
+ kind = 5;
+ jjstateSet[jjnewStateCnt++] = 1;
+ break;
+ case 3:
+ if (curChar == 45)
+ jjAddStates(3, 6);
+ break;
+ case 5:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 6)
+ kind = 6;
+ jjCheckNAdd(5);
+ break;
+ case 6:
+ if (curChar == 48 && kind > 6)
+ kind = 6;
+ break;
+ case 7:
+ if ((0x3fe000000000000L & l) != 0L)
+ jjCheckNAddTwoStates(8, 9);
+ break;
+ case 8:
+ if ((0x3ff000000000000L & l) != 0L)
+ jjCheckNAddTwoStates(8, 9);
+ break;
+ case 9:
+ if (curChar == 46)
+ jjCheckNAdd(10);
+ break;
+ case 10:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 7)
+ kind = 7;
+ jjCheckNAdd(10);
+ break;
+ case 11:
+ if (curChar == 48)
+ jjCheckNAdd(9);
+ break;
+ case 12:
+ if ((0x3fe000000000000L & l) == 0L)
+ break;
+ if (kind > 6)
+ kind = 6;
+ jjCheckNAddStates(0, 2);
+ break;
+ case 13:
+ if (curChar != 48)
+ break;
+ if (kind > 6)
+ kind = 6;
+ jjCheckNAdd(9);
+ break;
+ default : break;
+ }
+ } while(i != startsAt);
+ }
+ else if (curChar < 128)
+ {
+ long l = 1L << (curChar & 077);
+ do
+ {
+ switch(jjstateSet[--i])
+ {
+ case 0:
+ if (curChar == 113)
+ jjstateSet[jjnewStateCnt++] = 1;
+ break;
+ default : break;
+ }
+ } while(i != startsAt);
+ }
+ else
+ {
+ int hiByte = (int)(curChar >> 8);
+ int i1 = hiByte >> 6;
+ long l1 = 1L << (hiByte & 077);
+ int i2 = (curChar & 0xff) >> 6;
+ long l2 = 1L << (curChar & 077);
+ do
+ {
+ switch(jjstateSet[--i])
+ {
+ default : break;
+ }
+ } while(i != startsAt);
+ }
+ if (kind != 0x7fffffff)
+ {
+ jjmatchedKind = kind;
+ jjmatchedPos = curPos;
+ kind = 0x7fffffff;
+ }
+ ++curPos;
+ if ((i = jjnewStateCnt) == (startsAt = 14 - (jjnewStateCnt = startsAt)))
+ return curPos;
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) { return curPos; }
+ }
+}
+static final int[] jjnextStates = {
+ 5, 8, 9, 4, 6, 7, 11,
+};
+
+/** Token literal values. */
+public static final String[] jjstrLiteralImages = {
+"", null, null, null, null, null, null, null, "\44\156", "\53", "\55", "\52",
+"\57", "\136", "\50", "\51", };
+
+/** Lexer state names. */
+public static final String[] lexStateNames = {
+ "DEFAULT",
+};
+static final long[] jjtoToken = {
+ 0xffe1L,
+};
+static final long[] jjtoSkip = {
+ 0x1eL,
+};
+protected SimpleCharStream input_stream;
+private final int[] jjrounds = new int[14];
+private final int[] jjstateSet = new int[28];
+protected char curChar;
+/** Constructor. */
+public MtasFunctionParserTokenManager(SimpleCharStream stream){
+ if (SimpleCharStream.staticFlag)
+ throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.");
+ input_stream = stream;
+}
+
+/** Constructor. */
+public MtasFunctionParserTokenManager(SimpleCharStream stream, int lexState){
+ this(stream);
+ SwitchTo(lexState);
+}
+
+/** Reinitialise parser. */
+public void ReInit(SimpleCharStream stream)
+{
+ jjmatchedPos = jjnewStateCnt = 0;
+ curLexState = defaultLexState;
+ input_stream = stream;
+ ReInitRounds();
+}
+private void ReInitRounds()
+{
+ int i;
+ jjround = 0x80000001;
+ for (i = 14; i-- > 0;)
+ jjrounds[i] = 0x80000000;
+}
+
+/** Reinitialise parser. */
+public void ReInit(SimpleCharStream stream, int lexState)
+{
+ ReInit(stream);
+ SwitchTo(lexState);
+}
+
+/** Switch to specified lex state. */
+public void SwitchTo(int lexState)
+{
+ if (lexState >= 1 || lexState < 0)
+ throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE);
+ else
+ curLexState = lexState;
+}
+
+protected Token jjFillToken()
+{
+ final Token t;
+ final String curTokenImage;
+ final int beginLine;
+ final int endLine;
+ final int beginColumn;
+ final int endColumn;
+ String im = jjstrLiteralImages[jjmatchedKind];
+ curTokenImage = (im == null) ? input_stream.GetImage() : im;
+ beginLine = input_stream.getBeginLine();
+ beginColumn = input_stream.getBeginColumn();
+ endLine = input_stream.getEndLine();
+ endColumn = input_stream.getEndColumn();
+ t = Token.newToken(jjmatchedKind, curTokenImage);
+
+ t.beginLine = beginLine;
+ t.endLine = endLine;
+ t.beginColumn = beginColumn;
+ t.endColumn = endColumn;
+
+ return t;
+}
+
+int curLexState = 0;
+int defaultLexState = 0;
+int jjnewStateCnt;
+int jjround;
+int jjmatchedPos;
+int jjmatchedKind;
+
+/** Get the next Token. */
+public Token getNextToken()
+{
+ Token matchedToken;
+ int curPos = 0;
+
+ EOFLoop :
+ for (;;)
+ {
+ try
+ {
+ curChar = input_stream.BeginToken();
+ }
+ catch(java.io.IOException e)
+ {
+ jjmatchedKind = 0;
+ matchedToken = jjFillToken();
+ return matchedToken;
+ }
+
+ try { input_stream.backup(0);
+ while (curChar <= 32 && (0x100002600L & (1L << curChar)) != 0L)
+ curChar = input_stream.BeginToken();
+ }
+ catch (java.io.IOException e1) { continue EOFLoop; }
+ jjmatchedKind = 0x7fffffff;
+ jjmatchedPos = 0;
+ curPos = jjMoveStringLiteralDfa0_0();
+ if (jjmatchedKind != 0x7fffffff)
+ {
+ if (jjmatchedPos + 1 < curPos)
+ input_stream.backup(curPos - jjmatchedPos - 1);
+ if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L)
+ {
+ matchedToken = jjFillToken();
+ return matchedToken;
+ }
+ else
+ {
+ continue EOFLoop;
+ }
+ }
+ int error_line = input_stream.getEndLine();
+ int error_column = input_stream.getEndColumn();
+ String error_after = null;
+ boolean EOFSeen = false;
+ try { input_stream.readChar(); input_stream.backup(1); }
+ catch (java.io.IOException e1) {
+ EOFSeen = true;
+ error_after = curPos <= 1 ? "" : input_stream.GetImage();
+ if (curChar == '\n' || curChar == '\r') {
+ error_line++;
+ error_column = 0;
+ }
+ else
+ error_column++;
+ }
+ if (!EOFSeen) {
+ input_stream.backup(1);
+ error_after = curPos <= 1 ? "" : input_stream.GetImage();
+ }
+ throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
+ }
+}
+
+private void jjCheckNAdd(int state)
+{
+ if (jjrounds[state] != jjround)
+ {
+ jjstateSet[jjnewStateCnt++] = state;
+ jjrounds[state] = jjround;
+ }
+}
+private void jjAddStates(int start, int end)
+{
+ do {
+ jjstateSet[jjnewStateCnt++] = jjnextStates[start];
+ } while (start++ != end);
+}
+private void jjCheckNAddTwoStates(int state1, int state2)
+{
+ jjCheckNAdd(state1);
+ jjCheckNAdd(state2);
+}
+
+private void jjCheckNAddStates(int start, int end)
+{
+ do {
+ jjCheckNAdd(jjnextStates[start]);
+ } while (start++ != end);
+}
+
+}
diff --git a/src/mtas/parser/function/ParseException.java b/src/mtas/parser/function/ParseException.java
new file mode 100644
index 0000000..967b022
--- /dev/null
+++ b/src/mtas/parser/function/ParseException.java
@@ -0,0 +1,187 @@
+/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 5.0 */
+/* JavaCCOptions:KEEP_LINE_COL=null */
+package mtas.parser.function;
+
+/**
+ * This exception is thrown when parse errors are encountered.
+ * You can explicitly create objects of this exception type by
+ * calling the method generateParseException in the generated
+ * parser.
+ *
+ * You can modify this class to customize your error reporting
+ * mechanisms so long as you retain the public fields.
+ */
+public class ParseException extends Exception {
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * This constructor is used by the method "generateParseException"
+ * in the generated parser. Calling this constructor generates
+ * a new object of this type with the fields "currentToken",
+ * "expectedTokenSequences", and "tokenImage" set.
+ */
+ public ParseException(Token currentTokenVal,
+ int[][] expectedTokenSequencesVal,
+ String[] tokenImageVal
+ )
+ {
+ super(initialise(currentTokenVal, expectedTokenSequencesVal, tokenImageVal));
+ currentToken = currentTokenVal;
+ expectedTokenSequences = expectedTokenSequencesVal;
+ tokenImage = tokenImageVal;
+ }
+
+ /**
+ * The following constructors are for use by you for whatever
+ * purpose you can think of. Constructing the exception in this
+ * manner makes the exception behave in the normal way - i.e., as
+ * documented in the class "Throwable". The fields "errorToken",
+ * "expectedTokenSequences", and "tokenImage" do not contain
+ * relevant information. The JavaCC generated code does not use
+ * these constructors.
+ */
+
+ public ParseException() {
+ super();
+ }
+
+ /** Constructor with message. */
+ public ParseException(String message) {
+ super(message);
+ }
+
+
+ /**
+ * This is the last token that has been consumed successfully. If
+ * this object has been created due to a parse error, the token
+ * followng this token will (therefore) be the first error token.
+ */
+ public Token currentToken;
+
+ /**
+ * Each entry in this array is an array of integers. Each array
+ * of integers represents a sequence of tokens (by their ordinal
+ * values) that is expected at this point of the parse.
+ */
+ public int[][] expectedTokenSequences;
+
+ /**
+ * This is a reference to the "tokenImage" array of the generated
+ * parser within which the parse error occurred. This array is
+ * defined in the generated ...Constants interface.
+ */
+ public String[] tokenImage;
+
+ /**
+ * It uses "currentToken" and "expectedTokenSequences" to generate a parse
+ * error message and returns it. If this object has been created
+ * due to a parse error, and you do not catch it (it gets thrown
+ * from the parser) the correct error message
+ * gets displayed.
+ */
+ private static String initialise(Token currentToken,
+ int[][] expectedTokenSequences,
+ String[] tokenImage) {
+ String eol = System.getProperty("line.separator", "\n");
+ StringBuffer expected = new StringBuffer();
+ int maxSize = 0;
+ for (int i = 0; i < expectedTokenSequences.length; i++) {
+ if (maxSize < expectedTokenSequences[i].length) {
+ maxSize = expectedTokenSequences[i].length;
+ }
+ for (int j = 0; j < expectedTokenSequences[i].length; j++) {
+ expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' ');
+ }
+ if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
+ expected.append("...");
+ }
+ expected.append(eol).append(" ");
+ }
+ String retval = "Encountered \"";
+ Token tok = currentToken.next;
+ for (int i = 0; i < maxSize; i++) {
+ if (i != 0) retval += " ";
+ if (tok.kind == 0) {
+ retval += tokenImage[0];
+ break;
+ }
+ retval += " " + tokenImage[tok.kind];
+ retval += " \"";
+ retval += add_escapes(tok.image);
+ retval += " \"";
+ tok = tok.next;
+ }
+ retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn;
+ retval += "." + eol;
+ if (expectedTokenSequences.length == 1) {
+ retval += "Was expecting:" + eol + " ";
+ } else {
+ retval += "Was expecting one of:" + eol + " ";
+ }
+ retval += expected.toString();
+ return retval;
+ }
+
+ /**
+ * The end of line string for this machine.
+ */
+ protected String eol = System.getProperty("line.separator", "\n");
+
+ /**
+ * Used to convert raw characters to their escaped version
+ * when these raw version cannot be used as part of an ASCII
+ * string literal.
+ */
+ static String add_escapes(String str) {
+ StringBuffer retval = new StringBuffer();
+ char ch;
+ for (int i = 0; i < str.length(); i++) {
+ switch (str.charAt(i))
+ {
+ case 0 :
+ continue;
+ case '\b':
+ retval.append("\\b");
+ continue;
+ case '\t':
+ retval.append("\\t");
+ continue;
+ case '\n':
+ retval.append("\\n");
+ continue;
+ case '\f':
+ retval.append("\\f");
+ continue;
+ case '\r':
+ retval.append("\\r");
+ continue;
+ case '\"':
+ retval.append("\\\"");
+ continue;
+ case '\'':
+ retval.append("\\\'");
+ continue;
+ case '\\':
+ retval.append("\\\\");
+ continue;
+ default:
+ if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+ } else {
+ retval.append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.toString();
+ }
+
+}
+/* JavaCC - OriginalChecksum=03241e67478474b014438ca8f89ad5a8 (do not edit this line) */
diff --git a/src/mtas/parser/function/SimpleCharStream.java b/src/mtas/parser/function/SimpleCharStream.java
new file mode 100644
index 0000000..36c84dc
--- /dev/null
+++ b/src/mtas/parser/function/SimpleCharStream.java
@@ -0,0 +1,471 @@
+/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 5.0 */
+/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
+package mtas.parser.function;
+
+/**
+ * An implementation of interface CharStream, where the stream is assumed to
+ * contain only ASCII characters (without unicode processing).
+ */
+
+public class SimpleCharStream
+{
+/** Whether parser is static. */
+ public static final boolean staticFlag = false;
+ int bufsize;
+ int available;
+ int tokenBegin;
+/** Position in buffer. */
+ public int bufpos = -1;
+ protected int bufline[];
+ protected int bufcolumn[];
+
+ protected int column = 0;
+ protected int line = 1;
+
+ protected boolean prevCharIsCR = false;
+ protected boolean prevCharIsLF = false;
+
+ protected java.io.Reader inputStream;
+
+ protected char[] buffer;
+ protected int maxNextCharInd = 0;
+ protected int inBuf = 0;
+ protected int tabSize = 8;
+
+ protected void setTabSize(int i) { tabSize = i; }
+ protected int getTabSize(int i) { return tabSize; }
+
+
+ protected void ExpandBuff(boolean wrapAround)
+ {
+ char[] newbuffer = new char[bufsize + 2048];
+ int newbufline[] = new int[bufsize + 2048];
+ int newbufcolumn[] = new int[bufsize + 2048];
+
+ try
+ {
+ if (wrapAround)
+ {
+ System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+ System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos);
+ buffer = newbuffer;
+
+ System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+ System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos);
+ bufline = newbufline;
+
+ System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+ System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos);
+ bufcolumn = newbufcolumn;
+
+ maxNextCharInd = (bufpos += (bufsize - tokenBegin));
+ }
+ else
+ {
+ System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+ buffer = newbuffer;
+
+ System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+ bufline = newbufline;
+
+ System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+ bufcolumn = newbufcolumn;
+
+ maxNextCharInd = (bufpos -= tokenBegin);
+ }
+ }
+ catch (Throwable t)
+ {
+ throw new Error(t.getMessage());
+ }
+
+
+ bufsize += 2048;
+ available = bufsize;
+ tokenBegin = 0;
+ }
+
+ protected void FillBuff() throws java.io.IOException
+ {
+ if (maxNextCharInd == available)
+ {
+ if (available == bufsize)
+ {
+ if (tokenBegin > 2048)
+ {
+ bufpos = maxNextCharInd = 0;
+ available = tokenBegin;
+ }
+ else if (tokenBegin < 0)
+ bufpos = maxNextCharInd = 0;
+ else
+ ExpandBuff(false);
+ }
+ else if (available > tokenBegin)
+ available = bufsize;
+ else if ((tokenBegin - available) < 2048)
+ ExpandBuff(true);
+ else
+ available = tokenBegin;
+ }
+
+ int i;
+ try {
+ if ((i = inputStream.read(buffer, maxNextCharInd, available - maxNextCharInd)) == -1)
+ {
+ inputStream.close();
+ throw new java.io.IOException();
+ }
+ else
+ maxNextCharInd += i;
+ return;
+ }
+ catch(java.io.IOException e) {
+ --bufpos;
+ backup(0);
+ if (tokenBegin == -1)
+ tokenBegin = bufpos;
+ throw e;
+ }
+ }
+
+/** Start. */
+ public char BeginToken() throws java.io.IOException
+ {
+ tokenBegin = -1;
+ char c = readChar();
+ tokenBegin = bufpos;
+
+ return c;
+ }
+
+ protected void UpdateLineColumn(char c)
+ {
+ column++;
+
+ if (prevCharIsLF)
+ {
+ prevCharIsLF = false;
+ line += (column = 1);
+ }
+ else if (prevCharIsCR)
+ {
+ prevCharIsCR = false;
+ if (c == '\n')
+ {
+ prevCharIsLF = true;
+ }
+ else
+ line += (column = 1);
+ }
+
+ switch (c)
+ {
+ case '\r' :
+ prevCharIsCR = true;
+ break;
+ case '\n' :
+ prevCharIsLF = true;
+ break;
+ case '\t' :
+ column--;
+ column += (tabSize - (column % tabSize));
+ break;
+ default :
+ break;
+ }
+
+ bufline[bufpos] = line;
+ bufcolumn[bufpos] = column;
+ }
+
+/** Read a character. */
+ public char readChar() throws java.io.IOException
+ {
+ if (inBuf > 0)
+ {
+ --inBuf;
+
+ if (++bufpos == bufsize)
+ bufpos = 0;
+
+ return buffer[bufpos];
+ }
+
+ if (++bufpos >= maxNextCharInd)
+ FillBuff();
+
+ char c = buffer[bufpos];
+
+ UpdateLineColumn(c);
+ return c;
+ }
+
+ @Deprecated
+ /**
+ * @deprecated
+ * @see #getEndColumn
+ */
+
+ public int getColumn() {
+ return bufcolumn[bufpos];
+ }
+
+ @Deprecated
+ /**
+ * @deprecated
+ * @see #getEndLine
+ */
+
+ public int getLine() {
+ return bufline[bufpos];
+ }
+
+ /** Get token end column number. */
+ public int getEndColumn() {
+ return bufcolumn[bufpos];
+ }
+
+ /** Get token end line number. */
+ public int getEndLine() {
+ return bufline[bufpos];
+ }
+
+ /** Get token beginning column number. */
+ public int getBeginColumn() {
+ return bufcolumn[tokenBegin];
+ }
+
+ /** Get token beginning line number. */
+ public int getBeginLine() {
+ return bufline[tokenBegin];
+ }
+
+/** Backup a number of characters. */
+ public void backup(int amount) {
+
+ inBuf += amount;
+ if ((bufpos -= amount) < 0)
+ bufpos += bufsize;
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.Reader dstream, int startline,
+ int startcolumn, int buffersize)
+ {
+ inputStream = dstream;
+ line = startline;
+ column = startcolumn - 1;
+
+ available = bufsize = buffersize;
+ buffer = new char[buffersize];
+ bufline = new int[buffersize];
+ bufcolumn = new int[buffersize];
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.Reader dstream, int startline,
+ int startcolumn)
+ {
+ this(dstream, startline, startcolumn, 4096);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.Reader dstream)
+ {
+ this(dstream, 1, 1, 4096);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.Reader dstream, int startline,
+ int startcolumn, int buffersize)
+ {
+ inputStream = dstream;
+ line = startline;
+ column = startcolumn - 1;
+
+ if (buffer == null || buffersize != buffer.length)
+ {
+ available = bufsize = buffersize;
+ buffer = new char[buffersize];
+ bufline = new int[buffersize];
+ bufcolumn = new int[buffersize];
+ }
+ prevCharIsLF = prevCharIsCR = false;
+ tokenBegin = inBuf = maxNextCharInd = 0;
+ bufpos = -1;
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.Reader dstream, int startline,
+ int startcolumn)
+ {
+ ReInit(dstream, startline, startcolumn, 4096);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.Reader dstream)
+ {
+ ReInit(dstream, 1, 1, 4096);
+ }
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
+ {
+ this(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream, int startline,
+ int startcolumn, int buffersize)
+ {
+ this(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn) throws java.io.UnsupportedEncodingException
+ {
+ this(dstream, encoding, startline, startcolumn, 4096);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream, int startline,
+ int startcolumn)
+ {
+ this(dstream, startline, startcolumn, 4096);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
+ {
+ this(dstream, encoding, 1, 1, 4096);
+ }
+
+ /** Constructor. */
+ public SimpleCharStream(java.io.InputStream dstream)
+ {
+ this(dstream, 1, 1, 4096);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
+ {
+ ReInit(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, int startline,
+ int startcolumn, int buffersize)
+ {
+ ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
+ {
+ ReInit(dstream, encoding, 1, 1, 4096);
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream)
+ {
+ ReInit(dstream, 1, 1, 4096);
+ }
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn) throws java.io.UnsupportedEncodingException
+ {
+ ReInit(dstream, encoding, startline, startcolumn, 4096);
+ }
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, int startline,
+ int startcolumn)
+ {
+ ReInit(dstream, startline, startcolumn, 4096);
+ }
+ /** Get token literal value. */
+ public String GetImage()
+ {
+ if (bufpos >= tokenBegin)
+ return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
+ else
+ return new String(buffer, tokenBegin, bufsize - tokenBegin) +
+ new String(buffer, 0, bufpos + 1);
+ }
+
+ /** Get the suffix. */
+ public char[] GetSuffix(int len)
+ {
+ char[] ret = new char[len];
+
+ if ((bufpos + 1) >= len)
+ System.arraycopy(buffer, bufpos - len + 1, ret, 0, len);
+ else
+ {
+ System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0,
+ len - bufpos - 1);
+ System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1);
+ }
+
+ return ret;
+ }
+
+ /** Reset buffer when finished. */
+ public void Done()
+ {
+ buffer = null;
+ bufline = null;
+ bufcolumn = null;
+ }
+
+ /**
+ * Method to adjust line and column numbers for the start of a token.
+ */
+ public void adjustBeginLineColumn(int newLine, int newCol)
+ {
+ int start = tokenBegin;
+ int len;
+
+ if (bufpos >= tokenBegin)
+ {
+ len = bufpos - tokenBegin + inBuf + 1;
+ }
+ else
+ {
+ len = bufsize - tokenBegin + bufpos + 1 + inBuf;
+ }
+
+ int i = 0, j = 0, k = 0;
+ int nextColDiff = 0, columnDiff = 0;
+
+ while (i < len && bufline[j = start % bufsize] == bufline[k = ++start % bufsize])
+ {
+ bufline[j] = newLine;
+ nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j];
+ bufcolumn[j] = newCol + columnDiff;
+ columnDiff = nextColDiff;
+ i++;
+ }
+
+ if (i < len)
+ {
+ bufline[j] = newLine++;
+ bufcolumn[j] = newCol + columnDiff;
+
+ while (i++ < len)
+ {
+ if (bufline[j = start % bufsize] != bufline[++start % bufsize])
+ bufline[j] = newLine++;
+ else
+ bufline[j] = newLine;
+ }
+ }
+
+ line = bufline[j];
+ column = bufcolumn[j];
+ }
+
+}
+/* JavaCC - OriginalChecksum=351c902e220c6aaab99db9e1060ff515 (do not edit this line) */
diff --git a/src/mtas/parser/function/Token.java b/src/mtas/parser/function/Token.java
new file mode 100644
index 0000000..cc91e01
--- /dev/null
+++ b/src/mtas/parser/function/Token.java
@@ -0,0 +1,131 @@
+/* Generated By:JavaCC: Do not edit this line. Token.java Version 5.0 */
+/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
+package mtas.parser.function;
+
+/**
+ * Describes the input token stream.
+ */
+
+public class Token implements java.io.Serializable {
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * An integer that describes the kind of this token. This numbering
+ * system is determined by JavaCCParser, and a table of these numbers is
+ * stored in the file ...Constants.java.
+ */
+ public int kind;
+
+ /** The line number of the first character of this Token. */
+ public int beginLine;
+ /** The column number of the first character of this Token. */
+ public int beginColumn;
+ /** The line number of the last character of this Token. */
+ public int endLine;
+ /** The column number of the last character of this Token. */
+ public int endColumn;
+
+ /**
+ * The string image of the token.
+ */
+ public String image;
+
+ /**
+ * A reference to the next regular (non-special) token from the input
+ * stream. If this is the last token from the input stream, or if the
+ * token manager has not read tokens beyond this one, this field is
+ * set to null. This is true only if this token is also a regular
+ * token. Otherwise, see below for a description of the contents of
+ * this field.
+ */
+ public Token next;
+
+ /**
+ * This field is used to access special tokens that occur prior to this
+ * token, but after the immediately preceding regular (non-special) token.
+ * If there are no such special tokens, this field is set to null.
+ * When there are more than one such special token, this field refers
+ * to the last of these special tokens, which in turn refers to the next
+ * previous special token through its specialToken field, and so on
+ * until the first special token (whose specialToken field is null).
+ * The next fields of special tokens refer to other special tokens that
+ * immediately follow it (without an intervening regular token). If there
+ * is no such token, this field is null.
+ */
+ public Token specialToken;
+
+ /**
+ * An optional attribute value of the Token.
+ * Tokens which are not used as syntactic sugar will often contain
+ * meaningful values that will be used later on by the compiler or
+ * interpreter. This attribute value is often different from the image.
+ * Any subclass of Token that actually wants to return a non-null value can
+ * override this method as appropriate.
+ */
+ public Object getValue() {
+ return null;
+ }
+
+ /**
+ * No-argument constructor
+ */
+ public Token() {}
+
+ /**
+ * Constructs a new token for the specified Image.
+ */
+ public Token(int kind)
+ {
+ this(kind, null);
+ }
+
+ /**
+ * Constructs a new token for the specified Image and Kind.
+ */
+ public Token(int kind, String image)
+ {
+ this.kind = kind;
+ this.image = image;
+ }
+
+ /**
+ * Returns the image.
+ */
+ public String toString()
+ {
+ return image;
+ }
+
+ /**
+ * Returns a new Token object, by default. However, if you want, you
+ * can create and return subclass objects based on the value of ofKind.
+ * Simply add the cases to the switch for all those special cases.
+ * For example, if you have a subclass of Token called IDToken that
+ * you want to create if ofKind is ID, simply add something like :
+ *
+ * case MyParserConstants.ID : return new IDToken(ofKind, image);
+ *
+ * to the following switch statement. Then you can cast matchedToken
+ * variable to the appropriate type and use sit in your lexical actions.
+ */
+ public static Token newToken(int ofKind, String image)
+ {
+ switch(ofKind)
+ {
+ default : return new Token(ofKind, image);
+ }
+ }
+
+ public static Token newToken(int ofKind)
+ {
+ return newToken(ofKind, null);
+ }
+
+}
+/* JavaCC - OriginalChecksum=1ff051f734b3b825a428890df3aeb019 (do not edit this line) */
diff --git a/src/mtas/parser/function/TokenMgrError.java b/src/mtas/parser/function/TokenMgrError.java
new file mode 100644
index 0000000..aa9bc90
--- /dev/null
+++ b/src/mtas/parser/function/TokenMgrError.java
@@ -0,0 +1,147 @@
+/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 5.0 */
+/* JavaCCOptions: */
+package mtas.parser.function;
+
+/** Token Manager Error. */
+public class TokenMgrError extends Error
+{
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /*
+ * Ordinals for various reasons why an Error of this type can be thrown.
+ */
+
+ /**
+ * Lexical error occurred.
+ */
+ static final int LEXICAL_ERROR = 0;
+
+ /**
+ * An attempt was made to create a second instance of a static token manager.
+ */
+ static final int STATIC_LEXER_ERROR = 1;
+
+ /**
+ * Tried to change to an invalid lexical state.
+ */
+ static final int INVALID_LEXICAL_STATE = 2;
+
+ /**
+ * Detected (and bailed out of) an infinite loop in the token manager.
+ */
+ static final int LOOP_DETECTED = 3;
+
+ /**
+ * Indicates the reason why the exception is thrown. It will have
+ * one of the above 4 values.
+ */
+ int errorCode;
+
+ /**
+ * Replaces unprintable characters by their escaped (or unicode escaped)
+ * equivalents in the given string
+ */
+ protected static final String addEscapes(String str) {
+ StringBuffer retval = new StringBuffer();
+ char ch;
+ for (int i = 0; i < str.length(); i++) {
+ switch (str.charAt(i))
+ {
+ case 0 :
+ continue;
+ case '\b':
+ retval.append("\\b");
+ continue;
+ case '\t':
+ retval.append("\\t");
+ continue;
+ case '\n':
+ retval.append("\\n");
+ continue;
+ case '\f':
+ retval.append("\\f");
+ continue;
+ case '\r':
+ retval.append("\\r");
+ continue;
+ case '\"':
+ retval.append("\\\"");
+ continue;
+ case '\'':
+ retval.append("\\\'");
+ continue;
+ case '\\':
+ retval.append("\\\\");
+ continue;
+ default:
+ if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+ } else {
+ retval.append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.toString();
+ }
+
+ /**
+ * Returns a detailed message for the Error when it is thrown by the
+ * token manager to indicate a lexical error.
+ * Parameters :
+ * EOFSeen : indicates if EOF caused the lexical error
+ * curLexState : lexical state in which this error occurred
+ * errorLine : line number when the error occurred
+ * errorColumn : column number when the error occurred
+ * errorAfter : prefix that was seen before this error occurred
+ * curchar : the offending character
+ * Note: You can customize the lexical error message by modifying this method.
+ */
+ protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) {
+ return("Lexical error at line " +
+ errorLine + ", column " +
+ errorColumn + ". Encountered: " +
+ (EOFSeen ? "<EOF> " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") +
+ "after : \"" + addEscapes(errorAfter) + "\"");
+ }
+
+ /**
+ * You can also modify the body of this method to customize your error messages.
+ * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
+ * of end-users concern, so you can return something like :
+ *
+ * "Internal Error : Please file a bug report .... "
+ *
+ * from this method for such cases in the release version of your parser.
+ */
+ public String getMessage() {
+ return super.getMessage();
+ }
+
+ /*
+ * Constructors of various flavors follow.
+ */
+
+ /** No arg constructor. */
+ public TokenMgrError() {
+ }
+
+ /** Constructor with message and reason. */
+ public TokenMgrError(String message, int reason) {
+ super(message);
+ errorCode = reason;
+ }
+
+ /** Full Constructor. */
+ public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) {
+ this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
+ }
+}
+/* JavaCC - OriginalChecksum=c8e52d6b54c7ac5de1d496ea7ce2fcd9 (do not edit this line) */
diff --git a/src/mtas/parser/function/util/MtasFunctionParserFunction.java b/src/mtas/parser/function/util/MtasFunctionParserFunction.java
index 373be44..2b36d3b 100644
--- a/src/mtas/parser/function/util/MtasFunctionParserFunction.java
+++ b/src/mtas/parser/function/util/MtasFunctionParserFunction.java
@@ -13,28 +13,28 @@ public abstract class MtasFunctionParserFunction {
/** The parser doubles. */
protected MtasFunctionParserFunction[] parserDoubles;
-
+
/** The parser longs. */
protected MtasFunctionParserFunction[] parserLongs;
-
+
/** The constant doubles. */
protected Double[] constantDoubles;
-
+
/** The constant longs. */
protected long[] constantLongs;
/** The data type. */
protected String dataType = null;
-
+
/** The sum rule. */
protected boolean sumRule = false;
-
+
/** The need positions. */
protected boolean needPositions = false;
-
+
/** The degree. */
protected Integer degree = null;
-
+
/** The need argument. */
protected HashSet<Integer> needArgument = new HashSet<Integer>();
@@ -44,19 +44,22 @@ public abstract class MtasFunctionParserFunction {
/**
* Gets the response.
*
- * @param args the args
- * @param n the n
+ * @param args
+ * the args
+ * @param n
+ * the n
* @return the response
*/
- public final MtasFunctionParserFunctionResponse getResponse(long[] args, long n) {
- if(dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
+ public final MtasFunctionParserFunctionResponse getResponse(long[] args,
+ long n) {
+ if (dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
try {
long l = getValueLong(args, n);
return new MtasFunctionParserFunctionResponseLong(l, true);
} catch (IOException e) {
return new MtasFunctionParserFunctionResponseLong(0, false);
}
- } else if(dataType.equals(CodecUtil.DATA_TYPE_DOUBLE)) {
+ } else if (dataType.equals(CodecUtil.DATA_TYPE_DOUBLE)) {
try {
double d = getValueDouble(args, n);
return new MtasFunctionParserFunctionResponseDouble(d, true);
@@ -71,27 +74,34 @@ public abstract class MtasFunctionParserFunction {
/**
* Gets the value double.
*
- * @param args the args
- * @param n the n
+ * @param args
+ * the args
+ * @param n
+ * the n
* @return the value double
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public abstract double getValueDouble(long[] args, long n) throws IOException;
-
+
/**
* Gets the value long.
*
- * @param args the args
- * @param n the n
+ * @param args
+ * the args
+ * @param n
+ * the n
* @return the value long
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public abstract long getValueLong(long[] args, long n) throws IOException;
/**
* Close.
*
- * @throws ParseException the parse exception
+ * @throws ParseException
+ * the parse exception
*/
public void close() throws ParseException {
defined = true;
@@ -123,17 +133,18 @@ public abstract class MtasFunctionParserFunction {
public final Boolean needPositions() {
return needPositions;
}
-
+
/**
* Need argument.
*
- * @param i the i
+ * @param i
+ * the i
* @return the boolean
*/
public final Boolean needArgument(int i) {
return needArgument.contains(i);
}
-
+
/**
* Need arguments number.
*
@@ -141,21 +152,21 @@ public abstract class MtasFunctionParserFunction {
*/
public final int needArgumentsNumber() {
int number = 0;
- for(int i: needArgument) {
- number = Math.max(number, (i+1));
+ for (int i : needArgument) {
+ number = Math.max(number, (i + 1));
}
return number;
}
-
+
/**
- * Need arguments.
+ * Need argument.
*
- * @return the integer[]
+ * @return the hash set
*/
- public final Integer[] needArguments() {
- return needArgument.toArray(new Integer[needArgument.size()]);
+ public final HashSet<Integer> needArgument() {
+ return needArgument;
}
-
+
/**
* Defined.
*
diff --git a/src/mtas/parser/function/util/MtasFunctionParserFunctionBasic.java b/src/mtas/parser/function/util/MtasFunctionParserFunctionBasic.java
index 0551306..f0f35ac 100644
--- a/src/mtas/parser/function/util/MtasFunctionParserFunctionBasic.java
+++ b/src/mtas/parser/function/util/MtasFunctionParserFunctionBasic.java
@@ -14,70 +14,72 @@ public class MtasFunctionParserFunctionBasic
/** The first type. */
private String firstType;
-
+
/** The first id. */
private int firstId;
/** The tmp parser longs. */
private ArrayList<MtasFunctionParserFunction> tmpParserLongs = new ArrayList<MtasFunctionParserFunction>();
-
+
/** The tmp parser doubles. */
private ArrayList<MtasFunctionParserFunction> tmpParserDoubles = new ArrayList<MtasFunctionParserFunction>();
-
+
/** The tmp constant longs. */
private ArrayList<Long> tmpConstantLongs = new ArrayList<Long>();
-
+
/** The tmp constant doubles. */
private ArrayList<Double> tmpConstantDoubles = new ArrayList<Double>();
/** The number. */
private int number;
-
+
/** The operator list. */
private String[] operatorList;
-
+
/** The type list. */
private String[] typeList;
-
+
/** The id list. */
private int[] idList;
/** The tmp operator list. */
private ArrayList<String> tmpOperatorList = new ArrayList<String>();
-
+
/** The tmp type list. */
private ArrayList<String> tmpTypeList = new ArrayList<String>();
-
+
/** The tmp id list. */
private ArrayList<Integer> tmpIdList = new ArrayList<Integer>();
/** The Constant BASIC_OPERATOR_ADD. */
public final static String BASIC_OPERATOR_ADD = "add";
-
+
/** The Constant BASIC_OPERATOR_SUBTRACT. */
public final static String BASIC_OPERATOR_SUBTRACT = "subtract";
-
+
/** The Constant BASIC_OPERATOR_MULTIPLY. */
public final static String BASIC_OPERATOR_MULTIPLY = "multiply";
-
+
/** The Constant BASIC_OPERATOR_DIVIDE. */
public final static String BASIC_OPERATOR_DIVIDE = "divide";
-
+
/** The Constant BASIC_OPERATOR_POWER. */
public final static String BASIC_OPERATOR_POWER = "power";
/**
* Instantiates a new mtas function parser function basic.
*
- * @param item the item
- * @throws ParseException the parse exception
+ * @param item
+ * the item
+ * @throws ParseException
+ * the parse exception
*/
public MtasFunctionParserFunctionBasic(MtasFunctionParserItem item)
throws ParseException {
- sumRule=true;
+ sumRule = true;
String type = item.getType();
MtasFunctionParserFunction parser;
- firstType = type;
+ firstType = type;
degree = item.getDegree();
switch (type) {
case MtasFunctionParserItem.TYPE_N:
@@ -109,7 +111,7 @@ public class MtasFunctionParserFunctionBasic
tmpParserLongs.add(parser);
sumRule = parser.sumRule();
dataType = CodecUtil.DATA_TYPE_LONG;
- needPositions = needPositions?needPositions:parser.needPositions();
+ needPositions = needPositions ? needPositions : parser.needPositions();
needArgument.addAll(parser.needArgument);
} else {
throw new ParseException("incorrect dataType");
@@ -123,7 +125,7 @@ public class MtasFunctionParserFunctionBasic
tmpParserDoubles.add(parser);
sumRule = parser.sumRule();
dataType = CodecUtil.DATA_TYPE_DOUBLE;
- needPositions = needPositions?needPositions:parser.needPositions();
+ needPositions = needPositions ? needPositions : parser.needPositions();
needArgument.addAll(parser.needArgument);
} else {
throw new ParseException("incorrect dataType");
@@ -131,10 +133,12 @@ public class MtasFunctionParserFunctionBasic
break;
default:
throw new ParseException("unknown type");
- }
+ }
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.parser.function.util.MtasFunctionParserFunction#close()
*/
@Override
@@ -142,8 +146,7 @@ public class MtasFunctionParserFunctionBasic
if (!defined()) {
super.close();
if (tmpParserLongs.size() > 0) {
- parserLongs = new MtasFunctionParserFunction[tmpParserLongs
- .size()];
+ parserLongs = new MtasFunctionParserFunction[tmpParserLongs.size()];
parserLongs = tmpParserLongs.toArray(parserLongs);
}
if (tmpParserDoubles.size() > 0) {
@@ -177,7 +180,7 @@ public class MtasFunctionParserFunctionBasic
idList = new int[number];
for (int i = 0; i < number; i++) {
idList[i] = tmpIdList.get(i).intValue();
- }
+ }
}
} else {
number = 0;
@@ -191,8 +194,10 @@ public class MtasFunctionParserFunctionBasic
/**
* Adds the.
*
- * @param item the item
- * @throws ParseException the parse exception
+ * @param item
+ * the item
+ * @throws ParseException
+ * the parse exception
*/
public void add(MtasFunctionParserItem item) throws ParseException {
basic(BASIC_OPERATOR_ADD, item);
@@ -201,8 +206,10 @@ public class MtasFunctionParserFunctionBasic
/**
* Subtract.
*
- * @param item the item
- * @throws ParseException the parse exception
+ * @param item
+ * the item
+ * @throws ParseException
+ * the parse exception
*/
public void subtract(MtasFunctionParserItem item) throws ParseException {
basic(BASIC_OPERATOR_SUBTRACT, item);
@@ -211,8 +218,10 @@ public class MtasFunctionParserFunctionBasic
/**
* Multiply.
*
- * @param item the item
- * @throws ParseException the parse exception
+ * @param item
+ * the item
+ * @throws ParseException
+ * the parse exception
*/
public void multiply(MtasFunctionParserItem item) throws ParseException {
basic(BASIC_OPERATOR_MULTIPLY, item);
@@ -221,8 +230,10 @@ public class MtasFunctionParserFunctionBasic
/**
* Divide.
*
- * @param item the item
- * @throws ParseException the parse exception
+ * @param item
+ * the item
+ * @throws ParseException
+ * the parse exception
*/
public void divide(MtasFunctionParserItem item) throws ParseException {
basic(BASIC_OPERATOR_DIVIDE, item);
@@ -231,8 +242,10 @@ public class MtasFunctionParserFunctionBasic
/**
* Power.
*
- * @param item the item
- * @throws ParseException the parse exception
+ * @param item
+ * the item
+ * @throws ParseException
+ * the parse exception
*/
public void power(MtasFunctionParserItem item) throws ParseException {
basic(BASIC_OPERATOR_POWER, item);
@@ -241,9 +254,12 @@ public class MtasFunctionParserFunctionBasic
/**
* Basic.
*
- * @param operator the operator
- * @param item the item
- * @throws ParseException the parse exception
+ * @param operator
+ * the operator
+ * @param item
+ * the item
+ * @throws ParseException
+ * the parse exception
*/
private void basic(String operator, MtasFunctionParserItem item)
throws ParseException {
@@ -259,18 +275,19 @@ public class MtasFunctionParserFunctionBasic
tmpTypeList.add(type);
tmpIdList.add(0);
needPositions = true;
- if(sumRule && degree!=null) {
- if(operator.equals(BASIC_OPERATOR_ADD)||operator.equals(BASIC_OPERATOR_SUBTRACT)) {
- if(degree<0) {
- sumRule=false;
- degree=null;
- } else if(degree>0) {
- sumRule=false;
+ if (sumRule && degree != null) {
+ if (operator.equals(BASIC_OPERATOR_ADD)
+ || operator.equals(BASIC_OPERATOR_SUBTRACT)) {
+ if (degree < 0) {
+ sumRule = false;
+ degree = null;
+ } else if (degree > 0) {
+ sumRule = false;
}
- } else if(operator.equals(BASIC_OPERATOR_POWER)) {
- if(degree!=0) {
- sumRule=false;
- degree=null;
+ } else if (operator.equals(BASIC_OPERATOR_POWER)) {
+ if (degree != 0) {
+ sumRule = false;
+ degree = null;
}
}
}
@@ -279,46 +296,48 @@ public class MtasFunctionParserFunctionBasic
tmpTypeList.add(type);
tmpIdList.add(item.getId());
needArgument.add(item.getId());
- if(sumRule && degree!=null) {
- if(operator.equals(BASIC_OPERATOR_ADD)||operator.equals(BASIC_OPERATOR_SUBTRACT)) {
- if(degree!=1) {
- sumRule=false;
+ if (sumRule && degree != null) {
+ if (operator.equals(BASIC_OPERATOR_ADD)
+ || operator.equals(BASIC_OPERATOR_SUBTRACT)) {
+ if (degree != 1) {
+ sumRule = false;
}
- if(degree>=0) {
- degree=Math.max(degree, 1);
+ if (degree >= 0) {
+ degree = Math.max(degree, 1);
} else {
- degree=null;
+ degree = null;
}
- } else if(operator.equals(BASIC_OPERATOR_MULTIPLY)) {
- if(degree!=0) {
- sumRule=false;
+ } else if (operator.equals(BASIC_OPERATOR_MULTIPLY)) {
+ if (degree != 0) {
+ sumRule = false;
}
- degree+=1;
- } else if(operator.equals(BASIC_OPERATOR_DIVIDE)) {
- sumRule=false;
- degree-=1;
- } else if(operator.equals(BASIC_OPERATOR_POWER)) {
- sumRule=false;
- degree=null;
- }
+ degree += 1;
+ } else if (operator.equals(BASIC_OPERATOR_DIVIDE)) {
+ sumRule = false;
+ degree -= 1;
+ } else if (operator.equals(BASIC_OPERATOR_POWER)) {
+ sumRule = false;
+ degree = null;
+ }
}
break;
case MtasFunctionParserItem.TYPE_CONSTANT_LONG:
tmpTypeList.add(type);
tmpIdList.add(tmpConstantLongs.size());
tmpConstantLongs.add(item.getValueLong());
- if(sumRule && degree!=null) {
- if(operator.equals(BASIC_OPERATOR_ADD)||operator.equals(BASIC_OPERATOR_SUBTRACT)) {
- if(degree<0) {
- sumRule=false;
- degree=null;
- } else if(degree>0) {
- sumRule=false;
+ if (sumRule && degree != null) {
+ if (operator.equals(BASIC_OPERATOR_ADD)
+ || operator.equals(BASIC_OPERATOR_SUBTRACT)) {
+ if (degree < 0) {
+ sumRule = false;
+ degree = null;
+ } else if (degree > 0) {
+ sumRule = false;
}
- } else if(operator.equals(BASIC_OPERATOR_POWER)) {
- if(degree!=0) {
+ } else if (operator.equals(BASIC_OPERATOR_POWER)) {
+ if (degree != 0) {
sumRule = false;
- degree=null;
+ degree = null;
}
}
}
@@ -328,18 +347,19 @@ public class MtasFunctionParserFunctionBasic
tmpIdList.add(tmpConstantDoubles.size());
dataType = CodecUtil.DATA_TYPE_DOUBLE;
tmpConstantDoubles.add(item.getValueDouble());
- if(sumRule && degree!=null) {
- if(operator.equals(BASIC_OPERATOR_ADD)||operator.equals(BASIC_OPERATOR_SUBTRACT)) {
- if(degree<0) {
- sumRule=false;
- degree=null;
- } else if(degree>0) {
- sumRule=false;
+ if (sumRule && degree != null) {
+ if (operator.equals(BASIC_OPERATOR_ADD)
+ || operator.equals(BASIC_OPERATOR_SUBTRACT)) {
+ if (degree < 0) {
+ sumRule = false;
+ degree = null;
+ } else if (degree > 0) {
+ sumRule = false;
}
- } else if(operator.equals(BASIC_OPERATOR_POWER)) {
- if(degree!=0) {
+ } else if (operator.equals(BASIC_OPERATOR_POWER)) {
+ if (degree != 0) {
sumRule = false;
- degree=null;
+ degree = null;
}
}
}
@@ -350,28 +370,29 @@ public class MtasFunctionParserFunctionBasic
parser = item.getParser();
parser.close();
tmpParserLongs.add(parser);
- sumRule = sumRule?parser.sumRule():false;
- needPositions = needPositions?needPositions:parser.needPositions();
+ sumRule = sumRule ? parser.sumRule() : false;
+ needPositions = needPositions ? needPositions : parser.needPositions();
needArgument.addAll(parser.needArgument);
- if(sumRule && degree!=null) {
- if(operator.equals(BASIC_OPERATOR_ADD)||operator.equals(BASIC_OPERATOR_SUBTRACT)) {
- if(parser.degree!=degree) {
- sumRule=false;
- if(degree<0) {
- degree=null;
+ if (sumRule && degree != null) {
+ if (operator.equals(BASIC_OPERATOR_ADD)
+ || operator.equals(BASIC_OPERATOR_SUBTRACT)) {
+ if (parser.degree != degree) {
+ sumRule = false;
+ if (degree < 0) {
+ degree = null;
} else {
degree = Math.max(degree, parser.degree);
}
}
- } else if(operator.equals(BASIC_OPERATOR_MULTIPLY)) {
- if(degree!=0 || parser.degree!=0) {
- sumRule = false;
+ } else if (operator.equals(BASIC_OPERATOR_MULTIPLY)) {
+ if (degree != 0 || parser.degree != 0) {
+ sumRule = false;
}
- degree+=parser.degree;
- } else if(operator.equals(BASIC_OPERATOR_POWER)) {
- if(degree!=0) {
+ degree += parser.degree;
+ } else if (operator.equals(BASIC_OPERATOR_POWER)) {
+ if (degree != 0) {
sumRule = false;
- degree = null;
+ degree = null;
}
}
}
@@ -383,28 +404,29 @@ public class MtasFunctionParserFunctionBasic
parser = item.getParser();
parser.close();
tmpParserDoubles.add(parser);
- sumRule = sumRule?parser.sumRule():false;
- needPositions = needPositions?needPositions:parser.needPositions();
+ sumRule = sumRule ? parser.sumRule() : false;
+ needPositions = needPositions ? needPositions : parser.needPositions();
needArgument.addAll(parser.needArgument);
- if(sumRule && degree!=null) {
- if(operator.equals(BASIC_OPERATOR_ADD)||operator.equals(BASIC_OPERATOR_SUBTRACT)) {
- if(parser.degree!=degree) {
- sumRule=false;
- if(degree<0) {
- degree=null;
+ if (sumRule && degree != null) {
+ if (operator.equals(BASIC_OPERATOR_ADD)
+ || operator.equals(BASIC_OPERATOR_SUBTRACT)) {
+ if (parser.degree != degree) {
+ sumRule = false;
+ if (degree < 0) {
+ degree = null;
} else {
degree = Math.max(degree, parser.degree);
}
}
- } else if(operator.equals(BASIC_OPERATOR_MULTIPLY)) {
- if(degree!=0 || parser.degree!=0) {
- sumRule = false;
+ } else if (operator.equals(BASIC_OPERATOR_MULTIPLY)) {
+ if (degree != 0 || parser.degree != 0) {
+ sumRule = false;
}
- degree+=parser.degree;
- } else if(operator.equals(BASIC_OPERATOR_POWER)) {
- if(degree!=0) {
+ degree += parser.degree;
+ } else if (operator.equals(BASIC_OPERATOR_POWER)) {
+ if (degree != 0) {
sumRule = false;
- degree = null;
+ degree = null;
}
}
}
@@ -417,8 +439,12 @@ public class MtasFunctionParserFunctionBasic
}
}
- /* (non-Javadoc)
- * @see mtas.parser.function.util.MtasFunctionParserFunction#getValueDouble(long[], long)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.parser.function.util.MtasFunctionParserFunction#getValueDouble(long[],
+ * long)
*/
@Override
public double getValueDouble(long[] args, long n) throws IOException {
@@ -485,7 +511,7 @@ public class MtasFunctionParserFunctionBasic
case MtasFunctionParserItem.TYPE_CONSTANT_DOUBLE:
sum -= constantDoubles[idList[i]];
break;
- case MtasFunctionParserItem.TYPE_CONSTANT_LONG:
+ case MtasFunctionParserItem.TYPE_CONSTANT_LONG:
sum -= constantLongs[idList[i]];
break;
case MtasFunctionParserItem.TYPE_N:
@@ -552,22 +578,22 @@ public class MtasFunctionParserFunctionBasic
case BASIC_OPERATOR_POWER:
switch (typeList[i]) {
case MtasFunctionParserItem.TYPE_ARGUMENT:
- sum = Math.pow(sum,args[idList[i]]);
+ sum = Math.pow(sum, args[idList[i]]);
break;
case MtasFunctionParserItem.TYPE_PARSER_DOUBLE:
- sum = Math.pow(sum,parserDoubles[idList[i]].getValueDouble(args, n));
+ sum = Math.pow(sum, parserDoubles[idList[i]].getValueDouble(args, n));
break;
case MtasFunctionParserItem.TYPE_PARSER_LONG:
- sum = Math.pow(sum,parserLongs[idList[i]].getValueLong(args, n));
+ sum = Math.pow(sum, parserLongs[idList[i]].getValueLong(args, n));
break;
case MtasFunctionParserItem.TYPE_CONSTANT_DOUBLE:
- sum = Math.pow(sum,constantDoubles[idList[i]]);
+ sum = Math.pow(sum, constantDoubles[idList[i]]);
break;
case MtasFunctionParserItem.TYPE_CONSTANT_LONG:
- sum = Math.pow(sum,constantLongs[idList[i]]);
+ sum = Math.pow(sum, constantLongs[idList[i]]);
break;
case MtasFunctionParserItem.TYPE_N:
- sum = Math.pow(sum,n);
+ sum = Math.pow(sum, n);
break;
default:
throw new IOException("unknown type");
@@ -580,8 +606,12 @@ public class MtasFunctionParserFunctionBasic
return sum;
}
- /* (non-Javadoc)
- * @see mtas.parser.function.util.MtasFunctionParserFunction#getValueLong(long[], long)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.parser.function.util.MtasFunctionParserFunction#getValueLong(long[],
+ * long)
*/
@Override
public long getValueLong(long[] args, long n) throws IOException {
@@ -707,7 +737,7 @@ public class MtasFunctionParserFunctionBasic
default:
throw new IOException("unknown type");
}
- if(v!=0) {
+ if (v != 0) {
sum /= v;
} else {
throw new IOException("division by zero");
@@ -716,22 +746,22 @@ public class MtasFunctionParserFunctionBasic
case BASIC_OPERATOR_POWER:
switch (typeList[i]) {
case MtasFunctionParserItem.TYPE_ARGUMENT:
- sum = sum^args[idList[i]];
+ sum = sum ^ args[idList[i]];
break;
case MtasFunctionParserItem.TYPE_PARSER_DOUBLE:
- sum = sum^(long) parserDoubles[idList[i]].getValueDouble(args, n);
+ sum = sum ^ (long) parserDoubles[idList[i]].getValueDouble(args, n);
break;
case MtasFunctionParserItem.TYPE_PARSER_LONG:
- sum = sum^parserLongs[idList[i]].getValueLong(args, n);
+ sum = sum ^ parserLongs[idList[i]].getValueLong(args, n);
break;
case MtasFunctionParserItem.TYPE_CONSTANT_DOUBLE:
- sum = sum^constantDoubles[idList[i]].longValue();
+ sum = sum ^ constantDoubles[idList[i]].longValue();
break;
case MtasFunctionParserItem.TYPE_CONSTANT_LONG:
- sum = sum^constantLongs[idList[i]];
+ sum = sum ^ constantLongs[idList[i]];
break;
case MtasFunctionParserItem.TYPE_N:
- sum = sum^n;
+ sum = sum ^ n;
break;
default:
throw new IOException("unknown type");
@@ -746,57 +776,61 @@ public class MtasFunctionParserFunctionBasic
throw new IOException(e.getMessage());
}
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
String text = "?";
- if(firstType!=null) {
+ if (firstType != null) {
text = toString(firstType, firstId);
- for(int i=0; i<tmpOperatorList.size(); i++) {
+ for (int i = 0; i < tmpOperatorList.size(); i++) {
String operator = tmpOperatorList.get(i);
- if(operator.equals(BASIC_OPERATOR_ADD)) {
- text+=" + ";
- } else if(operator.equals(BASIC_OPERATOR_SUBTRACT)) {
- text+=" - ";
- } else if(operator.equals(BASIC_OPERATOR_MULTIPLY)) {
- text+=" * ";
- } else if(operator.equals(BASIC_OPERATOR_DIVIDE)) {
- text+=" / ";
- } else if(operator.equals(BASIC_OPERATOR_POWER)) {
- text+=" ^ ";
+ if (operator.equals(BASIC_OPERATOR_ADD)) {
+ text += " + ";
+ } else if (operator.equals(BASIC_OPERATOR_SUBTRACT)) {
+ text += " - ";
+ } else if (operator.equals(BASIC_OPERATOR_MULTIPLY)) {
+ text += " * ";
+ } else if (operator.equals(BASIC_OPERATOR_DIVIDE)) {
+ text += " / ";
+ } else if (operator.equals(BASIC_OPERATOR_POWER)) {
+ text += " ^ ";
} else {
- text+=" ? ";
+ text += " ? ";
}
- text+=toString(tmpTypeList.get(i), tmpIdList.get(i));
+ text += toString(tmpTypeList.get(i), tmpIdList.get(i));
}
}
return text;
}
-
+
/**
* To string.
*
- * @param type the type
- * @param id the id
+ * @param type
+ * the type
+ * @param id
+ * the id
* @return the string
*/
private String toString(String type, int id) {
- if(type.equals(MtasFunctionParserItem.TYPE_CONSTANT_LONG)) {
+ if (type.equals(MtasFunctionParserItem.TYPE_CONSTANT_LONG)) {
return tmpConstantLongs.get(id).toString();
- } else if(type.equals(MtasFunctionParserItem.TYPE_CONSTANT_DOUBLE)) {
+ } else if (type.equals(MtasFunctionParserItem.TYPE_CONSTANT_DOUBLE)) {
return tmpConstantDoubles.get(id).toString();
- } else if(type.equals(MtasFunctionParserItem.TYPE_PARSER_LONG)) {
- return "("+tmpParserLongs.get(id).toString()+")";
- } else if(type.equals(MtasFunctionParserItem.TYPE_PARSER_DOUBLE)) {
- return "("+tmpParserDoubles.get(id).toString()+")";
- } else if(type.equals(MtasFunctionParserItem.TYPE_ARGUMENT)) {
- return "$q"+id;
- } else if(type.equals(MtasFunctionParserItem.TYPE_N)) {
+ } else if (type.equals(MtasFunctionParserItem.TYPE_PARSER_LONG)) {
+ return "(" + tmpParserLongs.get(id).toString() + ")";
+ } else if (type.equals(MtasFunctionParserItem.TYPE_PARSER_DOUBLE)) {
+ return "(" + tmpParserDoubles.get(id).toString() + ")";
+ } else if (type.equals(MtasFunctionParserItem.TYPE_ARGUMENT)) {
+ return "$q" + id;
+ } else if (type.equals(MtasFunctionParserItem.TYPE_N)) {
return "$n";
- } else{
+ } else {
return "..";
}
}
diff --git a/src/mtas/parser/function/util/MtasFunctionParserFunctionDefault.java b/src/mtas/parser/function/util/MtasFunctionParserFunctionDefault.java
index 694a469..1c7d4e8 100644
--- a/src/mtas/parser/function/util/MtasFunctionParserFunctionDefault.java
+++ b/src/mtas/parser/function/util/MtasFunctionParserFunctionDefault.java
@@ -7,49 +7,58 @@ import mtas.codec.util.CodecUtil;
/**
* The Class MtasFunctionParserFunctionDefault.
*/
-public class MtasFunctionParserFunctionDefault extends MtasFunctionParserFunction {
+public class MtasFunctionParserFunctionDefault
+ extends MtasFunctionParserFunction {
-
/**
* Instantiates a new mtas function parser function default.
*
- * @param numberOfArguments the number of arguments
+ * @param numberOfArguments
+ * the number of arguments
*/
public MtasFunctionParserFunctionDefault(int numberOfArguments) {
this.dataType = CodecUtil.DATA_TYPE_LONG;
this.needPositions = false;
this.sumRule = true;
- this.degree = numberOfArguments>0?1:0;
- for(int i=0; i<numberOfArguments; i++) {
+ this.degree = numberOfArguments > 0 ? 1 : 0;
+ for (int i = 0; i < numberOfArguments; i++) {
this.needArgument.add(i);
}
}
-
- /* (non-Javadoc)
- * @see mtas.parser.function.util.MtasFunctionParserFunction#getValueDouble(long[], long)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.parser.function.util.MtasFunctionParserFunction#getValueDouble(long[],
+ * long)
*/
@Override
public double getValueDouble(long[] args, long n) throws IOException {
double value = 0;
- if(args!=null) {
- for(long a : args) {
- value+=a;
+ if (args != null) {
+ for (long a : args) {
+ value += a;
}
- }
+ }
return value;
}
- /* (non-Javadoc)
- * @see mtas.parser.function.util.MtasFunctionParserFunction#getValueLong(long[], long)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.parser.function.util.MtasFunctionParserFunction#getValueLong(long[],
+ * long)
*/
@Override
public long getValueLong(long[] args, long n) throws IOException {
long value = 0;
- if(args!=null) {
- for(long a : args) {
- value+=a;
+ if (args != null) {
+ for (long a : args) {
+ value += a;
}
- }
+ }
return value;
}
diff --git a/src/mtas/parser/function/util/MtasFunctionParserFunctionResponse.java b/src/mtas/parser/function/util/MtasFunctionParserFunctionResponse.java
index c8dd524..5a2fbfc 100644
--- a/src/mtas/parser/function/util/MtasFunctionParserFunctionResponse.java
+++ b/src/mtas/parser/function/util/MtasFunctionParserFunctionResponse.java
@@ -7,26 +7,29 @@ abstract public class MtasFunctionParserFunctionResponse {
/** The defined. */
boolean defined;
-
+
/** The value double. */
double valueDouble;
-
+
/** The value int. */
int valueInt;
-
+
/**
* Instantiates a new mtas function parser function response.
*
- * @param s the s
+ * @param s
+ * the s
*/
protected MtasFunctionParserFunctionResponse(boolean s) {
defined = s;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
abstract public boolean equals(Object obj);
-
+
}
diff --git a/src/mtas/parser/function/util/MtasFunctionParserFunctionResponseDouble.java b/src/mtas/parser/function/util/MtasFunctionParserFunctionResponseDouble.java
index 1fe5211..b2c5bdd 100644
--- a/src/mtas/parser/function/util/MtasFunctionParserFunctionResponseDouble.java
+++ b/src/mtas/parser/function/util/MtasFunctionParserFunctionResponseDouble.java
@@ -5,38 +5,46 @@ import java.io.IOException;
/**
* The Class MtasFunctionParserFunctionResponseDouble.
*/
-public class MtasFunctionParserFunctionResponseDouble extends MtasFunctionParserFunctionResponse {
+public class MtasFunctionParserFunctionResponseDouble
+ extends MtasFunctionParserFunctionResponse {
/** The value. */
private double value;
-
+
/**
* Instantiates a new mtas function parser function response double.
*
- * @param d the d
- * @param s the s
+ * @param d
+ * the d
+ * @param s
+ * the s
*/
public MtasFunctionParserFunctionResponseDouble(double d, boolean s) {
- super(s);
- value = d;
+ super(s);
+ value = d;
}
/**
* Gets the value.
*
* @return the value
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public double getValue() throws IOException {
- if(defined) {
+ if (defined) {
return value;
} else {
throw new IOException("undefined");
}
}
-
- /* (non-Javadoc)
- * @see mtas.parser.function.util.MtasFunctionParserFunctionResponse#equals(java.lang.Object)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.parser.function.util.MtasFunctionParserFunctionResponse#equals(java.
+ * lang.Object)
*/
@Override
public boolean equals(Object obj) {
@@ -48,8 +56,8 @@ public class MtasFunctionParserFunctionResponseDouble extends MtasFunctionParser
return false;
MtasFunctionParserFunctionResponseDouble other = (MtasFunctionParserFunctionResponseDouble) obj;
try {
- if(value == other.getValue()) {
- if(defined)
+ if (value == other.getValue()) {
+ if (defined)
return true;
else
return false;
@@ -57,19 +65,21 @@ public class MtasFunctionParserFunctionResponseDouble extends MtasFunctionParser
return false;
}
} catch (IOException e) {
- if(!defined)
+ if (!defined)
return true;
else
return false;
}
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
- return defined?"double:"+String.valueOf(value):"double:undefined";
+ return defined ? "double:" + String.valueOf(value) : "double:undefined";
}
-
+
}
diff --git a/src/mtas/parser/function/util/MtasFunctionParserFunctionResponseLong.java b/src/mtas/parser/function/util/MtasFunctionParserFunctionResponseLong.java
index 0eab790..8a6453f 100644
--- a/src/mtas/parser/function/util/MtasFunctionParserFunctionResponseLong.java
+++ b/src/mtas/parser/function/util/MtasFunctionParserFunctionResponseLong.java
@@ -5,19 +5,22 @@ import java.io.IOException;
/**
* The Class MtasFunctionParserFunctionResponseLong.
*/
-public class MtasFunctionParserFunctionResponseLong extends MtasFunctionParserFunctionResponse {
+public class MtasFunctionParserFunctionResponseLong
+ extends MtasFunctionParserFunctionResponse {
/** The value. */
private long value;
-
+
/**
* Instantiates a new mtas function parser function response long.
*
- * @param l the l
- * @param s the s
+ * @param l
+ * the l
+ * @param s
+ * the s
*/
public MtasFunctionParserFunctionResponseLong(long l, boolean s) {
- super(s);
+ super(s);
value = l;
}
@@ -25,18 +28,23 @@ public class MtasFunctionParserFunctionResponseLong extends MtasFunctionParserFu
* Gets the value.
*
* @return the value
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public long getValue() throws IOException {
- if(defined) {
+ if (defined) {
return value;
} else {
throw new IOException("undefined");
}
}
-
- /* (non-Javadoc)
- * @see mtas.parser.function.util.MtasFunctionParserFunctionResponse#equals(java.lang.Object)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.parser.function.util.MtasFunctionParserFunctionResponse#equals(java.
+ * lang.Object)
*/
@Override
public boolean equals(Object obj) {
@@ -48,8 +56,8 @@ public class MtasFunctionParserFunctionResponseLong extends MtasFunctionParserFu
return false;
MtasFunctionParserFunctionResponseLong other = (MtasFunctionParserFunctionResponseLong) obj;
try {
- if(value == other.getValue()) {
- if(defined)
+ if (value == other.getValue()) {
+ if (defined)
return true;
else
return false;
@@ -57,19 +65,21 @@ public class MtasFunctionParserFunctionResponseLong extends MtasFunctionParserFu
return false;
}
} catch (IOException e) {
- if(!defined)
+ if (!defined)
return true;
else
return false;
}
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
- return defined?"long:"+String.valueOf(value):"long:undefined";
+ return defined ? "long:" + String.valueOf(value) : "long:undefined";
}
-
+
}
diff --git a/src/mtas/parser/function/util/MtasFunctionParserItem.java b/src/mtas/parser/function/util/MtasFunctionParserItem.java
index 909782f..cd808ef 100644
--- a/src/mtas/parser/function/util/MtasFunctionParserItem.java
+++ b/src/mtas/parser/function/util/MtasFunctionParserItem.java
@@ -9,127 +9,142 @@ public class MtasFunctionParserItem {
/** The type. */
private String type = null;
-
+
/** The id. */
private Integer id = null;
-
+
/** The value long. */
private Long valueLong = null;
-
+
/** The value double. */
private Double valueDouble = null;
-
+
/** The degree. */
private Integer degree = null;
-
+
/** The parser. */
private MtasFunctionParserFunction parser = null;
-
+
/** The Constant TYPE_CONSTANT_LONG. */
public final static String TYPE_CONSTANT_LONG = "constantLong";
-
+
/** The Constant TYPE_CONSTANT_DOUBLE. */
public final static String TYPE_CONSTANT_DOUBLE = "constantDouble";
-
+
/** The Constant TYPE_PARSER_LONG. */
public final static String TYPE_PARSER_LONG = "parserLong";
-
+
/** The Constant TYPE_PARSER_DOUBLE. */
public final static String TYPE_PARSER_DOUBLE = "parserDouble";
-
+
/** The Constant TYPE_ARGUMENT. */
public final static String TYPE_ARGUMENT = "argument";
-
+
/** The Constant TYPE_N. */
public final static String TYPE_N = "n";
-
+
/**
* Instantiates a new mtas function parser item.
*
- * @param t the t
- * @throws ParseException the parse exception
+ * @param t
+ * the t
+ * @throws ParseException
+ * the parse exception
*/
public MtasFunctionParserItem(String t) throws ParseException {
- if(t.equals(TYPE_N)) {
+ if (t.equals(TYPE_N)) {
type = t;
degree = 0;
} else {
- throw new ParseException("unknown type "+t);
+ throw new ParseException("unknown type " + t);
}
}
-
+
/**
* Instantiates a new mtas function parser item.
*
- * @param t the t
- * @param i the i
- * @throws ParseException the parse exception
+ * @param t
+ * the t
+ * @param i
+ * the i
+ * @throws ParseException
+ * the parse exception
*/
public MtasFunctionParserItem(String t, int i) throws ParseException {
- if(t.equals(TYPE_ARGUMENT)) {
+ if (t.equals(TYPE_ARGUMENT)) {
type = t;
id = i;
degree = 1;
} else {
- throw new ParseException("unknown type "+t);
+ throw new ParseException("unknown type " + t);
}
}
-
+
/**
* Instantiates a new mtas function parser item.
*
- * @param t the t
- * @param l the l
- * @throws ParseException the parse exception
+ * @param t
+ * the t
+ * @param l
+ * the l
+ * @throws ParseException
+ * the parse exception
*/
public MtasFunctionParserItem(String t, long l) throws ParseException {
- if(t.equals(TYPE_CONSTANT_LONG)) {
+ if (t.equals(TYPE_CONSTANT_LONG)) {
type = t;
valueLong = l;
degree = 0;
} else {
- throw new ParseException("unknown type "+t);
+ throw new ParseException("unknown type " + t);
}
}
-
+
/**
* Instantiates a new mtas function parser item.
*
- * @param t the t
- * @param d the d
- * @throws ParseException the parse exception
+ * @param t
+ * the t
+ * @param d
+ * the d
+ * @throws ParseException
+ * the parse exception
*/
public MtasFunctionParserItem(String t, double d) throws ParseException {
- if(t.equals(TYPE_CONSTANT_DOUBLE)) {
+ if (t.equals(TYPE_CONSTANT_DOUBLE)) {
type = t;
valueDouble = d;
degree = 0;
} else {
- throw new ParseException("unknown type "+t);
- }
+ throw new ParseException("unknown type " + t);
+ }
}
-
+
/**
* Instantiates a new mtas function parser item.
*
- * @param t the t
- * @param p the p
- * @throws ParseException the parse exception
+ * @param t
+ * the t
+ * @param p
+ * the p
+ * @throws ParseException
+ * the parse exception
*/
- public MtasFunctionParserItem(String t, MtasFunctionParserFunction p) throws ParseException {
- if(t.equals(TYPE_PARSER_LONG)) {
+ public MtasFunctionParserItem(String t, MtasFunctionParserFunction p)
+ throws ParseException {
+ if (t.equals(TYPE_PARSER_LONG)) {
type = t;
parser = p;
degree = parser.degree;
- } else if(t.equals(TYPE_PARSER_DOUBLE)) {
+ } else if (t.equals(TYPE_PARSER_DOUBLE)) {
type = t;
parser = p;
degree = parser.degree;
} else {
- throw new ParseException("unknown type "+t);
+ throw new ParseException("unknown type " + t);
}
}
-
+
/**
* Gets the type.
*
@@ -138,7 +153,7 @@ public class MtasFunctionParserItem {
public String getType() {
return type;
}
-
+
/**
* Gets the id.
*
@@ -147,7 +162,7 @@ public class MtasFunctionParserItem {
public int getId() {
return id.intValue();
}
-
+
/**
* Gets the degree.
*
@@ -156,7 +171,7 @@ public class MtasFunctionParserItem {
public Integer getDegree() {
return degree;
}
-
+
/**
* Gets the value long.
*
@@ -165,7 +180,7 @@ public class MtasFunctionParserItem {
public long getValueLong() {
return valueLong.longValue();
}
-
+
/**
* Gets the value double.
*
@@ -174,7 +189,7 @@ public class MtasFunctionParserItem {
public double getValueDouble() {
return valueDouble.doubleValue();
}
-
+
/**
* Gets the parser.
*
@@ -183,5 +198,5 @@ public class MtasFunctionParserItem {
public MtasFunctionParserFunction getParser() {
return parser;
}
-
+
}
diff --git a/src/mtas/queries/MtasScoreProvider.java b/src/mtas/queries/MtasScoreProvider.java
index def2ff3..e72cdb7 100644
--- a/src/mtas/queries/MtasScoreProvider.java
+++ b/src/mtas/queries/MtasScoreProvider.java
@@ -5,7 +5,6 @@ import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.CustomScoreProvider;
-
/**
* The Class MtasScoreProvider.
*/
@@ -14,36 +13,44 @@ public class MtasScoreProvider extends CustomScoreProvider {
/**
* Instantiates a new mtas score provider.
*
- * @param context the context
+ * @param context
+ * the context
*/
public MtasScoreProvider(LeafReaderContext context) {
super(context);
-
+
}
- /* (non-Javadoc)
- * @see org.apache.lucene.queries.CustomScoreProvider#customScore(int, float, float)
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.queries.CustomScoreProvider#customScore(int, float,
+ * float)
*/
@Override
public float customScore(int doc, float subQueryScore, float valSrcScore) {
- System.out.println("* Init MTAS scorerprovider doc "+doc+" - "+subQueryScore+" - "+valSrcScore);
+ System.out.println("* Init MTAS scorerprovider doc " + doc + " - "
+ + subQueryScore + " - " + valSrcScore);
return (float) 3.0;
}
-
- /* (non-Javadoc)
- * @see org.apache.lucene.queries.CustomScoreProvider#customScore(int, float, float[])
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.queries.CustomScoreProvider#customScore(int, float,
+ * float[])
*/
@Override
- public float customScore(int doc, float subQueryScore, float valSrcScores[]) throws IOException {
- System.out.print("** Init MTAS scorerprovider doc "+doc+" - "+subQueryScore+" - ");
- System.out.print(valSrcScores.length+":");
- for(int i=0; i<valSrcScores.length ; i++ ) {
- System.out.print(valSrcScores[i]+",");
+ public float customScore(int doc, float subQueryScore, float valSrcScores[])
+ throws IOException {
+ System.out.print("** Init MTAS scorerprovider doc " + doc + " - "
+ + subQueryScore + " - ");
+ System.out.print(valSrcScores.length + ":");
+ for (int i = 0; i < valSrcScores.length; i++) {
+ System.out.print(valSrcScores[i] + ",");
}
System.out.println(" voor veld ");
return (float) 4.0;
}
-
-
-
+
}
diff --git a/src/mtas/queries/MtasScoreQuery.java b/src/mtas/queries/MtasScoreQuery.java
index 97ee305..0f6eab1 100644
--- a/src/mtas/queries/MtasScoreQuery.java
+++ b/src/mtas/queries/MtasScoreQuery.java
@@ -6,7 +6,6 @@ import org.apache.lucene.queries.CustomScoreQuery;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.search.Query;
-
/**
* The Class MtasScoreQuery.
*/
@@ -15,40 +14,49 @@ public class MtasScoreQuery extends CustomScoreQuery {
/**
* Instantiates a new mtas score query.
*
- * @param subQuery the sub query
+ * @param subQuery
+ * the sub query
*/
public MtasScoreQuery(Query subQuery) {
super(subQuery);
- System.out.println("* Init MTAS scorer "+subQuery.toString());
+ System.out.println("* Init MTAS scorer " + subQuery.toString());
}
-
+
/**
* Instantiates a new mtas score query.
*
- * @param subQuery the sub query
- * @param scoringQuery the scoring query
+ * @param subQuery
+ * the sub query
+ * @param scoringQuery
+ * the scoring query
*/
public MtasScoreQuery(Query subQuery, FunctionQuery scoringQuery) {
super(subQuery, scoringQuery);
- System.out.println("** Init MTAS scorer "+subQuery.toString());
+ System.out.println("** Init MTAS scorer " + subQuery.toString());
}
-
+
/**
* Instantiates a new mtas score query.
*
- * @param subQuery the sub query
- * @param scoringQueries the scoring queries
+ * @param subQuery
+ * the sub query
+ * @param scoringQueries
+ * the scoring queries
*/
public MtasScoreQuery(Query subQuery, FunctionQuery... scoringQueries) {
super(subQuery, scoringQueries);
- System.out.println("*** Init MTAS scorer "+subQuery.toString());
+ System.out.println("*** Init MTAS scorer " + subQuery.toString());
}
-
- /* (non-Javadoc)
- * @see org.apache.lucene.queries.CustomScoreQuery#getCustomScoreProvider(org.apache.lucene.index.LeafReaderContext)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.queries.CustomScoreQuery#getCustomScoreProvider(org.
+ * apache.lucene.index.LeafReaderContext)
*/
@Override
- public CustomScoreProvider getCustomScoreProvider(final LeafReaderContext context) {
+ public CustomScoreProvider getCustomScoreProvider(
+ final LeafReaderContext context) {
return new MtasScoreProvider(context);
}
diff --git a/src/mtas/search/MtasCollector.java b/src/mtas/search/MtasCollector.java
index 25d6602..702a4a9 100644
--- a/src/mtas/search/MtasCollector.java
+++ b/src/mtas/search/MtasCollector.java
@@ -5,13 +5,14 @@ import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.SimpleCollector;
-
/**
* The Class MtasCollector.
*/
public class MtasCollector extends SimpleCollector {
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Collector#needsScores()
*/
@Override
@@ -19,23 +20,26 @@ public class MtasCollector extends SimpleCollector {
return false;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.SimpleCollector#doSetNextReader(org.apache.lucene.index.LeafReaderContext)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.SimpleCollector#doSetNextReader(org.apache.lucene.
+ * index.LeafReaderContext)
*/
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
-
+
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.SimpleCollector#collect(int)
*/
@Override
public void collect(int doc) throws IOException {
- System.out.println("Mtas collector voor doc "+doc);
-
+ // System.out.println("Mtas collector voor doc "+doc);
}
-
-
}
diff --git a/src/mtas/search/similarities/MtasSimScorer.java b/src/mtas/search/similarities/MtasSimScorer.java
index 316db5c..5a7fca5 100644
--- a/src/mtas/search/similarities/MtasSimScorer.java
+++ b/src/mtas/search/similarities/MtasSimScorer.java
@@ -8,28 +8,37 @@ import org.apache.lucene.util.BytesRef;
*/
public class MtasSimScorer extends SimScorer {
- /* (non-Javadoc)
- * @see org.apache.lucene.search.similarities.Similarity.SimScorer#score(int, float)
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.search.similarities.Similarity.SimScorer#score(int,
+ * float)
*/
@Override
public float score(int doc, float freq) {
return 0;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.similarities.Similarity.SimScorer#computeSlopFactor(int)
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.search.similarities.Similarity.SimScorer#
+ * computeSlopFactor(int)
*/
@Override
public float computeSlopFactor(int distance) {
return 0;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.similarities.Similarity.SimScorer#computePayloadFactor(int, int, int, org.apache.lucene.util.BytesRef)
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.search.similarities.Similarity.SimScorer#
+ * computePayloadFactor(int, int, int, org.apache.lucene.util.BytesRef)
*/
@Override
public float computePayloadFactor(int doc, int start, int end,
- BytesRef payload) {
+ BytesRef payload) {
return 0;
}
diff --git a/src/mtas/search/spans/MtasEndSpans.java b/src/mtas/search/spans/MtasEndSpans.java
index 3a7fb54..02714e2 100644
--- a/src/mtas/search/spans/MtasEndSpans.java
+++ b/src/mtas/search/spans/MtasEndSpans.java
@@ -8,45 +8,54 @@ import org.apache.lucene.search.spans.Spans;
* The Class MtasEndSpans.
*/
public class MtasEndSpans extends Spans {
-
-/** The spans. */
-Spans spans;
-
+
+ /** The spans. */
+ Spans spans;
+
/**
* Instantiates a new mtas end spans.
*
- * @param spans the spans
+ * @param spans
+ * the spans
*/
public MtasEndSpans(Spans spans) {
super();
this.spans = spans;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#nextStartPosition()
*/
@Override
public int nextStartPosition() throws IOException {
- return (spans==null)?NO_MORE_POSITIONS:spans.nextStartPosition();
+ return (spans == null) ? NO_MORE_POSITIONS : spans.nextStartPosition();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#startPosition()
*/
@Override
public int startPosition() {
- return (spans==null)?-1:spans.endPosition();
+ return (spans == null) ? -1 : spans.endPosition();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#endPosition()
*/
@Override
public int endPosition() {
- return (spans==null)?-1:spans.endPosition();
+ return (spans == null) ? -1 : spans.endPosition();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#width()
*/
@Override
@@ -54,54 +63,68 @@ Spans spans;
return 0;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans.SpanCollector)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans
+ * .SpanCollector)
*/
@Override
public void collect(SpanCollector collector) throws IOException {
- if(spans!=null) {
- spans.collect(collector);
+ if (spans != null) {
+ spans.collect(collector);
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#docID()
*/
@Override
public int docID() {
- return (spans==null)?NO_MORE_DOCS:spans.docID();
+ return (spans == null) ? NO_MORE_DOCS : spans.docID();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#nextDoc()
*/
@Override
public int nextDoc() throws IOException {
- return (spans==null)?NO_MORE_DOCS:spans.nextDoc();
+ return (spans == null) ? NO_MORE_DOCS : spans.nextDoc();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#advance(int)
*/
@Override
public int advance(int target) throws IOException {
- return (spans==null)?NO_MORE_DOCS:spans.advance(target);
+ return (spans == null) ? NO_MORE_DOCS : spans.advance(target);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#cost()
*/
@Override
public long cost() {
- return (spans==null)?0:spans.cost();
+ return (spans == null) ? 0 : spans.cost();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#positionsCost()
*/
@Override
public float positionsCost() {
- return (spans==null)?0:spans.positionsCost();
- }
+ return (spans == null) ? 0 : spans.positionsCost();
+ }
}
diff --git a/src/mtas/search/spans/MtasSpanAndQuery.java b/src/mtas/search/spans/MtasSpanAndQuery.java
index e91d71e..fb08e28 100644
--- a/src/mtas/search/spans/MtasSpanAndQuery.java
+++ b/src/mtas/search/spans/MtasSpanAndQuery.java
@@ -14,25 +14,29 @@ public class MtasSpanAndQuery extends SpanNearQuery {
/** The clauses. */
private List<SpanQuery> clauses;
-
+
/** The query name. */
private static String QUERY_NAME = "mtasSpanAndQuery";
/**
* Instantiates a new mtas span and query.
*
- * @param clauses the clauses
+ * @param clauses
+ * the clauses
*/
public MtasSpanAndQuery(SpanQuery... clauses) {
super(clauses, -1 * (clauses.length - 1), false);
this.clauses = new ArrayList<>(clauses.length);
for (SpanQuery clause : clauses) {
this.clauses.add(clause);
- }
+ }
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanNearQuery#toString(java.lang.String)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanNearQuery#toString(java.lang.String)
*/
@Override
public String toString(String field) {
@@ -50,22 +54,26 @@ public class MtasSpanAndQuery extends SpanNearQuery {
return buffer.toString();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanNearQuery#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
- if (obj== null)
+ if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final MtasSpanAndQuery that = (MtasSpanAndQuery) obj;
- return clauses.equals(that.clauses);
+ return clauses.equals(that.clauses);
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanNearQuery#hashCode()
*/
@Override
@@ -75,6 +83,4 @@ public class MtasSpanAndQuery extends SpanNearQuery {
return h;
}
-
-
}
diff --git a/src/mtas/search/spans/MtasSpanEndQuery.java b/src/mtas/search/spans/MtasSpanEndQuery.java
index 20ccb4f..b09a5df 100644
--- a/src/mtas/search/spans/MtasSpanEndQuery.java
+++ b/src/mtas/search/spans/MtasSpanEndQuery.java
@@ -21,27 +21,31 @@ public class MtasSpanEndQuery extends SpanQuery {
/** The query. */
private SpanQuery query;
-
+
/** The query name. */
private static String QUERY_NAME = "mtasSpanEndQuery";
/**
* Instantiates a new mtas span end query.
*
- * @param query the query
+ * @param query
+ * the query
*/
public MtasSpanEndQuery(SpanQuery query) {
super();
- this.query = query;
+ this.query = query;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
*/
@Override
public Query rewrite(IndexReader reader) throws IOException {
query.rewrite(reader);
- return this;
+ return this;
}
/*
@@ -53,13 +57,15 @@ public class MtasSpanEndQuery extends SpanQuery {
@Override
public String toString(String field) {
StringBuilder buffer = new StringBuilder();
- buffer.append(QUERY_NAME+"([");
- buffer.append(this.query.toString(field));
+ buffer.append(QUERY_NAME + "([");
+ buffer.append(this.query.toString(field));
buffer.append("])");
return buffer.toString();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
@@ -67,17 +73,21 @@ public class MtasSpanEndQuery extends SpanQuery {
return query.getField();
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
throws IOException {
- SpanWeight spanWeight = ((SpanQuery) searcher.rewrite(query)).createWeight(searcher,
- needsScores);
+ SpanWeight spanWeight = ((SpanQuery) searcher.rewrite(query))
+ .createWeight(searcher, needsScores);
return new SpanTermWeight(spanWeight, searcher);
}
-
+
/**
* The Class SpanTermWeight.
*/
@@ -85,62 +95,81 @@ public class MtasSpanEndQuery extends SpanQuery {
/** The span weight. */
SpanWeight spanWeight;
-
+
/**
* Instantiates a new span term weight.
*
- * @param spanWeight the span weight
- * @param searcher the searcher
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param spanWeight
+ * the span weight
+ * @param searcher
+ * the searcher
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- public SpanTermWeight(SpanWeight spanWeight, IndexSearcher searcher) throws IOException {
+ public SpanTermWeight(SpanWeight spanWeight, IndexSearcher searcher)
+ throws IOException {
super(MtasSpanEndQuery.this, searcher, null);
this.spanWeight = spanWeight;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.Map)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.
+ * Map)
*/
@Override
public void extractTermContexts(Map<Term, TermContext> contexts) {
- spanWeight.extractTermContexts(contexts);
+ spanWeight.extractTermContexts(contexts);
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.index.LeafReaderContext, org.apache.lucene.search.spans.SpanWeight.Postings)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.
+ * index.LeafReaderContext,
+ * org.apache.lucene.search.spans.SpanWeight.Postings)
*/
@Override
public Spans getSpans(LeafReaderContext context, Postings requiredPostings)
throws IOException {
- return new MtasEndSpans(spanWeight.getSpans(context, requiredPostings));
+ return new MtasEndSpans(spanWeight.getSpans(context, requiredPostings));
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Weight#extractTerms(java.util.Set)
*/
@Override
public void extractTerms(Set<Term> terms) {
spanWeight.extractTerms(terms);
}
-
+
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
- if (obj== null)
+ if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final MtasSpanEndQuery that = (MtasSpanEndQuery) obj;
- return query.equals(that.query);
+ return query.equals(that.query);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
diff --git a/src/mtas/search/spans/MtasSpanMatchAll.java b/src/mtas/search/spans/MtasSpanMatchAll.java
index 27259f1..ab00c1a 100644
--- a/src/mtas/search/spans/MtasSpanMatchAll.java
+++ b/src/mtas/search/spans/MtasSpanMatchAll.java
@@ -15,50 +15,57 @@ public class MtasSpanMatchAll extends Spans {
/** The field. */
private String field;
-
+
/** The doc id. */
- private int minPosition, maxPosition, currentStartPosition, currentEndPosition, docId;
-
+ private int minPosition, maxPosition, currentStartPosition,
+ currentEndPosition, docId;
+
/** The mtas codec info. */
private CodecInfo mtasCodecInfo;
-
+
/**
* Instantiates a new mtas span match all.
*
- * @param mtasCodecInfo the mtas codec info
- * @param field the field
+ * @param mtasCodecInfo
+ * the mtas codec info
+ * @param field
+ * the field
*/
public MtasSpanMatchAll(CodecInfo mtasCodecInfo, String field) {
- super();
+ super();
this.mtasCodecInfo = mtasCodecInfo;
this.field = field;
minPosition = NO_MORE_POSITIONS;
maxPosition = NO_MORE_POSITIONS;
currentStartPosition = NO_MORE_POSITIONS;
currentEndPosition = NO_MORE_POSITIONS;
- docId = -1;
+ docId = -1;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#nextStartPosition()
*/
@Override
public int nextStartPosition() throws IOException {
- if(currentStartPosition < minPosition) {
+ if (currentStartPosition < minPosition) {
currentStartPosition = minPosition;
currentEndPosition = currentStartPosition + 1;
} else {
currentStartPosition++;
currentEndPosition = currentStartPosition + 1;
- if(currentStartPosition > maxPosition) {
+ if (currentStartPosition > maxPosition) {
currentStartPosition = NO_MORE_POSITIONS;
currentEndPosition = NO_MORE_POSITIONS;
- }
+ }
}
return currentStartPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#startPosition()
*/
@Override
@@ -66,7 +73,9 @@ public class MtasSpanMatchAll extends Spans {
return currentStartPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#endPosition()
*/
@Override
@@ -74,7 +83,9 @@ public class MtasSpanMatchAll extends Spans {
return currentEndPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#width()
*/
@Override
@@ -82,15 +93,21 @@ public class MtasSpanMatchAll extends Spans {
return 0;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans.SpanCollector)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans
+ * .SpanCollector)
*/
@Override
public void collect(SpanCollector collector) throws IOException {
-
+
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#docID()
*/
@Override
@@ -98,13 +115,15 @@ public class MtasSpanMatchAll extends Spans {
return docId;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#nextDoc()
*/
@Override
public int nextDoc() throws IOException {
IndexDoc indexDoc = mtasCodecInfo.getNextDoc(field, docId);
- if(indexDoc!=null) {
+ if (indexDoc != null) {
docId = indexDoc.docId;
minPosition = indexDoc.minPosition;
maxPosition = indexDoc.maxPosition;
@@ -120,13 +139,15 @@ public class MtasSpanMatchAll extends Spans {
return docId;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#advance(int)
*/
@Override
- public int advance(int target) throws IOException {
- IndexDoc indexDoc = mtasCodecInfo.getNextDoc(field, (target-1));
- if(indexDoc!=null) {
+ public int advance(int target) throws IOException {
+ IndexDoc indexDoc = mtasCodecInfo.getNextDoc(field, (target - 1));
+ if (indexDoc != null) {
docId = indexDoc.docId;
minPosition = indexDoc.minPosition;
maxPosition = indexDoc.maxPosition;
@@ -137,25 +158,29 @@ public class MtasSpanMatchAll extends Spans {
minPosition = NO_MORE_POSITIONS;
maxPosition = NO_MORE_POSITIONS;
currentStartPosition = NO_MORE_POSITIONS;
- currentEndPosition = NO_MORE_POSITIONS;
+ currentEndPosition = NO_MORE_POSITIONS;
}
return docId;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#cost()
*/
@Override
public long cost() {
return 0;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#positionsCost()
*/
@Override
public float positionsCost() {
return 0;
- }
+ }
}
diff --git a/src/mtas/search/spans/MtasSpanMatchAllQuery.java b/src/mtas/search/spans/MtasSpanMatchAllQuery.java
index bc5a107..e058e51 100644
--- a/src/mtas/search/spans/MtasSpanMatchAllQuery.java
+++ b/src/mtas/search/spans/MtasSpanMatchAllQuery.java
@@ -25,20 +25,23 @@ public class MtasSpanMatchAllQuery extends SpanQuery {
/** The field. */
private String field;
-
+
/** The query name. */
private static String QUERY_NAME = "mtasSpanMatchAllQuery";
/**
* Instantiates a new mtas span match all query.
*
- * @param field the field
+ * @param field
+ * the field
*/
public MtasSpanMatchAllQuery(String field) {
this.field = field;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
@@ -46,8 +49,12 @@ public class MtasSpanMatchAllQuery extends SpanQuery {
return field;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
@@ -63,24 +70,36 @@ public class MtasSpanMatchAllQuery extends SpanQuery {
/**
* Instantiates a new span all weight.
*
- * @param searcher the searcher
- * @param termContexts the term contexts
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param searcher
+ * the searcher
+ * @param termContexts
+ * the term contexts
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public SpanAllWeight(IndexSearcher searcher,
Map<Term, TermContext> termContexts) throws IOException {
super(MtasSpanMatchAllQuery.this, searcher, termContexts);
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.Map)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.
+ * Map)
*/
@Override
public void extractTermContexts(Map<Term, TermContext> contexts) {
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.index.LeafReaderContext, org.apache.lucene.search.spans.SpanWeight.Postings)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.
+ * index.LeafReaderContext,
+ * org.apache.lucene.search.spans.SpanWeight.Postings)
*/
@Override
public Spans getSpans(LeafReaderContext context, Postings requiredPostings)
@@ -119,15 +138,21 @@ public class MtasSpanMatchAllQuery extends SpanQuery {
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Weight#extractTerms(java.util.Set)
*/
@Override
public void extractTerms(Set<Term> terms) {
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#getSimScorer(org.apache.lucene.index.LeafReaderContext)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#getSimScorer(org.apache.lucene.
+ * index.LeafReaderContext)
*/
@Override
public SimScorer getSimScorer(LeafReaderContext context) {
@@ -149,7 +174,9 @@ public class MtasSpanMatchAllQuery extends SpanQuery {
return buffer.toString();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
@@ -164,7 +191,9 @@ public class MtasSpanMatchAllQuery extends SpanQuery {
return field.equals(that.field);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
diff --git a/src/mtas/search/spans/MtasSpanMatchNone.java b/src/mtas/search/spans/MtasSpanMatchNone.java
index b7dfe15..f8e8816 100644
--- a/src/mtas/search/spans/MtasSpanMatchNone.java
+++ b/src/mtas/search/spans/MtasSpanMatchNone.java
@@ -12,19 +12,22 @@ public class MtasSpanMatchNone extends Spans {
/** The doc id. */
private int currentStartPosition, currentEndPosition, docId;
-
+
/**
* Instantiates a new mtas span match none.
*
- * @param field the field
+ * @param field
+ * the field
*/
public MtasSpanMatchNone(String field) {
currentStartPosition = NO_MORE_POSITIONS;
currentEndPosition = NO_MORE_POSITIONS;
docId = -1;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#nextStartPosition()
*/
@Override
@@ -32,7 +35,9 @@ public class MtasSpanMatchNone extends Spans {
return currentStartPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#startPosition()
*/
@Override
@@ -40,7 +45,9 @@ public class MtasSpanMatchNone extends Spans {
return currentStartPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#endPosition()
*/
@Override
@@ -48,7 +55,9 @@ public class MtasSpanMatchNone extends Spans {
return currentEndPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#width()
*/
@Override
@@ -56,15 +65,21 @@ public class MtasSpanMatchNone extends Spans {
return 0;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans.SpanCollector)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans
+ * .SpanCollector)
*/
@Override
public void collect(SpanCollector collector) throws IOException {
-
+
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#docID()
*/
@Override
@@ -72,7 +87,9 @@ public class MtasSpanMatchNone extends Spans {
return docId;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#nextDoc()
*/
@Override
@@ -83,7 +100,9 @@ public class MtasSpanMatchNone extends Spans {
return docId;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#advance(int)
*/
@Override
@@ -94,7 +113,9 @@ public class MtasSpanMatchNone extends Spans {
return docId;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#cost()
*/
@Override
@@ -102,7 +123,9 @@ public class MtasSpanMatchNone extends Spans {
return 0;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#positionsCost()
*/
@Override
diff --git a/src/mtas/search/spans/MtasSpanOrQuery.java b/src/mtas/search/spans/MtasSpanOrQuery.java
index 023bcf7..c8f6e09 100644
--- a/src/mtas/search/spans/MtasSpanOrQuery.java
+++ b/src/mtas/search/spans/MtasSpanOrQuery.java
@@ -11,33 +11,37 @@ import org.apache.lucene.search.spans.SpanQuery;
* The Class MtasSpanOrQuery.
*/
public class MtasSpanOrQuery extends MtasSpanUniquePositionQuery {
-
+
/** The clauses. */
private List<SpanQuery> clauses;
-
+
/** The query name. */
private static String QUERY_NAME = "mtasSpanOrQuery";
-
+
/**
* Instantiates a new mtas span or query.
*
- * @param clauses the clauses
+ * @param clauses
+ * the clauses
*/
public MtasSpanOrQuery(SpanQuery... clauses) {
- super(new SpanOrQuery(clauses));
+ super(new SpanOrQuery(clauses));
this.clauses = new ArrayList<>(clauses.length);
for (SpanQuery clause : clauses) {
this.clauses.add(clause);
}
}
-
- /* (non-Javadoc)
- * @see mtas.search.spans.MtasSpanUniquePositionQuery#toString(java.lang.String)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * mtas.search.spans.MtasSpanUniquePositionQuery#toString(java.lang.String)
*/
@Override
public String toString(String field) {
StringBuilder buffer = new StringBuilder();
- buffer.append(QUERY_NAME+"([");
+ buffer.append(QUERY_NAME + "([");
Iterator<SpanQuery> i = clauses.iterator();
while (i.hasNext()) {
SpanQuery clause = i.next();
@@ -49,23 +53,27 @@ public class MtasSpanOrQuery extends MtasSpanUniquePositionQuery {
buffer.append("])");
return buffer.toString();
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.search.spans.MtasSpanUniquePositionQuery#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
- if (obj== null)
+ if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final MtasSpanOrQuery that = (MtasSpanOrQuery) obj;
- return clauses.equals(that.clauses);
+ return clauses.equals(that.clauses);
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see mtas.search.spans.MtasSpanUniquePositionQuery#hashCode()
*/
@Override
diff --git a/src/mtas/search/spans/MtasSpanPosition.java b/src/mtas/search/spans/MtasSpanPosition.java
index 3012e69..2a18bdc 100644
--- a/src/mtas/search/spans/MtasSpanPosition.java
+++ b/src/mtas/search/spans/MtasSpanPosition.java
@@ -15,21 +15,25 @@ public class MtasSpanPosition extends Spans {
/** The field. */
private String field;
-
+
/** The doc id. */
private int start, end, minPosition, maxPosition, currentStartPosition,
currentEndPosition, docId;
-
+
/** The mtas codec info. */
private CodecInfo mtasCodecInfo;
/**
* Instantiates a new mtas span position.
*
- * @param mtasCodecInfo the mtas codec info
- * @param field the field
- * @param start the start
- * @param end the end
+ * @param mtasCodecInfo
+ * the mtas codec info
+ * @param field
+ * the field
+ * @param start
+ * the start
+ * @param end
+ * the end
*/
public MtasSpanPosition(CodecInfo mtasCodecInfo, String field, int start,
int end) {
@@ -45,7 +49,9 @@ public class MtasSpanPosition extends Spans {
docId = -1;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#nextStartPosition()
*/
@Override
@@ -64,7 +70,9 @@ public class MtasSpanPosition extends Spans {
return currentStartPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#startPosition()
*/
@Override
@@ -72,7 +80,9 @@ public class MtasSpanPosition extends Spans {
return currentStartPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#endPosition()
*/
@Override
@@ -80,7 +90,9 @@ public class MtasSpanPosition extends Spans {
return currentEndPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#width()
*/
@Override
@@ -88,15 +100,21 @@ public class MtasSpanPosition extends Spans {
return 0;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans.SpanCollector)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans
+ * .SpanCollector)
*/
@Override
public void collect(SpanCollector collector) throws IOException {
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#docID()
*/
@Override
@@ -104,12 +122,14 @@ public class MtasSpanPosition extends Spans {
return docId;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#nextDoc()
*/
@Override
public int nextDoc() throws IOException {
- do {
+ do {
IndexDoc indexDoc = mtasCodecInfo.getNextDoc(field, docId);
if (indexDoc != null) {
docId = indexDoc.docId;
@@ -123,12 +143,14 @@ public class MtasSpanPosition extends Spans {
maxPosition = NO_MORE_POSITIONS;
currentStartPosition = NO_MORE_POSITIONS;
currentEndPosition = NO_MORE_POSITIONS;
- }
+ }
} while (docId != NO_MORE_DOCS && (minPosition > maxPosition));
return docId;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#advance(int)
*/
@Override
@@ -149,12 +171,14 @@ public class MtasSpanPosition extends Spans {
currentStartPosition = NO_MORE_POSITIONS;
currentEndPosition = NO_MORE_POSITIONS;
}
- tmpTarget = docId;
- } while(docId!=NO_MORE_DOCS && minPosition>maxPosition);
+ tmpTarget = docId;
+ } while (docId != NO_MORE_DOCS && minPosition > maxPosition);
return docId;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#cost()
*/
@Override
@@ -162,7 +186,9 @@ public class MtasSpanPosition extends Spans {
return 0;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#positionsCost()
*/
@Override
diff --git a/src/mtas/search/spans/MtasSpanPositionQuery.java b/src/mtas/search/spans/MtasSpanPositionQuery.java
index 2361296..8b5f51f 100644
--- a/src/mtas/search/spans/MtasSpanPositionQuery.java
+++ b/src/mtas/search/spans/MtasSpanPositionQuery.java
@@ -25,18 +25,20 @@ public class MtasSpanPositionQuery extends SpanQuery {
/** The field. */
private String field;
-
+
/** The end. */
private int start, end;
-
+
/** The query name. */
private static String QUERY_NAME = "mtasSpanPositionQuery";
/**
* Instantiates a new mtas span position query.
*
- * @param field the field
- * @param position the position
+ * @param field
+ * the field
+ * @param position
+ * the position
*/
public MtasSpanPositionQuery(String field, int position) {
this.field = field;
@@ -47,9 +49,12 @@ public class MtasSpanPositionQuery extends SpanQuery {
/**
* Instantiates a new mtas span position query.
*
- * @param field the field
- * @param start the start
- * @param end the end
+ * @param field
+ * the field
+ * @param start
+ * the start
+ * @param end
+ * the end
*/
public MtasSpanPositionQuery(String field, int start, int end) {
this.field = field;
@@ -57,7 +62,9 @@ public class MtasSpanPositionQuery extends SpanQuery {
this.end = end;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
@@ -65,8 +72,12 @@ public class MtasSpanPositionQuery extends SpanQuery {
return field;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
@@ -82,24 +93,36 @@ public class MtasSpanPositionQuery extends SpanQuery {
/**
* Instantiates a new span all weight.
*
- * @param searcher the searcher
- * @param termContexts the term contexts
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param searcher
+ * the searcher
+ * @param termContexts
+ * the term contexts
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public SpanAllWeight(IndexSearcher searcher,
Map<Term, TermContext> termContexts) throws IOException {
super(MtasSpanPositionQuery.this, searcher, termContexts);
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.Map)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.
+ * Map)
*/
@Override
public void extractTermContexts(Map<Term, TermContext> contexts) {
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.index.LeafReaderContext, org.apache.lucene.search.spans.SpanWeight.Postings)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.
+ * index.LeafReaderContext,
+ * org.apache.lucene.search.spans.SpanWeight.Postings)
*/
@Override
public Spans getSpans(LeafReaderContext context, Postings requiredPostings)
@@ -138,15 +161,21 @@ public class MtasSpanPositionQuery extends SpanQuery {
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Weight#extractTerms(java.util.Set)
*/
@Override
public void extractTerms(Set<Term> terms) {
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#getSimScorer(org.apache.lucene.index.LeafReaderContext)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#getSimScorer(org.apache.lucene.
+ * index.LeafReaderContext)
*/
@Override
public SimScorer getSimScorer(LeafReaderContext context) {
@@ -169,7 +198,9 @@ public class MtasSpanPositionQuery extends SpanQuery {
return buffer.toString();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
@@ -184,7 +215,9 @@ public class MtasSpanPositionQuery extends SpanQuery {
return field.equals(that.field) && start == that.start && end == that.end;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
diff --git a/src/mtas/search/spans/MtasSpanPrefixQuery.java b/src/mtas/search/spans/MtasSpanPrefixQuery.java
index 4b7f006..21856bc 100644
--- a/src/mtas/search/spans/MtasSpanPrefixQuery.java
+++ b/src/mtas/search/spans/MtasSpanPrefixQuery.java
@@ -28,10 +28,9 @@ public class MtasSpanPrefixQuery extends SpanQuery {
/** The single position. */
private boolean singlePosition;
-
+
/** The query name. */
private static String QUERY_NAME = "mtasSpanPrefixQuery";
-
/** The term. */
private Term term;
@@ -42,7 +41,8 @@ public class MtasSpanPrefixQuery extends SpanQuery {
/**
* Instantiates a new mtas span prefix query.
*
- * @param term the term
+ * @param term
+ * the term
*/
public MtasSpanPrefixQuery(Term term) {
this(term, true);
@@ -51,8 +51,10 @@ public class MtasSpanPrefixQuery extends SpanQuery {
/**
* Instantiates a new mtas span prefix query.
*
- * @param term the term
- * @param singlePosition the single position
+ * @param term
+ * the term
+ * @param singlePosition
+ * the single position
*/
public MtasSpanPrefixQuery(Term term, boolean singlePosition) {
super();
@@ -71,8 +73,11 @@ public class MtasSpanPrefixQuery extends SpanQuery {
}
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
*/
@Override
public Query rewrite(IndexReader reader) throws IOException {
@@ -103,17 +108,19 @@ public class MtasSpanPrefixQuery extends SpanQuery {
@Override
public String toString(String field) {
StringBuilder buffer = new StringBuilder();
- buffer.append(QUERY_NAME+"([");
+ buffer.append(QUERY_NAME + "([");
if (value == null) {
buffer.append(this.query.getField() + ":" + prefix);
} else {
buffer.append(this.query.getField() + ":" + prefix + "=" + value);
}
- buffer.append(","+singlePosition+"])");
+ buffer.append("," + singlePosition + "])");
return buffer.toString();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
@@ -121,8 +128,12 @@ public class MtasSpanPrefixQuery extends SpanQuery {
return term.field();
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
@@ -130,12 +141,14 @@ public class MtasSpanPrefixQuery extends SpanQuery {
return ((SpanQuery) searcher.rewrite(query)).createWeight(searcher,
needsScores);
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
- public boolean equals(Object obj) {
+ public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
@@ -143,17 +156,19 @@ public class MtasSpanPrefixQuery extends SpanQuery {
if (getClass() != obj.getClass())
return false;
MtasSpanPrefixQuery other = (MtasSpanPrefixQuery) obj;
- return other.term.equals(term) && (other.singlePosition==singlePosition);
+ return other.term.equals(term) && (other.singlePosition == singlePosition);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
public int hashCode() {
int h = QUERY_NAME.hashCode();
h = (h * 7) ^ term.hashCode();
- h += (singlePosition?1:0);
+ h += (singlePosition ? 1 : 0);
return h;
}
diff --git a/src/mtas/search/spans/MtasSpanRecurrence.java b/src/mtas/search/spans/MtasSpanRecurrence.java
index f32dcec..9685762 100644
--- a/src/mtas/search/spans/MtasSpanRecurrence.java
+++ b/src/mtas/search/spans/MtasSpanRecurrence.java
@@ -13,38 +13,42 @@ public class MtasSpanRecurrence extends Spans {
/** The spans. */
Spans spans;
-
+
/** The minimum recurrence. */
int minimumRecurrence;
-
+
/** The maximum recurrence. */
int maximumRecurrence;
/** The queue spans. */
List<Match> queueSpans;
-
+
/** The queue matches. */
List<Match> queueMatches;
-
+
/** The current match. */
Match currentMatch;
-
+
/** The no more positions. */
boolean noMorePositions;
-
+
/** The last start position. */
int lastStartPosition; // startPosition of last retrieved span
-
+
/** The last span. */
boolean lastSpan; // last span for this document added to queue
/**
* Instantiates a new mtas span recurrence.
*
- * @param mtasSpanRecurrenceQuery the mtas span recurrence query
- * @param spans the spans
- * @param minimumRecurrence the minimum recurrence
- * @param maximumRecurrence the maximum recurrence
+ * @param mtasSpanRecurrenceQuery
+ * the mtas span recurrence query
+ * @param spans
+ * the spans
+ * @param minimumRecurrence
+ * the minimum recurrence
+ * @param maximumRecurrence
+ * the maximum recurrence
*/
public MtasSpanRecurrence(MtasSpanRecurrenceQuery mtasSpanRecurrenceQuery,
Spans spans, int minimumRecurrence, int maximumRecurrence) {
@@ -58,7 +62,9 @@ public class MtasSpanRecurrence extends Spans {
resetQueue();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#nextStartPosition()
*/
@Override
@@ -75,7 +81,9 @@ public class MtasSpanRecurrence extends Spans {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#startPosition()
*/
@Override
@@ -84,7 +92,9 @@ public class MtasSpanRecurrence extends Spans {
: currentMatch.startPosition();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#endPosition()
*/
@Override
@@ -93,7 +103,9 @@ public class MtasSpanRecurrence extends Spans {
: currentMatch.endPosition();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#width()
*/
@Override
@@ -101,15 +113,21 @@ public class MtasSpanRecurrence extends Spans {
return 1;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans.SpanCollector)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans
+ * .SpanCollector)
*/
@Override
public void collect(SpanCollector collector) throws IOException {
spans.collect(collector);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#docID()
*/
@Override
@@ -117,7 +135,9 @@ public class MtasSpanRecurrence extends Spans {
return spans.docID();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#nextDoc()
*/
@Override
@@ -126,7 +146,9 @@ public class MtasSpanRecurrence extends Spans {
return (spans.nextDoc() == NO_MORE_DOCS) ? NO_MORE_DOCS : toMatchDoc();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#advance(int)
*/
@Override
@@ -152,14 +174,15 @@ public class MtasSpanRecurrence extends Spans {
* To match doc.
*
* @return the int
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
int toMatchDoc() throws IOException {
while (true) {
if (findMatches()) {
return docID();
- }
- resetQueue();
+ }
+ resetQueue();
if (spans.nextDoc() == NO_MORE_DOCS) {
return NO_MORE_DOCS;
}
@@ -170,7 +193,8 @@ public class MtasSpanRecurrence extends Spans {
* Collect span.
*
* @return true, if successful
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
// try to get something in the queue of spans
private boolean collectSpan() throws IOException {
@@ -190,7 +214,8 @@ public class MtasSpanRecurrence extends Spans {
* Find matches.
*
* @return true, if successful
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private boolean findMatches() throws IOException {
// check for something in queue of matches
@@ -212,13 +237,14 @@ public class MtasSpanRecurrence extends Spans {
while (!lastSpan && (lastStartPosition == firstMatch.startPosition())) {
collectSpan();
}
- while (!queueSpans.isEmpty()
- && (queueSpans.get(0).startPosition() == firstMatch.startPosition())) {
+ while (!queueSpans.isEmpty() && (queueSpans.get(0)
+ .startPosition() == firstMatch.startPosition())) {
matches.add(queueSpans.remove(0));
}
// construct all matches for this startPosition
for (Match match : matches) {
- for (int n = (minimumRecurrence - 1); n <= (maximumRecurrence - 1); n++) {
+ for (int n = (minimumRecurrence - 1); n <= (maximumRecurrence
+ - 1); n++) {
findMatches(match, n);
}
}
@@ -233,9 +259,12 @@ public class MtasSpanRecurrence extends Spans {
/**
* Find matches.
*
- * @param match the match
- * @param n the n
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param match
+ * the match
+ * @param n
+ * the n
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private void findMatches(Match match, int n) throws IOException {
if (n > 0) {
@@ -282,18 +311,20 @@ public class MtasSpanRecurrence extends Spans {
* The Class Match.
*/
private class Match {
-
+
/** The start position. */
private int startPosition;
-
+
/** The end position. */
private int endPosition;
/**
* Instantiates a new match.
*
- * @param startPosition the start position
- * @param endPosition the end position
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
*/
Match(int startPosition, int endPosition) {
this.startPosition = startPosition;
@@ -318,7 +349,9 @@ public class MtasSpanRecurrence extends Spans {
return endPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
@@ -334,7 +367,9 @@ public class MtasSpanRecurrence extends Spans {
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#cost()
*/
@Override
@@ -342,7 +377,9 @@ public class MtasSpanRecurrence extends Spans {
return (spans == null) ? 0 : spans.cost();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#positionsCost()
*/
@Override
diff --git a/src/mtas/search/spans/MtasSpanRecurrenceQuery.java b/src/mtas/search/spans/MtasSpanRecurrenceQuery.java
index d681a2e..7dad191 100644
--- a/src/mtas/search/spans/MtasSpanRecurrenceQuery.java
+++ b/src/mtas/search/spans/MtasSpanRecurrenceQuery.java
@@ -22,13 +22,13 @@ public class MtasSpanRecurrenceQuery extends SpanQuery implements Cloneable {
/** The clause. */
private SpanQuery clause;
-
+
/** The minimum recurrence. */
private int minimumRecurrence;
-
+
/** The maximum recurrence. */
private int maximumRecurrence;
-
+
/** The field. */
private String field;
@@ -38,9 +38,12 @@ public class MtasSpanRecurrenceQuery extends SpanQuery implements Cloneable {
/**
* Instantiates a new mtas span recurrence query.
*
- * @param clause the clause
- * @param minimumRecurrence the minimum recurrence
- * @param maximumRecurrence the maximum recurrence
+ * @param clause
+ * the clause
+ * @param minimumRecurrence
+ * the minimum recurrence
+ * @param maximumRecurrence
+ * the maximum recurrence
*/
public MtasSpanRecurrenceQuery(SpanQuery clause, int minimumRecurrence,
int maximumRecurrence) {
@@ -67,7 +70,9 @@ public class MtasSpanRecurrenceQuery extends SpanQuery implements Cloneable {
return clause;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
@@ -83,8 +88,11 @@ public class MtasSpanRecurrenceQuery extends SpanQuery implements Cloneable {
// return soq;
// }
- /* (non-Javadoc)
- * @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
*/
@Override
public Query rewrite(IndexReader reader) throws IOException {
@@ -97,7 +105,9 @@ public class MtasSpanRecurrenceQuery extends SpanQuery implements Cloneable {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#toString(java.lang.String)
*/
@Override
@@ -110,7 +120,9 @@ public class MtasSpanRecurrenceQuery extends SpanQuery implements Cloneable {
return buffer.toString();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
@@ -127,7 +139,9 @@ public class MtasSpanRecurrenceQuery extends SpanQuery implements Cloneable {
&& maximumRecurrence == that.maximumRecurrence;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
@@ -139,8 +153,12 @@ public class MtasSpanRecurrenceQuery extends SpanQuery implements Cloneable {
return h;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
@@ -161,10 +179,14 @@ public class MtasSpanRecurrenceQuery extends SpanQuery implements Cloneable {
/**
* Instantiates a new span recurrence weight.
*
- * @param subWeight the sub weight
- * @param searcher the searcher
- * @param terms the terms
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param subWeight
+ * the sub weight
+ * @param searcher
+ * the searcher
+ * @param terms
+ * the terms
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public SpanRecurrenceWeight(SpanWeight subWeight, IndexSearcher searcher,
Map<Term, TermContext> terms) throws IOException {
@@ -172,16 +194,25 @@ public class MtasSpanRecurrenceQuery extends SpanQuery implements Cloneable {
this.subWeight = subWeight;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.Map)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.
+ * Map)
*/
@Override
public void extractTermContexts(Map<Term, TermContext> contexts) {
subWeight.extractTermContexts(contexts);
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.index.LeafReaderContext, org.apache.lucene.search.spans.SpanWeight.Postings)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.
+ * index.LeafReaderContext,
+ * org.apache.lucene.search.spans.SpanWeight.Postings)
*/
@Override
public Spans getSpans(LeafReaderContext context, Postings requiredPostings)
@@ -203,7 +234,9 @@ public class MtasSpanRecurrenceQuery extends SpanQuery implements Cloneable {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Weight#extractTerms(java.util.Set)
*/
@Override
diff --git a/src/mtas/search/spans/MtasSpanRegexQuery.java b/src/mtas/search/spans/MtasSpanRegexQuery.java
index 619f2b0..22c5944 100644
--- a/src/mtas/search/spans/MtasSpanRegexQuery.java
+++ b/src/mtas/search/spans/MtasSpanRegexQuery.java
@@ -40,11 +40,12 @@ public class MtasSpanRegexQuery extends SpanQuery {
/** The query name. */
private static String QUERY_NAME = "mtasSpanRegexQuery";
-
-
- /* (non-Javadoc)
- * @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
*/
@Override
public Query rewrite(IndexReader reader) throws IOException {
@@ -82,7 +83,7 @@ public class MtasSpanRegexQuery extends SpanQuery {
@Override
public String toString(String field) {
StringBuilder buffer = new StringBuilder();
- buffer.append(QUERY_NAME+"([");
+ buffer.append(QUERY_NAME + "([");
if (value == null) {
buffer.append(this.query.getField() + ":" + prefix);
} else {
@@ -92,7 +93,9 @@ public class MtasSpanRegexQuery extends SpanQuery {
return buffer.toString();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
@@ -100,8 +103,12 @@ public class MtasSpanRegexQuery extends SpanQuery {
return term.field();
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
@@ -110,7 +117,9 @@ public class MtasSpanRegexQuery extends SpanQuery {
needsScores);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
@@ -122,19 +131,20 @@ public class MtasSpanRegexQuery extends SpanQuery {
if (getClass() != obj.getClass())
return false;
MtasSpanRegexQuery that = (MtasSpanRegexQuery) obj;
- return term.equals(that.term) && singlePosition==that.singlePosition;
+ return term.equals(that.term) && singlePosition == that.singlePosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
public int hashCode() {
int h = QUERY_NAME.hashCode();
h = (h * 7) ^ term.hashCode();
- h += (singlePosition?1:0);
+ h += (singlePosition ? 1 : 0);
return h;
}
-
}
diff --git a/src/mtas/search/spans/MtasSpanRegexpQuery.java b/src/mtas/search/spans/MtasSpanRegexpQuery.java
index 82c88b0..120eb31 100644
--- a/src/mtas/search/spans/MtasSpanRegexpQuery.java
+++ b/src/mtas/search/spans/MtasSpanRegexpQuery.java
@@ -41,11 +41,12 @@ public class MtasSpanRegexpQuery extends SpanQuery {
/** The query name. */
private static String QUERY_NAME = "mtasSpanRegexpQuery";
-
+
/**
* Instantiates a new mtas span regexp query.
*
- * @param term the term
+ * @param term
+ * the term
*/
public MtasSpanRegexpQuery(Term term) {
this(term, true);
@@ -54,8 +55,10 @@ public class MtasSpanRegexpQuery extends SpanQuery {
/**
* Instantiates a new mtas span regexp query.
*
- * @param term the term
- * @param singlePosition the single position
+ * @param term
+ * the term
+ * @param singlePosition
+ * the single position
*/
public MtasSpanRegexpQuery(Term term, boolean singlePosition) {
super();
@@ -74,8 +77,11 @@ public class MtasSpanRegexpQuery extends SpanQuery {
}
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
*/
@Override
public Query rewrite(IndexReader reader) throws IOException {
@@ -113,7 +119,7 @@ public class MtasSpanRegexpQuery extends SpanQuery {
@Override
public String toString(String field) {
StringBuilder buffer = new StringBuilder();
- buffer.append(QUERY_NAME+"([");
+ buffer.append(QUERY_NAME + "([");
if (value == null) {
buffer.append(this.query.getField() + ":" + prefix);
} else {
@@ -123,7 +129,9 @@ public class MtasSpanRegexpQuery extends SpanQuery {
return buffer.toString();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
@@ -131,8 +139,12 @@ public class MtasSpanRegexpQuery extends SpanQuery {
return term.field();
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
@@ -140,8 +152,10 @@ public class MtasSpanRegexpQuery extends SpanQuery {
return ((SpanQuery) searcher.rewrite(query)).createWeight(searcher,
needsScores);
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
@@ -153,19 +167,20 @@ public class MtasSpanRegexpQuery extends SpanQuery {
if (getClass() != obj.getClass())
return false;
MtasSpanRegexpQuery that = (MtasSpanRegexpQuery) obj;
- return term.equals(that.term) && singlePosition==that.singlePosition;
+ return term.equals(that.term) && singlePosition == that.singlePosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
public int hashCode() {
int h = QUERY_NAME.hashCode();
h = (h * 7) ^ term.hashCode();
- h += (singlePosition?1:0);
+ h += (singlePosition ? 1 : 0);
return h;
}
-
}
diff --git a/src/mtas/search/spans/MtasSpanSequence.java b/src/mtas/search/spans/MtasSpanSequence.java
index d2ddb44..a640c40 100644
--- a/src/mtas/search/spans/MtasSpanSequence.java
+++ b/src/mtas/search/spans/MtasSpanSequence.java
@@ -19,21 +19,23 @@ public class MtasSpanSequence extends Spans {
/** The queue spans. */
private List<QueueItem> queueSpans;
-
+
/** The queue matches. */
private List<Match> queueMatches;
-
+
/** The current position. */
private int docId, currentPosition;
-
+
/** The current match. */
Match currentMatch;
/**
* Instantiates a new mtas span sequence.
*
- * @param mtasSpanSequenceQuery the mtas span sequence query
- * @param setSequenceSpans the set sequence spans
+ * @param mtasSpanSequenceQuery
+ * the mtas span sequence query
+ * @param setSequenceSpans
+ * the set sequence spans
*/
public MtasSpanSequence(MtasSpanSequenceQuery mtasSpanSequenceQuery,
List<MtasSpanSequenceSpans> setSequenceSpans) {
@@ -47,7 +49,9 @@ public class MtasSpanSequence extends Spans {
resetQueue();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#nextStartPosition()
*/
@Override
@@ -64,7 +68,9 @@ public class MtasSpanSequence extends Spans {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#startPosition()
*/
@Override
@@ -76,7 +82,9 @@ public class MtasSpanSequence extends Spans {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#endPosition()
*/
@Override
@@ -88,7 +96,9 @@ public class MtasSpanSequence extends Spans {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#width()
*/
@Override
@@ -96,14 +106,20 @@ public class MtasSpanSequence extends Spans {
return 0;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans.SpanCollector)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans
+ * .SpanCollector)
*/
@Override
public void collect(SpanCollector collector) throws IOException {
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#docID()
*/
@Override
@@ -111,7 +127,9 @@ public class MtasSpanSequence extends Spans {
return docId;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#nextDoc()
*/
@Override
@@ -126,7 +144,8 @@ public class MtasSpanSequence extends Spans {
* Go to next doc.
*
* @return true, if successful
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private boolean goToNextDoc() throws IOException {
if (docId == NO_MORE_DOCS) {
@@ -139,17 +158,17 @@ public class MtasSpanSequence extends Spans {
if (newDocId == null) {
spanDocId = item.sequenceSpans.spans.nextDoc();
} else {
- if(!item.sequenceSpans.optional) {
- spanDocId = item.sequenceSpans.spans.advance(newDocId);
+ if (!item.sequenceSpans.optional) {
+ spanDocId = item.sequenceSpans.spans.advance(newDocId);
} else {
- if(item.sequenceSpans.spans.docID()==-1) {
- spanDocId = item.sequenceSpans.spans.advance(newDocId);
- } else if(newDocId>item.sequenceSpans.spans.docID()) {
+ if (item.sequenceSpans.spans.docID() == -1) {
+ spanDocId = item.sequenceSpans.spans.advance(newDocId);
+ } else if (newDocId > item.sequenceSpans.spans.docID()) {
spanDocId = item.sequenceSpans.spans.advance(newDocId);
} else {
spanDocId = item.sequenceSpans.spans.docID();
}
- }
+ }
}
if (spanDocId.equals(NO_MORE_DOCS)) {
item.noMoreDocs = true;
@@ -185,7 +204,7 @@ public class MtasSpanSequence extends Spans {
}
}
}
- }
+ }
}
// nothing found
if (newDocId == null) {
@@ -223,9 +242,11 @@ public class MtasSpanSequence extends Spans {
/**
* Advance to doc.
*
- * @param target the target
+ * @param target
+ * the target
* @return the integer
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private Integer advanceToDoc(int target) throws IOException {
if (docId == NO_MORE_DOCS) {
@@ -236,7 +257,7 @@ public class MtasSpanSequence extends Spans {
Integer spanDocId, newDocId = target;
for (QueueItem item : queueSpans) {
if (item.sequenceSpans.spans.docID() < newDocId) {
- spanDocId = item.sequenceSpans.spans.advance(newDocId);
+ spanDocId = item.sequenceSpans.spans.advance(newDocId);
if (spanDocId.equals(NO_MORE_DOCS)) {
item.noMoreDocs = true;
if (!item.sequenceSpans.optional) {
@@ -263,12 +284,12 @@ public class MtasSpanSequence extends Spans {
}
}
// find match
- docId = newDocId;
+ docId = newDocId;
// try and glue together
if (findMatches()) {
return null;
// no matches
- } else {
+ } else {
resetQueue();
// try next document
return (newDocId + 1);
@@ -280,7 +301,8 @@ public class MtasSpanSequence extends Spans {
* Find matches.
*
* @return true, if successful
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private boolean findMatches() throws IOException {
Boolean status = _findMatches();
@@ -294,7 +316,8 @@ public class MtasSpanSequence extends Spans {
* _find matches.
*
* @return true, if successful
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private boolean _findMatches() throws IOException {
// queue not empty
@@ -329,14 +352,15 @@ public class MtasSpanSequence extends Spans {
// used as lower boundary on endPosition next
minStartPositionNext = null;
for (int j = (i + 1); j < queueSpans.size(); j++) {
- if(queueSpans.get(j).sequenceSpans.optional) {
+ if (queueSpans.get(j).sequenceSpans.optional) {
break;
} else {
// check for available lowestPosition
if (queueSpans.get(j).lowestPosition != null) {
- minStartPositionNext = (minStartPositionNext == null) ? queueSpans
- .get(j).lowestPosition : Math.min(minStartPositionNext,
- queueSpans.get(j).lowestPosition);
+ minStartPositionNext = (minStartPositionNext == null)
+ ? queueSpans.get(j).lowestPosition
+ : Math.min(minStartPositionNext,
+ queueSpans.get(j).lowestPosition);
// computing restrictions not possible
} else {
minStartPositionNext = null;
@@ -345,11 +369,11 @@ public class MtasSpanSequence extends Spans {
}
}
// fill queue
- if ((minStartPositionPrevious == null)||subMatchesOptional) {
+ if ((minStartPositionPrevious == null) || subMatchesOptional) {
fillQueue(item, null, maxEndPositionPrevious, minStartPositionNext);
} else {
- fillQueue(item, minStartPositionPrevious,
- maxEndPositionPrevious, minStartPositionNext);
+ fillQueue(item, minStartPositionPrevious, maxEndPositionPrevious,
+ minStartPositionNext);
}
// check for available positions
if (!item.sequenceSpans.optional && item.noMorePositions
@@ -384,15 +408,17 @@ public class MtasSpanSequence extends Spans {
subMatchesStartPosition = subMatchesQueue.get(0).startPosition;
}
// compute minimum startPosition for next span
- if(item.lowestPosition!=null) {
- minStartPositionPrevious = (minStartPositionPrevious == null) ? item.lowestPosition
- : Math.min(minStartPositionPrevious, item.lowestPosition);
- }
+ if (item.lowestPosition != null) {
+ minStartPositionPrevious = (minStartPositionPrevious == null)
+ ? item.lowestPosition
+ : Math.min(minStartPositionPrevious, item.lowestPosition);
+ }
// for optional spans
if (item.sequenceSpans.optional) {
// update stats
if (item.lowestPosition != null) {
- minOptionalStartPosition = (minOptionalStartPosition == null) ? item.lowestPosition
+ minOptionalStartPosition = (minOptionalStartPosition == null)
+ ? item.lowestPosition
: Math.min(minOptionalStartPosition, item.lowestPosition);
}
// for not optional spans
@@ -404,38 +430,38 @@ public class MtasSpanSequence extends Spans {
maxEndPositionPrevious = null;
}
// compute maximum endPosition for next span
- if(item.lowestPosition!=null) {
- for (Integer endPosition : item.queue.get(item.lowestPosition)) {
- maxEndPositionPrevious = (maxEndPositionPrevious == null) ? endPosition
- : Math.max(maxEndPositionPrevious, endPosition);
+ if (item.lowestPosition != null) {
+ for (Integer endPosition : item.queue.get(item.lowestPosition)) {
+ maxEndPositionPrevious = (maxEndPositionPrevious == null)
+ ? endPosition : Math.max(maxEndPositionPrevious, endPosition);
}
- }
+ }
}
}
if (subMatchesQueue.isEmpty()) {
- //condition has only optional parts
- if(subMatchesOptional) {
- //check for
+ // condition has only optional parts
+ if (subMatchesOptional) {
+ // check for
boolean allFinished = true;
for (int i = 0; i < queueSpans.size(); i++) {
- if(!queueSpans.get(i).noMorePositions) {
+ if (!queueSpans.get(i).noMorePositions) {
allFinished = false;
break;
}
}
- if(allFinished) {
+ if (allFinished) {
currentPosition = NO_MORE_POSITIONS;
- }
+ }
}
return false;
} else if ((minOptionalStartPosition != null)
&& (minOptionalStartPosition < subMatchesStartPosition)) {
for (int i = 0; i < queueSpans.size(); i++) {
- if(!queueSpans.get(i).sequenceSpans.optional) {
+ if (!queueSpans.get(i).sequenceSpans.optional) {
break;
} else {
queueSpans.get(i).del(minOptionalStartPosition);
- }
+ }
}
return false;
} else {
@@ -455,15 +481,18 @@ public class MtasSpanSequence extends Spans {
/**
* _glue.
*
- * @param subMatchesQueue the sub matches queue
- * @param subMatchesOptional the sub matches optional
- * @param item the item
+ * @param subMatchesQueue
+ * the sub matches queue
+ * @param subMatchesOptional
+ * the sub matches optional
+ * @param item
+ * the item
* @return the list
*/
private List<Match> _glue(List<Match> subMatchesQueue,
Boolean subMatchesOptional, QueueItem item) {
- List<Match> newSubMatchesQueue = new ArrayList<Match>();
- //no previous queue, only use current item
+ List<Match> newSubMatchesQueue = new ArrayList<Match>();
+ // no previous queue, only use current item
if (subMatchesQueue.isEmpty()) {
if (item.filledPosition) {
for (Integer endPosition : item.queue.get(item.lowestPosition)) {
@@ -472,13 +501,13 @@ public class MtasSpanSequence extends Spans {
newSubMatchesQueue.add(m);
}
}
- }
+ }
return newSubMatchesQueue;
- //previous queue
+ // previous queue
} else {
- //startposition from queue
+ // startposition from queue
int startPosition = subMatchesQueue.get(0).startPosition;
- //previous queue optional, current item optional
+ // previous queue optional, current item optional
if (subMatchesOptional && item.sequenceSpans.optional) {
// forget previous, because current has lower startposition
if (item.filledPosition && item.lowestPosition < startPosition) {
@@ -510,11 +539,11 @@ public class MtasSpanSequence extends Spans {
}
}
}
- //no filled position
+ // no filled position
} else {
newSubMatchesQueue.addAll(subMatchesQueue);
}
- //previous queue optional, current item not optional
+ // previous queue optional, current item not optional
} else if (subMatchesOptional && !item.sequenceSpans.optional) {
assert item.filledPosition : "span not optional, should contain items";
// forget previous
@@ -547,11 +576,11 @@ public class MtasSpanSequence extends Spans {
}
}
}
- //previous queue not optional, current item optional
+ // previous queue not optional, current item optional
} else if (!subMatchesOptional && item.sequenceSpans.optional) {
newSubMatchesQueue.addAll(subMatchesQueue);
// merge with previous
- if (item.filledPosition) {
+ if (item.filledPosition) {
for (Match m : subMatchesQueue) {
if (item.queue.containsKey(m.endPosition)) {
for (Integer endPosition : item.queue.get(m.endPosition)) {
@@ -562,8 +591,8 @@ public class MtasSpanSequence extends Spans {
}
}
}
- }
- //previous queue not optional, current item not optional
+ }
+ // previous queue not optional, current item not optional
} else if (!subMatchesOptional && !item.sequenceSpans.optional) {
if (item.filledPosition) {
for (Match m : subMatchesQueue) {
@@ -585,11 +614,16 @@ public class MtasSpanSequence extends Spans {
/**
* Fill queue.
*
- * @param item the item
- * @param minStartPosition the min start position
- * @param maxStartPosition the max start position
- * @param minEndPosition the min end position
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param item
+ * the item
+ * @param minStartPosition
+ * the min start position
+ * @param maxStartPosition
+ * the max start position
+ * @param minEndPosition
+ * the min end position
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private void fillQueue(QueueItem item, Integer minStartPosition,
Integer maxStartPosition, Integer minEndPosition) throws IOException {
@@ -619,7 +653,7 @@ public class MtasSpanSequence extends Spans {
while (!item.noMorePositions) {
newStartPosition = item.sequenceSpans.spans.nextStartPosition();
if (newStartPosition == NO_MORE_POSITIONS) {
- if(!item.queue.isEmpty()) {
+ if (!item.queue.isEmpty()) {
item.filledPosition = true;
item.lastFilledPosition = item.lastRetrievedPosition;
}
@@ -655,7 +689,9 @@ public class MtasSpanSequence extends Spans {
currentMatch = null;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#asTwoPhaseIterator()
*/
@Override
@@ -667,23 +703,24 @@ public class MtasSpanSequence extends Spans {
* The Class QueueItem.
*/
private class QueueItem {
-
+
/** The filled position. */
private boolean noMoreDocs, noMorePositions, filledPosition;
-
+
/** The last retrieved position. */
private Integer lowestPosition, lastFilledPosition, lastRetrievedPosition;
-
+
/** The queue. */
private HashMap<Integer, List<Integer>> queue;
-
+
/** The sequence spans. */
public MtasSpanSequenceSpans sequenceSpans;
/**
* Instantiates a new queue item.
*
- * @param sequenceSpans the sequence spans
+ * @param sequenceSpans
+ * the sequence spans
*/
QueueItem(MtasSpanSequenceSpans sequenceSpans) {
noMoreDocs = false;
@@ -707,8 +744,10 @@ public class MtasSpanSequence extends Spans {
/**
* Adds the.
*
- * @param startPosition the start position
- * @param endPosition the end position
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
*/
public void add(int startPosition, int endPosition) {
if (!queue.keySet().contains(startPosition)) {
@@ -719,7 +758,7 @@ public class MtasSpanSequence extends Spans {
queue.put(startPosition, new ArrayList<Integer>());
}
queue.get(startPosition).add(endPosition);
- if ((lowestPosition == null)||(lowestPosition>startPosition)) {
+ if ((lowestPosition == null) || (lowestPosition > startPosition)) {
lowestPosition = startPosition;
}
lastRetrievedPosition = startPosition;
@@ -728,7 +767,8 @@ public class MtasSpanSequence extends Spans {
/**
* Del.
*
- * @param position the position
+ * @param position
+ * the position
*/
public void del(int position) {
ArrayList<Integer> removePositions = new ArrayList<Integer>();
@@ -747,7 +787,7 @@ public class MtasSpanSequence extends Spans {
lastFilledPosition = null;
filledPosition = false;
} else {
- lowestPosition = Collections.min(queue.keySet());
+ lowestPosition = Collections.min(queue.keySet());
if (filledPosition) {
if (!queue.keySet().contains(lastFilledPosition)) {
lastFilledPosition = null;
@@ -763,18 +803,20 @@ public class MtasSpanSequence extends Spans {
* The Class Match.
*/
private class Match {
-
+
/** The start position. */
private int startPosition;
-
+
/** The end position. */
private int endPosition;
/**
* Instantiates a new match.
*
- * @param startPosition the start position
- * @param endPosition the end position
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
*/
Match(int startPosition, int endPosition) {
this.startPosition = startPosition;
@@ -799,7 +841,9 @@ public class MtasSpanSequence extends Spans {
return endPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
@@ -814,8 +858,10 @@ public class MtasSpanSequence extends Spans {
}
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#cost()
*/
@Override
@@ -823,14 +869,14 @@ public class MtasSpanSequence extends Spans {
return 0;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#positionsCost()
*/
@Override
public float positionsCost() {
return 0;
- }
-
-
+ }
}
diff --git a/src/mtas/search/spans/MtasSpanSequenceItem.java b/src/mtas/search/spans/MtasSpanSequenceItem.java
index d44ccf0..2d4a6ce 100644
--- a/src/mtas/search/spans/MtasSpanSequenceItem.java
+++ b/src/mtas/search/spans/MtasSpanSequenceItem.java
@@ -6,24 +6,26 @@ import org.apache.lucene.search.spans.SpanQuery;
* The Class MtasSpanSequenceItem.
*/
public class MtasSpanSequenceItem {
-
+
/** The span query. */
private SpanQuery spanQuery;
-
+
/** The optional. */
private boolean optional;
-
+
/**
* Instantiates a new mtas span sequence item.
*
- * @param spanQuery the span query
- * @param optional the optional
+ * @param spanQuery
+ * the span query
+ * @param optional
+ * the optional
*/
public MtasSpanSequenceItem(SpanQuery spanQuery, boolean optional) {
this.spanQuery = spanQuery;
this.optional = optional;
}
-
+
/**
* Gets the query.
*
@@ -32,16 +34,17 @@ public class MtasSpanSequenceItem {
public SpanQuery getQuery() {
return spanQuery;
}
-
+
/**
* Sets the query.
*
- * @param spanQuery the new query
+ * @param spanQuery
+ * the new query
*/
public void setQuery(SpanQuery spanQuery) {
this.spanQuery = spanQuery;
}
-
+
/**
* Checks if is optional.
*
@@ -50,8 +53,10 @@ public class MtasSpanSequenceItem {
public boolean isOptional() {
return optional;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#clone()
*/
@Override
@@ -59,18 +64,21 @@ public class MtasSpanSequenceItem {
MtasSpanSequenceItem item = new MtasSpanSequenceItem(spanQuery, optional);
return item;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
public boolean equals(Object o) {
- if(o instanceof MtasSpanSequenceItem) {
+ if (o instanceof MtasSpanSequenceItem) {
final MtasSpanSequenceItem that = (MtasSpanSequenceItem) o;
- return spanQuery.equals(that.getQuery()) && (optional==that.isOptional());
+ return spanQuery.equals(that.getQuery())
+ && (optional == that.isOptional());
} else {
return false;
}
}
-
+
}
diff --git a/src/mtas/search/spans/MtasSpanSequenceQuery.java b/src/mtas/search/spans/MtasSpanSequenceQuery.java
index 47c73c4..c37003e 100644
--- a/src/mtas/search/spans/MtasSpanSequenceQuery.java
+++ b/src/mtas/search/spans/MtasSpanSequenceQuery.java
@@ -25,37 +25,45 @@ public class MtasSpanSequenceQuery extends SpanQuery implements Cloneable {
/** The items. */
private List<MtasSpanSequenceItem> items;
-
+
/** The field. */
private String field;
-
+
/** The query name. */
- private static String QUERY_NAME = "mtasSpanSequenceQuery";
-
+ private static String QUERY_NAME = "mtasSpanSequenceQuery";
+
/**
* Instantiates a new mtas span sequence query.
*
- * @param items the items
+ * @param items
+ * the items
*/
public MtasSpanSequenceQuery(List<MtasSpanSequenceItem> items) {
- this.items = items;
- //get field and do checks
+ this.items = items;
+ // get field and do checks
for (MtasSpanSequenceItem item : items) {
if (field == null) {
field = item.getQuery().getField();
- } else if (item.getQuery().getField() != null && !item.getQuery().getField().equals(field)) {
+ } else if (item.getQuery().getField() != null
+ && !item.getQuery().getField().equals(field)) {
throw new IllegalArgumentException("Clauses must have same field.");
}
- }
+ }
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
- public String getField() { return field; }
-
- /* (non-Javadoc)
+ public String getField() {
+ return field;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#clone()
*/
@Override
@@ -68,42 +76,47 @@ public class MtasSpanSequenceQuery extends SpanQuery implements Cloneable {
MtasSpanSequenceQuery soq = new MtasSpanSequenceQuery(newItems);
return soq;
}
-
- /* (non-Javadoc)
- * @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
*/
@Override
public Query rewrite(IndexReader reader) throws IOException {
MtasSpanSequenceQuery clone = null;
- for (int i = 0 ; i < items.size(); i++) {
+ for (int i = 0; i < items.size(); i++) {
SpanQuery c = items.get(i).getQuery();
SpanQuery query = (SpanQuery) c.rewrite(reader);
- if (query != c) { // clause rewrote: must clone
+ if (query != c) { // clause rewrote: must clone
if (clone == null)
clone = this.clone();
clone.items.get(i).setQuery(query);
}
}
if (clone != null) {
- return clone; // some clauses rewrote
+ return clone; // some clauses rewrote
} else {
- return this; // no clauses rewrote
+ return this; // no clauses rewrote
}
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#toString(java.lang.String)
*/
@Override
public String toString(String field) {
StringBuilder buffer = new StringBuilder();
- buffer.append(QUERY_NAME+"([");
+ buffer.append(QUERY_NAME + "([");
Iterator<MtasSpanSequenceItem> i = items.iterator();
while (i.hasNext()) {
MtasSpanSequenceItem item = i.next();
SpanQuery clause = item.getQuery();
buffer.append(clause.toString(field));
- if(item.isOptional()) {
+ if (item.isOptional()) {
buffer.append("{OPTIONAL}");
}
if (i.hasNext()) {
@@ -113,8 +126,10 @@ public class MtasSpanSequenceQuery extends SpanQuery implements Cloneable {
buffer.append("])");
return buffer.toString();
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
@@ -126,10 +141,12 @@ public class MtasSpanSequenceQuery extends SpanQuery implements Cloneable {
if (getClass() != obj.getClass())
return false;
MtasSpanSequenceQuery other = (MtasSpanSequenceQuery) obj;
- return field.equals(other.field) && items.equals(other.items);
+ return field.equals(other.field) && items.equals(other.items);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
@@ -139,35 +156,41 @@ public class MtasSpanSequenceQuery extends SpanQuery implements Cloneable {
return h;
}
-
-
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
- throws IOException {
+ throws IOException {
List<MtasSpanSequenceWeight> subWeights = new ArrayList<MtasSpanSequenceWeight>();
- for(MtasSpanSequenceItem item : items) {
- subWeights.add(new MtasSpanSequenceWeight(item.getQuery().createWeight(searcher, false), item.isOptional()) );
+ for (MtasSpanSequenceItem item : items) {
+ subWeights.add(new MtasSpanSequenceWeight(
+ item.getQuery().createWeight(searcher, false), item.isOptional()));
}
- return new SpanSequenceWeight(subWeights, searcher, needsScores ? getTermContexts(subWeights) : null);
+ return new SpanSequenceWeight(subWeights, searcher,
+ needsScores ? getTermContexts(subWeights) : null);
}
-
+
/**
* Gets the term contexts.
*
- * @param items the items
+ * @param items
+ * the items
* @return the term contexts
*/
- protected Map<Term, TermContext> getTermContexts(List<MtasSpanSequenceWeight> items) {
+ protected Map<Term, TermContext> getTermContexts(
+ List<MtasSpanSequenceWeight> items) {
List<SpanWeight> weights = new ArrayList<SpanWeight>();
- for(MtasSpanSequenceWeight item : items) {
+ for (MtasSpanSequenceWeight item : items) {
weights.add(item.spanWeight);
}
return getTermContexts(weights);
}
-
+
/**
* The Class SpanSequenceWeight.
*/
@@ -179,18 +202,28 @@ public class MtasSpanSequenceQuery extends SpanQuery implements Cloneable {
/**
* Instantiates a new span sequence weight.
*
- * @param subWeights the sub weights
- * @param searcher the searcher
- * @param terms the terms
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param subWeights
+ * the sub weights
+ * @param searcher
+ * the searcher
+ * @param terms
+ * the terms
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- public SpanSequenceWeight(List<MtasSpanSequenceWeight> subWeights, IndexSearcher searcher, Map<Term, TermContext> terms) throws IOException {
+ public SpanSequenceWeight(List<MtasSpanSequenceWeight> subWeights,
+ IndexSearcher searcher, Map<Term, TermContext> terms)
+ throws IOException {
super(MtasSpanSequenceQuery.this, searcher, terms);
- this.subWeights = subWeights;
+ this.subWeights = subWeights;
}
-
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.Map)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.
+ * Map)
*/
@Override
public void extractTermContexts(Map<Term, TermContext> contexts) {
@@ -199,8 +232,13 @@ public class MtasSpanSequenceQuery extends SpanQuery implements Cloneable {
}
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.index.LeafReaderContext, org.apache.lucene.search.spans.SpanWeight.Postings)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.
+ * index.LeafReaderContext,
+ * org.apache.lucene.search.spans.SpanWeight.Postings)
*/
@Override
public Spans getSpans(LeafReaderContext context, Postings requiredPostings)
@@ -209,28 +247,32 @@ public class MtasSpanSequenceQuery extends SpanQuery implements Cloneable {
if (terms == null) {
return null; // field does not exist
}
- List<MtasSpanSequenceSpans> setSequenceSpans = new ArrayList<>(items.size());
+ List<MtasSpanSequenceSpans> setSequenceSpans = new ArrayList<>(
+ items.size());
boolean allSpansEmpty = true;
- for (MtasSpanSequenceWeight w : subWeights) {
- Spans sequenceSpans = w.spanWeight.getSpans(context, requiredPostings);
+ for (MtasSpanSequenceWeight w : subWeights) {
+ Spans sequenceSpans = w.spanWeight.getSpans(context, requiredPostings);
if (sequenceSpans != null) {
- setSequenceSpans.add(new MtasSpanSequenceSpans(sequenceSpans, w.optional));
+ setSequenceSpans
+ .add(new MtasSpanSequenceSpans(sequenceSpans, w.optional));
allSpansEmpty = false;
} else {
- if(w.optional) {
+ if (w.optional) {
setSequenceSpans.add(new MtasSpanSequenceSpans(null, w.optional));
} else {
return null;
}
}
}
- if(allSpansEmpty) {
+ if (allSpansEmpty) {
return null; // at least one required
}
- return new MtasSpanSequence(MtasSpanSequenceQuery.this, setSequenceSpans);
+ return new MtasSpanSequence(MtasSpanSequenceQuery.this, setSequenceSpans);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Weight#extractTerms(java.util.Set)
*/
@Override
@@ -239,48 +281,52 @@ public class MtasSpanSequenceQuery extends SpanQuery implements Cloneable {
w.spanWeight.extractTerms(terms);
}
}
-
+
}
-
+
/**
* The Class MtasSpanSequenceSpans.
*/
public class MtasSpanSequenceSpans {
-
+
/** The spans. */
public Spans spans;
-
+
/** The optional. */
public boolean optional;
-
+
/**
* Instantiates a new mtas span sequence spans.
*
- * @param spans the spans
- * @param optional the optional
+ * @param spans
+ * the spans
+ * @param optional
+ * the optional
*/
public MtasSpanSequenceSpans(Spans spans, boolean optional) {
this.spans = spans;
this.optional = optional;
}
}
-
+
/**
* The Class MtasSpanSequenceWeight.
*/
- public class MtasSpanSequenceWeight {
-
+ public class MtasSpanSequenceWeight {
+
/** The span weight. */
public SpanWeight spanWeight;
-
+
/** The optional. */
- public boolean optional;
-
+ public boolean optional;
+
/**
* Instantiates a new mtas span sequence weight.
*
- * @param spanWeight the span weight
- * @param optional the optional
+ * @param spanWeight
+ * the span weight
+ * @param optional
+ * the optional
*/
public MtasSpanSequenceWeight(SpanWeight spanWeight, boolean optional) {
this.spanWeight = spanWeight;
diff --git a/src/mtas/search/spans/MtasSpanStartQuery.java b/src/mtas/search/spans/MtasSpanStartQuery.java
index 0a2cd26..311197d 100644
--- a/src/mtas/search/spans/MtasSpanStartQuery.java
+++ b/src/mtas/search/spans/MtasSpanStartQuery.java
@@ -21,27 +21,31 @@ public class MtasSpanStartQuery extends SpanQuery {
/** The query. */
private SpanQuery query;
-
+
/** The query name. */
private static String QUERY_NAME = "mtasSpanStartQuery";
/**
* Instantiates a new mtas span start query.
*
- * @param query the query
+ * @param query
+ * the query
*/
public MtasSpanStartQuery(SpanQuery query) {
super();
- this.query = query;
+ this.query = query;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
*/
@Override
public Query rewrite(IndexReader reader) throws IOException {
query.rewrite(reader);
- return this;
+ return this;
}
/*
@@ -53,13 +57,15 @@ public class MtasSpanStartQuery extends SpanQuery {
@Override
public String toString(String field) {
StringBuilder buffer = new StringBuilder();
- buffer.append(QUERY_NAME+"([");
- buffer.append(this.query.toString(field));
+ buffer.append(QUERY_NAME + "([");
+ buffer.append(this.query.toString(field));
buffer.append("])");
return buffer.toString();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
@@ -67,17 +73,21 @@ public class MtasSpanStartQuery extends SpanQuery {
return query.getField();
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
throws IOException {
- SpanWeight spanWeight = ((SpanQuery) searcher.rewrite(query)).createWeight(searcher,
- needsScores);
+ SpanWeight spanWeight = ((SpanQuery) searcher.rewrite(query))
+ .createWeight(searcher, needsScores);
return new SpanTermWeight(spanWeight, searcher);
}
-
+
/**
* The Class SpanTermWeight.
*/
@@ -85,62 +95,81 @@ public class MtasSpanStartQuery extends SpanQuery {
/** The span weight. */
SpanWeight spanWeight;
-
+
/**
* Instantiates a new span term weight.
*
- * @param spanWeight the span weight
- * @param searcher the searcher
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param spanWeight
+ * the span weight
+ * @param searcher
+ * the searcher
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- public SpanTermWeight(SpanWeight spanWeight, IndexSearcher searcher) throws IOException {
+ public SpanTermWeight(SpanWeight spanWeight, IndexSearcher searcher)
+ throws IOException {
super(MtasSpanStartQuery.this, searcher, null);
this.spanWeight = spanWeight;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.Map)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.
+ * Map)
*/
@Override
public void extractTermContexts(Map<Term, TermContext> contexts) {
- spanWeight.extractTermContexts(contexts);
+ spanWeight.extractTermContexts(contexts);
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.index.LeafReaderContext, org.apache.lucene.search.spans.SpanWeight.Postings)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.
+ * index.LeafReaderContext,
+ * org.apache.lucene.search.spans.SpanWeight.Postings)
*/
@Override
public Spans getSpans(LeafReaderContext ctx, Postings requiredPostings)
throws IOException {
- return new MtasStartSpans(spanWeight.getSpans(ctx, requiredPostings));
+ return new MtasStartSpans(spanWeight.getSpans(ctx, requiredPostings));
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Weight#extractTerms(java.util.Set)
*/
@Override
public void extractTerms(Set<Term> terms) {
spanWeight.extractTerms(terms);
}
-
+
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
- if (obj== null)
+ if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final MtasSpanStartQuery that = (MtasSpanStartQuery) obj;
- return query.equals(that.query);
+ return query.equals(that.query);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
diff --git a/src/mtas/search/spans/MtasSpanTermQuery.java b/src/mtas/search/spans/MtasSpanTermQuery.java
index 1783379..c18010f 100644
--- a/src/mtas/search/spans/MtasSpanTermQuery.java
+++ b/src/mtas/search/spans/MtasSpanTermQuery.java
@@ -44,7 +44,8 @@ public class MtasSpanTermQuery extends SpanTermQuery {
/**
* Instantiates a new mtas span term query.
*
- * @param term the term
+ * @param term
+ * the term
*/
public MtasSpanTermQuery(Term term) {
this(term, true);
@@ -53,8 +54,10 @@ public class MtasSpanTermQuery extends SpanTermQuery {
/**
* Instantiates a new mtas span term query.
*
- * @param term the term
- * @param singlePosition the single position
+ * @param term
+ * the term
+ * @param singlePosition
+ * the single position
*/
public MtasSpanTermQuery(Term term, boolean singlePosition) {
this(new SpanTermQuery(term), true);
@@ -63,8 +66,10 @@ public class MtasSpanTermQuery extends SpanTermQuery {
/**
* Instantiates a new mtas span term query.
*
- * @param query the query
- * @param singlePosition the single position
+ * @param query
+ * the query
+ * @param singlePosition
+ * the single position
*/
public MtasSpanTermQuery(SpanTermQuery query, boolean singlePosition) {
super(query.getTerm());
@@ -81,8 +86,12 @@ public class MtasSpanTermQuery extends SpanTermQuery {
}
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanTermQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanTermQuery#createWeight(org.apache.lucene
+ * .search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
@@ -109,10 +118,14 @@ public class MtasSpanTermQuery extends SpanTermQuery {
/**
* Instantiates a new span term weight.
*
- * @param termContext the term context
- * @param searcher the searcher
- * @param terms the terms
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param termContext
+ * the term context
+ * @param searcher
+ * the searcher
+ * @param terms
+ * the terms
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
public SpanTermWeight(TermContext termContext, IndexSearcher searcher,
Map<Term, TermContext> terms) throws IOException {
@@ -200,7 +213,7 @@ public class MtasSpanTermQuery extends SpanTermQuery {
}
FieldInfo fieldInfo = r.getFieldInfos().fieldInfo(field);
-
+
if (CodecUtil.isSinglePositionPrefix(fieldInfo, prefix)) {
postings = termsEnum.postings(null,
requiredPostings.getRequiredPostings());
@@ -231,8 +244,11 @@ public class MtasSpanTermQuery extends SpanTermQuery {
}
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanTermQuery#toString(java.lang.String)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanTermQuery#toString(java.lang.String)
*/
@Override
public String toString(String field) {
@@ -247,7 +263,9 @@ public class MtasSpanTermQuery extends SpanTermQuery {
return buffer.toString();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanTermQuery#equals(java.lang.Object)
*/
@Override
diff --git a/src/mtas/search/spans/MtasSpanUniquePosition.java b/src/mtas/search/spans/MtasSpanUniquePosition.java
index 415156a..0a9e2a6 100644
--- a/src/mtas/search/spans/MtasSpanUniquePosition.java
+++ b/src/mtas/search/spans/MtasSpanUniquePosition.java
@@ -14,33 +14,35 @@ public class MtasSpanUniquePosition extends Spans {
/** The spans. */
Spans spans;
-
+
/** The queue spans. */
List<Match> queueSpans;
-
+
/** The queue matches. */
List<Match> queueMatches;
-
+
/** The current match. */
Match currentMatch;
-
+
/** The last start position. */
int lastStartPosition; // startPosition of last retrieved span
-
+
/** The last span. */
boolean lastSpan; // last span for this document added to queue
-
+
/** The no more positions. */
boolean noMorePositions;
-
+
/**
* Instantiates a new mtas span unique position.
*
- * @param mtasSpanUniquePositionQuery the mtas span unique position query
- * @param spans the spans
+ * @param mtasSpanUniquePositionQuery
+ * the mtas span unique position query
+ * @param spans
+ * the spans
*/
- public MtasSpanUniquePosition(MtasSpanUniquePositionQuery mtasSpanUniquePositionQuery,
- Spans spans) {
+ public MtasSpanUniquePosition(
+ MtasSpanUniquePositionQuery mtasSpanUniquePositionQuery, Spans spans) {
super();
this.spans = spans;
queueSpans = new ArrayList<Match>();
@@ -48,7 +50,9 @@ public class MtasSpanUniquePosition extends Spans {
resetQueue();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#nextStartPosition()
*/
@Override
@@ -65,40 +69,54 @@ public class MtasSpanUniquePosition extends Spans {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#startPosition()
*/
@Override
public int startPosition() {
- return (currentMatch==null)?(noMorePositions?NO_MORE_POSITIONS:-1):currentMatch.startPosition();
+ return (currentMatch == null) ? (noMorePositions ? NO_MORE_POSITIONS : -1)
+ : currentMatch.startPosition();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#endPosition()
*/
@Override
public int endPosition() {
- return (currentMatch==null)?(noMorePositions?NO_MORE_POSITIONS:-1):currentMatch.endPosition();
+ return (currentMatch == null) ? (noMorePositions ? NO_MORE_POSITIONS : -1)
+ : currentMatch.endPosition();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#width()
*/
@Override
public int width() {
- //return (currentMatch.endPosition() - currentMatch.startPosition());
+ // return (currentMatch.endPosition() - currentMatch.startPosition());
return 1;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans.SpanCollector)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans
+ * .SpanCollector)
*/
@Override
public void collect(SpanCollector collector) throws IOException {
spans.collect(collector);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#docID()
*/
@Override
@@ -106,7 +124,9 @@ public class MtasSpanUniquePosition extends Spans {
return spans.docID();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#nextDoc()
*/
@Override
@@ -116,7 +136,9 @@ public class MtasSpanUniquePosition extends Spans {
return (spans.nextDoc() == NO_MORE_DOCS) ? NO_MORE_DOCS : toMatchDoc();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#advance(int)
*/
@Override
@@ -142,7 +164,8 @@ public class MtasSpanUniquePosition extends Spans {
* To match doc.
*
* @return the int
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
int toMatchDoc() throws IOException {
while (true) {
@@ -159,7 +182,8 @@ public class MtasSpanUniquePosition extends Spans {
* Collect span.
*
* @return true, if successful
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
// try to get something in the queue of spans
private boolean collectSpan() throws IOException {
@@ -179,7 +203,8 @@ public class MtasSpanUniquePosition extends Spans {
* Find matches.
*
* @return true, if successful
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private boolean findMatches() throws IOException {
// check for something in queue of matches
@@ -202,15 +227,15 @@ public class MtasSpanUniquePosition extends Spans {
while (!lastSpan && (lastStartPosition == firstMatch.startPosition())) {
collectSpan();
}
- while (!queueSpans.isEmpty()
- && (queueSpans.get(0).startPosition() == firstMatch.startPosition())) {
+ while (!queueSpans.isEmpty() && (queueSpans.get(0)
+ .startPosition() == firstMatch.startPosition())) {
matches.add(queueSpans.get(0));
queueSpans.remove(0);
}
// construct all matches for this startposition
for (Match match : matches) {
- //only unique spans
- if(!queueMatches.contains(match)) {
+ // only unique spans
+ if (!queueMatches.contains(match)) {
queueMatches.add(match);
}
}
@@ -222,23 +247,24 @@ public class MtasSpanUniquePosition extends Spans {
}
}
-
/**
* The Class Match.
*/
private class Match {
-
+
/** The start position. */
private int startPosition;
-
+
/** The end position. */
private int endPosition;
/**
* Instantiates a new match.
*
- * @param startPosition the start position
- * @param endPosition the end position
+ * @param startPosition
+ * the start position
+ * @param endPosition
+ * the end position
*/
Match(int startPosition, int endPosition) {
this.startPosition = startPosition;
@@ -263,7 +289,9 @@ public class MtasSpanUniquePosition extends Spans {
return endPosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
@@ -279,21 +307,24 @@ public class MtasSpanUniquePosition extends Spans {
}
-
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#cost()
*/
@Override
public long cost() {
- return (spans==null)?0:spans.cost();
+ return (spans == null) ? 0 : spans.cost();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#positionsCost()
*/
@Override
public float positionsCost() {
- return (spans==null)?0:spans.positionsCost();
- }
+ return (spans == null) ? 0 : spans.positionsCost();
+ }
}
diff --git a/src/mtas/search/spans/MtasSpanUniquePositionQuery.java b/src/mtas/search/spans/MtasSpanUniquePositionQuery.java
index aef5dec..ba63543 100644
--- a/src/mtas/search/spans/MtasSpanUniquePositionQuery.java
+++ b/src/mtas/search/spans/MtasSpanUniquePositionQuery.java
@@ -20,20 +20,21 @@ import org.apache.lucene.search.spans.Spans;
* The Class MtasSpanUniquePositionQuery.
*/
abstract public class MtasSpanUniquePositionQuery extends SpanQuery {
-
+
/** The clause. */
private SpanQuery clause;
-
+
/** The field. */
private String field;
-
+
/** The query name. */
private static String QUERY_NAME = "mtasSpanUniquePositionQuery";
-
+
/**
* Instantiates a new mtas span unique position query.
*
- * @param clause the clause
+ * @param clause
+ * the clause
*/
public MtasSpanUniquePositionQuery(SpanQuery clause) {
field = clause.getField();
@@ -48,29 +49,37 @@ abstract public class MtasSpanUniquePositionQuery extends SpanQuery {
public SpanQuery getClause() {
return clause;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
- public String getField() { return field; }
+ public String getField() {
+ return field;
+ }
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
- if (obj== null)
+ if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final MtasSpanUniquePositionQuery that = (MtasSpanUniquePositionQuery) obj;
- return clause.equals(that.clause);
- }
+ return clause.equals(that.clause);
+ }
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
@@ -79,30 +88,36 @@ abstract public class MtasSpanUniquePositionQuery extends SpanQuery {
h = (h * 7) ^ clause.hashCode();
return h;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#toString(java.lang.String)
*/
@Override
public String toString(String field) {
StringBuilder buffer = new StringBuilder();
- buffer.append(QUERY_NAME+"([");
+ buffer.append(QUERY_NAME + "([");
buffer.append(clause.toString(field));
buffer.append("])");
return buffer.toString();
}
-
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
throws IOException {
- SpanWeight subWeight = clause.createWeight(searcher, false);
- return new SpanUniquePositionWeight(subWeight, searcher, needsScores ? getTermContexts(subWeight) : null);
+ SpanWeight subWeight = clause.createWeight(searcher, false);
+ return new SpanUniquePositionWeight(subWeight, searcher,
+ needsScores ? getTermContexts(subWeight) : null);
}
-
/**
* The Class SpanUniquePositionWeight.
*/
@@ -114,26 +129,41 @@ abstract public class MtasSpanUniquePositionQuery extends SpanQuery {
/**
* Instantiates a new span unique position weight.
*
- * @param subWeight the sub weight
- * @param searcher the searcher
- * @param terms the terms
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param subWeight
+ * the sub weight
+ * @param searcher
+ * the searcher
+ * @param terms
+ * the terms
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
- public SpanUniquePositionWeight(SpanWeight subWeight, IndexSearcher searcher, Map<Term, TermContext> terms) throws IOException {
+ public SpanUniquePositionWeight(SpanWeight subWeight,
+ IndexSearcher searcher, Map<Term, TermContext> terms)
+ throws IOException {
super(MtasSpanUniquePositionQuery.this, searcher, terms);
this.subWeight = subWeight;
}
-
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.Map)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.
+ * Map)
*/
@Override
public void extractTermContexts(Map<Term, TermContext> contexts) {
subWeight.extractTermContexts(contexts);
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.index.LeafReaderContext, org.apache.lucene.search.spans.SpanWeight.Postings)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.
+ * index.LeafReaderContext,
+ * org.apache.lucene.search.spans.SpanWeight.Postings)
*/
@Override
public Spans getSpans(LeafReaderContext context, Postings requiredPostings)
@@ -148,23 +178,24 @@ abstract public class MtasSpanUniquePositionQuery extends SpanQuery {
return null;
} else {
SimScorer scorer = getSimScorer(context);
- if(scorer==null) {
+ if (scorer == null) {
scorer = new MtasSimScorer();
- }
- return new MtasSpanUniquePosition(MtasSpanUniquePositionQuery.this, subSpan);
+ }
+ return new MtasSpanUniquePosition(MtasSpanUniquePositionQuery.this,
+ subSpan);
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Weight#extractTerms(java.util.Set)
*/
@Override
public void extractTerms(Set<Term> terms) {
subWeight.extractTerms(terms);
}
-
+
}
-
-
}
diff --git a/src/mtas/search/spans/MtasSpanWildcardQuery.java b/src/mtas/search/spans/MtasSpanWildcardQuery.java
index 852ccbe..7d0a1ea 100644
--- a/src/mtas/search/spans/MtasSpanWildcardQuery.java
+++ b/src/mtas/search/spans/MtasSpanWildcardQuery.java
@@ -45,7 +45,8 @@ public class MtasSpanWildcardQuery extends SpanQuery {
/**
* Instantiates a new mtas span wildcard query.
*
- * @param term the term
+ * @param term
+ * the term
*/
public MtasSpanWildcardQuery(Term term) {
this(term, true);
@@ -54,8 +55,10 @@ public class MtasSpanWildcardQuery extends SpanQuery {
/**
* Instantiates a new mtas span wildcard query.
*
- * @param term the term
- * @param singlePosition the single position
+ * @param term
+ * the term
+ * @param singlePosition
+ * the single position
*/
public MtasSpanWildcardQuery(Term term, boolean singlePosition) {
super();
@@ -74,8 +77,11 @@ public class MtasSpanWildcardQuery extends SpanQuery {
}
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader)
*/
@Override
public Query rewrite(IndexReader reader) throws IOException {
@@ -123,7 +129,9 @@ public class MtasSpanWildcardQuery extends SpanQuery {
return buffer.toString();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.SpanQuery#getField()
*/
@Override
@@ -131,8 +139,12 @@ public class MtasSpanWildcardQuery extends SpanQuery {
return term.field();
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.search.IndexSearcher, boolean)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
+ * search.IndexSearcher, boolean)
*/
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
@@ -141,7 +153,9 @@ public class MtasSpanWildcardQuery extends SpanQuery {
needsScores);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#equals(java.lang.Object)
*/
@Override
@@ -156,7 +170,9 @@ public class MtasSpanWildcardQuery extends SpanQuery {
return term.equals(that.term) && singlePosition == that.singlePosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.Query#hashCode()
*/
@Override
diff --git a/src/mtas/search/spans/MtasStartSpans.java b/src/mtas/search/spans/MtasStartSpans.java
index 897a0e1..24d1240 100644
--- a/src/mtas/search/spans/MtasStartSpans.java
+++ b/src/mtas/search/spans/MtasStartSpans.java
@@ -9,45 +9,54 @@ import org.apache.lucene.search.spans.Spans;
* The Class MtasStartSpans.
*/
public class MtasStartSpans extends Spans {
-
+
/** The spans. */
Spans spans;
-
+
/**
* Instantiates a new mtas start spans.
*
- * @param spans the spans
+ * @param spans
+ * the spans
*/
public MtasStartSpans(Spans spans) {
super();
this.spans = spans;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#nextStartPosition()
*/
@Override
public int nextStartPosition() throws IOException {
- return (spans==null)?NO_MORE_POSITIONS:spans.nextStartPosition();
+ return (spans == null) ? NO_MORE_POSITIONS : spans.nextStartPosition();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#startPosition()
*/
@Override
public int startPosition() {
- return (spans==null)?-1:spans.startPosition();
+ return (spans == null) ? -1 : spans.startPosition();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#endPosition()
*/
@Override
public int endPosition() {
- return (spans==null)?-1:spans.startPosition();
+ return (spans == null) ? -1 : spans.startPosition();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#width()
*/
@Override
@@ -55,54 +64,68 @@ public class MtasStartSpans extends Spans {
return 0;
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans.SpanCollector)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.spans.Spans#collect(org.apache.lucene.search.spans
+ * .SpanCollector)
*/
@Override
public void collect(SpanCollector collector) throws IOException {
- if(spans!=null) {
- spans.collect(collector);
+ if (spans != null) {
+ spans.collect(collector);
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#docID()
*/
@Override
public int docID() {
- return (spans==null)?NO_MORE_DOCS:spans.docID();
+ return (spans == null) ? NO_MORE_DOCS : spans.docID();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#nextDoc()
*/
@Override
public int nextDoc() throws IOException {
- return (spans==null)?NO_MORE_DOCS:spans.nextDoc();
+ return (spans == null) ? NO_MORE_DOCS : spans.nextDoc();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#advance(int)
*/
@Override
public int advance(int target) throws IOException {
- return (spans==null)?NO_MORE_DOCS:spans.advance(target);
+ return (spans == null) ? NO_MORE_DOCS : spans.advance(target);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.DocIdSetIterator#cost()
*/
@Override
public long cost() {
- return (spans==null)?0:spans.cost();
+ return (spans == null) ? 0 : spans.cost();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.Spans#positionsCost()
*/
@Override
public float positionsCost() {
- return (spans==null)?0:spans.positionsCost();
- }
+ return (spans == null) ? 0 : spans.positionsCost();
+ }
}
diff --git a/src/mtas/search/spans/MtasTermSpans.java b/src/mtas/search/spans/MtasTermSpans.java
index 62084cb..b7b3453 100644
--- a/src/mtas/search/spans/MtasTermSpans.java
+++ b/src/mtas/search/spans/MtasTermSpans.java
@@ -2,7 +2,6 @@ package mtas.search.spans;
import java.io.IOException;
import java.util.Collection;
-import java.util.TreeSet;
import mtas.analysis.token.MtasPosition;
import mtas.codec.payload.MtasPayloadDecoder;
@@ -16,104 +15,120 @@ import org.apache.lucene.search.spans.TermSpans;
* The Class MtasTermSpans.
*/
public class MtasTermSpans extends TermSpans {
-
+
/** The mtas position. */
protected MtasPosition mtasPosition = null;
-
+
/** The assume single position. */
private boolean assumeSinglePosition;
-
+
/** The payload span collector. */
private PayloadSpanCollector payloadSpanCollector;
-
+
/**
* Instantiates a new mtas term spans.
*
- * @param postings the postings
- * @param term the term
+ * @param postings
+ * the postings
+ * @param term
+ * the term
*/
public MtasTermSpans(PostingsEnum postings, Term term) {
- this(postings,term,false);
+ this(postings, term, false);
}
-
+
/**
* Instantiates a new mtas term spans.
*
- * @param postings the postings
- * @param term the term
- * @param assumeSinglePosition the assume single position
+ * @param postings
+ * the postings
+ * @param term
+ * the term
+ * @param assumeSinglePosition
+ * the assume single position
*/
- public MtasTermSpans(PostingsEnum postings, Term term, boolean assumeSinglePosition) {
- super(null,postings, term,1);
+ public MtasTermSpans(PostingsEnum postings, Term term,
+ boolean assumeSinglePosition) {
+ super(null, postings, term, 1);
payloadSpanCollector = new PayloadSpanCollector();
- this.assumeSinglePosition = assumeSinglePosition;
+ this.assumeSinglePosition = assumeSinglePosition;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.lucene.search.spans.TermSpans#endPosition()
*/
@Override
public int endPosition() {
- if(assumeSinglePosition) {
+ if (assumeSinglePosition) {
return super.endPosition();
} else {
- int status = super.endPosition();
+ int status = super.endPosition();
try {
processEncodedPayload();
- return (status != NO_MORE_POSITIONS) ? (mtasPosition.getEnd()+1) : NO_MORE_POSITIONS;
+ return (status != NO_MORE_POSITIONS) ? (mtasPosition.getEnd() + 1)
+ : NO_MORE_POSITIONS;
} catch (IOException e) {
return NO_MORE_POSITIONS;
}
- }
+ }
}
-
+
/**
* Gets the positions.
*
* @return the positions
*/
- public TreeSet<Integer> getPositions() {
- TreeSet<Integer> list;
- if(assumeSinglePosition) {
- list = new TreeSet<Integer>();
- list.add(super.startPosition());
+ public int[] getPositions() {
+ int[] list;
+ if (assumeSinglePosition) {
+ list = new int[1];
+ list[0] = super.startPosition();
return list;
} else {
try {
processEncodedPayload();
- return mtasPosition.getPositions();
- } catch(IOException e) {
- list = new TreeSet<Integer>();
- for(int i = super.endPosition(); i < super.endPosition(); i++)
- list.add(i);
- return list;
+ list = mtasPosition.getPositions();
+ if (list != null) {
+ return mtasPosition.getPositions();
+ }
+ } catch (IOException e) {
+ // do nothing
}
+ int start = super.startPosition();
+ int end = super.endPosition();
+ list = new int[end - start];
+ for (int i = start; i < end; i++)
+ list[i - start] = i;
+ return list;
}
}
-
/**
* Process encoded payload.
*
- * @throws IOException Signals that an I/O exception has occurred.
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
private void processEncodedPayload() throws IOException {
- if(!readPayload) {
+ if (!readPayload) {
payloadSpanCollector.reset();
- collect(payloadSpanCollector);
- Collection<byte[]> originalPayloadCollection = payloadSpanCollector.getPayloads();
- if(originalPayloadCollection.iterator().hasNext()) {
+ collect(payloadSpanCollector);
+ Collection<byte[]> originalPayloadCollection = payloadSpanCollector
+ .getPayloads();
+ if (originalPayloadCollection.iterator().hasNext()) {
byte[] payload = originalPayloadCollection.iterator().next();
- if(payload==null) {
+ if (payload == null) {
throw new IOException("no payload");
- }
+ }
MtasPayloadDecoder payloadDecoder = new MtasPayloadDecoder();
- payloadDecoder.init(startPosition(), payload);
+ payloadDecoder.init(startPosition(), payload);
mtasPosition = payloadDecoder.getMtasPosition();
} else {
throw new IOException("no payload");
- }
- }
+ }
+ }
}
}
diff --git a/src/mtas/solr/handler/MtasRequestHandler.java b/src/mtas/solr/handler/MtasRequestHandler.java
index 33fad38..d4d2882 100644
--- a/src/mtas/solr/handler/MtasRequestHandler.java
+++ b/src/mtas/solr/handler/MtasRequestHandler.java
@@ -28,36 +28,37 @@ public class MtasRequestHandler extends RequestHandlerBase {
/** The error. */
private static String ERROR = "error";
-
+
/** The action config files. */
private static String ACTION_CONFIG_FILES = "files";
-
+
/** The action config file. */
private static String ACTION_CONFIG_FILE = "file";
-
+
/** The action mapping. */
private static String ACTION_MAPPING = "mapping";
-
- /** The action segments. */
- private static String ACTION_SEGMENTS = "segments";
-
+
/** The param action. */
private static String PARAM_ACTION = "action";
-
+
/** The param config file. */
private static String PARAM_CONFIG_FILE = "file";
-
+
/** The param mapping configuration. */
private static String PARAM_MAPPING_CONFIGURATION = "configuration";
-
+
/** The param mapping document. */
private static String PARAM_MAPPING_DOCUMENT = "document";
-
+
/** The param mapping document url. */
private static String PARAM_MAPPING_DOCUMENT_URL = "url";
- /* (non-Javadoc)
- * @see org.apache.solr.handler.RequestHandlerBase#handleRequestBody(org.apache.solr.request.SolrQueryRequest, org.apache.solr.response.SolrQueryResponse)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.solr.handler.RequestHandlerBase#handleRequestBody(org.apache.
+ * solr.request.SolrQueryRequest, org.apache.solr.response.SolrQueryResponse)
*/
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
@@ -76,7 +77,7 @@ public class MtasRequestHandler extends RequestHandlerBase {
InputStream is;
try {
is = req.getCore().getResourceLoader().openResource(file);
- rsp.add(ACTION_CONFIG_FILE, IOUtils.toString(is));
+ rsp.add(ACTION_CONFIG_FILE, IOUtils.toString(is, "UTF-8"));
} catch (IOException e) {
rsp.add(ERROR, e.getMessage());
}
@@ -85,7 +86,7 @@ public class MtasRequestHandler extends RequestHandlerBase {
} else if (req.getParams().get(PARAM_ACTION, "false")
.equals(ACTION_MAPPING)) {
String configuration = null, document = null, documentUrl = null;
- if(req.getContentStreams()!=null) {
+ if (req.getContentStreams() != null) {
Iterator<ContentStream> it = req.getContentStreams().iterator();
if (it.hasNext()) {
ContentStream cs = it.next();
@@ -94,32 +95,32 @@ public class MtasRequestHandler extends RequestHandlerBase {
configuration = params.get(PARAM_MAPPING_CONFIGURATION);
document = params.get(PARAM_MAPPING_DOCUMENT);
documentUrl = params.get(PARAM_MAPPING_DOCUMENT_URL);
- }
+ }
} else {
configuration = req.getParams().get(PARAM_MAPPING_CONFIGURATION);
document = req.getParams().get(PARAM_MAPPING_DOCUMENT);
documentUrl = req.getParams().get(PARAM_MAPPING_DOCUMENT_URL);
- }
- if (configuration != null && documentUrl != null) {
+ }
+ if (configuration != null && documentUrl != null) {
try {
- MtasTokenizer tokenizer = new MtasTokenizer(
- IOUtils.toInputStream(configuration));
+ MtasTokenizer<String> tokenizer = new MtasTokenizer<String>(
+ IOUtils.toInputStream(configuration, "UTF-8"));
MtasFetchData fetchData = new MtasFetchData(
new StringReader(documentUrl));
- rsp.add(ACTION_MAPPING, tokenizer.getList(fetchData.getUrl(null)));
+ rsp.add(ACTION_MAPPING, tokenizer.getList(fetchData.getUrl(null, null)));
tokenizer.close();
} catch (IOException | MtasParserException e) {
rsp.add(ERROR, e.getMessage());
}
} else if (configuration != null && document != null) {
try {
- MtasTokenizer tokenizer = new MtasTokenizer(
- IOUtils.toInputStream(configuration));
+ MtasTokenizer<String> tokenizer = new MtasTokenizer<String>(
+ IOUtils.toInputStream(configuration, "UTF-8"));
rsp.add(ACTION_MAPPING,
tokenizer.getList(new StringReader(document)));
tokenizer.close();
} catch (IOException | MtasParserException e) {
- rsp.add(ERROR, e.getMessage());
+ rsp.add(ERROR, e.getMessage());
}
}
}
@@ -128,8 +129,10 @@ public class MtasRequestHandler extends RequestHandlerBase {
/**
* Gets the files.
*
- * @param dir the dir
- * @param subDir the sub dir
+ * @param dir
+ * the dir
+ * @param subDir
+ * the sub dir
* @return the files
*/
private ArrayList<String> getFiles(String dir, String subDir) {
@@ -148,7 +151,9 @@ public class MtasRequestHandler extends RequestHandlerBase {
return files;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.solr.handler.RequestHandlerBase#getDescription()
*/
@Override
@@ -159,10 +164,13 @@ public class MtasRequestHandler extends RequestHandlerBase {
/**
* Gets the params from json.
*
- * @param params the params
- * @param json the json
+ * @param params
+ * the params
+ * @param json
+ * the json
* @return the params from json
*/
+ @SuppressWarnings("unchecked")
private static void getParamsFromJSON(Map<String, String> params,
String json) {
JSONParser parser = new JSONParser(json);
@@ -195,6 +203,5 @@ public class MtasRequestHandler extends RequestHandlerBase {
return;
}
}
-
}
diff --git a/src/mtas/solr/handler/component/MtasSolrSearchComponent.java b/src/mtas/solr/handler/component/MtasSolrSearchComponent.java
index 0797edd..58c9256 100644
--- a/src/mtas/solr/handler/component/MtasSolrSearchComponent.java
+++ b/src/mtas/solr/handler/component/MtasSolrSearchComponent.java
@@ -1,68 +1,35 @@
package mtas.solr.handler.component;
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Reader;
-import java.io.Serializable;
-import java.io.StringReader;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import mtas.analysis.token.MtasToken;
import mtas.codec.MtasCodecPostingsFormat;
-import mtas.codec.util.DataCollector;
-import mtas.codec.util.DataCollector.MtasDataCollector;
-import mtas.codec.util.collector.MtasDataItem;
import mtas.codec.util.CodecComponent.ComponentFacet;
-import mtas.codec.util.CodecComponent.ComponentField;
import mtas.codec.util.CodecComponent.ComponentFields;
import mtas.codec.util.CodecComponent.ComponentGroup;
import mtas.codec.util.CodecComponent.ComponentKwic;
import mtas.codec.util.CodecComponent.ComponentList;
import mtas.codec.util.CodecComponent.ComponentPosition;
-import mtas.codec.util.CodecComponent.ComponentPrefix;
import mtas.codec.util.CodecComponent.ComponentSpan;
import mtas.codec.util.CodecComponent.ComponentTermVector;
import mtas.codec.util.CodecComponent.ComponentToken;
-import mtas.codec.util.CodecComponent.KwicHit;
-import mtas.codec.util.CodecComponent.KwicToken;
-import mtas.codec.util.CodecComponent.ListHit;
-import mtas.codec.util.CodecComponent.ListToken;
import mtas.codec.util.CodecUtil;
-import mtas.parser.cql.MtasCQLParser;
-import mtas.parser.function.ParseException;
+import mtas.solr.handler.component.util.MtasSolrResultMerge;
+import mtas.solr.handler.component.util.MtasSolrComponentFacet;
+import mtas.solr.handler.component.util.MtasSolrComponentGroup;
+import mtas.solr.handler.component.util.MtasSolrComponentKwic;
+import mtas.solr.handler.component.util.MtasSolrComponentList;
+import mtas.solr.handler.component.util.MtasSolrComponentPrefix;
+import mtas.solr.handler.component.util.MtasSolrComponentStats;
+import mtas.solr.handler.component.util.MtasSolrComponentTermvector;
-import org.apache.lucene.document.FieldType.LegacyNumericType;
-import org.apache.lucene.search.spans.SpanQuery;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.handler.component.ShardRequest;
-import org.apache.solr.handler.component.ShardResponse;
-import org.apache.solr.schema.FieldType;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocList;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SolrIndexSearcher;
@@ -72,284 +39,66 @@ import org.apache.solr.search.SolrIndexSearcher;
*/
public class MtasSolrSearchComponent extends SearchComponent {
- /** The Constant QUERY_TYPE_CQL. */
- public static final String QUERY_TYPE_CQL = "cql";
+ /** The search component. */
+ MtasSolrSearchComponent searchComponent;
/** The Constant PARAM_MTAS. */
public static final String PARAM_MTAS = "mtas";
- /** The Constant PARAM_MTAS_FACET. */
- public static final String PARAM_MTAS_FACET = PARAM_MTAS + ".facet";
-
- /** The Constant NAME_MTAS_FACET_KEY. */
- public static final String NAME_MTAS_FACET_KEY = "key";
-
- /** The Constant NAME_MTAS_FACET_FIELD. */
- public static final String NAME_MTAS_FACET_FIELD = "field";
-
- /** The Constant NAME_MTAS_FACET_QUERY. */
- private static final String NAME_MTAS_FACET_QUERY = "query";
-
- /** The Constant NAME_MTAS_FACET_BASE. */
- private static final String NAME_MTAS_FACET_BASE = "base";
-
- /** The Constant SUBNAME_MTAS_FACET_QUERY_TYPE. */
- public static final String SUBNAME_MTAS_FACET_QUERY_TYPE = "type";
-
- /** The Constant SUBNAME_MTAS_FACET_QUERY_VALUE. */
- public static final String SUBNAME_MTAS_FACET_QUERY_VALUE = "value";
-
- /** The Constant SUBNAME_MTAS_FACET_BASE_FIELD. */
- public static final String SUBNAME_MTAS_FACET_BASE_FIELD = "field";
-
- /** The Constant SUBNAME_MTAS_FACET_BASE_TYPE. */
- public static final String SUBNAME_MTAS_FACET_BASE_TYPE = "type";
-
- /** The Constant SUBNAME_MTAS_FACET_BASE_SORT_TYPE. */
- public static final String SUBNAME_MTAS_FACET_BASE_SORT_TYPE = "sort.type";
-
- /** The Constant SUBNAME_MTAS_FACET_BASE_SORT_DIRECTION. */
- public static final String SUBNAME_MTAS_FACET_BASE_SORT_DIRECTION = "sort.direction";
-
- /** The Constant SUBNAME_MTAS_FACET_BASE_NUMBER. */
- public static final String SUBNAME_MTAS_FACET_BASE_NUMBER = "number";
-
- /** The Constant SUBNAME_MTAS_FACET_BASE_MINIMUM. */
- public static final String SUBNAME_MTAS_FACET_BASE_MINIMUM = "minimum";
-
- /** The Constant SUBNAME_MTAS_FACET_BASE_MAXIMUM. */
- public static final String SUBNAME_MTAS_FACET_BASE_MAXIMUM = "maximum";
-
- /** The Constant SUBNAME_MTAS_FACET_BASE_FUNCTION. */
- public static final String SUBNAME_MTAS_FACET_BASE_FUNCTION = "function";
-
- /** The Constant PARAM_MTAS_KWIC. */
- public static final String PARAM_MTAS_KWIC = PARAM_MTAS + ".kwic";
-
- /** The Constant NAME_MTAS_KWIC_FIELD. */
- public static final String NAME_MTAS_KWIC_FIELD = "field";
-
- /** The Constant NAME_MTAS_KWIC_QUERY_TYPE. */
- public static final String NAME_MTAS_KWIC_QUERY_TYPE = "query.type";
-
- /** The Constant NAME_MTAS_KWIC_QUERY_VALUE. */
- public static final String NAME_MTAS_KWIC_QUERY_VALUE = "query.value";
-
- /** The Constant NAME_MTAS_KWIC_KEY. */
- public static final String NAME_MTAS_KWIC_KEY = "key";
-
- /** The Constant NAME_MTAS_KWIC_PREFIX. */
- public static final String NAME_MTAS_KWIC_PREFIX = "prefix";
-
- /** The Constant NAME_MTAS_KWIC_NUMBER. */
- public static final String NAME_MTAS_KWIC_NUMBER = "number";
-
- /** The Constant NAME_MTAS_KWIC_START. */
- public static final String NAME_MTAS_KWIC_START = "start";
-
- /** The Constant NAME_MTAS_KWIC_LEFT. */
- public static final String NAME_MTAS_KWIC_LEFT = "left";
-
- /** The Constant NAME_MTAS_KWIC_RIGHT. */
- public static final String NAME_MTAS_KWIC_RIGHT = "right";
-
- /** The Constant NAME_MTAS_KWIC_OUTPUT. */
- public static final String NAME_MTAS_KWIC_OUTPUT = "output";
-
- /** The Constant PARAM_MTAS_LIST. */
- public static final String PARAM_MTAS_LIST = PARAM_MTAS + ".list";
-
- /** The Constant NAME_MTAS_LIST_FIELD. */
- public static final String NAME_MTAS_LIST_FIELD = "field";
-
- /** The Constant NAME_MTAS_LIST_QUERY_TYPE. */
- public static final String NAME_MTAS_LIST_QUERY_TYPE = "query.type";
-
- /** The Constant NAME_MTAS_LIST_QUERY_VALUE. */
- public static final String NAME_MTAS_LIST_QUERY_VALUE = "query.value";
-
- /** The Constant NAME_MTAS_LIST_KEY. */
- public static final String NAME_MTAS_LIST_KEY = "key";
-
- /** The Constant NAME_MTAS_LIST_PREFIX. */
- public static final String NAME_MTAS_LIST_PREFIX = "prefix";
-
- /** The Constant NAME_MTAS_LIST_START. */
- public static final String NAME_MTAS_LIST_START = "start";
-
- /** The Constant NAME_MTAS_LIST_NUMBER. */
- public static final String NAME_MTAS_LIST_NUMBER = "number";
-
- /** The Constant NAME_MTAS_LIST_LEFT. */
- public static final String NAME_MTAS_LIST_LEFT = "left";
-
- /** The Constant NAME_MTAS_LIST_RIGHT. */
- public static final String NAME_MTAS_LIST_RIGHT = "right";
-
- /** The Constant NAME_MTAS_LIST_OUTPUT. */
- public static final String NAME_MTAS_LIST_OUTPUT = "output";
-
- /** The Constant PARAM_MTAS_GROUP. */
- public static final String PARAM_MTAS_GROUP = PARAM_MTAS + ".group";
-
- /** The Constant NAME_MTAS_GROUP_FIELD. */
- public static final String NAME_MTAS_GROUP_FIELD = "field";
-
- /** The Constant NAME_MTAS_GROUP_QUERY_TYPE. */
- public static final String NAME_MTAS_GROUP_QUERY_TYPE = "query.type";
-
- /** The Constant NAME_MTAS_GROUP_QUERY_VALUE. */
- public static final String NAME_MTAS_GROUP_QUERY_VALUE = "query.value";
-
- /** The Constant NAME_MTAS_GROUP_KEY. */
- public static final String NAME_MTAS_GROUP_KEY = "key";
-
- /** The Constant NAME_MTAS_GROUP_GROUPING_LEFT. */
- public static final String NAME_MTAS_GROUP_GROUPING_LEFT = "grouping.left";
-
- /** The Constant NAME_MTAS_GROUP_GROUPING_RIGHT. */
- public static final String NAME_MTAS_GROUP_GROUPING_RIGHT = "grouping.right";
-
- /** The Constant NAME_MTAS_GROUP_GROUPING_HIT_INSIDE. */
- public static final String NAME_MTAS_GROUP_GROUPING_HIT_INSIDE = "grouping.hit.inside";
-
- /** The Constant NAME_MTAS_GROUP_GROUPING_HIT_LEFT. */
- public static final String NAME_MTAS_GROUP_GROUPING_HIT_LEFT = "grouping.hit.left";
-
- /** The Constant NAME_MTAS_GROUP_GROUPING_HIT_RIGHT. */
- public static final String NAME_MTAS_GROUP_GROUPING_HIT_RIGHT = "grouping.hit.right";
-
- /** The Constant NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_LEFT. */
- public static final String NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_LEFT = "grouping.hit.insideLeft";
-
- /** The Constant NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_RIGHT. */
- public static final String NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_RIGHT = "grouping.hit.insideRight";
-
- /** The Constant NAME_MTAS_GROUP_GROUPING_POSITION. */
- public static final String NAME_MTAS_GROUP_GROUPING_POSITION = "position";
-
- /** The Constant NAME_MTAS_GROUP_GROUPING_PREFIXES. */
- public static final String NAME_MTAS_GROUP_GROUPING_PREFIXES = "prefixes";
-
- /** The Constant PARAM_MTAS_TERMVECTOR. */
- public static final String PARAM_MTAS_TERMVECTOR = PARAM_MTAS + ".termvector";
-
- /** The Constant NAME_MTAS_TERMVECTOR_FIELD. */
- public static final String NAME_MTAS_TERMVECTOR_FIELD = "field";
-
- /** The Constant NAME_MTAS_TERMVECTOR_KEY. */
- public static final String NAME_MTAS_TERMVECTOR_KEY = "key";
-
- /** The Constant NAME_MTAS_TERMVECTOR_PREFIX. */
- public static final String NAME_MTAS_TERMVECTOR_PREFIX = "prefix";
-
- /** The Constant NAME_MTAS_TERMVECTOR_REGEXP. */
- public static final String NAME_MTAS_TERMVECTOR_REGEXP = "regexp";
-
- /** The Constant NAME_MTAS_TERMVECTOR_TYPE. */
- public static final String NAME_MTAS_TERMVECTOR_TYPE = "type";
-
- /** The Constant NAME_MTAS_TERMVECTOR_SORT_TYPE. */
- public static final String NAME_MTAS_TERMVECTOR_SORT_TYPE = "sort.type";
-
- /** The Constant NAME_MTAS_TERMVECTOR_SORT_DIRECTION. */
- public static final String NAME_MTAS_TERMVECTOR_SORT_DIRECTION = "sort.direction";
-
- /** The Constant NAME_MTAS_TERMVECTOR_START. */
- public static final String NAME_MTAS_TERMVECTOR_START = "start";
-
- /** The Constant NAME_MTAS_TERMVECTOR_NUMBER. */
- public static final String NAME_MTAS_TERMVECTOR_NUMBER = "number";
-
- /** The Constant NAME_MTAS_TERMVECTOR_MINIMUM. */
- public static final String NAME_MTAS_TERMVECTOR_MINIMUM = "minimum";
-
- /** The Constant NAME_MTAS_TERMVECTOR_MAXIMUM. */
- public static final String NAME_MTAS_TERMVECTOR_MAXIMUM = "maximum";
-
- /** The Constant NAME_MTAS_TERMVECTOR_FUNCTION. */
- public static final String NAME_MTAS_TERMVECTOR_FUNCTION = "function";
+ /** The Constant STAGE_TERMVECTOR_MISSING_TOP. */
+ public static final int STAGE_TERMVECTOR_MISSING_TOP = ResponseBuilder.STAGE_EXECUTE_QUERY
+ + 10;
- /** The Constant PARAM_MTAS_PREFIX. */
- public static final String PARAM_MTAS_PREFIX = PARAM_MTAS + ".prefix";
+ /** The Constant STAGE_TERMVECTOR_MISSING_KEY. */
+ public static final int STAGE_TERMVECTOR_MISSING_KEY = ResponseBuilder.STAGE_EXECUTE_QUERY
+ + 11;
- /** The Constant NAME_MTAS_PREFIX_FIELD. */
- public static final String NAME_MTAS_PREFIX_FIELD = "field";
+ /** The Constant STAGE_TERMVECTOR_FINISH. */
+ public static final int STAGE_TERMVECTOR_FINISH = ResponseBuilder.STAGE_EXECUTE_QUERY
+ + 12;
- /** The Constant NAME_MTAS_PREFIX_KEY. */
- public static final String NAME_MTAS_PREFIX_KEY = "key";
+ /** The Constant STAGE_LIST. */
+ public static final int STAGE_LIST = ResponseBuilder.STAGE_EXECUTE_QUERY + 20;
- /** The Constant PARAM_MTAS_STATS. */
- public static final String PARAM_MTAS_STATS = PARAM_MTAS + ".stats";
+ /** The Constant STAGE_PREFIX. */
+ public static final int STAGE_PREFIX = ResponseBuilder.STAGE_EXECUTE_QUERY
+ + 30;
- /** The Constant PARAM_MTAS_STATS_POSITIONS. */
- public static final String PARAM_MTAS_STATS_POSITIONS = PARAM_MTAS_STATS
- + ".positions";
+ /** The Constant STAGE_STATS. */
+ public static final int STAGE_STATS = ResponseBuilder.STAGE_EXECUTE_QUERY
+ + 40;
- /** The Constant NAME_MTAS_STATS_POSITIONS_FIELD. */
- public static final String NAME_MTAS_STATS_POSITIONS_FIELD = "field";
+ /** The Constant STAGE_FACET. */
+ public static final int STAGE_FACET = ResponseBuilder.STAGE_EXECUTE_QUERY
+ + 50;
- /** The Constant NAME_MTAS_STATS_POSITIONS_KEY. */
- public static final String NAME_MTAS_STATS_POSITIONS_KEY = "key";
+ /** The Constant STAGE_GROUP. */
+ public static final int STAGE_GROUP = ResponseBuilder.STAGE_EXECUTE_QUERY
+ + 60;
- /** The Constant NAME_MTAS_STATS_POSITIONS_TYPE. */
- public static final String NAME_MTAS_STATS_POSITIONS_TYPE = "type";
+ /** The mtas solr result merge. */
+ private MtasSolrResultMerge mtasSolrResultMerge;
- /** The Constant NAME_MTAS_STATS_POSITIONS_MINIMUM. */
- public static final String NAME_MTAS_STATS_POSITIONS_MINIMUM = "minimum";
+ /** The search stats. */
+ private MtasSolrComponentStats searchStats;
- /** The Constant NAME_MTAS_STATS_POSITIONS_MAXIMUM. */
- public static final String NAME_MTAS_STATS_POSITIONS_MAXIMUM = "maximum";
+ /** The search termvector. */
+ private MtasSolrComponentTermvector searchTermvector;
- /** The Constant PARAM_MTAS_STATS_TOKENS. */
- public static final String PARAM_MTAS_STATS_TOKENS = PARAM_MTAS_STATS
- + ".tokens";
+ /** The search prefix. */
+ private MtasSolrComponentPrefix searchPrefix;
- /** The Constant NAME_MTAS_STATS_TOKENS_FIELD. */
- public static final String NAME_MTAS_STATS_TOKENS_FIELD = "field";
+ /** The search facet. */
+ private MtasSolrComponentFacet searchFacet;
- /** The Constant NAME_MTAS_STATS_TOKENS_KEY. */
- public static final String NAME_MTAS_STATS_TOKENS_KEY = "key";
+ /** The search group. */
+ private MtasSolrComponentGroup searchGroup;
- /** The Constant NAME_MTAS_STATS_TOKENS_TYPE. */
- public static final String NAME_MTAS_STATS_TOKENS_TYPE = "type";
+ /** The search list. */
+ private MtasSolrComponentList searchList;
- /** The Constant NAME_MTAS_STATS_TOKENS_MINIMUM. */
- public static final String NAME_MTAS_STATS_TOKENS_MINIMUM = "minimum";
-
- /** The Constant NAME_MTAS_STATS_TOKENS_MAXIMUM. */
- public static final String NAME_MTAS_STATS_TOKENS_MAXIMUM = "maximum";
-
- /** The Constant PARAM_MTAS_STATS_SPANS. */
- public static final String PARAM_MTAS_STATS_SPANS = PARAM_MTAS_STATS
- + ".spans";
-
- /** The Constant NAME_MTAS_STATS_SPANS_FIELD. */
- public static final String NAME_MTAS_STATS_SPANS_FIELD = "field";
-
- /** The Constant NAME_MTAS_STATS_SPANS_QUERY. */
- public static final String NAME_MTAS_STATS_SPANS_QUERY = "query";
-
- /** The Constant NAME_MTAS_STATS_SPANS_KEY. */
- public static final String NAME_MTAS_STATS_SPANS_KEY = "key";
-
- /** The Constant NAME_MTAS_STATS_SPANS_TYPE. */
- public static final String NAME_MTAS_STATS_SPANS_TYPE = "type";
-
- /** The Constant NAME_MTAS_STATS_SPANS_MINIMUM. */
- public static final String NAME_MTAS_STATS_SPANS_MINIMUM = "minimum";
-
- /** The Constant NAME_MTAS_STATS_SPANS_MAXIMUM. */
- public static final String NAME_MTAS_STATS_SPANS_MAXIMUM = "maximum";
-
- /** The Constant NAME_MTAS_STATS_SPANS_FUNCTION. */
- public static final String NAME_MTAS_STATS_SPANS_FUNCTION = "function";
-
- /** The Constant SUBNAME_MTAS_STATS_SPANS_QUERY_TYPE. */
- public static final String SUBNAME_MTAS_STATS_SPANS_QUERY_TYPE = "type";
-
- /** The Constant SUBNAME_MTAS_STATS_SPANS_QUERY_VALUE. */
- public static final String SUBNAME_MTAS_STATS_SPANS_QUERY_VALUE = "value";
+ /** The search kwic. */
+ private MtasSolrComponentKwic searchKwic;
/*
* (non-Javadoc)
@@ -385,1163 +134,56 @@ public class MtasSolrSearchComponent extends SearchComponent {
// + rb.req.getParams().getBool("isShard", false) + " PREPARE " + rb.stage
// + " " + rb.req.getParamString());
if (rb.req.getParams().getBool(PARAM_MTAS, false)) {
+
+ mtasSolrResultMerge = new MtasSolrResultMerge();
+
ComponentFields mtasFields = new ComponentFields();
// get settings kwic
- if (rb.req.getParams().getBool(PARAM_MTAS_KWIC, false)) {
- prepareKwic(rb, mtasFields);
+ if (rb.req.getParams().getBool(MtasSolrComponentKwic.PARAM_MTAS_KWIC,
+ false)) {
+ searchKwic = new MtasSolrComponentKwic(this);
+ searchKwic.prepare(rb, mtasFields);
}
// get settings list
- if (rb.req.getParams().getBool(PARAM_MTAS_LIST, false)) {
- prepareList(rb, mtasFields);
+ if (rb.req.getParams().getBool(MtasSolrComponentList.PARAM_MTAS_LIST,
+ false)) {
+ searchList = new MtasSolrComponentList(this);
+ searchList.prepare(rb, mtasFields);
}
// get settings group
- if (rb.req.getParams().getBool(PARAM_MTAS_GROUP, false)) {
- prepareGroup(rb, mtasFields);
+ if (rb.req.getParams().getBool(MtasSolrComponentGroup.PARAM_MTAS_GROUP,
+ false)) {
+ searchGroup = new MtasSolrComponentGroup(this);
+ searchGroup.prepare(rb, mtasFields);
}
// get settings termvector
- if (rb.req.getParams().getBool(PARAM_MTAS_TERMVECTOR, false)) {
- prepareTermVector(rb, mtasFields);
+ if (rb.req.getParams()
+ .getBool(MtasSolrComponentTermvector.PARAM_MTAS_TERMVECTOR, false)) {
+ searchTermvector = new MtasSolrComponentTermvector(this);
+ searchTermvector.prepare(rb, mtasFields);
}
// get settings prefix
- if (rb.req.getParams().getBool(PARAM_MTAS_PREFIX, false)) {
- preparePrefix(rb, mtasFields);
+ if (rb.req.getParams().getBool(MtasSolrComponentPrefix.PARAM_MTAS_PREFIX,
+ false)) {
+ searchPrefix = new MtasSolrComponentPrefix(this);
+ searchPrefix.prepare(rb, mtasFields);
}
// get settings stats
- if (rb.req.getParams().getBool(PARAM_MTAS_STATS, false)) {
- prepareStats(rb, mtasFields);
+ if (rb.req.getParams().getBool(MtasSolrComponentStats.PARAM_MTAS_STATS,
+ false)) {
+ searchStats = new MtasSolrComponentStats(this);
+ searchStats.prepare(rb, mtasFields);
}
// get settings facet
- if (rb.req.getParams().getBool(PARAM_MTAS_FACET, false)) {
- prepareFacet(rb, mtasFields);
+ if (rb.req.getParams().getBool(MtasSolrComponentFacet.PARAM_MTAS_FACET,
+ false)) {
+ searchFacet = new MtasSolrComponentFacet(this);
+ searchFacet.prepare(rb, mtasFields);
}
rb.req.getContext().put(ComponentFields.class, mtasFields);
}
}
- /**
- * Gets the field type.
- *
- * @param schema the schema
- * @param field the field
- * @return the field type
- */
- private String getFieldType(IndexSchema schema, String field) {
- SchemaField sf = schema.getField(field);
- FieldType ft = sf.getType();
- if (ft != null) {
- if (ft.getNumericType() != null) {
- LegacyNumericType nt = ft.getNumericType();
- if (nt.equals(LegacyNumericType.INT)) {
- return ComponentFacet.TYPE_INTEGER;
- } else if (nt.equals(LegacyNumericType.DOUBLE)) {
- return ComponentFacet.TYPE_DOUBLE;
- } else if (nt.equals(LegacyNumericType.LONG)) {
- return ComponentFacet.TYPE_LONG;
- } else if (nt.equals(LegacyNumericType.FLOAT)) {
- return ComponentFacet.TYPE_FLOAT;
- }
- }
- }
- return ComponentFacet.TYPE_STRING;
- }
-
- /**
- * Prepare kwic.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void prepareKwic(ResponseBuilder rb, ComponentFields mtasFields)
- throws IOException {
- Set<String> ids = getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_KWIC);
- if (ids.size() > 0) {
- int tmpCounter = 0;
- String[] fields = new String[ids.size()];
- String[] queryTypes = new String[ids.size()];
- String[] queryValues = new String[ids.size()];
- String[] keys = new String[ids.size()];
- String[] prefixes = new String[ids.size()];
- String[] numbers = new String[ids.size()];
- String[] starts = new String[ids.size()];
- String[] lefts = new String[ids.size()];
- String[] rights = new String[ids.size()];
- String[] outputs = new String[ids.size()];
- for (String id : ids) {
- fields[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_FIELD, null);
- queryTypes[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_QUERY_TYPE, null);
- queryValues[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_QUERY_VALUE,
- null);
- keys[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_KEY,
- String.valueOf(tmpCounter))
- .trim();
- prefixes[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_PREFIX, null);
- numbers[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_NUMBER, null);
- starts[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_START, null);
- lefts[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_LEFT, null);
- rights[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_RIGHT, null);
- starts[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_START, null);
- outputs[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_OUTPUT, null);
- tmpCounter++;
- }
- String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
- mtasFields.doKwic = true;
- rb.setNeedDocList(true);
- for (String field : fields) {
- if (field == null || field.isEmpty()) {
- throw new IOException("no (valid) field in mtas kwic");
- } else if (!mtasFields.list.containsKey(field)) {
- mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
- }
- }
- compareAndCheck(keys, fields, NAME_MTAS_KWIC_KEY, NAME_MTAS_KWIC_FIELD,
- true);
- compareAndCheck(queryValues, fields, NAME_MTAS_KWIC_QUERY_VALUE,
- NAME_MTAS_KWIC_FIELD, false);
- compareAndCheck(queryTypes, fields, NAME_MTAS_KWIC_QUERY_TYPE,
- NAME_MTAS_KWIC_FIELD, false);
- compareAndCheck(prefixes, fields, NAME_MTAS_KWIC_PREFIX,
- NAME_MTAS_KWIC_FIELD, false);
- compareAndCheck(numbers, fields, NAME_MTAS_KWIC_NUMBER,
- NAME_MTAS_KWIC_FIELD, false);
- compareAndCheck(starts, fields, NAME_MTAS_KWIC_START,
- NAME_MTAS_KWIC_FIELD, false);
- compareAndCheck(lefts, fields, NAME_MTAS_KWIC_LEFT, NAME_MTAS_KWIC_FIELD,
- false);
- compareAndCheck(rights, fields, NAME_MTAS_KWIC_RIGHT,
- NAME_MTAS_KWIC_FIELD, false);
- compareAndCheck(outputs, fields, NAME_MTAS_KWIC_OUTPUT,
- NAME_MTAS_KWIC_FIELD, false);
- for (int i = 0; i < fields.length; i++) {
- ComponentField cf = mtasFields.list.get(fields[i]);
- SpanQuery q = constructQuery(queryValues[i], queryTypes[i], fields[i]);
- // minimize number of queries
- if (cf.spanQueryList.contains(q)) {
- q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
- } else {
- cf.spanQueryList.add(q);
- }
- String key = (keys[i] == null) || (keys[i].isEmpty())
- ? String.valueOf(i) + ":" + fields[i] + ":" + queryValues[i]
- : keys[i].trim();
- String prefix = prefixes[i];
- Integer number = (numbers[i] != null) ? getPositiveInteger(numbers[i])
- : null;
- int start = getPositiveInteger(starts[i]);
- int left = getPositiveInteger(lefts[i]);
- int right = getPositiveInteger(rights[i]);
- String output = outputs[i];
- mtasFields.list.get(fields[i]).kwicList.add(new ComponentKwic(q, key,
- prefix, number, start, left, right, output));
- }
- }
- }
-
- /**
- * Prepare facet.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void prepareFacet(ResponseBuilder rb, ComponentFields mtasFields)
- throws IOException {
- Set<String> ids = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_FACET);
- if (ids.size() > 0) {
- int tmpCounter = 0;
- String tmpValue;
- String[] fields = new String[ids.size()];
- String[] keys = new String[ids.size()];
- String[][] queryTypes = new String[ids.size()][];
- String[][] queryValues = new String[ids.size()][];
- String[][] baseFields = new String[ids.size()][];
- String[][] baseFieldTypes = new String[ids.size()][];
- String[][] baseTypes = new String[ids.size()][];
- String[][] baseSortTypes = new String[ids.size()][];
- String[][] baseSortDirections = new String[ids.size()][];
- Integer[][] baseNumbers = new Integer[ids.size()][];
- Double[][] baseMinima = new Double[ids.size()][];
- Double[][] baseMaxima = new Double[ids.size()][];
- String[][] baseFunctions = new String[ids.size()][];
- for (String id : ids) {
- fields[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_FIELD, null);
- keys[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_KEY,
- String.valueOf(tmpCounter))
- .trim();
- Set<String> qIds = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_QUERY);
- if (qIds.size() > 0) {
- int tmpQCounter = 0;
- queryTypes[tmpCounter] = new String[qIds.size()];
- queryValues[tmpCounter] = new String[qIds.size()];
- for (String qId : qIds) {
- queryTypes[tmpCounter][tmpQCounter] = rb.req.getParams()
- .get(
- PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_QUERY
- + "." + qId + "." + SUBNAME_MTAS_FACET_QUERY_TYPE,
- null);
- queryValues[tmpCounter][tmpQCounter] = rb.req.getParams()
- .get(
- PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_QUERY
- + "." + qId + "." + SUBNAME_MTAS_FACET_QUERY_VALUE,
- null);
- tmpQCounter++;
- }
- } else {
- throw new IOException(
- "no " + NAME_MTAS_FACET_QUERY + " for mtas facet " + id);
- }
- Set<String> bIds = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE);
- if (bIds.size() > 0) {
- int tmpBCounter = 0;
- baseFields[tmpCounter] = new String[bIds.size()];
- baseFieldTypes[tmpCounter] = new String[bIds.size()];
- baseTypes[tmpCounter] = new String[bIds.size()];
- baseSortTypes[tmpCounter] = new String[bIds.size()];
- baseSortDirections[tmpCounter] = new String[bIds.size()];
- baseNumbers[tmpCounter] = new Integer[bIds.size()];
- baseMinima[tmpCounter] = new Double[bIds.size()];
- baseMaxima[tmpCounter] = new Double[bIds.size()];
- baseFunctions[tmpCounter] = new String[bIds.size()];
- for (String bId : bIds) {
- baseFields[tmpCounter][tmpBCounter] = rb.req.getParams()
- .get(
- PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
- + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_FIELD,
- null);
- baseFieldTypes[tmpCounter][tmpBCounter] = getFieldType(
- rb.req.getSchema(), baseFields[tmpCounter][tmpBCounter]);
- baseTypes[tmpCounter][tmpBCounter] = rb.req.getParams()
- .get(PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
- + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_TYPE, null);
- baseSortTypes[tmpCounter][tmpBCounter] = rb.req.getParams()
- .get(
- PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
- + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_SORT_TYPE,
- null);
- baseSortDirections[tmpCounter][tmpBCounter] = rb.req.getParams()
- .get(PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
- + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_SORT_DIRECTION,
- null);
- tmpValue = rb.req.getParams()
- .get(
- PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
- + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_NUMBER,
- null);
- baseNumbers[tmpCounter][tmpBCounter] = tmpValue != null
- ? getPositiveInteger(tmpValue) : null;
- tmpValue = rb.req.getParams()
- .get(
- PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
- + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_MINIMUM,
- null);
- baseMinima[tmpCounter][tmpBCounter] = tmpValue != null
- ? getDouble(tmpValue) : null;
- tmpValue = rb.req.getParams()
- .get(
- PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
- + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_MAXIMUM,
- null);
- baseMaxima[tmpCounter][tmpBCounter] = tmpValue != null
- ? getDouble(tmpValue) : null;
- baseFunctions[tmpCounter][tmpBCounter] = rb.req.getParams()
- .get(
- PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
- + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_FUNCTION,
- null);
- tmpBCounter++;
- }
- } else {
- throw new IOException(
- "no " + NAME_MTAS_FACET_BASE + " for mtas facet " + id);
- }
- tmpCounter++;
- }
- String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
- mtasFields.doFacet = true;
- rb.setNeedDocSet(true);
- for (String field : fields) {
- if (field == null || field.isEmpty()) {
- throw new IOException("no (valid) field in mtas facet");
- } else if (!mtasFields.list.containsKey(field)) {
- mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
- }
- }
- compareAndCheck(keys, fields, NAME_MTAS_FACET_KEY, NAME_MTAS_FACET_FIELD,
- true);
- for (int i = 0; i < fields.length; i++) {
- ComponentField cf = mtasFields.list.get(fields[i]);
- int queryNumber = queryValues[i].length;
- SpanQuery ql[] = new SpanQuery[queryNumber];
- for (int j = 0; j < queryNumber; j++) {
- SpanQuery q = constructQuery(queryValues[i][j], queryTypes[i][j],
- fields[i]);
- // minimize number of queries
- if (cf.spanQueryList.contains(q)) {
- q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
- } else {
- cf.spanQueryList.add(q);
- }
- ql[j] = q;
- }
- String key = (keys[i] == null) || (keys[i].isEmpty())
- ? String.valueOf(i) + ":" + fields[i] : keys[i].trim();
- try {
- mtasFields.list.get(fields[i]).facetList.add(new ComponentFacet(ql,
- fields[i], key, baseFields[i], baseFieldTypes[i], baseTypes[i],
- baseSortTypes[i], baseSortDirections[i], baseNumbers[i],
- baseMinima[i], baseMaxima[i], baseFunctions[i]));
- } catch (ParseException e) {
- throw new IOException(e.getMessage());
- }
- }
- }
- }
-
- /**
- * Prepare list.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void prepareList(ResponseBuilder rb, ComponentFields mtasFields)
- throws IOException {
- Set<String> ids = getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_LIST);
- if (ids.size() > 0) {
- int tmpCounter = 0;
- String[] fields = new String[ids.size()];
- String[] queryTypes = new String[ids.size()];
- String[] queryValues = new String[ids.size()];
- String[] keys = new String[ids.size()];
- String[] prefixes = new String[ids.size()];
- String[] starts = new String[ids.size()];
- String[] numbers = new String[ids.size()];
- String[] lefts = new String[ids.size()];
- String[] rights = new String[ids.size()];
- String[] outputs = new String[ids.size()];
- for (String id : ids) {
- fields[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_FIELD, null);
- keys[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_KEY,
- String.valueOf(tmpCounter))
- .trim();
- queryTypes[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_QUERY_TYPE, null);
- queryValues[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_QUERY_VALUE,
- null);
- prefixes[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_PREFIX, null);
- starts[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_START, null);
- numbers[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_NUMBER, null);
- lefts[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_LEFT, null);
- rights[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_RIGHT, null);
- outputs[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_OUTPUT, null);
- tmpCounter++;
- }
- String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
- mtasFields.doList = true;
- rb.setNeedDocSet(true);
- for (String field : fields) {
- if (field == null || field.isEmpty()) {
- throw new IOException("no (valid) field in mtas list");
- } else if (!mtasFields.list.containsKey(field)) {
- mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
- }
- }
- compareAndCheck(keys, fields, NAME_MTAS_LIST_KEY, NAME_MTAS_LIST_FIELD,
- true);
- compareAndCheck(prefixes, queryValues, NAME_MTAS_LIST_QUERY_VALUE,
- NAME_MTAS_LIST_FIELD, false);
- compareAndCheck(prefixes, queryTypes, NAME_MTAS_LIST_QUERY_TYPE,
- NAME_MTAS_LIST_FIELD, false);
- compareAndCheck(prefixes, fields, NAME_MTAS_LIST_PREFIX,
- NAME_MTAS_LIST_FIELD, false);
- compareAndCheck(starts, fields, NAME_MTAS_LIST_START,
- NAME_MTAS_LIST_FIELD, false);
- compareAndCheck(numbers, fields, NAME_MTAS_LIST_NUMBER,
- NAME_MTAS_LIST_FIELD, false);
- compareAndCheck(lefts, fields, NAME_MTAS_LIST_LEFT, NAME_MTAS_LIST_FIELD,
- false);
- compareAndCheck(rights, fields, NAME_MTAS_LIST_RIGHT,
- NAME_MTAS_LIST_FIELD, false);
- compareAndCheck(outputs, fields, NAME_MTAS_LIST_OUTPUT,
- NAME_MTAS_LIST_FIELD, false);
- for (int i = 0; i < fields.length; i++) {
- ComponentField cf = mtasFields.list.get(fields[i]);
- SpanQuery q = constructQuery(queryValues[i], queryTypes[i], fields[i]);
- // minimize number of queries
- if (cf.spanQueryList.contains(q)) {
- q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
- } else {
- cf.spanQueryList.add(q);
- }
- String key = (keys[i] == null) || (keys[i].isEmpty())
- ? String.valueOf(i) + ":" + fields[i] + ":" + queryValues[i]
- : keys[i].trim();
- String prefix = prefixes[i];
- int start = (starts[i] == null) || (starts[i].isEmpty()) ? 0
- : Integer.parseInt(starts[i]);
- int number = (numbers[i] == null) || (numbers[i].isEmpty()) ? 10
- : Integer.parseInt(numbers[i]);
- int left = (lefts[i] == null) || lefts[i].isEmpty() ? 0
- : Integer.parseInt(lefts[i]);
- int right = (rights[i] == null) || rights[i].isEmpty() ? 0
- : Integer.parseInt(rights[i]);
- String output = outputs[i];
- mtasFields.list.get(fields[i]).listList
- .add(new ComponentList(q, fields[i], queryValues[i], queryTypes[i],
- key, prefix, start, number, left, right, output));
- }
- }
- }
-
- /**
- * Prepare group.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void prepareGroup(ResponseBuilder rb, ComponentFields mtasFields)
- throws IOException {
- Set<String> ids = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_GROUP);
- if (ids.size() > 0) {
- int tmpCounter = 0;
- String[] fields = new String[ids.size()];
- String[] queryTypes = new String[ids.size()];
- String[] queryValues = new String[ids.size()];
- String[] keys = new String[ids.size()];
- String[][] groupingLeftPosition = new String[ids.size()][];
- String[][] groupingLeftPrefixes = new String[ids.size()][];
- String[][] groupingRightPosition = new String[ids.size()][];
- String[][] groupingRightPrefixes = new String[ids.size()][];
- String[] groupingHitInsidePrefixes = new String[ids.size()];
- String[][] groupingHitLeftPosition = new String[ids.size()][];
- String[][] groupingHitLeftPrefixes = new String[ids.size()][];
- String[][] groupingHitRightPosition = new String[ids.size()][];
- String[][] groupingHitRightPrefixes = new String[ids.size()][];
- String[][] groupingHitInsideLeftPosition = new String[ids.size()][];
- String[][] groupingHitInsideLeftPrefixes = new String[ids.size()][];
- String[][] groupingHitInsideRightPosition = new String[ids.size()][];
- String[][] groupingHitInsideRightPrefixes = new String[ids.size()][];
- for (String id : ids) {
- fields[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_GROUP + "." + id + "." + NAME_MTAS_GROUP_FIELD, null);
- keys[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_GROUP + "." + id + "." + NAME_MTAS_GROUP_KEY,
- String.valueOf(tmpCounter))
- .trim();
- queryTypes[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_GROUP + "." + id + "." + NAME_MTAS_GROUP_QUERY_TYPE,
- null);
- queryValues[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_GROUP + "." + id + "." + NAME_MTAS_GROUP_QUERY_VALUE,
- null);
- groupingHitInsidePrefixes[tmpCounter] = null;
- // collect
- SortedSet<String> gids;
- String tmpName;
- // collect grouping inside
- tmpName = PARAM_MTAS_GROUP + "." + id + "."
- + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE;
- groupingHitInsidePrefixes[tmpCounter] = rb.req.getParams()
- .get(tmpName + "." + NAME_MTAS_GROUP_GROUPING_PREFIXES);
- // collect grouping left
- tmpName = PARAM_MTAS_GROUP + "." + id + "."
- + NAME_MTAS_GROUP_GROUPING_LEFT;
- gids = getIdsFromParameters(rb.req.getParams(), tmpName);
- groupingLeftPosition[tmpCounter] = new String[gids.size()];
- groupingLeftPrefixes[tmpCounter] = new String[gids.size()];
- prepareGroup(rb.req.getParams(), gids, tmpName,
- groupingLeftPosition[tmpCounter], groupingLeftPrefixes[tmpCounter]);
- // collect grouping right
- tmpName = PARAM_MTAS_GROUP + "." + id + "."
- + NAME_MTAS_GROUP_GROUPING_RIGHT;
- gids = getIdsFromParameters(rb.req.getParams(), tmpName);
- groupingRightPosition[tmpCounter] = new String[gids.size()];
- groupingRightPrefixes[tmpCounter] = new String[gids.size()];
- prepareGroup(rb.req.getParams(), gids, tmpName,
- groupingRightPosition[tmpCounter],
- groupingRightPrefixes[tmpCounter]);
- // collect grouping hit left
- tmpName = PARAM_MTAS_GROUP + "." + id + "."
- + NAME_MTAS_GROUP_GROUPING_HIT_LEFT;
- gids = getIdsFromParameters(rb.req.getParams(), tmpName);
- groupingHitLeftPosition[tmpCounter] = new String[gids.size()];
- groupingHitLeftPrefixes[tmpCounter] = new String[gids.size()];
- prepareGroup(rb.req.getParams(), gids, tmpName,
- groupingHitLeftPosition[tmpCounter],
- groupingHitLeftPrefixes[tmpCounter]);
- // collect grouping hit right
- tmpName = PARAM_MTAS_GROUP + "." + id + "."
- + NAME_MTAS_GROUP_GROUPING_HIT_RIGHT;
- gids = getIdsFromParameters(rb.req.getParams(), tmpName);
- groupingHitRightPosition[tmpCounter] = new String[gids.size()];
- groupingHitRightPrefixes[tmpCounter] = new String[gids.size()];
- prepareGroup(rb.req.getParams(), gids, tmpName,
- groupingHitRightPosition[tmpCounter],
- groupingHitRightPrefixes[tmpCounter]);
- // collect grouping hit inside left
- tmpName = PARAM_MTAS_GROUP + "." + id + "."
- + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_LEFT;
- gids = getIdsFromParameters(rb.req.getParams(), tmpName);
- groupingHitInsideLeftPosition[tmpCounter] = new String[gids.size()];
- groupingHitInsideLeftPrefixes[tmpCounter] = new String[gids.size()];
- prepareGroup(rb.req.getParams(), gids, tmpName,
- groupingHitInsideLeftPosition[tmpCounter],
- groupingHitInsideLeftPrefixes[tmpCounter]);
- // collect grouping hit inside right
- tmpName = PARAM_MTAS_GROUP + "." + id + "."
- + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_RIGHT;
- gids = getIdsFromParameters(rb.req.getParams(), tmpName);
- groupingHitInsideRightPosition[tmpCounter] = new String[gids.size()];
- groupingHitInsideRightPrefixes[tmpCounter] = new String[gids.size()];
- prepareGroup(rb.req.getParams(), gids, tmpName,
- groupingHitInsideRightPosition[tmpCounter],
- groupingHitInsideRightPrefixes[tmpCounter]);
-
- tmpCounter++;
- }
- String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
- mtasFields.doGroup = true;
- rb.setNeedDocSet(true);
- for (String field : fields) {
- if (field == null || field.isEmpty()) {
- throw new IOException("no (valid) field in mtas group");
- } else if (!mtasFields.list.containsKey(field)) {
- mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
- }
- }
- compareAndCheck(keys, fields, NAME_MTAS_GROUP_KEY, NAME_MTAS_GROUP_FIELD,
- true);
- compareAndCheck(queryValues, fields, NAME_MTAS_GROUP_QUERY_VALUE,
- NAME_MTAS_GROUP_FIELD, false);
- compareAndCheck(queryTypes, fields, NAME_MTAS_GROUP_QUERY_TYPE,
- NAME_MTAS_GROUP_FIELD, false);
- for (int i = 0; i < fields.length; i++) {
- ComponentField cf = mtasFields.list.get(fields[i]);
- SpanQuery q = constructQuery(queryValues[i], queryTypes[i], fields[i]);
- // minimize number of queries
- if (cf.spanQueryList.contains(q)) {
- q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
- } else {
- cf.spanQueryList.add(q);
- }
- String key = (keys[i] == null) || (keys[i].isEmpty())
- ? String.valueOf(i) + ":" + fields[i] + ":" + queryValues[i]
- : keys[i].trim();
- mtasFields.list.get(fields[i]).groupList.add(new ComponentGroup(q,
- fields[i], queryValues[i], queryTypes[i], key,
- groupingHitInsidePrefixes[i], groupingHitInsideLeftPosition[i],
- groupingHitInsideLeftPrefixes[i], groupingHitInsideRightPosition[i],
- groupingHitInsideRightPrefixes[i], groupingHitLeftPosition[i],
- groupingHitLeftPrefixes[i], groupingHitRightPosition[i],
- groupingHitRightPrefixes[i], groupingLeftPosition[i],
- groupingLeftPrefixes[i], groupingRightPosition[i],
- groupingRightPrefixes[i]));
- }
- }
- }
-
- /**
- * Prepare group.
- *
- * @param solrParams the solr params
- * @param gids the gids
- * @param name the name
- * @param positions the positions
- * @param prefixes the prefixes
- */
- private void prepareGroup(SolrParams solrParams, SortedSet<String> gids,
- String name, String[] positions, String[] prefixes) {
- SortedSet<String> sgids;
- if (gids.size() > 0) {
- int tmpSubCounter = 0;
- for (String gid : gids) {
- positions[tmpSubCounter] = solrParams.get(
- name + "." + gid + "." + NAME_MTAS_GROUP_GROUPING_POSITION, null);
- prefixes[tmpSubCounter] = solrParams.get(
- name + "." + gid + "." + NAME_MTAS_GROUP_GROUPING_PREFIXES, null);
- tmpSubCounter++;
- }
- }
- }
-
- /**
- * Prepare term vector.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void prepareTermVector(ResponseBuilder rb, ComponentFields mtasFields)
- throws IOException {
- Set<String> ids = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_TERMVECTOR);
- if (ids.size() > 0) {
- int tmpCounter = 0;
- String[] fields = new String[ids.size()];
- String[] keys = new String[ids.size()];
- String[] prefixes = new String[ids.size()];
- String[] regexps = new String[ids.size()];
- String[] sortTypes = new String[ids.size()];
- String[] sortDirections = new String[ids.size()];
- String[] types = new String[ids.size()];
- String[] startValues = new String[ids.size()];
- String[] numbers = new String[ids.size()];
- String[] functions = new String[ids.size()];
- for (String id : ids) {
- fields[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_TERMVECTOR + "." + id + "." + NAME_MTAS_TERMVECTOR_FIELD,
- null);
- keys[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_TERMVECTOR + "." + id + "." + NAME_MTAS_TERMVECTOR_KEY,
- String.valueOf(tmpCounter)).trim();
- prefixes[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_TERMVECTOR
- + "." + id + "." + NAME_MTAS_TERMVECTOR_PREFIX, null);
- regexps[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_TERMVECTOR + "."
- + id + "." + NAME_MTAS_TERMVECTOR_REGEXP, null);
- sortTypes[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_TERMVECTOR
- + "." + id + "." + NAME_MTAS_TERMVECTOR_SORT_TYPE, null);
- sortDirections[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_TERMVECTOR + "." + id + "."
- + NAME_MTAS_TERMVECTOR_SORT_DIRECTION, null);
- startValues[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_TERMVECTOR + "." + id + "." + NAME_MTAS_TERMVECTOR_START,
- null);
- numbers[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_TERMVECTOR + "."
- + id + "." + NAME_MTAS_TERMVECTOR_NUMBER, null);
- types[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_TERMVECTOR + "." + id + "." + NAME_MTAS_TERMVECTOR_TYPE,
- null);
- functions[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_TERMVECTOR
- + "." + id + "." + NAME_MTAS_TERMVECTOR_FUNCTION, null);
- tmpCounter++;
- }
- String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
- mtasFields.doTermVector = true;
- rb.setNeedDocSet(true);
- // init and checks
- for (String field : fields) {
- if (field == null || field.isEmpty()) {
- throw new IOException("no (valid) field in mtas termvector");
- } else if (!mtasFields.list.containsKey(field)) {
- mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
- }
- }
- compareAndCheck(keys, fields, NAME_MTAS_TERMVECTOR_KEY,
- NAME_MTAS_TERMVECTOR_FIELD, true);
- compareAndCheck(prefixes, fields, NAME_MTAS_TERMVECTOR_PREFIX,
- NAME_MTAS_TERMVECTOR_FIELD, false);
- compareAndCheck(regexps, fields, NAME_MTAS_TERMVECTOR_REGEXP,
- NAME_MTAS_TERMVECTOR_FIELD, false);
- compareAndCheck(types, fields, NAME_MTAS_TERMVECTOR_TYPE,
- NAME_MTAS_TERMVECTOR_FIELD, false);
- compareAndCheck(sortTypes, fields, NAME_MTAS_TERMVECTOR_SORT_TYPE,
- NAME_MTAS_TERMVECTOR_FIELD, false);
- compareAndCheck(sortDirections, fields,
- NAME_MTAS_TERMVECTOR_SORT_DIRECTION, NAME_MTAS_TERMVECTOR_FIELD,
- false);
- compareAndCheck(startValues, fields, NAME_MTAS_TERMVECTOR_START,
- NAME_MTAS_TERMVECTOR_FIELD, false);
- compareAndCheck(numbers, fields, NAME_MTAS_TERMVECTOR_NUMBER,
- NAME_MTAS_TERMVECTOR_FIELD, false);
- compareAndCheck(functions, fields, NAME_MTAS_TERMVECTOR_FUNCTION,
- NAME_MTAS_TERMVECTOR_FIELD, false);
- for (int i = 0; i < fields.length; i++) {
- String field = fields[i];
- String prefix = (prefixes[i] == null) || (prefixes[i].isEmpty()) ? null
- : prefixes[i].trim();
- String key = (keys[i] == null) || (keys[i].isEmpty())
- ? String.valueOf(i) + ":" + field + ":" + prefix : keys[i].trim();
- String regexp = (regexps[i] == null) || (regexps[i].isEmpty()) ? null
- : regexps[i].trim();
- String startValue = (startValues[i] == null)
- || (startValues[i].isEmpty()) ? null : startValues[i].trim();
- int number = (numbers[i] == null) || (numbers[i].isEmpty()) ? 0
- : Integer.parseInt(numbers[i]);
- String type = (types[i] == null) || (types[i].isEmpty()) ? null
- : types[i].trim();
- String sortType = (sortTypes[i] == null) || (sortTypes[i].isEmpty())
- ? CodecUtil.SORT_TERM : sortTypes[i].trim();
- String sortDirection = (sortDirections[i] == null)
- || (sortDirections[i].isEmpty()) ? null : sortDirections[i].trim();
- String function = functions[i];
- if (prefix == null || prefix.isEmpty()) {
- throw new IOException("no (valid) prefix in mtas termvector");
- } else {
- try {
- mtasFields.list.get(field).termVectorList
- .add(new ComponentTermVector(key, prefix, regexp, type,
- sortType, sortDirection, startValue, number, function));
- } catch (ParseException e) {
- throw new IOException(e.getMessage());
- }
- }
- }
-
- }
- }
-
- /**
- * Prepare prefix.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void preparePrefix(ResponseBuilder rb, ComponentFields mtasFields)
- throws IOException {
- Set<String> ids = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_PREFIX);
- if (ids.size() > 0) {
- int tmpCounter = 0;
- String[] fields = new String[ids.size()];
- String[] keys = new String[ids.size()];
- for (String id : ids) {
- fields[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_PREFIX + "." + id + "." + NAME_MTAS_PREFIX_FIELD, null);
- keys[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_PREFIX + "." + id + "." + NAME_MTAS_PREFIX_KEY,
- String.valueOf(tmpCounter))
- .trim();
- String uniqueKeyField = rb.req.getSchema().getUniqueKeyField()
- .getName();
- mtasFields.doPrefix = true;
- // init and checks
- for (String field : fields) {
- if (field == null || field.isEmpty()) {
- throw new IOException("no (valid) field in mtas prefix");
- } else if (!mtasFields.list.containsKey(field)) {
- mtasFields.list.put(field,
- new ComponentField(field, uniqueKeyField));
- }
- }
- compareAndCheck(keys, fields, NAME_MTAS_PREFIX_KEY,
- NAME_MTAS_PREFIX_FIELD, true);
- for (int i = 0; i < fields.length; i++) {
- String field = fields[i];
- String key = ((keys == null) || (keys[i] == null)
- || (keys[i].isEmpty())) ? String.valueOf(i) + ":" + field
- : keys[i].trim();
- mtasFields.list.get(field).prefix = new ComponentPrefix(key);
- }
- }
- }
- }
-
- /**
- * Prepare stats.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void prepareStats(ResponseBuilder rb, ComponentFields mtasFields)
- throws IOException {
- if (rb.req.getParams().getBool(PARAM_MTAS_STATS_POSITIONS, false)) {
- prepareStatsPositions(rb, mtasFields);
- }
- if (rb.req.getParams().getBool(PARAM_MTAS_STATS_TOKENS, false)) {
- prepareStatsTokens(rb, mtasFields);
- }
- if (rb.req.getParams().getBool(PARAM_MTAS_STATS_SPANS, false)) {
- prepareStatsSpans(rb, mtasFields);
- }
- }
-
- /**
- * Prepare stats positions.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void prepareStatsPositions(ResponseBuilder rb,
- ComponentFields mtasFields) throws IOException {
- Set<String> ids = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_STATS_POSITIONS);
- if (ids.size() > 0) {
- int tmpCounter = 0;
- String[] fields = new String[ids.size()];
- String[] keys = new String[ids.size()];
- String[] minima = new String[ids.size()];
- String[] maxima = new String[ids.size()];
- String[] types = new String[ids.size()];
- for (String id : ids) {
- fields[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_POSITIONS
- + "." + id + "." + NAME_MTAS_STATS_POSITIONS_FIELD, null);
- keys[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_STATS_POSITIONS + "." + id + "."
- + NAME_MTAS_STATS_POSITIONS_KEY, String.valueOf(tmpCounter))
- .trim();
- minima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_POSITIONS
- + "." + id + "." + NAME_MTAS_STATS_POSITIONS_MINIMUM, null);
- maxima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_POSITIONS
- + "." + id + "." + NAME_MTAS_STATS_POSITIONS_MAXIMUM, null);
- types[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_POSITIONS
- + "." + id + "." + NAME_MTAS_STATS_POSITIONS_TYPE, null);
- tmpCounter++;
- }
- String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
- mtasFields.doStats = true;
- mtasFields.doStatsPositions = true;
- rb.setNeedDocSet(true);
- for (String field : fields) {
- if (field == null || field.isEmpty()) {
- throw new IOException("no (valid) field in mtas stats positions");
- } else if (!mtasFields.list.containsKey(field)) {
- mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
- }
- }
- compareAndCheck(keys, fields, NAME_MTAS_STATS_POSITIONS_KEY,
- NAME_MTAS_STATS_POSITIONS_FIELD, true);
- compareAndCheck(minima, fields, NAME_MTAS_STATS_POSITIONS_MINIMUM,
- NAME_MTAS_STATS_POSITIONS_FIELD, false);
- compareAndCheck(maxima, fields, NAME_MTAS_STATS_POSITIONS_MAXIMUM,
- NAME_MTAS_STATS_POSITIONS_FIELD, false);
- compareAndCheck(types, fields, NAME_MTAS_STATS_POSITIONS_TYPE,
- NAME_MTAS_STATS_POSITIONS_FIELD, false);
- for (int i = 0; i < fields.length; i++) {
- String field = fields[i];
- String key = keys[i];
- String type = (types[i] == null) || (types[i].isEmpty()) ? null
- : types[i].trim();
- Double minimum = (minima[i] == null) || (minima[i].isEmpty()) ? null
- : Double.parseDouble(minima[i]);
- Double maximum = (maxima[i] == null) || (maxima[i].isEmpty()) ? null
- : Double.parseDouble(maxima[i]);
- try {
- mtasFields.list.get(field).statsPositionList
- .add(new ComponentPosition(field, key, minimum, maximum, type));
- } catch (ParseException e) {
- throw new IOException(e.getMessage());
- }
- }
- }
- }
-
- /**
- * Prepare stats tokens.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void prepareStatsTokens(ResponseBuilder rb,
- ComponentFields mtasFields) throws IOException {
- Set<String> ids = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_STATS_TOKENS);
- if (ids.size() > 0) {
- int tmpCounter = 0;
- String[] fields = new String[ids.size()];
- String[] keys = new String[ids.size()];
- String[] minima = new String[ids.size()];
- String[] maxima = new String[ids.size()];
- String[] types = new String[ids.size()];
- for (String id : ids) {
- fields[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_TOKENS
- + "." + id + "." + NAME_MTAS_STATS_TOKENS_FIELD, null);
- keys[tmpCounter] = rb.req.getParams()
- .get(PARAM_MTAS_STATS_TOKENS + "." + id + "."
- + NAME_MTAS_STATS_TOKENS_KEY, String.valueOf(tmpCounter))
- .trim();
- minima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_TOKENS
- + "." + id + "." + NAME_MTAS_STATS_TOKENS_MINIMUM, null);
- maxima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_TOKENS
- + "." + id + "." + NAME_MTAS_STATS_TOKENS_MAXIMUM, null);
- types[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_TOKENS + "."
- + id + "." + NAME_MTAS_STATS_TOKENS_TYPE, null);
- tmpCounter++;
- }
- String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
- mtasFields.doStats = true;
- mtasFields.doStatsTokens = true;
- rb.setNeedDocSet(true);
- for (String field : fields) {
- if (field == null || field.isEmpty()) {
- throw new IOException("no (valid) field in mtas stats tokens");
- } else if (!mtasFields.list.containsKey(field)) {
- mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
- }
- }
- compareAndCheck(keys, fields, NAME_MTAS_STATS_TOKENS_KEY,
- NAME_MTAS_STATS_TOKENS_FIELD, true);
- compareAndCheck(minima, fields, NAME_MTAS_STATS_TOKENS_MINIMUM,
- NAME_MTAS_STATS_TOKENS_FIELD, false);
- compareAndCheck(maxima, fields, NAME_MTAS_STATS_TOKENS_MAXIMUM,
- NAME_MTAS_STATS_TOKENS_FIELD, false);
- compareAndCheck(types, fields, NAME_MTAS_STATS_TOKENS_TYPE,
- NAME_MTAS_STATS_TOKENS_FIELD, false);
- for (int i = 0; i < fields.length; i++) {
- String field = fields[i];
- String key = keys[i];
- String type = (types[i] == null) || (types[i].isEmpty()) ? null
- : types[i].trim();
- Double minimum = (minima[i] == null) || (minima[i].isEmpty()) ? null
- : Double.parseDouble(minima[i]);
- Double maximum = (maxima[i] == null) || (maxima[i].isEmpty()) ? null
- : Double.parseDouble(maxima[i]);
- try {
- mtasFields.list.get(field).statsTokenList
- .add(new ComponentToken(field, key, minimum, maximum, type));
- } catch (ParseException e) {
- throw new IOException(e.getMessage());
- }
- }
- }
- }
-
- /**
- * Prepare stats spans.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void prepareStatsSpans(ResponseBuilder rb, ComponentFields mtasFields)
- throws IOException {
- SortedSet<String> ids = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_STATS_SPANS);
- if (ids.size() > 0) {
- int tmpCounter = 0;
- String[] fields = new String[ids.size()];
- String[] keys = new String[ids.size()];
- String[] minima = new String[ids.size()];
- String[] maxima = new String[ids.size()];
- String[] types = new String[ids.size()];
- String[] functions = new String[ids.size()];
- String[][] queryTypes = new String[ids.size()][];
- String[][] queryValues = new String[ids.size()][];
- for (String id : ids) {
- fields[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_SPANS + "."
- + id + "." + NAME_MTAS_STATS_SPANS_FIELD, null);
- keys[tmpCounter] = rb.req.getParams().get(
- PARAM_MTAS_STATS_SPANS + "." + id + "." + NAME_MTAS_STATS_SPANS_KEY,
- String.valueOf(tmpCounter)).trim();
- minima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_SPANS + "."
- + id + "." + NAME_MTAS_STATS_SPANS_MINIMUM, null);
- maxima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_SPANS + "."
- + id + "." + NAME_MTAS_STATS_SPANS_MAXIMUM, null);
- types[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_SPANS + "."
- + id + "." + NAME_MTAS_STATS_SPANS_TYPE, null);
- functions[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_SPANS
- + "." + id + "." + NAME_MTAS_STATS_SPANS_FUNCTION, null);
-
- Set<String> qIds = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_STATS_SPANS + "." + id + "."
- + NAME_MTAS_STATS_SPANS_QUERY);
- if (qIds.size() > 0) {
- int tmpQCounter = 0;
- queryTypes[tmpCounter] = new String[qIds.size()];
- queryValues[tmpCounter] = new String[qIds.size()];
- for (String qId : qIds) {
- queryTypes[tmpCounter][tmpQCounter] = rb.req.getParams()
- .get(PARAM_MTAS_STATS_SPANS + "." + id + "."
- + NAME_MTAS_STATS_SPANS_QUERY + "." + qId + "."
- + SUBNAME_MTAS_STATS_SPANS_QUERY_TYPE, null);
- queryValues[tmpCounter][tmpQCounter] = rb.req.getParams()
- .get(PARAM_MTAS_STATS_SPANS + "." + id + "."
- + NAME_MTAS_STATS_SPANS_QUERY + "." + qId + "."
- + SUBNAME_MTAS_STATS_SPANS_QUERY_VALUE, null);
- tmpQCounter++;
- }
- } else {
- throw new IOException("no " + NAME_MTAS_STATS_SPANS_QUERY
- + " for mtas stats span " + id);
- }
- tmpCounter++;
- }
- String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
- mtasFields.doStats = true;
- mtasFields.doStatsSpans = true;
- rb.setNeedDocSet(true);
- for (String field : fields) {
- if (field == null || field.isEmpty()) {
- throw new IOException("no (valid) field in mtas stats spans");
- } else if (!mtasFields.list.containsKey(field)) {
- mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
- }
- }
- compareAndCheck(keys, fields, NAME_MTAS_STATS_SPANS_KEY,
- NAME_MTAS_STATS_SPANS_FIELD, true);
- compareAndCheck(minima, fields, NAME_MTAS_STATS_SPANS_MINIMUM,
- NAME_MTAS_STATS_SPANS_FIELD, false);
- compareAndCheck(maxima, fields, NAME_MTAS_STATS_SPANS_MAXIMUM,
- NAME_MTAS_STATS_SPANS_FIELD, false);
- compareAndCheck(types, fields, NAME_MTAS_STATS_SPANS_TYPE,
- NAME_MTAS_STATS_SPANS_FIELD, false);
- compareAndCheck(types, fields, NAME_MTAS_STATS_SPANS_FUNCTION,
- NAME_MTAS_STATS_SPANS_FIELD, false);
-
- for (int i = 0; i < fields.length; i++) {
- ComponentField cf = mtasFields.list.get(fields[i]);
- int queryNumber = queryValues[i].length;
- SpanQuery ql[] = new SpanQuery[queryNumber];
- for (int j = 0; j < queryNumber; j++) {
- SpanQuery q = constructQuery(queryValues[i][j], queryTypes[i][j],
- fields[i]);
- // minimize number of queries
- if (cf.spanQueryList.contains(q)) {
- q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
- } else {
- cf.spanQueryList.add(q);
- }
- ql[j] = q;
- }
- Double minimum = (minima[i] == null) || (minima[i].isEmpty()) ? null
- : Double.parseDouble(minima[i]);
- Double maximum = (maxima[i] == null) || (maxima[i].isEmpty()) ? null
- : Double.parseDouble(maxima[i]);
- String key = (keys[i] == null) || (keys[i].isEmpty())
- ? String.valueOf(i) + ":" + fields[i] + ":" + queryNumber
- : keys[i].trim();
- String type = (types[i] == null) || (types[i].isEmpty()) ? null
- : types[i].trim();
- String function = (functions[i] == null) ? null : functions[i];
- mtasFields.list.get(fields[i]).statsSpanList
- .add(new ComponentSpan(ql, key, minimum, maximum, type, function));
- }
- } else {
- throw new IOException("missing parameters stats spans");
- }
- }
-
- /**
- * Gets the ids from parameters.
- *
- * @param params the params
- * @param prefix the prefix
- * @return the ids from parameters
- */
- private SortedSet<String> getIdsFromParameters(SolrParams params,
- String prefix) {
- SortedSet<String> ids = new TreeSet<String>();
- Iterator<String> it = params.getParameterNamesIterator();
- Pattern pattern = Pattern
- .compile("^" + Pattern.quote(prefix) + "\\.([^\\.]+)(\\..*|$)");
- while (it.hasNext()) {
- String item = it.next();
- Matcher m = pattern.matcher(item);
- if (m.matches()) {
- ids.add(m.group(1));
- }
- }
- return ids;
- }
-
- /**
- * Construct query.
- *
- * @param queryValue the query value
- * @param queryType the query type
- * @param field the field
- * @return the span query
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private SpanQuery constructQuery(String queryValue, String queryType,
- String field) throws IOException {
- if (queryType == null || queryType.isEmpty()) {
- throw new IOException("no (valid) type for query " + queryValue);
- } else if (queryValue == null || queryValue.isEmpty()) {
- throw new IOException("no (valid) value for " + queryType + " query");
- }
- Reader reader = new BufferedReader(new StringReader(queryValue));
- if (queryType.equals(QUERY_TYPE_CQL)) {
- MtasCQLParser p = new MtasCQLParser(reader);
- try {
- return p.parse(field);
- } catch (mtas.parser.cql.ParseException e) {
- throw new IOException(
- "couldn't parse " + queryType + " query " + queryValue);
- }
- } else {
- throw new IOException(
- "unknown queryType " + queryType + " for query " + queryValue);
- }
- }
-
- /**
- * Gets the positive integer.
- *
- * @param number the number
- * @return the positive integer
- */
- private int getPositiveInteger(String number) {
- try {
- return Math.max(0, Integer.parseInt(number));
- } catch (NumberFormatException e) {
- return 0;
- }
- }
-
- /**
- * Gets the double.
- *
- * @param number the number
- * @return the double
- */
- private Double getDouble(String number) {
- try {
- return Double.parseDouble(number);
- } catch (NumberFormatException e) {
- return null;
- }
- }
-
- /**
- * Compare and check.
- *
- * @param list the list
- * @param original the original
- * @param nameNew the name new
- * @param nameOriginal the name original
- * @param unique the unique
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void compareAndCheck(String[] list, String[] original, String nameNew,
- String nameOriginal, Boolean unique) throws IOException {
- if (list != null) {
- if (list.length != original.length) {
- throw new IOException(
- "unequal size " + nameNew + " and " + nameOriginal);
- }
- if (unique) {
- Set<String> set = new HashSet<String>();
- for (int i = 0; i < list.length; i++) {
- set.add(list[i]);
- }
- if (set.size() < list.length) {
- throw new IOException("duplicate " + nameNew);
- }
- }
- }
- }
-
/*
* (non-Javadoc)
*
@@ -1551,10 +193,10 @@ public class MtasSolrSearchComponent extends SearchComponent {
*/
@Override
public void process(ResponseBuilder rb) throws IOException {
- // System.out.println(System.nanoTime() + " - "
- // + Thread.currentThread().getId() + " - "
- // + rb.req.getParams().getBool("isShard", false) + " PROCESS " + rb.stage
- // + " " + rb.req.getParamString());
+ // System.out
+ // .println(System.nanoTime() + " - " + Thread.currentThread().getId()
+ // + " - " + rb.req.getParams().getBool("isShard", false) + " PROCESS "
+ // + rb.stage + " " + rb.req.getParamString());
ComponentFields mtasFields = getMtasFields(rb);
if (mtasFields != null) {
DocSet docSet = rb.getResults().docSet;
@@ -1592,23 +234,23 @@ public class MtasSolrSearchComponent extends SearchComponent {
}
NamedList<Object> mtasResponse = new SimpleOrderedMap<>();
if (mtasFields.doKwic) {
- ArrayList<NamedList> mtasKwicResponses = new ArrayList<NamedList>();
+ ArrayList<NamedList<?>> mtasKwicResponses = new ArrayList<NamedList<?>>();
for (String field : mtasFields.list.keySet()) {
for (ComponentKwic kwic : mtasFields.list.get(field).kwicList) {
- mtasKwicResponses.add(createKwic(kwic));
+ mtasKwicResponses.add(searchKwic.create(kwic));
}
}
// add to response
mtasResponse.add("kwic", mtasKwicResponses);
}
if (mtasFields.doFacet) {
- ArrayList<NamedList> mtasFacetResponses = new ArrayList<NamedList>();
+ ArrayList<NamedList<?>> mtasFacetResponses = new ArrayList<NamedList<?>>();
for (String field : mtasFields.list.keySet()) {
for (ComponentFacet facet : mtasFields.list.get(field).facetList) {
if (rb.req.getParams().getBool("isShard", false)) {
- mtasFacetResponses.add(createFacet(facet, true));
+ mtasFacetResponses.add(searchFacet.create(facet, true));
} else {
- mtasFacetResponses.add(createFacet(facet, false));
+ mtasFacetResponses.add(searchFacet.create(facet, false));
}
}
}
@@ -1616,23 +258,23 @@ public class MtasSolrSearchComponent extends SearchComponent {
mtasResponse.add("facet", mtasFacetResponses);
}
if (mtasFields.doList) {
- ArrayList<NamedList> mtasListResponses = new ArrayList<NamedList>();
+ ArrayList<NamedList<?>> mtasListResponses = new ArrayList<NamedList<?>>();
for (String field : mtasFields.list.keySet()) {
for (ComponentList list : mtasFields.list.get(field).listList) {
- mtasListResponses.add(createList(list));
+ mtasListResponses.add(searchList.create(list));
}
}
// add to response
mtasResponse.add("list", mtasListResponses);
}
if (mtasFields.doGroup) {
- ArrayList<NamedList> mtasGroupResponses = new ArrayList<NamedList>();
+ ArrayList<NamedList<?>> mtasGroupResponses = new ArrayList<NamedList<?>>();
for (String field : mtasFields.list.keySet()) {
for (ComponentGroup group : mtasFields.list.get(field).groupList) {
if (rb.req.getParams().getBool("isShard", false)) {
- mtasGroupResponses.add(createGroup(group, true));
+ mtasGroupResponses.add(searchGroup.create(group, true));
} else {
- mtasGroupResponses.add(createGroup(group, false));
+ mtasGroupResponses.add(searchGroup.create(group, false));
}
}
}
@@ -1640,15 +282,16 @@ public class MtasSolrSearchComponent extends SearchComponent {
mtasResponse.add("group", mtasGroupResponses);
}
if (mtasFields.doTermVector) {
- ArrayList<NamedList> mtasTermVectorResponses = new ArrayList<NamedList>();
+ ArrayList<NamedList<?>> mtasTermVectorResponses = new ArrayList<NamedList<?>>();
for (String field : mtasFields.list.keySet()) {
for (ComponentTermVector termVector : mtasFields.list
.get(field).termVectorList) {
if (rb.req.getParams().getBool("isShard", false)) {
- mtasTermVectorResponses.add(createTermVector(termVector, true));
+ mtasTermVectorResponses
+ .add(searchTermvector.create(termVector, true));
} else {
mtasTermVectorResponses
- .add(createTermVector(termVector, false));
+ .add(searchTermvector.create(termVector, false));
}
}
}
@@ -1656,15 +299,15 @@ public class MtasSolrSearchComponent extends SearchComponent {
mtasResponse.add("termvector", mtasTermVectorResponses);
}
if (mtasFields.doPrefix) {
- ArrayList<NamedList> mtasPrefixResponses = new ArrayList<NamedList>();
+ ArrayList<NamedList<?>> mtasPrefixResponses = new ArrayList<NamedList<?>>();
for (String field : mtasFields.list.keySet()) {
if (mtasFields.list.get(field).prefix != null) {
if (rb.req.getParams().getBool("isShard", false)) {
- mtasPrefixResponses
- .add(createPrefix(mtasFields.list.get(field).prefix, true));
+ mtasPrefixResponses.add(searchPrefix
+ .create(mtasFields.list.get(field).prefix, true));
} else {
- mtasPrefixResponses.add(
- createPrefix(mtasFields.list.get(field).prefix, false));
+ mtasPrefixResponses.add(searchPrefix
+ .create(mtasFields.list.get(field).prefix, false));
}
}
}
@@ -1680,10 +323,11 @@ public class MtasSolrSearchComponent extends SearchComponent {
for (ComponentToken token : mtasFields.list
.get(field).statsTokenList) {
if (rb.req.getParams().getBool("isShard", false)) {
- mtasStatsTokensResponses.add(createStatsToken(token, true));
+ mtasStatsTokensResponses
+ .add(searchStats.createToken(token, true));
} else {
mtasStatsTokensResponses
- .add(createStatsToken(token, false));
+ .add(searchStats.createToken(token, false));
}
}
}
@@ -1696,10 +340,10 @@ public class MtasSolrSearchComponent extends SearchComponent {
.get(field).statsPositionList) {
if (rb.req.getParams().getBool("isShard", false)) {
mtasStatsPositionsResponses
- .add(createStatsPosition(position, true));
+ .add(searchStats.createPosition(position, true));
} else {
mtasStatsPositionsResponses
- .add(createStatsPosition(position, false));
+ .add(searchStats.createPosition(position, false));
}
}
}
@@ -1711,9 +355,11 @@ public class MtasSolrSearchComponent extends SearchComponent {
for (ComponentSpan span : mtasFields.list
.get(field).statsSpanList) {
if (rb.req.getParams().getBool("isShard", false)) {
- mtasStatsSpansResponses.add(createStatsSpan(span, true));
+ mtasStatsSpansResponses
+ .add(searchStats.createSpan(span, true));
} else {
- mtasStatsSpansResponses.add(createStatsSpan(span, false));
+ mtasStatsSpansResponses
+ .add(searchStats.createSpan(span, false));
}
}
}
@@ -1745,104 +391,28 @@ public class MtasSolrSearchComponent extends SearchComponent {
// + rb.req.getParams().getBool("isShard", false) + " MODIFY REQUEST "
// + rb.stage + " " + rb.req.getParamString());
if (sreq.params.getBool(PARAM_MTAS, false)) {
- if (sreq.params.getBool(PARAM_MTAS_STATS, false)) {
- if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
- // do nothing
- } else {
- // remove stats for other requests
- sreq.params.remove(PARAM_MTAS_STATS);
- sreq.params.remove(PARAM_MTAS_STATS_POSITIONS);
- sreq.params.remove(PARAM_MTAS_STATS_SPANS);
- }
+ if (sreq.params.getBool(MtasSolrComponentStats.PARAM_MTAS_STATS, false)) {
+ searchStats.modifyRequest(rb, who, sreq);
}
- if (sreq.params.getBool(PARAM_MTAS_KWIC, false)) {
- if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
- // do nothing
- } else {
- Set<String> keys = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_LIST);
- sreq.params.remove(PARAM_MTAS_KWIC);
- for (String key : keys) {
- sreq.params.remove(
- PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_FIELD);
- sreq.params.remove(
- PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_QUERY_VALUE);
- sreq.params
- .remove(PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_KEY);
- sreq.params.remove(
- PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_PREFIX);
- sreq.params.remove(
- PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_NUMBER);
- sreq.params.remove(
- PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_LEFT);
- sreq.params.remove(
- PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_RIGHT);
- sreq.params.remove(
- PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_OUTPUT);
- }
- }
+ if (sreq.params.getBool(MtasSolrComponentTermvector.PARAM_MTAS_TERMVECTOR,
+ false)) {
+ searchTermvector.modifyRequest(rb, who, sreq);
}
- if (sreq.params.getBool(PARAM_MTAS_LIST, false)) {
- // compute keys
- Set<String> keys = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_LIST);
- if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
- for (String key : keys) {
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_PREFIX);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_START);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_NUMBER);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_LEFT);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_RIGHT);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_OUTPUT);
- // don't get data
- sreq.params.add(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_NUMBER, "0");
- }
- } else {
- sreq.params.remove(PARAM_MTAS_LIST);
- for (String key : keys) {
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_FIELD);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_QUERY_VALUE);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_QUERY_TYPE);
- sreq.params
- .remove(PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_KEY);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_PREFIX);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_START);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_NUMBER);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_LEFT);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_RIGHT);
- sreq.params.remove(
- PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_OUTPUT);
- }
- }
- }
- if (sreq.params.getBool(PARAM_MTAS_TERMVECTOR, false)) {
- // compute keys
- Set<String> keys = getIdsFromParameters(rb.req.getParams(),
- PARAM_MTAS_TERMVECTOR);
- if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
- for (String key : keys) {
- System.out.println("Normal request termvector "+key);
- }
- } else {
- for (String key : keys) {
- System.out.println("Special request termvector "+key);
- }
- }
+ if (sreq.params.getBool(MtasSolrComponentPrefix.PARAM_MTAS_PREFIX,
+ false)) {
+ searchPrefix.modifyRequest(rb, who, sreq);
+ }
+ if (sreq.params.getBool(MtasSolrComponentFacet.PARAM_MTAS_FACET, false)) {
+ searchFacet.modifyRequest(rb, who, sreq);
+ }
+ if (sreq.params.getBool(MtasSolrComponentGroup.PARAM_MTAS_GROUP, false)) {
+ searchGroup.modifyRequest(rb, who, sreq);
+ }
+ if (sreq.params.getBool(MtasSolrComponentList.PARAM_MTAS_LIST, false)) {
+ searchList.modifyRequest(rb, who, sreq);
+ }
+ if (sreq.params.getBool(MtasSolrComponentKwic.PARAM_MTAS_KWIC, false)) {
+ searchKwic.modifyRequest(rb, who, sreq);
}
}
}
@@ -1856,155 +426,10 @@ public class MtasSolrSearchComponent extends SearchComponent {
*/
@Override
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
- // System.out.println(System.nanoTime()+" - "+Thread.currentThread().getId()
- // + " - "
- // + rb.req.getParams().getBool("isShard", false) + " HANDLERESPONSES "
- // + rb.stage + " " + rb.req.getParamString());
- }
-
- /**
- * Merge responses tree set.
- *
- * @param originalList the original list
- * @param shardList the shard list
- */
- private void mergeResponsesTreeSet(TreeSet<Object> originalList,
- TreeSet<Object> shardList) {
- for (Object item : shardList) {
- originalList.add(item);
- }
- }
-
- /**
- * Merge responses array list.
- *
- * @param originalList the original list
- * @param shardList the shard list
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void mergeResponsesArrayList(ArrayList<Object> originalList,
- ArrayList<Object> shardList) throws IOException {
- // get keys from original
- HashMap<String, Object> originalKeyList = new HashMap<String, Object>();
- for (Object item : originalList) {
- if (item instanceof NamedList<?>) {
- NamedList<Object> itemList = (NamedList<Object>) item;
- Object key = itemList.get("key");
- if ((key != null) && (key instanceof String)) {
- originalKeyList.put((String) key, item);
- }
- }
- }
- for (Object item : shardList) {
- if (item instanceof NamedList<?>) {
- NamedList<Object> itemList = (NamedList<Object>) item;
- Object key = itemList.get("key");
- // item with key
- if ((key != null) && (key instanceof String)) {
- // merge
- if (originalKeyList.containsKey(key)) {
- Object originalItem = originalKeyList.get(key);
- if (originalItem.getClass().equals(item.getClass())) {
- mergeResponsesNamedList((NamedList<Object>) originalItem,
- (NamedList<Object>) item);
- } else {
- // ignore?
- }
- // add
- } else {
- originalList.add(adjustablePartsCloned(item));
- }
- } else {
- originalList.add(item);
- }
- } else {
- originalList.add(item);
- }
- }
- }
-
- /**
- * Merge responses named list.
- *
- * @param mainResponse the main response
- * @param shardResponse the shard response
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void mergeResponsesNamedList(NamedList<Object> mainResponse,
- NamedList<Object> shardResponse) throws IOException {
- Iterator<Entry<String, Object>> it = shardResponse.iterator();
- while (it.hasNext()) {
- Entry<String, Object> entry = it.next();
- String name = entry.getKey();
- Object shardValue = entry.getValue();
- int originalId = mainResponse.indexOf(name, 0);
- if (originalId < 0) {
- mainResponse.add(name, adjustablePartsCloned(shardValue));
- } else {
- Object original = mainResponse.getVal(originalId);
- if (original == null) {
- original = adjustablePartsCloned(shardValue);
- } else if (original.getClass().equals(shardValue.getClass())) {
- // merge ArrayList
- if (original instanceof ArrayList) {
- ArrayList originalList = (ArrayList) original;
- ArrayList shardList = (ArrayList) shardValue;
- mergeResponsesArrayList(originalList, shardList);
- // merge Namedlist
- } else if (original instanceof NamedList<?>) {
- mergeResponsesNamedList((NamedList<Object>) original,
- (NamedList<Object>) shardValue);
- // merge TreeSet
- } else if (original instanceof TreeSet<?>) {
- mergeResponsesTreeSet((TreeSet<Object>) original,
- (TreeSet<Object>) shardValue);
- } else if (original instanceof ComponentSortSelect) {
- ComponentSortSelect originalComponentSortSelect = (ComponentSortSelect) original;
- originalComponentSortSelect.merge((ComponentSortSelect) shardValue);
- } else if (original instanceof String) {
- // ignore?
- } else if (original instanceof Integer) {
- original = (Integer) original + ((Integer) shardValue);
- } else if (original instanceof Long) {
- original = (Long) original + ((Long) shardValue);
- } else {
- // ignore?
- }
- mainResponse.setVal(originalId, original);
- } else {
- // ignore?
- }
- }
- }
- }
-
- /**
- * Adjustable parts cloned.
- *
- * @param original the original
- * @return the object
- */
- private Object adjustablePartsCloned(Object original) {
- if (original instanceof NamedList) {
- NamedList<Object> newObject = new SimpleOrderedMap();
- NamedList<Object> originalObject = (NamedList<Object>) original;
- for (int i = 0; i < originalObject.size(); i++) {
- newObject.add(originalObject.getName(i),
- adjustablePartsCloned(originalObject.getVal(i)));
- }
- return newObject;
- } else if (original instanceof ArrayList) {
- ArrayList<Object> newObject = new ArrayList<Object>();
- ArrayList<Object> originalObject = (ArrayList<Object>) original;
- for (int i = 0; i < originalObject.size(); i++) {
- newObject.add(adjustablePartsCloned(originalObject.get(i)));
- }
- return newObject;
- } else if (original instanceof Integer) {
- Integer originalObject = (Integer) original;
- return new Integer(originalObject.intValue());
- }
- return original;
+ // System.out
+ // .println(System.nanoTime() + " - " + Thread.currentThread().getId()
+ // + " - " + rb.req.getParams().getBool("isShard", false)
+ // + " HANDLERESPONSES " + rb.stage + " " + rb.req.getParamString());
}
/*
@@ -2016,170 +441,40 @@ public class MtasSolrSearchComponent extends SearchComponent {
*/
@Override
public void finishStage(ResponseBuilder rb) {
- // System.out.println(System.nanoTime()+" - "+Thread.currentThread().getId()
- // + " - "
- // + rb.req.getParams().getBool("isShard", false) + " FINISHRESPONSES "
- // + rb.stage + " " + rb.req.getParamString());
+ // System.out
+ // .println(System.nanoTime() + " - " + Thread.currentThread().getId()
+ // + " - " + rb.req.getParams().getBool("isShard", false)
+ // + " FINISHRESPONSES " + rb.stage + " " + rb.req.getParamString());
if (rb.req.getParams().getBool(PARAM_MTAS, false)) {
- // mtas response
- NamedList<Object> mtasResponse = null;
- try {
- mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
- } catch (ClassCastException e) {
- mtasResponse = null;
+ if (rb.req.getParams().getBool(MtasSolrComponentStats.PARAM_MTAS_STATS,
+ false)) {
+ searchStats.finishStage(rb);
}
- if (mtasResponse == null) {
- mtasResponse = new SimpleOrderedMap<>();
- rb.rsp.add("mtas", mtasResponse);
+ if (rb.req.getParams()
+ .getBool(MtasSolrComponentTermvector.PARAM_MTAS_TERMVECTOR, false)) {
+ searchTermvector.finishStage(rb);
}
- // get fields stage
- if ((rb.stage == ResponseBuilder.STAGE_GET_FIELDS)) {
- if (rb.req.getParams().getBool(PARAM_MTAS_KWIC, false)) {
- finishStageArrayList(rb, mtasResponse, "kwic", null);
- }
- if (rb.req.getParams().getBool(PARAM_MTAS_LIST, false)) {
- finishStageArrayList(rb, mtasResponse, "list",
- ShardRequest.PURPOSE_PRIVATE);
- }
- // execute query stage
- } else if ((rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY)) {
- if (rb.req.getParams().getBool(PARAM_MTAS_STATS, false)) {
- finishStageNamedList(rb, mtasResponse, "stats", null);
- }
- if (rb.req.getParams().getBool(PARAM_MTAS_LIST, false)) {
- finishStageArrayList(rb, mtasResponse, "list", null);
- }
- if (rb.req.getParams().getBool(PARAM_MTAS_GROUP, false)) {
- finishStageArrayList(rb, mtasResponse, "group", null);
- }
- if (rb.req.getParams().getBool(PARAM_MTAS_TERMVECTOR, false)) {
- finishStageArrayList(rb, mtasResponse, "termvector", null);
- }
- if (rb.req.getParams().getBool(PARAM_MTAS_FACET, false)) {
- finishStageArrayList(rb, mtasResponse, "facet", null);
- }
- if (rb.req.getParams().getBool(PARAM_MTAS_PREFIX, false)) {
- finishStageArrayList(rb, mtasResponse, "prefix", null);
- // repair prefix lists
- try {
- ArrayList<NamedList> list = (ArrayList<NamedList>) mtasResponse
- .findRecursive("prefix");
- if (list != null) {
- for (NamedList item : list) {
- TreeSet<String> singlePosition = (TreeSet<String>) item
- .get("singlePosition");
- TreeSet<String> multiplePosition = (TreeSet<String>) item
- .get("multiplePosition");
- for (String prefix : multiplePosition) {
- if (singlePosition.contains(prefix)) {
- singlePosition.remove(prefix);
- }
- }
- }
- }
- } catch (ClassCastException e) {
-
- }
- }
+ if (rb.req.getParams().getBool(MtasSolrComponentPrefix.PARAM_MTAS_PREFIX,
+ false)) {
+ searchPrefix.finishStage(rb);
}
- }
-
- }
-
- /**
- * Finish stage array list.
- *
- * @param rb the rb
- * @param mtasResponse the mtas response
- * @param key the key
- * @param preferredPurpose the preferred purpose
- */
- private void finishStageArrayList(ResponseBuilder rb,
- NamedList<Object> mtasResponse, String key, Integer preferredPurpose) {
- // create new response for key
- ArrayList<Object> mtasListResponse = new ArrayList<Object>();
- mtasResponse.removeAll(key);
- mtasResponse.add(key, mtasListResponse);
- // collect responses for each shard
- HashMap<String, ArrayList<Object>> mtasListShardResponses = new HashMap<String, ArrayList<Object>>();
- for (ShardRequest sreq : rb.finished) {
- for (ShardResponse response : sreq.responses) {
- // only continue if new shard or preferred purpose
- if (mtasListShardResponses.containsKey(response.getShard())
- && ((preferredPurpose == null)
- || (sreq.purpose != preferredPurpose))) {
- break;
- }
- // update
- try {
- NamedList<Object> result = response.getSolrResponse().getResponse();
- ArrayList<Object> data = (ArrayList<Object>) result
- .findRecursive("mtas", key);
- if (data != null) {
- mtasListShardResponses.put(response.getShard(), decode(data));
- }
- } catch (ClassCastException e) {
-
- }
+ if (rb.req.getParams().getBool(MtasSolrComponentFacet.PARAM_MTAS_FACET,
+ false)) {
+ searchFacet.finishStage(rb);
}
- }
-
- try {
- for (ArrayList<Object> mtasListShardResponse : mtasListShardResponses
- .values()) {
- mergeResponsesArrayList(mtasListResponse, mtasListShardResponse);
+ if (rb.req.getParams().getBool(MtasSolrComponentGroup.PARAM_MTAS_GROUP,
+ false)) {
+ searchGroup.finishStage(rb);
}
- rewrite(mtasListResponse);
- } catch (IOException e) {
- mtasListResponse.add(e.getMessage());
- }
- }
-
- /**
- * Finish stage named list.
- *
- * @param rb the rb
- * @param mtasResponse the mtas response
- * @param key the key
- * @param preferredPurpose the preferred purpose
- */
- private void finishStageNamedList(ResponseBuilder rb,
- NamedList<Object> mtasResponse, String key, Integer preferredPurpose) {
- // create new response for key
- NamedList<Object> mtasListResponse = new SimpleOrderedMap<>();
- mtasResponse.removeAll(key);
- mtasResponse.add(key, mtasListResponse);
- // collect responses for each shard
- HashMap<String, NamedList<Object>> mtasListShardResponses = new HashMap<String, NamedList<Object>>();
- for (ShardRequest sreq : rb.finished) {
- for (ShardResponse response : sreq.responses) {
- // only continue if new shard or preferred purpose
- if (mtasListShardResponses.containsKey(response.getShard())
- && ((preferredPurpose == null)
- || (sreq.purpose != preferredPurpose))) {
- break;
- }
- // update
- try {
- NamedList<Object> result = response.getSolrResponse().getResponse();
- NamedList<Object> data = (NamedList<Object>) result
- .findRecursive("mtas", key);
- if (data != null) {
- mtasListShardResponses.put(response.getShard(), decode(data));
- }
- } catch (ClassCastException e) {
-
- }
+ if (rb.req.getParams().getBool(MtasSolrComponentList.PARAM_MTAS_LIST,
+ false)) {
+ searchList.finishStage(rb);
}
- try {
- for (NamedList<Object> mtasListShardResponse : mtasListShardResponses
- .values()) {
- mergeResponsesNamedList(mtasListResponse, mtasListShardResponse);
- }
- rewrite(mtasListResponse);
- } catch (IOException e) {
- e.printStackTrace();
+ if (rb.req.getParams().getBool(MtasSolrComponentKwic.PARAM_MTAS_KWIC,
+ false)) {
+ searchKwic.finishStage(rb);
}
+ mtasSolrResultMerge.merge(rb);
}
}
@@ -2192,1119 +487,90 @@ public class MtasSolrSearchComponent extends SearchComponent {
*/
@Override
public int distributedProcess(ResponseBuilder rb) throws IOException {
-// System.out.println(Thread.currentThread().getId() + " - "
-// + rb.req.getParams().getBool("isShard", false) + " DISTIRBUTEDPROCESS "
-// + rb.stage + " " + rb.req.getParamString());
+ // System.out.println(Thread.currentThread().getId() + " - "
+ // + rb.req.getParams().getBool("isShard", false) + " DISTIRBUTEDPROCESS "
+ // + rb.stage + " " + rb.req.getParamString());
+ // distributed processes
if (rb.req.getParams().getBool(PARAM_MTAS, false)) {
- ComponentFields mtasFields = getMtasFields(rb);
- if (rb.req.getParams().getBool(PARAM_MTAS_LIST, false)) {
- distributedProcessList(rb, mtasFields);
- }
- if (rb.req.getParams().getBool(PARAM_MTAS_TERMVECTOR, false)) {
- distributedProcessTermvector(rb, mtasFields);
- }
- }
- return ResponseBuilder.STAGE_DONE;
- }
-
- /**
- * Distributed process termvector.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void distributedProcessTermvector(ResponseBuilder rb,
- ComponentFields mtasFields) throws IOException {
- System.out.println(Thread.currentThread().getId() + "\t" + rb.stage);
- if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
- HashMap<String, HashMap<String, Integer>> listShardTotals = new HashMap<String, HashMap<String, Integer>>();
- for (ShardRequest sreq : rb.finished) {
- if (sreq.params.getBool(PARAM_MTAS, false)
- && sreq.params.getBool(PARAM_MTAS_TERMVECTOR, false)) {
- HashMap<String, HashMap<String, HashSet<String>>> missingKeys = computeMissingItemsPerShard(
- sreq.responses, "mtas", "termvector");
- System.out
- .println(Thread.currentThread().getId() + "\t" + missingKeys);
+ if (rb.stage == STAGE_TERMVECTOR_MISSING_TOP) {
+ ComponentFields mtasFields = getMtasFields(rb);
+ searchTermvector.distributedProcessMissingTop(rb, mtasFields);
+ } else if (rb.stage == STAGE_TERMVECTOR_MISSING_KEY) {
+ ComponentFields mtasFields = getMtasFields(rb);
+ searchTermvector.distributedProcessMissingKey(rb, mtasFields);
+ } else if (rb.stage == STAGE_TERMVECTOR_FINISH) {
+ ComponentFields mtasFields = getMtasFields(rb);
+ searchTermvector.distributedProcessFinish(rb, mtasFields);
+ } else if (rb.stage == STAGE_LIST) {
+ ComponentFields mtasFields = getMtasFields(rb);
+ searchList.distributedProcess(rb, mtasFields);
+ } else if (rb.stage == STAGE_PREFIX) {
+ ComponentFields mtasFields = getMtasFields(rb);
+ searchPrefix.distributedProcess(rb, mtasFields);
+ } else if (rb.stage == STAGE_STATS) {
+ ComponentFields mtasFields = getMtasFields(rb);
+ searchStats.distributedProcess(rb, mtasFields);
+ } else if (rb.stage == STAGE_FACET) {
+ ComponentFields mtasFields = getMtasFields(rb);
+ searchFacet.distributedProcess(rb, mtasFields);
+ } else if (rb.stage == STAGE_GROUP) {
+ ComponentFields mtasFields = getMtasFields(rb);
+ searchGroup.distributedProcess(rb, mtasFields);
+ }
+ // compute new stage and return if not finished
+ if (rb.stage >= ResponseBuilder.STAGE_EXECUTE_QUERY
+ && rb.stage <= ResponseBuilder.STAGE_GET_FIELDS) {
+ if (rb.stage < STAGE_TERMVECTOR_MISSING_TOP
+ && rb.req.getParams().getBool(
+ MtasSolrComponentTermvector.PARAM_MTAS_TERMVECTOR, false)) {
+ return STAGE_TERMVECTOR_MISSING_TOP;
+ } else if (rb.stage < STAGE_TERMVECTOR_MISSING_KEY
+ && rb.req.getParams().getBool(
+ MtasSolrComponentTermvector.PARAM_MTAS_TERMVECTOR, false)) {
+ return STAGE_TERMVECTOR_MISSING_KEY;
+ } else if (rb.stage < STAGE_TERMVECTOR_FINISH
+ && rb.req.getParams().getBool(
+ MtasSolrComponentTermvector.PARAM_MTAS_TERMVECTOR, false)) {
+ return STAGE_TERMVECTOR_FINISH;
+ } else if (rb.stage < STAGE_LIST && rb.req.getParams()
+ .getBool(MtasSolrComponentList.PARAM_MTAS_LIST, false)) {
+ return STAGE_LIST;
+ } else if (rb.stage < STAGE_PREFIX && rb.req.getParams()
+ .getBool(MtasSolrComponentPrefix.PARAM_MTAS_PREFIX, false)) {
+ return STAGE_PREFIX;
+ } else if (rb.stage < STAGE_STATS && rb.req.getParams()
+ .getBool(MtasSolrComponentStats.PARAM_MTAS_STATS, false)) {
+ return STAGE_STATS;
+ } else if (rb.stage < STAGE_FACET && rb.req.getParams()
+ .getBool(MtasSolrComponentFacet.PARAM_MTAS_FACET, false)) {
+ return STAGE_FACET;
+ } else if (rb.stage < STAGE_GROUP && rb.req.getParams()
+ .getBool(MtasSolrComponentGroup.PARAM_MTAS_GROUP, false)) {
+ return STAGE_GROUP;
}
}
- }
- }
- /**
- * Distributed process list.
- *
- * @param rb the rb
- * @param mtasFields the mtas fields
- */
- private void distributedProcessList(ResponseBuilder rb,
- ComponentFields mtasFields) {
- if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
- if (mtasFields.doList) {
- // compute total from shards
- HashMap<String, HashMap<String, Integer>> listShardTotals = new HashMap<String, HashMap<String, Integer>>();
- for (ShardRequest sreq : rb.finished) {
- if (sreq.params.getBool(PARAM_MTAS, false)
- && sreq.params.getBool(PARAM_MTAS_LIST, false)) {
- for (ShardResponse response : sreq.responses) {
- NamedList<Object> result = response.getSolrResponse()
- .getResponse();
- try {
- ArrayList<NamedList<Object>> data = (ArrayList<NamedList<Object>>) result
- .findRecursive("mtas", "list");
- if (data != null) {
- for (NamedList<Object> dataItem : data) {
- Object key = dataItem.get("key");
- Object total = dataItem.get("total");
- if ((key != null) && (key instanceof String)
- && (total != null) && (total instanceof Integer)) {
- if (!listShardTotals.containsKey(key)) {
- listShardTotals.put((String) key,
- new HashMap<String, Integer>());
- }
- HashMap<String, Integer> listShardTotal = listShardTotals
- .get(key);
- listShardTotal.put(response.getShard(), (Integer) total);
- }
- }
- }
- } catch (ClassCastException e) {
- }
- }
- }
- }
- // compute shard requests
- HashMap<String, ModifiableSolrParams> shardRequests = new HashMap<String, ModifiableSolrParams>();
- int requestId = 0;
- for (String field : mtasFields.list.keySet()) {
- for (ComponentList list : mtasFields.list.get(field).listList) {
- requestId++;
- if (listShardTotals.containsKey(list.key) && (list.number > 0)) {
- Integer position = 0, total = 0;
- Integer start = list.start;
- Integer number = list.number;
- HashMap<String, Integer> totals = listShardTotals.get(list.key);
- for (int i = 0; i < rb.shards.length; i++) {
- if (number < 0) {
- break;
- }
- int subTotal = totals.get(rb.shards[i]);
- // System.out.println(i + " : " + rb.shards[i] + " : "
- // + totals.get(rb.shards[i]) + " - " + start + " " + number);
- if ((start >= 0) && (start < subTotal)) {
- ModifiableSolrParams params;
- if (!shardRequests.containsKey(rb.shards[i])) {
- shardRequests.put(rb.shards[i], new ModifiableSolrParams());
- }
- params = shardRequests.get(rb.shards[i]);
- params.add(PARAM_MTAS_LIST + "." + requestId + "."
- + NAME_MTAS_LIST_FIELD, list.field);
- params.add(PARAM_MTAS_LIST + "." + requestId + "."
- + NAME_MTAS_LIST_QUERY_VALUE, list.queryValue);
- params.add(PARAM_MTAS_LIST + "." + requestId + "."
- + NAME_MTAS_LIST_QUERY_TYPE, list.queryType);
- params.add(PARAM_MTAS_LIST + "." + requestId + "."
- + NAME_MTAS_LIST_KEY, list.key);
- params.add(PARAM_MTAS_LIST + "." + requestId + "."
- + NAME_MTAS_LIST_PREFIX, list.prefix);
- params.add(PARAM_MTAS_LIST + "." + requestId + "."
- + NAME_MTAS_LIST_START, Integer.toString(start));
- params.add(
- PARAM_MTAS_LIST + "." + requestId + "."
- + NAME_MTAS_LIST_NUMBER,
- Integer.toString(Math.min(number, (subTotal - start))));
- params.add(PARAM_MTAS_LIST + "." + requestId + "."
- + NAME_MTAS_LIST_LEFT, Integer.toString(list.left));
- params.add(PARAM_MTAS_LIST + "." + requestId + "."
- + NAME_MTAS_LIST_RIGHT, Integer.toString(list.right));
- params.add(PARAM_MTAS_LIST + "." + requestId + "."
- + NAME_MTAS_LIST_OUTPUT, list.output);
- number -= (subTotal - start);
- start = 0;
- } else {
- start -= subTotal;
- }
- position += subTotal;
- }
- }
- }
- }
-
- for (String shardName : shardRequests.keySet()) {
- ShardRequest sreq = new ShardRequest();
- sreq.shards = new String[] { shardName };
- sreq.purpose = ShardRequest.PURPOSE_PRIVATE;
- sreq.params = new ModifiableSolrParams();
- sreq.params.add("fq", rb.req.getParams().getParams("fq"));
- sreq.params.add("q", rb.req.getParams().getParams("q"));
- sreq.params.add("cache", rb.req.getParams().getParams("cache"));
- sreq.params.add("rows", "0");
- sreq.params.add(PARAM_MTAS,
- rb.req.getOriginalParams().getParams(PARAM_MTAS));
- sreq.params.add(PARAM_MTAS_LIST,
- rb.req.getOriginalParams().getParams(PARAM_MTAS_LIST));
- sreq.params.add(shardRequests.get(shardName));
- rb.addRequest(this, sreq);
- }
- }
}
+ return ResponseBuilder.STAGE_DONE;
}
/**
- * Compute missing items per shard.
+ * Gets the mtas fields.
*
- * @param shardResponses the shard responses
- * @param args the args
- * @return the hash map
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param rb
+ * the rb
+ * @return the mtas fields
*/
- private HashMap<String, HashMap<String, HashSet<String>>> computeMissingItemsPerShard(
- List<ShardResponse> shardResponses, String... args) throws IOException {
- HashMap<String, HashMap<String, HashSet<String>>> result = new HashMap<String, HashMap<String, HashSet<String>>>();
- HashMap<String, HashMap<String, HashSet<String>>> itemsPerShardSets = new HashMap<String, HashMap<String, HashSet<String>>>();
- HashMap<String, HashSet<String>> itemSets = new HashMap<String, HashSet<String>>();
- // loop over responses different shards
- for (ShardResponse shardResponse : shardResponses) {
- NamedList<Object> response = shardResponse.getSolrResponse()
- .getResponse();
- try {
- ArrayList<NamedList<Object>> data = (ArrayList<NamedList<Object>>) response
- .findRecursive(args);
- if (data != null) {
- // loop over different keys
- for (NamedList<Object> dataItem : data) {
- Object oKey = dataItem.get("key");
- Object oList = dataItem.get("list");
- if (oKey != null && oList != null && oKey instanceof String
- && oList instanceof ComponentSortSelect) {
- ComponentSortSelect list = (ComponentSortSelect) oList;
- if (list.sortType.equals(CodecUtil.SORT_TERM)) {
- break;
- } else {
- String key = (String) oKey;
- HashMap<String, HashSet<String>> itemsPerShardSet;
- HashSet<String> itemSet, tmpItemSet = new HashSet<String>();
- if (itemsPerShardSets.containsKey(key)) {
- itemsPerShardSet = itemsPerShardSets.get(key);
- itemSet = itemSets.get(key);
- } else {
- itemsPerShardSet = new HashMap<String, HashSet<String>>();
- itemSet = new HashSet<String>();
- itemsPerShardSets.put(key, itemsPerShardSet);
- itemSets.put(key, itemSet);
- }
- itemsPerShardSet.put(shardResponse.getShardAddress(),
- tmpItemSet);
- Iterator<Entry<String, Object>> it = list.getNamedList()
- .iterator();
- String item;
- while (it.hasNext()) {
- item = it.next().getKey();
- tmpItemSet.add(item);
- itemSet.add(item);
- }
- }
- }
- }
- }
- } catch (ClassCastException e) {
- }
- }
- // construct result
- for (String key : itemSets.keySet()) {
- if (itemsPerShardSets.containsKey(key)) {
- HashMap<String, HashSet<String>> itemsPerShardSet = itemsPerShardSets
- .get(key);
- for (String shardAddress : itemsPerShardSet.keySet()) {
- HashMap<String, HashSet<String>> tmpShardKeySet;
- if (result.containsKey(shardAddress)) {
- tmpShardKeySet = result.get(shardAddress);
- } else {
- tmpShardKeySet = new HashMap<String, HashSet<String>>();
- result.put(shardAddress, tmpShardKeySet);
- }
- HashSet<String> tmpResult = new HashSet<String>();
- tmpShardKeySet.put(key, tmpResult);
- HashSet<String> itemsSet = itemsPerShardSet.get(shardAddress);
- for (String item : itemSets.get(key)) {
- if (!itemsSet.contains(item)) {
- tmpResult.add(item);
- }
- }
- }
- }
- }
- return result;
- }
/**
* Gets the mtas fields.
*
- * @param rb the rb
+ * @param rb
+ * the rb
* @return the mtas fields
*/
private ComponentFields getMtasFields(ResponseBuilder rb) {
return (ComponentFields) rb.req.getContext().get(ComponentFields.class);
}
- /**
- * Encode.
- *
- * @param o the o
- * @return the string
- */
- private String encode(Object o) {
- ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
- ObjectOutputStream objectOutputStream;
- try {
- objectOutputStream = new ObjectOutputStream(byteArrayOutputStream);
- objectOutputStream.writeObject(o);
- objectOutputStream.close();
- return Base64.byteArrayToBase64(byteArrayOutputStream.toByteArray());
- } catch (IOException e) {
- e.printStackTrace();
- return null;
- }
- }
-
- /**
- * Decode.
- *
- * @param s the s
- * @return the object
- */
- private Object decode(String s) {
- byte[] bytes = Base64.base64ToByteArray(s);
- ObjectInputStream objectInputStream;
- try {
- objectInputStream = new ObjectInputStream(
- new ByteArrayInputStream(bytes));
- return objectInputStream.readObject();
- } catch (IOException | ClassNotFoundException e) {
- e.printStackTrace();
- return null;
- }
- }
-
- /**
- * Decode.
- *
- * @param l the l
- * @return the array list
- */
- private ArrayList decode(ArrayList l) {
- for (int i = 0; i < l.size(); i++) {
- if (l.get(i) instanceof NamedList) {
- l.set(i, decode((NamedList) l.get(i)));
- } else if (l.get(i) instanceof ArrayList) {
- l.set(i, decode((ArrayList) l.get(i)));
- }
- }
- return l;
- }
-
- /**
- * Decode.
- *
- * @param nl the nl
- * @return the named list
- */
- private NamedList<Object> decode(NamedList<Object> nl) {
- for (int i = 0; i < nl.size(); i++) {
- String key = nl.getName(i);
- Object o = nl.getVal(i);
- if (key.matches("^_encoded_.*$")) {
- if (o instanceof String) {
- Object decodedObject = decode((String) nl.getVal(i));
- String decodedKey = key.replaceFirst("^_encoded_", "");
- if (decodedKey.equals("")) {
- decodedKey = "_" + decodedObject.getClass().getSimpleName() + "_";
- }
- nl.setName(i, decodedKey);
- nl.setVal(i, decodedObject);
- } else if (o instanceof NamedList) {
- NamedList nl2 = (NamedList) o;
- for (int j = 0; j < nl2.size(); j++) {
- if (nl2.getVal(j) instanceof String) {
- nl2.setVal(j, decode((String) nl2.getVal(j)));
- }
- }
- } else {
- System.out.println("unknown type " + o.getClass().getCanonicalName());
- }
- } else {
- if (o instanceof NamedList) {
- nl.setVal(i, decode((NamedList<Object>) o));
- } else if (o instanceof ArrayList) {
- nl.setVal(i, decode((ArrayList<Object>) o));
- }
- }
- }
- return nl;
- }
-
- /**
- * Rewrite.
- *
- * @param al the al
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void rewrite(ArrayList<Object> al) throws IOException {
- for (int i = 0; i < al.size(); i++) {
- if (al.get(i) instanceof NamedList) {
- rewrite((NamedList) al.get(i));
- } else if (al.get(i) instanceof ArrayList) {
- rewrite((ArrayList) al.get(i));
- }
- }
- }
-
- /**
- * Rewrite.
- *
- * @param nl the nl
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void rewrite(NamedList<Object> nl) throws IOException {
- HashMap<String, NamedList<Object>> collapseNamedList = null;
- HashMap<String, String> mergeMappingList = new HashMap<String, String>();
- Pattern mergePattern = Pattern.compile("_merge_([^_]+)_([^_]+)+");
- int length = nl.size();
- for (int i = 0; i < length; i++) {
- if (nl.getVal(i) instanceof NamedList) {
- NamedList o = (NamedList) nl.getVal(i);
- rewrite(o);
- nl.setVal(i, o);
- } else if (nl.getVal(i) instanceof ArrayList) {
- ArrayList o = (ArrayList) nl.getVal(i);
- rewrite(o);
- nl.setVal(i, o);
- } else if (nl.getVal(i) instanceof MtasDataItem) {
- MtasDataItem dataItem = (MtasDataItem) nl.getVal(i);
- nl.setVal(i, dataItem.rewrite());
- } else if (nl.getVal(i) instanceof ComponentSortSelect) {
- ComponentSortSelect o = (ComponentSortSelect) nl.getVal(i);
- if (o.dataCollector.getCollectorType()
- .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
- NamedList<Object> nnl = o.getNamedList();
- rewrite(nnl);
- nl.setVal(i, nnl);
- Matcher mergeMatcher = mergePattern.matcher(nl.getName(i));
- if (mergeMatcher.find()) {
- mergeMappingList.put(mergeMatcher.group(1), mergeMatcher.group(2));
- }
- } else if (o.dataCollector.getCollectorType()
- .equals(DataCollector.COLLECTOR_TYPE_DATA)) {
- NamedList<Object> nnl = o.getData();
- if (nnl.size() > 0) {
- rewrite(nnl);
- collapseNamedList = new HashMap<String, NamedList<Object>>();
- collapseNamedList.put(nl.getName(i), nnl);
- nl.setVal(i, nnl);
- } else {
- nl.setVal(i, null);
- }
- }
- }
- }
- // merge
- if (mergeMappingList.size() > 0) {
- for (String key : mergeMappingList.keySet()) {
- Object so = nl
- .get("_merge_" + key + "_" + mergeMappingList.get(key) + "_");
- Object to = nl.get(mergeMappingList.get(key));
- if (to != null && so != null && to instanceof NamedList
- && so instanceof NamedList) {
- NamedList<Object> snl = (NamedList<Object>) so;
- NamedList<Object> tnl = (NamedList<Object>) to;
- for (int i = 0; i < tnl.size(); i++) {
- Object item = snl.get(tnl.getName(i));
- if (item != null && tnl.getVal(i) instanceof NamedList) {
- ((NamedList<Object>) tnl.getVal(i)).add(key, item);
- }
- }
- } else {
- throw new IOException(
- "can't merge " + key + " with " + mergeMappingList.get(key));
- }
- }
- for (String key : mergeMappingList.keySet()) {
- nl.remove("_merge_" + key + "_" + mergeMappingList.get(key) + "_");
- }
- }
- // collapse
- if (collapseNamedList != null) {
- for (String key : collapseNamedList.keySet()) {
- nl.remove(key);
- }
- for (NamedList<Object> items : collapseNamedList.values()) {
- nl.addAll(items);
- }
- }
- }
-
- /**
- * Creates the stats position.
- *
- * @param position the position
- * @param encode the encode
- * @return the simple ordered map
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private SimpleOrderedMap<Object> createStatsPosition(
- ComponentPosition position, Boolean encode) throws IOException {
- // System.out.println("Create stats position " + position.dataType + " "
- // + position.statsType + " " + position.statsItems + " --- " + encode);
- SimpleOrderedMap<Object> mtasPositionResponse = new SimpleOrderedMap<>();
- mtasPositionResponse.add("key", position.key);
- ComponentSortSelect data = new ComponentSortSelect(position.dataCollector,
- position.dataType, position.statsType, position.statsItems);
- if (encode) {
- mtasPositionResponse.add("_encoded_data", encode(data));
- } else {
- mtasPositionResponse.add(position.dataCollector.getCollectorType(), data);
- rewrite(mtasPositionResponse);
- }
- return mtasPositionResponse;
- }
-
- /**
- * Creates the stats token.
- *
- * @param token the token
- * @param encode the encode
- * @return the simple ordered map
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private SimpleOrderedMap<Object> createStatsToken(ComponentToken token,
- Boolean encode) throws IOException {
- // System.out.println("Create stats position " + position.dataType + " "
- // + position.statsType + " " + position.statsItems + " --- " + encode);
- SimpleOrderedMap<Object> mtasTokenResponse = new SimpleOrderedMap<>();
- mtasTokenResponse.add("key", token.key);
- ComponentSortSelect data = new ComponentSortSelect(token.dataCollector,
- token.dataType, token.statsType, token.statsItems);
- if (encode) {
- mtasTokenResponse.add("_encoded_data", encode(data));
- } else {
- mtasTokenResponse.add(token.dataCollector.getCollectorType(), data);
- rewrite(mtasTokenResponse);
- }
- return mtasTokenResponse;
- }
-
- /**
- * Creates the stats span.
- *
- * @param span the span
- * @param encode the encode
- * @return the simple ordered map
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private SimpleOrderedMap<Object> createStatsSpan(ComponentSpan span,
- Boolean encode) throws IOException {
- // System.out.println("Create stats span " + span.dataType + " "
- // + span.statsType + " " + span.statsItems + " --- " + encode);
- SimpleOrderedMap<Object> mtasSpanResponse = new SimpleOrderedMap<>();
- mtasSpanResponse.add("key", span.key);
- ComponentSortSelect data = new ComponentSortSelect(span.dataCollector,
- span.dataType, span.statsType, span.statsItems);
- if (encode) {
- mtasSpanResponse.add("_encoded_data", encode(data));
- } else {
- mtasSpanResponse.add(span.dataCollector.getCollectorType(), data);
- rewrite(mtasSpanResponse);
- }
- return mtasSpanResponse;
- }
-
- /**
- * Creates the term vector.
- *
- * @param termVector the term vector
- * @param encode the encode
- * @return the simple ordered map
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private SimpleOrderedMap<Object> createTermVector(
- ComponentTermVector termVector, Boolean encode) throws IOException {
- SimpleOrderedMap<Object> mtasTermVectorResponse = new SimpleOrderedMap<>();
- mtasTermVectorResponse.add("key", termVector.key);
- ComponentSortSelect defaultData = new ComponentSortSelect(
- termVector.dataDefaultCollector,
- new String[] { termVector.defaultDataType },
- new String[] { termVector.defaultStatsType },
- new TreeSet[] { termVector.defaultStatsItems },
- new String[] { termVector.sortType },
- new String[] { termVector.sortDirection },
- new Integer[] { termVector.start },
- new Integer[] { termVector.number });
- ComponentSortSelect functionData = null;
- if (termVector.dataFunctionCollector != null) {
- functionData = new ComponentSortSelect(termVector.dataFunctionCollector,
- new String[] { termVector.functionDataType },
- new String[] { termVector.functionStatsType },
- new TreeSet[] { termVector.functionStatsItems },
- new String[] { termVector.sortType },
- new String[] { termVector.sortDirection },
- new Integer[] { termVector.start },
- new Integer[] { termVector.number });
- }
- if (encode) {
- mtasTermVectorResponse.add("_encoded_list", encode(defaultData));
- if (functionData != null) {
- mtasTermVectorResponse.add("_encoded__merge_function_list_",
- encode(functionData));
- }
- } else {
- mtasTermVectorResponse.add("list", defaultData);
- if (functionData != null) {
- mtasTermVectorResponse.add("_merge_function_list_", functionData);
- }
- rewrite(mtasTermVectorResponse);
- }
-
- return mtasTermVectorResponse;
- }
-
- /**
- * Creates the prefix.
- *
- * @param prefix the prefix
- * @param encode the encode
- * @return the simple ordered map
- */
- private SimpleOrderedMap<Object> createPrefix(ComponentPrefix prefix,
- Boolean encode) {
- SimpleOrderedMap<Object> mtasPrefixResponse = new SimpleOrderedMap<Object>();
- mtasPrefixResponse.add("key", prefix.key);
- if (encode) {
- mtasPrefixResponse.add("_encoded_singlePosition",
- encode(prefix.singlePositionList));
- mtasPrefixResponse.add("_encoded_multiplePosition",
- encode(prefix.multiplePositionList));
- } else {
- mtasPrefixResponse.add("singlePosition", prefix.singlePositionList);
- mtasPrefixResponse.add("multiplePosition", prefix.multiplePositionList);
- }
- return mtasPrefixResponse;
- }
-
- /**
- * Creates the list.
- *
- * @param list the list
- * @return the simple ordered map
- */
- private SimpleOrderedMap<Object> createList(ComponentList list) {
- SimpleOrderedMap<Object> mtasListResponse = new SimpleOrderedMap<>();
- mtasListResponse.add("key", list.key);
- mtasListResponse.add("total", list.total);
- if (list.output != null) {
- ArrayList<NamedList<Object>> mtasListItemResponses = new ArrayList<NamedList<Object>>();
- if (list.output.equals(ComponentList.LIST_OUTPUT_HIT)) {
- mtasListResponse.add("number", list.hits.size());
- for (ListHit hit : list.hits) {
- NamedList<Object> mtasListItemResponse = new SimpleOrderedMap<>();
- mtasListItemResponse.add("documentKey",
- list.uniqueKey.get(hit.docId));
- mtasListItemResponse.add("documentHitPosition", hit.docPosition);
- mtasListItemResponse.add("documentHitTotal",
- list.subTotal.get(hit.docId));
- mtasListItemResponse.add("documentMinPosition",
- list.minPosition.get(hit.docId));
- mtasListItemResponse.add("documentMaxPosition",
- list.maxPosition.get(hit.docId));
- mtasListItemResponse.add("startPosition", hit.startPosition);
- mtasListItemResponse.add("endPosition", hit.endPosition);
-
- TreeMap<Integer, ArrayList<ArrayList<String>>> hitData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
- TreeMap<Integer, ArrayList<ArrayList<String>>> leftData = null,
- rightData = null;
- if (list.left > 0) {
- leftData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
- }
- if (list.right > 0) {
- rightData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
- }
- for (int position = Math.max(0,
- hit.startPosition - list.left); position <= (hit.endPosition
- + list.right); position++) {
- ArrayList<ArrayList<String>> hitDataItem = new ArrayList<ArrayList<String>>();
- if (hit.hits.containsKey(position)) {
- for (String term : hit.hits.get(position)) {
- ArrayList<String> hitDataSubItem = new ArrayList<String>();
- hitDataSubItem.add(CodecUtil.termPrefix(term));
- hitDataSubItem.add(CodecUtil.termValue(term));
- hitDataItem.add(hitDataSubItem);
- }
- }
- if (position < hit.startPosition) {
- leftData.put(position, hitDataItem);
- } else if (position > hit.endPosition) {
- rightData.put(position, hitDataItem);
- } else {
- hitData.put(position, hitDataItem);
- }
- }
- if (list.left > 0) {
- mtasListItemResponse.add("left", leftData);
- }
- mtasListItemResponse.add("hit", hitData);
- if (list.right > 0) {
- mtasListItemResponse.add("right", rightData);
- }
- mtasListItemResponses.add(mtasListItemResponse);
- }
- } else if (list.output.equals(ComponentList.LIST_OUTPUT_TOKEN)) {
- mtasListResponse.add("number", list.tokens.size());
- for (ListToken tokenHit : list.tokens) {
- NamedList<Object> mtasListItemResponse = new SimpleOrderedMap<>();
- mtasListItemResponse.add("documentKey",
- list.uniqueKey.get(tokenHit.docId));
- mtasListItemResponse.add("documentHitPosition", tokenHit.docPosition);
- mtasListItemResponse.add("documentHitTotal",
- list.subTotal.get(tokenHit.docId));
- mtasListItemResponse.add("documentMinPosition",
- list.minPosition.get(tokenHit.docId));
- mtasListItemResponse.add("documentMaxPosition",
- list.maxPosition.get(tokenHit.docId));
- mtasListItemResponse.add("startPosition", tokenHit.startPosition);
- mtasListItemResponse.add("endPosition", tokenHit.endPosition);
-
- ArrayList<NamedList<Object>> mtasListItemResponseItemTokens = new ArrayList<NamedList<Object>>();
- for (MtasToken<?> token : tokenHit.tokens) {
- NamedList<Object> mtasListItemResponseItemToken = new SimpleOrderedMap<>();
- if (token.getId() != null) {
- mtasListItemResponseItemToken.add("mtasId", token.getId());
- }
- mtasListItemResponseItemToken.add("prefix", token.getPrefix());
- mtasListItemResponseItemToken.add("value", token.getPostfix());
- if (token.getPositionStart() != null) {
- mtasListItemResponseItemToken.add("positionStart",
- token.getPositionStart());
- mtasListItemResponseItemToken.add("positionEnd",
- token.getPositionEnd());
- }
- if (token.getPositions() != null) {
- mtasListItemResponseItemToken.add("positions",
- token.getPositions());
- }
- if (token.getParentId() != null) {
- mtasListItemResponseItemToken.add("parentMtasId",
- token.getParentId());
- }
- if (token.getPayload() != null) {
- mtasListItemResponseItemToken.add("payload", token.getPayload());
- }
- if (token.getOffsetStart() != null) {
- mtasListItemResponseItemToken.add("offsetStart",
- token.getOffsetStart());
- mtasListItemResponseItemToken.add("offsetEnd",
- token.getOffsetEnd());
- }
- if (token.getRealOffsetStart() != null) {
- mtasListItemResponseItemToken.add("realOffsetStart",
- token.getRealOffsetStart());
- mtasListItemResponseItemToken.add("realOffsetEnd",
- token.getRealOffsetEnd());
- }
- mtasListItemResponseItemTokens.add(mtasListItemResponseItemToken);
- }
- mtasListItemResponse.add("tokens", mtasListItemResponseItemTokens);
- mtasListItemResponses.add(mtasListItemResponse);
- }
- }
- mtasListResponse.add("list", mtasListItemResponses);
- }
- return mtasListResponse;
- }
-
- /**
- * Creates the group.
- *
- * @param group the group
- * @param encode the encode
- * @return the simple ordered map
- */
- private SimpleOrderedMap<Object> createGroup(ComponentGroup group,
- Boolean encode) {
- SimpleOrderedMap<Object> mtasGroupResponse = new SimpleOrderedMap<>();
- mtasGroupResponse.add("key", group.key);
- ComponentSortSelect data = new ComponentSortSelect(group.dataCollector,
- new String[] { group.dataType }, new String[] { group.statsType },
- new TreeSet[] { group.statsItems }, new String[] { group.sortType },
- new String[] { group.sortDirection }, new Integer[] { group.start },
- new Integer[] { group.number });
- if (encode) {
- mtasGroupResponse.add("_encoded_list", encode(data));
- } else {
- try {
- mtasGroupResponse.add("list", data);
- rewrite(mtasGroupResponse);
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- return mtasGroupResponse;
- }
-
- /**
- * Creates the facet.
- *
- * @param facet the facet
- * @param encode the encode
- * @return the simple ordered map
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private SimpleOrderedMap<Object> createFacet(ComponentFacet facet,
- Boolean encode) throws IOException {
- SimpleOrderedMap<Object> mtasFacetResponse = new SimpleOrderedMap<>();
- mtasFacetResponse.add("key", facet.key);
- SimpleOrderedMap<Object> mtasFacetItemResponses = new SimpleOrderedMap();
- ComponentSortSelect data = new ComponentSortSelect(facet.dataCollector,
- facet.baseDataTypes, facet.baseStatsTypes, facet.baseStatsItems,
- facet.baseSortTypes, facet.baseSortDirections, null, facet.baseNumbers);
- if (encode) {
- mtasFacetResponse.add("_encoded_list", encode(data));
- } else {
- try {
- mtasFacetResponse.add("list", data);
- rewrite(mtasFacetResponse);
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- return mtasFacetResponse;
- }
-
- /**
- * Creates the kwic.
- *
- * @param kwic the kwic
- * @return the simple ordered map
- */
- private SimpleOrderedMap<Object> createKwic(ComponentKwic kwic) {
- SimpleOrderedMap<Object> mtasKwicResponse = new SimpleOrderedMap<>();
- mtasKwicResponse.add("key", kwic.key);
- ArrayList<NamedList<Object>> mtasKwicItemResponses = new ArrayList<NamedList<Object>>();
- if (kwic.output.equals(ComponentKwic.KWIC_OUTPUT_HIT)) {
- for (int docId : kwic.hits.keySet()) {
- NamedList<Object> mtasKwicItemResponse = new SimpleOrderedMap<>();
- ArrayList<KwicHit> list = kwic.hits.get(docId);
- ArrayList<NamedList<Object>> mtasKwicItemResponseItems = new ArrayList<NamedList<Object>>();
- for (KwicHit h : list) {
- NamedList<Object> mtasKwicItemResponseItem = new SimpleOrderedMap<>();
- TreeMap<Integer, ArrayList<ArrayList<String>>> hitData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
- TreeMap<Integer, ArrayList<ArrayList<String>>> leftData = null,
- rightData = null;
- if (kwic.left > 0) {
- leftData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
- }
- if (kwic.right > 0) {
- rightData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
- }
- for (int position = Math.max(0,
- h.startPosition - kwic.left); position <= (h.endPosition
- + kwic.right); position++) {
- if (h.hits.containsKey(position)) {
- ArrayList<ArrayList<String>> hitDataItem = new ArrayList<ArrayList<String>>();
- for (String term : h.hits.get(position)) {
- ArrayList<String> hitDataSubItem = new ArrayList<String>();
- hitDataSubItem.add(CodecUtil.termPrefix(term));
- hitDataSubItem.add(CodecUtil.termValue(term));
- hitDataItem.add(hitDataSubItem);
- }
- if (position < h.startPosition) {
- leftData.put(position, hitDataItem);
- } else if (position > h.endPosition) {
- rightData.put(position, hitDataItem);
- } else {
- hitData.put(position, hitDataItem);
- }
- }
- }
- if (kwic.left > 0) {
- mtasKwicItemResponseItem.add("left", leftData);
- }
- mtasKwicItemResponseItem.add("hit", hitData);
- if (kwic.right > 0) {
- mtasKwicItemResponseItem.add("right", rightData);
- }
- mtasKwicItemResponseItems.add(mtasKwicItemResponseItem);
- }
- mtasKwicItemResponse.add("documentKey", kwic.uniqueKey.get(docId));
- mtasKwicItemResponse.add("documentTotal", kwic.subTotal.get(docId));
- mtasKwicItemResponse.add("documentMinPosition",
- kwic.minPosition.get(docId));
- mtasKwicItemResponse.add("documentMaxPosition",
- kwic.maxPosition.get(docId));
- mtasKwicItemResponse.add("list", mtasKwicItemResponseItems);
- mtasKwicItemResponses.add(mtasKwicItemResponse);
- }
- } else if (kwic.output.equals(ComponentKwic.KWIC_OUTPUT_TOKEN)) {
- for (int docId : kwic.tokens.keySet()) {
- NamedList<Object> mtasKwicItemResponse = new SimpleOrderedMap<>();
- ArrayList<KwicToken> list = kwic.tokens.get(docId);
- ArrayList<NamedList<Object>> mtasKwicItemResponseItems = new ArrayList<NamedList<Object>>();
- for (KwicToken k : list) {
- NamedList<Object> mtasKwicItemResponseItem = new SimpleOrderedMap<>();
- mtasKwicItemResponseItem.add("startPosition", k.startPosition);
- mtasKwicItemResponseItem.add("endPosition", k.endPosition);
- ArrayList<NamedList<Object>> mtasKwicItemResponseItemTokens = new ArrayList<NamedList<Object>>();
- for (MtasToken<?> token : k.tokens) {
- NamedList<Object> mtasKwicItemResponseItemToken = new SimpleOrderedMap<>();
- if (token.getId() != null) {
- mtasKwicItemResponseItemToken.add("mtasId", token.getId());
- }
- mtasKwicItemResponseItemToken.add("prefix", token.getPrefix());
- mtasKwicItemResponseItemToken.add("value", token.getPostfix());
- if (token.getPositionStart() != null) {
- mtasKwicItemResponseItemToken.add("positionStart",
- token.getPositionStart());
- mtasKwicItemResponseItemToken.add("positionEnd",
- token.getPositionEnd());
- }
- if (token.getPositions() != null) {
- mtasKwicItemResponseItemToken.add("positions",
- token.getPositions());
- }
- if (token.getParentId() != null) {
- mtasKwicItemResponseItemToken.add("parentMtasId",
- token.getParentId());
- }
- if (token.getPayload() != null) {
- mtasKwicItemResponseItemToken.add("payload", token.getPayload());
- }
- if (token.getOffsetStart() != null) {
- mtasKwicItemResponseItemToken.add("offsetStart",
- token.getOffsetStart());
- mtasKwicItemResponseItemToken.add("offsetEnd",
- token.getOffsetEnd());
- }
- if (token.getRealOffsetStart() != null) {
- mtasKwicItemResponseItemToken.add("realOffsetStart",
- token.getRealOffsetStart());
- mtasKwicItemResponseItemToken.add("realOffsetEnd",
- token.getRealOffsetEnd());
- }
- mtasKwicItemResponseItemTokens.add(mtasKwicItemResponseItemToken);
- }
- mtasKwicItemResponseItem.add("tokens",
- mtasKwicItemResponseItemTokens);
- mtasKwicItemResponseItems.add(mtasKwicItemResponseItem);
- }
- mtasKwicItemResponse.add("documentKey", kwic.uniqueKey.get(docId));
- mtasKwicItemResponse.add("documentTotal", kwic.subTotal.get(docId));
- mtasKwicItemResponse.add("documentMinPosition",
- kwic.minPosition.get(docId));
- mtasKwicItemResponse.add("documentMaxPosition",
- kwic.maxPosition.get(docId));
- mtasKwicItemResponse.add("list", mtasKwicItemResponseItems);
- mtasKwicItemResponses.add(mtasKwicItemResponse);
- }
- }
- mtasKwicResponse.add("list", mtasKwicItemResponses);
- return mtasKwicResponse;
- }
-
- /**
- * The Class ComponentSortSelect.
- */
- public static class ComponentSortSelect implements Serializable {
-
- /** The Constant serialVersionUID. */
- private static final long serialVersionUID = 1L;
-
- /** The stats type. */
- public String dataType, statsType;
-
- /** The stats items. */
- public TreeSet<String> statsItems;
-
- /** The sort direction. */
- public String sortType, sortDirection;
-
- /** The number. */
- public Integer start, number;
-
- /** The data collector. */
- public MtasDataCollector<?, ?> dataCollector = null;
-
- /** The sub stats type. */
- private String[] subDataType, subStatsType;
-
- /** The sub stats items. */
- private TreeSet<String>[] subStatsItems;
-
- /** The sub sort direction. */
- private String[] subSortType, subSortDirection;
-
- /** The sub number. */
- private Integer[] subStart, subNumber;
-
- /**
- * Instantiates a new component sort select.
- *
- * @param dataCollector the data collector
- * @param dataType the data type
- * @param statsType the stats type
- * @param statsItems the stats items
- * @param sortType the sort type
- * @param sortDirection the sort direction
- * @param start the start
- * @param number the number
- */
- public ComponentSortSelect(MtasDataCollector<?, ?> dataCollector,
- String[] dataType, String[] statsType, TreeSet<String>[] statsItems,
- String[] sortType, String[] sortDirection, Integer[] start,
- Integer[] number) {
- this.dataCollector = dataCollector;
- this.dataType = (dataType == null) ? null : dataType[0];
- this.statsType = (statsType == null) ? null : statsType[0];
- this.statsItems = (statsItems == null) ? null : statsItems[0];
- this.sortType = (sortType == null) ? null : sortType[0];
- this.sortDirection = (sortDirection == null) ? null : sortDirection[0];
- this.start = (start == null) ? null : start[0];
- this.number = (number == null) ? null : number[0];
- if ((dataType != null) && (dataType.length > 1)) {
- subDataType = new String[dataType.length - 1];
- subStatsType = new String[dataType.length - 1];
- subStatsItems = new TreeSet[dataType.length - 1];
- subSortType = new String[dataType.length - 1];
- subSortDirection = new String[dataType.length - 1];
- System.arraycopy(dataType, 1, subDataType, 0, dataType.length - 1);
- System.arraycopy(statsType, 1, subStatsType, 0, dataType.length - 1);
- System.arraycopy(statsItems, 1, subStatsItems, 0, dataType.length - 1);
- System.arraycopy(sortType, 1, subSortType, 0, dataType.length - 1);
- System.arraycopy(sortDirection, 1, subSortDirection, 0,
- dataType.length - 1);
- } else {
- subDataType = null;
- subStatsType = null;
- subStatsItems = null;
- subSortType = null;
- subSortDirection = null;
- }
- }
-
- /**
- * Instantiates a new component sort select.
- *
- * @param dataCollector the data collector
- * @param dataType the data type
- * @param statsType the stats type
- * @param statsItems the stats items
- */
- public ComponentSortSelect(MtasDataCollector dataCollector, String dataType,
- String statsType, TreeSet<String> statsItems) {
- this(dataCollector, new String[] { dataType }, new String[] { statsType },
- new TreeSet[] { statsItems }, new String[] { null },
- new String[] { null }, new Integer[] { 0 }, new Integer[] { 1 });
- }
-
- /**
- * Merge.
- *
- * @param newItem the new item
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public void merge(ComponentSortSelect newItem) throws IOException {
- dataCollector.merge(newItem.dataCollector);
- }
-
- /**
- * Gets the total.
- *
- * @return the total
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public int getTotal() throws IOException {
- if (dataCollector.getCollectorType()
- .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
- return dataCollector.getSize();
- } else {
- throw new IOException(
- "only allowed for " + DataCollector.COLLECTOR_TYPE_LIST);
- }
- }
-
- /**
- * Gets the data.
- *
- * @return the data
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public NamedList<Object> getData() throws IOException {
- if (dataCollector.getCollectorType()
- .equals(DataCollector.COLLECTOR_TYPE_DATA)) {
- NamedList<Object> mtasResponse = new SimpleOrderedMap<>();
- MtasDataItem dataItem = dataCollector.getData();
- if (dataItem != null) {
- mtasResponse.addAll(dataItem.rewrite());
- }
- if ((subDataType != null) && (dataItem.getSub() != null)) {
- ComponentSortSelect css = new ComponentSortSelect(dataItem.getSub(),
- subDataType, subStatsType, subStatsItems, subSortType,
- subSortDirection, subStart, subNumber);
- if (dataItem.getSub().getCollectorType()
- .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
- mtasResponse.add(dataItem.getSub().getCollectorType(),
- css.getNamedList());
- } else if (dataItem.getSub().getCollectorType()
- .equals(DataCollector.COLLECTOR_TYPE_DATA)) {
- mtasResponse.add(dataItem.getSub().getCollectorType(),
- css.getData());
- }
- }
- return mtasResponse;
- } else {
- throw new IOException(
- "only allowed for " + DataCollector.COLLECTOR_TYPE_DATA);
- }
- }
-
- /**
- * Gets the named list.
- *
- * @return the named list
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public NamedList<Object> getNamedList() throws IOException {
- if (dataCollector.getCollectorType()
- .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
- SimpleOrderedMap<Object> mtasResponseList = new SimpleOrderedMap<>();
- Map<String, MtasDataItem<?>> dataList = (Map<String, MtasDataItem<?>>) dataCollector
- .getList();
- for (String key : dataList.keySet()) {
- SimpleOrderedMap<Object> mtasResponseListItem = new SimpleOrderedMap<>();
- MtasDataItem<?> dataItem = dataList.get(key);
- mtasResponseListItem.addAll(dataItem.rewrite());
- if ((subDataType != null) && (dataItem.getSub() != null)) {
- ComponentSortSelect css = new ComponentSortSelect(dataItem.getSub(),
- subDataType, subStatsType, subStatsItems, subSortType,
- subSortDirection, subStart, subNumber);
- if (dataItem.getSub().getCollectorType()
- .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
- mtasResponseListItem.add(dataItem.getSub().getCollectorType(),
- css.getNamedList());
- } else if (dataItem.getSub().getCollectorType()
- .equals(DataCollector.COLLECTOR_TYPE_DATA)) {
- mtasResponseListItem.add(dataItem.getSub().getCollectorType(),
- css.getData());
- }
- }
- mtasResponseList.add(key, mtasResponseListItem);
- }
- return mtasResponseList;
- } else {
- throw new IOException(
- "only allowed for " + DataCollector.COLLECTOR_TYPE_LIST);
- }
- }
-
- }
-
}
diff --git a/src/mtas/solr/handler/component/util/MtasSolrComponentFacet.java b/src/mtas/solr/handler/component/util/MtasSolrComponentFacet.java
new file mode 100644
index 0000000..c6c0b3d
--- /dev/null
+++ b/src/mtas/solr/handler/component/util/MtasSolrComponentFacet.java
@@ -0,0 +1,516 @@
+package mtas.solr.handler.component.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Set;
+
+import org.apache.lucene.document.FieldType.LegacyNumericType;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.handler.component.ShardResponse;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+
+import mtas.codec.util.CodecComponent.ComponentFacet;
+import mtas.codec.util.CodecComponent.ComponentField;
+import mtas.codec.util.CodecComponent.ComponentFields;
+import mtas.codec.util.CodecComponent.SubComponentFunction;
+import mtas.codec.util.collector.MtasDataCollector;
+import mtas.parser.function.ParseException;
+import mtas.solr.handler.component.MtasSolrSearchComponent;
+
+/**
+ * The Class MtasSolrComponentFacet.
+ */
+@SuppressWarnings("deprecation")
+public class MtasSolrComponentFacet {
+
+ /** The search component. */
+ MtasSolrSearchComponent searchComponent;
+
+ /** The Constant PARAM_MTAS_FACET. */
+ public static final String PARAM_MTAS_FACET = MtasSolrSearchComponent.PARAM_MTAS
+ + ".facet";
+
+ /** The Constant NAME_MTAS_FACET_KEY. */
+ public static final String NAME_MTAS_FACET_KEY = "key";
+
+ /** The Constant NAME_MTAS_FACET_FIELD. */
+ public static final String NAME_MTAS_FACET_FIELD = "field";
+
+ /** The Constant NAME_MTAS_FACET_QUERY. */
+ private static final String NAME_MTAS_FACET_QUERY = "query";
+
+ /** The Constant NAME_MTAS_FACET_BASE. */
+ private static final String NAME_MTAS_FACET_BASE = "base";
+
+ /** The Constant SUBNAME_MTAS_FACET_QUERY_TYPE. */
+ public static final String SUBNAME_MTAS_FACET_QUERY_TYPE = "type";
+
+ /** The Constant SUBNAME_MTAS_FACET_QUERY_VALUE. */
+ public static final String SUBNAME_MTAS_FACET_QUERY_VALUE = "value";
+
+ /** The Constant SUBNAME_MTAS_FACET_BASE_FIELD. */
+ public static final String SUBNAME_MTAS_FACET_BASE_FIELD = "field";
+
+ /** The Constant SUBNAME_MTAS_FACET_BASE_TYPE. */
+ public static final String SUBNAME_MTAS_FACET_BASE_TYPE = "type";
+
+ /** The Constant SUBNAME_MTAS_FACET_BASE_SORT_TYPE. */
+ public static final String SUBNAME_MTAS_FACET_BASE_SORT_TYPE = "sort.type";
+
+ /** The Constant SUBNAME_MTAS_FACET_BASE_SORT_DIRECTION. */
+ public static final String SUBNAME_MTAS_FACET_BASE_SORT_DIRECTION = "sort.direction";
+
+ /** The Constant SUBNAME_MTAS_FACET_BASE_NUMBER. */
+ public static final String SUBNAME_MTAS_FACET_BASE_NUMBER = "number";
+
+ /** The Constant SUBNAME_MTAS_FACET_BASE_MINIMUM. */
+ public static final String SUBNAME_MTAS_FACET_BASE_MINIMUM = "minimum";
+
+ /** The Constant SUBNAME_MTAS_FACET_BASE_MAXIMUM. */
+ public static final String SUBNAME_MTAS_FACET_BASE_MAXIMUM = "maximum";
+
+ /** The Constant SUBNAME_MTAS_FACET_BASE_FUNCTION. */
+ public static final String SUBNAME_MTAS_FACET_BASE_FUNCTION = "function";
+
+ /** The Constant SUBNAME_MTAS_FACET_BASE_FUNCTION_KEY. */
+ public static final String SUBNAME_MTAS_FACET_BASE_FUNCTION_KEY = "key";
+
+ /** The Constant SUBNAME_MTAS_FACET_BASE_FUNCTION_EXPRESSION. */
+ public static final String SUBNAME_MTAS_FACET_BASE_FUNCTION_EXPRESSION = "expression";
+
+ /** The Constant SUBNAME_MTAS_FACET_BASE_FUNCTION_TYPE. */
+ public static final String SUBNAME_MTAS_FACET_BASE_FUNCTION_TYPE = "type";
+
+ /**
+ * Instantiates a new mtas solr component facet.
+ *
+ * @param searchComponent
+ * the search component
+ */
+ public MtasSolrComponentFacet(MtasSolrSearchComponent searchComponent) {
+ this.searchComponent = searchComponent;
+ }
+
+ /**
+ * Prepare.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ Set<String> ids = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_FACET);
+ if (ids.size() > 0) {
+ int tmpCounter = 0;
+ String tmpValue;
+ String[] fields = new String[ids.size()];
+ String[] keys = new String[ids.size()];
+ String[][] queryTypes = new String[ids.size()][];
+ String[][] queryValues = new String[ids.size()][];
+ String[][] baseFields = new String[ids.size()][];
+ String[][] baseFieldTypes = new String[ids.size()][];
+ String[][] baseTypes = new String[ids.size()][];
+ String[][] baseSortTypes = new String[ids.size()][];
+ String[][] baseSortDirections = new String[ids.size()][];
+ Integer[][] baseNumbers = new Integer[ids.size()][];
+ Double[][] baseMinima = new Double[ids.size()][];
+ Double[][] baseMaxima = new Double[ids.size()][];
+ String[][][] baseFunctionExpressions = new String[ids.size()][][];
+ String[][][] baseFunctionKeys = new String[ids.size()][][];
+ String[][][] baseFunctionTypes = new String[ids.size()][][];
+ for (String id : ids) {
+ fields[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_FIELD, null);
+ keys[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_KEY,
+ String.valueOf(tmpCounter))
+ .trim();
+ Set<String> qIds = MtasSolrResultUtil.getIdsFromParameters(
+ rb.req.getParams(),
+ PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_QUERY);
+ if (qIds.size() > 0) {
+ int tmpQCounter = 0;
+ queryTypes[tmpCounter] = new String[qIds.size()];
+ queryValues[tmpCounter] = new String[qIds.size()];
+ for (String qId : qIds) {
+ queryTypes[tmpCounter][tmpQCounter] = rb.req.getParams()
+ .get(
+ PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_QUERY
+ + "." + qId + "." + SUBNAME_MTAS_FACET_QUERY_TYPE,
+ null);
+ queryValues[tmpCounter][tmpQCounter] = rb.req.getParams()
+ .get(
+ PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_QUERY
+ + "." + qId + "." + SUBNAME_MTAS_FACET_QUERY_VALUE,
+ null);
+ tmpQCounter++;
+ }
+ } else {
+ throw new IOException(
+ "no " + NAME_MTAS_FACET_QUERY + " for mtas facet " + id);
+ }
+ Set<String> bIds = MtasSolrResultUtil.getIdsFromParameters(
+ rb.req.getParams(),
+ PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE);
+ if (bIds.size() > 0) {
+ int tmpBCounter = 0;
+ baseFields[tmpCounter] = new String[bIds.size()];
+ baseFieldTypes[tmpCounter] = new String[bIds.size()];
+ baseTypes[tmpCounter] = new String[bIds.size()];
+ baseSortTypes[tmpCounter] = new String[bIds.size()];
+ baseSortDirections[tmpCounter] = new String[bIds.size()];
+ baseNumbers[tmpCounter] = new Integer[bIds.size()];
+ baseMinima[tmpCounter] = new Double[bIds.size()];
+ baseMaxima[tmpCounter] = new Double[bIds.size()];
+ baseFunctionKeys[tmpCounter] = new String[bIds.size()][];
+ baseFunctionExpressions[tmpCounter] = new String[bIds.size()][];
+ baseFunctionTypes[tmpCounter] = new String[bIds.size()][];
+ for (String bId : bIds) {
+ baseFields[tmpCounter][tmpBCounter] = rb.req.getParams()
+ .get(
+ PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
+ + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_FIELD,
+ null);
+ baseFieldTypes[tmpCounter][tmpBCounter] = getFieldType(
+ rb.req.getSchema(), baseFields[tmpCounter][tmpBCounter]);
+ baseTypes[tmpCounter][tmpBCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
+ + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_TYPE, null);
+ baseSortTypes[tmpCounter][tmpBCounter] = rb.req.getParams()
+ .get(
+ PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
+ + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_SORT_TYPE,
+ null);
+ baseSortDirections[tmpCounter][tmpBCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
+ + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_SORT_DIRECTION,
+ null);
+ tmpValue = rb.req.getParams()
+ .get(
+ PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
+ + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_NUMBER,
+ null);
+ baseNumbers[tmpCounter][tmpBCounter] = tmpValue != null
+ ? getPositiveInteger(tmpValue) : null;
+ tmpValue = rb.req.getParams()
+ .get(
+ PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
+ + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_MINIMUM,
+ null);
+ baseMinima[tmpCounter][tmpBCounter] = tmpValue != null
+ ? getDouble(tmpValue) : null;
+ tmpValue = rb.req.getParams()
+ .get(
+ PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
+ + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_MAXIMUM,
+ null);
+ baseMaxima[tmpCounter][tmpBCounter] = tmpValue != null
+ ? getDouble(tmpValue) : null;
+ Set<String> functionIds = MtasSolrResultUtil.getIdsFromParameters(
+ rb.req.getParams(),
+ PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE + "."
+ + bId + "." + SUBNAME_MTAS_FACET_BASE_FUNCTION);
+ baseFunctionExpressions[tmpCounter][tmpBCounter] = new String[functionIds
+ .size()];
+ baseFunctionKeys[tmpCounter][tmpBCounter] = new String[functionIds
+ .size()];
+ baseFunctionTypes[tmpCounter][tmpBCounter] = new String[functionIds
+ .size()];
+ int tmpSubCounter = 0;
+ for (String functionId : functionIds) {
+ baseFunctionKeys[tmpCounter][tmpBCounter][tmpSubCounter] = rb.req
+ .getParams()
+ .get(PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
+ + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_FUNCTION + "."
+ + functionId + "." + SUBNAME_MTAS_FACET_BASE_FUNCTION_KEY,
+ String.valueOf(tmpSubCounter))
+ .trim();
+ baseFunctionExpressions[tmpCounter][tmpBCounter][tmpSubCounter] = rb.req
+ .getParams()
+ .get(PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
+ + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_FUNCTION + "."
+ + functionId + "."
+ + SUBNAME_MTAS_FACET_BASE_FUNCTION_EXPRESSION, null);
+ baseFunctionTypes[tmpCounter][tmpBCounter][tmpSubCounter] = rb.req
+ .getParams()
+ .get(PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_BASE
+ + "." + bId + "." + SUBNAME_MTAS_FACET_BASE_FUNCTION + "."
+ + functionId + "."
+ + SUBNAME_MTAS_FACET_BASE_FUNCTION_TYPE, null);
+ tmpSubCounter++;
+ }
+ tmpBCounter++;
+ }
+ } else {
+ throw new IOException(
+ "no " + NAME_MTAS_FACET_BASE + " for mtas facet " + id);
+ }
+ tmpCounter++;
+ }
+ String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
+ mtasFields.doFacet = true;
+ rb.setNeedDocSet(true);
+ for (String field : fields) {
+ if (field == null || field.isEmpty()) {
+ throw new IOException("no (valid) field in mtas facet");
+ } else if (!mtasFields.list.containsKey(field)) {
+ mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
+ }
+ }
+ MtasSolrResultUtil.compareAndCheck(keys, fields, NAME_MTAS_FACET_KEY,
+ NAME_MTAS_FACET_FIELD, true);
+ for (int i = 0; i < fields.length; i++) {
+ ComponentField cf = mtasFields.list.get(fields[i]);
+ int queryNumber = queryValues[i].length;
+ SpanQuery ql[] = new SpanQuery[queryNumber];
+ for (int j = 0; j < queryNumber; j++) {
+ SpanQuery q = MtasSolrResultUtil.constructQuery(queryValues[i][j],
+ queryTypes[i][j], fields[i]);
+ // minimize number of queries
+ if (cf.spanQueryList.contains(q)) {
+ q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
+ } else {
+ cf.spanQueryList.add(q);
+ }
+ ql[j] = q;
+ }
+ String key = (keys[i] == null) || (keys[i].isEmpty())
+ ? String.valueOf(i) + ":" + fields[i] : keys[i].trim();
+ try {
+ mtasFields.list.get(fields[i]).facetList.add(new ComponentFacet(ql,
+ fields[i], key, baseFields[i], baseFieldTypes[i], baseTypes[i],
+ baseSortTypes[i], baseSortDirections[i], baseNumbers[i],
+ baseMinima[i], baseMaxima[i], baseFunctionKeys[i],
+ baseFunctionExpressions[i], baseFunctionTypes[i]));
+ } catch (ParseException e) {
+ throw new IOException(e.getMessage());
+ }
+ }
+ }
+ }
+
+ /**
+ * Modify request.
+ *
+ * @param rb
+ * the rb
+ * @param who
+ * the who
+ * @param sreq
+ * the sreq
+ */
+ public void modifyRequest(ResponseBuilder rb, SearchComponent who,
+ ShardRequest sreq) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (sreq.params.getBool(PARAM_MTAS_FACET, false)) {
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
+ // do nothing
+ } else {
+ // remove prefix for other requests
+ Set<String> keys = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_FACET);
+ sreq.params.remove(PARAM_MTAS_FACET);
+ for (String key : keys) {
+ sreq.params.remove(
+ PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_FIELD);
+ sreq.params.remove(
+ PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_KEY);
+ sreq.params.remove(
+ PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_QUERY);
+ sreq.params.remove(
+ PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Creates the.
+ *
+ * @param facet
+ * the facet
+ * @param encode
+ * the encode
+ * @return the simple ordered map
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public SimpleOrderedMap<Object> create(ComponentFacet facet, Boolean encode)
+ throws IOException {
+ SimpleOrderedMap<Object> mtasFacetResponse = new SimpleOrderedMap<>();
+ mtasFacetResponse.add("key", facet.key);
+ HashMap<MtasDataCollector<?, ?>, HashMap<String, MtasSolrResult>> functionData = new HashMap<MtasDataCollector<?, ?>, HashMap<String, MtasSolrResult>>();
+ for (int i = 0; i < facet.baseFields.length; i++) {
+ if (facet.baseFunctionList[i] != null) {
+ for (MtasDataCollector<?, ?> functionDataCollector : facet.baseFunctionList[i]
+ .keySet()) {
+ SubComponentFunction[] tmpSubComponentFunctionList = facet.baseFunctionList[i]
+ .get(functionDataCollector);
+ if (tmpSubComponentFunctionList != null) {
+ HashMap<String, MtasSolrResult> tmpList = new HashMap<String, MtasSolrResult>();
+ for (SubComponentFunction tmpSubComponentFunction : tmpSubComponentFunctionList) {
+ tmpList.put(tmpSubComponentFunction.key,
+ new MtasSolrResult(tmpSubComponentFunction.dataCollector,
+ tmpSubComponentFunction.dataType,
+ tmpSubComponentFunction.statsType,
+ tmpSubComponentFunction.statsItems, null));
+ }
+ functionData.put(functionDataCollector, tmpList);
+ }
+ }
+ }
+ }
+ MtasSolrResult data = new MtasSolrResult(facet.dataCollector,
+ facet.baseDataTypes, facet.baseStatsTypes, facet.baseStatsItems,
+ facet.baseSortTypes, facet.baseSortDirections, null, facet.baseNumbers,
+ functionData);
+
+ if (encode) {
+ mtasFacetResponse.add("_encoded_list", MtasSolrResultUtil.encode(data));
+ } else {
+ mtasFacetResponse.add("list", data);
+ MtasSolrResultUtil.rewrite(mtasFacetResponse);
+ }
+ return mtasFacetResponse;
+ }
+
+ /**
+ * Finish stage.
+ *
+ * @param rb
+ * the rb
+ */
+ @SuppressWarnings("unchecked")
+ public void finishStage(ResponseBuilder rb) {
+ if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (rb.stage >= ResponseBuilder.STAGE_EXECUTE_QUERY
+ && rb.stage < ResponseBuilder.STAGE_GET_FIELDS) {
+ for (ShardRequest sreq : rb.finished) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
+ && sreq.params.getBool(PARAM_MTAS_FACET, false)) {
+ for (ShardResponse shardResponse : sreq.responses) {
+ NamedList<Object> response = shardResponse.getSolrResponse()
+ .getResponse();
+ try {
+ ArrayList<NamedList<Object>> data = (ArrayList<NamedList<Object>>) response
+ .findRecursive("mtas", "facet");
+ if (data != null) {
+ MtasSolrResultUtil.decode(data);
+ }
+ } catch (ClassCastException e) {
+ // shouldn't happen
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Distributed process.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ public void distributedProcess(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ // rewrite
+ NamedList<Object> mtasResponse = null;
+ try {
+ mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
+ if (mtasResponse != null) {
+ ArrayList<Object> mtasResponseFacet;
+ try {
+ mtasResponseFacet = (ArrayList<Object>) mtasResponse.get("facet");
+ if (mtasResponseFacet != null) {
+ MtasSolrResultUtil.rewrite(mtasResponseFacet);
+ }
+ } catch (ClassCastException e) {
+ mtasResponseFacet = null;
+ }
+ }
+ } catch (ClassCastException e) {
+ mtasResponse = null;
+ }
+ }
+
+ /**
+ * Gets the field type.
+ *
+ * @param schema
+ * the schema
+ * @param field
+ * the field
+ * @return the field type
+ */
+ private String getFieldType(IndexSchema schema, String field) {
+ SchemaField sf = schema.getField(field);
+ FieldType ft = sf.getType();
+ if (ft != null) {
+ if (ft.getNumericType() != null) {
+ LegacyNumericType nt = ft.getNumericType();
+ if (nt.equals(LegacyNumericType.INT)) {
+ return ComponentFacet.TYPE_INTEGER;
+ } else if (nt.equals(LegacyNumericType.DOUBLE)) {
+ return ComponentFacet.TYPE_DOUBLE;
+ } else if (nt.equals(LegacyNumericType.LONG)) {
+ return ComponentFacet.TYPE_LONG;
+ } else if (nt.equals(LegacyNumericType.FLOAT)) {
+ return ComponentFacet.TYPE_FLOAT;
+ }
+ }
+ }
+ return ComponentFacet.TYPE_STRING;
+ }
+
+ /**
+ * Gets the positive integer.
+ *
+ * @param number
+ * the number
+ * @return the positive integer
+ */
+ private int getPositiveInteger(String number) {
+ try {
+ return Math.max(0, Integer.parseInt(number));
+ } catch (NumberFormatException e) {
+ return 0;
+ }
+ }
+
+ /**
+ * Gets the double.
+ *
+ * @param number
+ * the number
+ * @return the double
+ */
+ private Double getDouble(String number) {
+ try {
+ return Double.parseDouble(number);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
+}
diff --git a/src/mtas/solr/handler/component/util/MtasSolrComponentGroup.java b/src/mtas/solr/handler/component/util/MtasSolrComponentGroup.java
new file mode 100644
index 0000000..de60443
--- /dev/null
+++ b/src/mtas/solr/handler/component/util/MtasSolrComponentGroup.java
@@ -0,0 +1,459 @@
+package mtas.solr.handler.component.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.handler.component.ShardResponse;
+
+import mtas.codec.util.CodecComponent.ComponentField;
+import mtas.codec.util.CodecComponent.ComponentFields;
+import mtas.codec.util.CodecComponent.ComponentGroup;
+import mtas.solr.handler.component.MtasSolrSearchComponent;
+
+/**
+ * The Class MtasSolrComponentGroup.
+ */
+public class MtasSolrComponentGroup {
+
+ /** The search component. */
+ MtasSolrSearchComponent searchComponent;
+
+ /** The Constant PARAM_MTAS_GROUP. */
+ public static final String PARAM_MTAS_GROUP = MtasSolrSearchComponent.PARAM_MTAS
+ + ".group";
+
+ /** The Constant NAME_MTAS_GROUP_FIELD. */
+ public static final String NAME_MTAS_GROUP_FIELD = "field";
+
+ /** The Constant NAME_MTAS_GROUP_QUERY_TYPE. */
+ public static final String NAME_MTAS_GROUP_QUERY_TYPE = "query.type";
+
+ /** The Constant NAME_MTAS_GROUP_QUERY_VALUE. */
+ public static final String NAME_MTAS_GROUP_QUERY_VALUE = "query.value";
+
+ /** The Constant NAME_MTAS_GROUP_KEY. */
+ public static final String NAME_MTAS_GROUP_KEY = "key";
+
+ /** The Constant NAME_MTAS_GROUP_GROUPING_LEFT. */
+ public static final String NAME_MTAS_GROUP_GROUPING_LEFT = "grouping.left";
+
+ /** The Constant NAME_MTAS_GROUP_GROUPING_RIGHT. */
+ public static final String NAME_MTAS_GROUP_GROUPING_RIGHT = "grouping.right";
+
+ /** The Constant NAME_MTAS_GROUP_GROUPING_HIT_INSIDE. */
+ public static final String NAME_MTAS_GROUP_GROUPING_HIT_INSIDE = "grouping.hit.inside";
+
+ /** The Constant NAME_MTAS_GROUP_GROUPING_HIT_LEFT. */
+ public static final String NAME_MTAS_GROUP_GROUPING_HIT_LEFT = "grouping.hit.left";
+
+ /** The Constant NAME_MTAS_GROUP_GROUPING_HIT_RIGHT. */
+ public static final String NAME_MTAS_GROUP_GROUPING_HIT_RIGHT = "grouping.hit.right";
+
+ /** The Constant NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_LEFT. */
+ public static final String NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_LEFT = "grouping.hit.insideLeft";
+
+ /** The Constant NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_RIGHT. */
+ public static final String NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_RIGHT = "grouping.hit.insideRight";
+
+ /** The Constant NAME_MTAS_GROUP_GROUPING_POSITION. */
+ public static final String NAME_MTAS_GROUP_GROUPING_POSITION = "position";
+
+ /** The Constant NAME_MTAS_GROUP_GROUPING_PREFIXES. */
+ public static final String NAME_MTAS_GROUP_GROUPING_PREFIXES = "prefixes";
+
+ /**
+ * Instantiates a new mtas solr component group.
+ *
+ * @param searchComponent
+ * the search component
+ */
+ public MtasSolrComponentGroup(MtasSolrSearchComponent searchComponent) {
+ this.searchComponent = searchComponent;
+ }
+
+ /**
+ * Prepare.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ Set<String> ids = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_GROUP);
+ if (ids.size() > 0) {
+ int tmpCounter = 0;
+ String[] fields = new String[ids.size()];
+ String[] queryTypes = new String[ids.size()];
+ String[] queryValues = new String[ids.size()];
+ String[] keys = new String[ids.size()];
+ String[][] groupingLeftPosition = new String[ids.size()][];
+ String[][] groupingLeftPrefixes = new String[ids.size()][];
+ String[][] groupingRightPosition = new String[ids.size()][];
+ String[][] groupingRightPrefixes = new String[ids.size()][];
+ String[] groupingHitInsidePrefixes = new String[ids.size()];
+ String[][] groupingHitLeftPosition = new String[ids.size()][];
+ String[][] groupingHitLeftPrefixes = new String[ids.size()][];
+ String[][] groupingHitRightPosition = new String[ids.size()][];
+ String[][] groupingHitRightPrefixes = new String[ids.size()][];
+ String[][] groupingHitInsideLeftPosition = new String[ids.size()][];
+ String[][] groupingHitInsideLeftPrefixes = new String[ids.size()][];
+ String[][] groupingHitInsideRightPosition = new String[ids.size()][];
+ String[][] groupingHitInsideRightPrefixes = new String[ids.size()][];
+ for (String id : ids) {
+ fields[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_GROUP + "." + id + "." + NAME_MTAS_GROUP_FIELD, null);
+ keys[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_GROUP + "." + id + "." + NAME_MTAS_GROUP_KEY,
+ String.valueOf(tmpCounter))
+ .trim();
+ queryTypes[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_GROUP + "." + id + "." + NAME_MTAS_GROUP_QUERY_TYPE,
+ null);
+ queryValues[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_GROUP + "." + id + "." + NAME_MTAS_GROUP_QUERY_VALUE,
+ null);
+ groupingHitInsidePrefixes[tmpCounter] = null;
+ // collect
+ SortedSet<String> gids;
+ String tmpName;
+ // collect grouping inside
+ tmpName = PARAM_MTAS_GROUP + "." + id + "."
+ + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE;
+ groupingHitInsidePrefixes[tmpCounter] = rb.req.getParams()
+ .get(tmpName + "." + NAME_MTAS_GROUP_GROUPING_PREFIXES);
+ // collect grouping left
+ tmpName = PARAM_MTAS_GROUP + "." + id + "."
+ + NAME_MTAS_GROUP_GROUPING_LEFT;
+ gids = MtasSolrResultUtil.getIdsFromParameters(rb.req.getParams(),
+ tmpName);
+ groupingLeftPosition[tmpCounter] = new String[gids.size()];
+ groupingLeftPrefixes[tmpCounter] = new String[gids.size()];
+ prepare(rb.req.getParams(), gids, tmpName,
+ groupingLeftPosition[tmpCounter], groupingLeftPrefixes[tmpCounter]);
+ // collect grouping right
+ tmpName = PARAM_MTAS_GROUP + "." + id + "."
+ + NAME_MTAS_GROUP_GROUPING_RIGHT;
+ gids = MtasSolrResultUtil.getIdsFromParameters(rb.req.getParams(),
+ tmpName);
+ groupingRightPosition[tmpCounter] = new String[gids.size()];
+ groupingRightPrefixes[tmpCounter] = new String[gids.size()];
+ prepare(rb.req.getParams(), gids, tmpName,
+ groupingRightPosition[tmpCounter],
+ groupingRightPrefixes[tmpCounter]);
+ // collect grouping hit left
+ tmpName = PARAM_MTAS_GROUP + "." + id + "."
+ + NAME_MTAS_GROUP_GROUPING_HIT_LEFT;
+ gids = MtasSolrResultUtil.getIdsFromParameters(rb.req.getParams(),
+ tmpName);
+ groupingHitLeftPosition[tmpCounter] = new String[gids.size()];
+ groupingHitLeftPrefixes[tmpCounter] = new String[gids.size()];
+ prepare(rb.req.getParams(), gids, tmpName,
+ groupingHitLeftPosition[tmpCounter],
+ groupingHitLeftPrefixes[tmpCounter]);
+ // collect grouping hit right
+ tmpName = PARAM_MTAS_GROUP + "." + id + "."
+ + NAME_MTAS_GROUP_GROUPING_HIT_RIGHT;
+ gids = MtasSolrResultUtil.getIdsFromParameters(rb.req.getParams(),
+ tmpName);
+ groupingHitRightPosition[tmpCounter] = new String[gids.size()];
+ groupingHitRightPrefixes[tmpCounter] = new String[gids.size()];
+ prepare(rb.req.getParams(), gids, tmpName,
+ groupingHitRightPosition[tmpCounter],
+ groupingHitRightPrefixes[tmpCounter]);
+ // collect grouping hit inside left
+ tmpName = PARAM_MTAS_GROUP + "." + id + "."
+ + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_LEFT;
+ gids = MtasSolrResultUtil.getIdsFromParameters(rb.req.getParams(),
+ tmpName);
+ groupingHitInsideLeftPosition[tmpCounter] = new String[gids.size()];
+ groupingHitInsideLeftPrefixes[tmpCounter] = new String[gids.size()];
+ prepare(rb.req.getParams(), gids, tmpName,
+ groupingHitInsideLeftPosition[tmpCounter],
+ groupingHitInsideLeftPrefixes[tmpCounter]);
+ // collect grouping hit inside right
+ tmpName = PARAM_MTAS_GROUP + "." + id + "."
+ + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_RIGHT;
+ gids = MtasSolrResultUtil.getIdsFromParameters(rb.req.getParams(),
+ tmpName);
+ groupingHitInsideRightPosition[tmpCounter] = new String[gids.size()];
+ groupingHitInsideRightPrefixes[tmpCounter] = new String[gids.size()];
+ prepare(rb.req.getParams(), gids, tmpName,
+ groupingHitInsideRightPosition[tmpCounter],
+ groupingHitInsideRightPrefixes[tmpCounter]);
+
+ tmpCounter++;
+ }
+ String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
+ mtasFields.doGroup = true;
+ rb.setNeedDocSet(true);
+ for (String field : fields) {
+ if (field == null || field.isEmpty()) {
+ throw new IOException("no (valid) field in mtas group");
+ } else if (!mtasFields.list.containsKey(field)) {
+ mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
+ }
+ }
+ MtasSolrResultUtil.compareAndCheck(keys, fields, NAME_MTAS_GROUP_KEY,
+ NAME_MTAS_GROUP_FIELD, true);
+ MtasSolrResultUtil.compareAndCheck(queryValues, fields,
+ NAME_MTAS_GROUP_QUERY_VALUE, NAME_MTAS_GROUP_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(queryTypes, fields,
+ NAME_MTAS_GROUP_QUERY_TYPE, NAME_MTAS_GROUP_FIELD, false);
+ for (int i = 0; i < fields.length; i++) {
+ ComponentField cf = mtasFields.list.get(fields[i]);
+ SpanQuery q = MtasSolrResultUtil.constructQuery(queryValues[i],
+ queryTypes[i], fields[i]);
+ // minimize number of queries
+ if (cf.spanQueryList.contains(q)) {
+ q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
+ } else {
+ cf.spanQueryList.add(q);
+ }
+ String key = (keys[i] == null) || (keys[i].isEmpty())
+ ? String.valueOf(i) + ":" + fields[i] + ":" + queryValues[i]
+ : keys[i].trim();
+ mtasFields.list.get(fields[i]).groupList.add(new ComponentGroup(q,
+ fields[i], queryValues[i], queryTypes[i], key,
+ groupingHitInsidePrefixes[i], groupingHitInsideLeftPosition[i],
+ groupingHitInsideLeftPrefixes[i], groupingHitInsideRightPosition[i],
+ groupingHitInsideRightPrefixes[i], groupingHitLeftPosition[i],
+ groupingHitLeftPrefixes[i], groupingHitRightPosition[i],
+ groupingHitRightPrefixes[i], groupingLeftPosition[i],
+ groupingLeftPrefixes[i], groupingRightPosition[i],
+ groupingRightPrefixes[i]));
+ }
+ }
+ }
+
+ /**
+ * Prepare.
+ *
+ * @param solrParams
+ * the solr params
+ * @param gids
+ * the gids
+ * @param name
+ * the name
+ * @param positions
+ * the positions
+ * @param prefixes
+ * the prefixes
+ */
+ private void prepare(SolrParams solrParams, SortedSet<String> gids,
+ String name, String[] positions, String[] prefixes) {
+ if (gids.size() > 0) {
+ int tmpSubCounter = 0;
+ for (String gid : gids) {
+ positions[tmpSubCounter] = solrParams.get(
+ name + "." + gid + "." + NAME_MTAS_GROUP_GROUPING_POSITION, null);
+ prefixes[tmpSubCounter] = solrParams.get(
+ name + "." + gid + "." + NAME_MTAS_GROUP_GROUPING_PREFIXES, null);
+ tmpSubCounter++;
+ }
+ }
+ }
+
+ /**
+ * Modify request.
+ *
+ * @param rb
+ * the rb
+ * @param who
+ * the who
+ * @param sreq
+ * the sreq
+ */
+ public void modifyRequest(ResponseBuilder rb, SearchComponent who,
+ ShardRequest sreq) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (sreq.params.getBool(PARAM_MTAS_GROUP, false)) {
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
+ // do nothing
+ } else {
+ // remove prefix for other requests
+ Set<String> keys = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_GROUP);
+ sreq.params.remove(PARAM_MTAS_GROUP);
+ Set<String> subKeys;
+ for (String key : keys) {
+ sreq.params.remove(
+ PARAM_MTAS_GROUP + "." + key + "." + NAME_MTAS_GROUP_FIELD);
+ sreq.params.remove(
+ PARAM_MTAS_GROUP + "." + key + "." + NAME_MTAS_GROUP_KEY);
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_QUERY_TYPE);
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_QUERY_VALUE);
+ subKeys = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_GROUP + "."
+ + key + "." + NAME_MTAS_GROUP_GROUPING_LEFT);
+ for (String subKey : subKeys) {
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_GROUPING_LEFT + "." + subKey + "."
+ + NAME_MTAS_GROUP_GROUPING_POSITION);
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_GROUPING_LEFT + "." + subKey + "."
+ + NAME_MTAS_GROUP_GROUPING_PREFIXES);
+ }
+ subKeys = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_GROUP + "."
+ + key + "." + NAME_MTAS_GROUP_GROUPING_RIGHT);
+ for (String subKey : subKeys) {
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_GROUPING_RIGHT + "." + subKey + "."
+ + NAME_MTAS_GROUP_GROUPING_POSITION);
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_GROUPING_RIGHT + "." + subKey + "."
+ + NAME_MTAS_GROUP_GROUPING_PREFIXES);
+ }
+ subKeys = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_GROUP + "."
+ + key + "." + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE);
+ for (String subKey : subKeys) {
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE + "." + subKey + "."
+ + NAME_MTAS_GROUP_GROUPING_POSITION);
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE + "." + subKey + "."
+ + NAME_MTAS_GROUP_GROUPING_PREFIXES);
+ }
+ subKeys = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_GROUP + "."
+ + key + "." + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_LEFT);
+ for (String subKey : subKeys) {
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_LEFT + "." + subKey
+ + "." + NAME_MTAS_GROUP_GROUPING_POSITION);
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_LEFT + "." + subKey
+ + "." + NAME_MTAS_GROUP_GROUPING_PREFIXES);
+ }
+ subKeys = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_GROUP + "."
+ + key + "." + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_RIGHT);
+ for (String subKey : subKeys) {
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_RIGHT + "." + subKey
+ + "." + NAME_MTAS_GROUP_GROUPING_POSITION);
+ sreq.params.remove(PARAM_MTAS_GROUP + "." + key + "."
+ + NAME_MTAS_GROUP_GROUPING_HIT_INSIDE_RIGHT + "." + subKey
+ + "." + NAME_MTAS_GROUP_GROUPING_PREFIXES);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Creates the.
+ *
+ * @param group
+ * the group
+ * @param encode
+ * the encode
+ * @return the simple ordered map
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ public SimpleOrderedMap<Object> create(ComponentGroup group, Boolean encode)
+ throws IOException {
+ SimpleOrderedMap<Object> mtasGroupResponse = new SimpleOrderedMap<>();
+ mtasGroupResponse.add("key", group.key);
+ MtasSolrResult data = new MtasSolrResult(group.dataCollector,
+ new String[] { group.dataType }, new String[] { group.statsType },
+ new TreeSet[] { group.statsItems }, new String[] { group.sortType },
+ new String[] { group.sortDirection }, new Integer[] { group.start },
+ new Integer[] { group.number }, null);
+ if (encode) {
+ mtasGroupResponse.add("_encoded_list", MtasSolrResultUtil.encode(data));
+ } else {
+ mtasGroupResponse.add("list", data);
+ MtasSolrResultUtil.rewrite(mtasGroupResponse);
+ }
+ return mtasGroupResponse;
+ }
+
+ /**
+ * Finish stage.
+ *
+ * @param rb
+ * the rb
+ */
+ @SuppressWarnings("unchecked")
+ public void finishStage(ResponseBuilder rb) {
+ if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (rb.stage >= ResponseBuilder.STAGE_EXECUTE_QUERY
+ && rb.stage < ResponseBuilder.STAGE_GET_FIELDS) {
+ for (ShardRequest sreq : rb.finished) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
+ && sreq.params.getBool(PARAM_MTAS_GROUP, false)) {
+ for (ShardResponse shardResponse : sreq.responses) {
+ NamedList<Object> response = shardResponse.getSolrResponse()
+ .getResponse();
+ try {
+ ArrayList<NamedList<Object>> data = (ArrayList<NamedList<Object>>) response
+ .findRecursive("mtas", "group");
+ if (data != null) {
+ MtasSolrResultUtil.decode(data);
+ }
+ } catch (ClassCastException e) {
+ // shouldn't happen
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Distributed process.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ public void distributedProcess(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ // rewrite
+ NamedList<Object> mtasResponse = null;
+ try {
+ mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
+ if (mtasResponse != null) {
+ ArrayList<Object> mtasResponseGroup;
+ try {
+ mtasResponseGroup = (ArrayList<Object>) mtasResponse.get("group");
+ if (mtasResponseGroup != null) {
+ MtasSolrResultUtil.rewrite(mtasResponseGroup);
+ }
+ } catch (ClassCastException e) {
+ mtasResponseGroup = null;
+ }
+ }
+ } catch (ClassCastException e) {
+ mtasResponse = null;
+ }
+ }
+
+}
diff --git a/src/mtas/solr/handler/component/util/MtasSolrComponentKwic.java b/src/mtas/solr/handler/component/util/MtasSolrComponentKwic.java
new file mode 100644
index 0000000..c9bc197
--- /dev/null
+++ b/src/mtas/solr/handler/component/util/MtasSolrComponentKwic.java
@@ -0,0 +1,393 @@
+package mtas.solr.handler.component.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.handler.component.ShardRequest;
+
+import mtas.analysis.token.MtasToken;
+import mtas.codec.util.CodecUtil;
+import mtas.codec.util.CodecComponent.ComponentField;
+import mtas.codec.util.CodecComponent.ComponentFields;
+import mtas.codec.util.CodecComponent.ComponentKwic;
+import mtas.codec.util.CodecComponent.KwicHit;
+import mtas.codec.util.CodecComponent.KwicToken;
+import mtas.solr.handler.component.MtasSolrSearchComponent;
+
+/**
+ * The Class MtasSolrComponentKwic.
+ */
+public class MtasSolrComponentKwic {
+
+ /** The search component. */
+ MtasSolrSearchComponent searchComponent;
+
+ /** The Constant PARAM_MTAS_KWIC. */
+ public static final String PARAM_MTAS_KWIC = MtasSolrSearchComponent.PARAM_MTAS
+ + ".kwic";
+
+ /** The Constant NAME_MTAS_KWIC_FIELD. */
+ public static final String NAME_MTAS_KWIC_FIELD = "field";
+
+ /** The Constant NAME_MTAS_KWIC_QUERY_TYPE. */
+ public static final String NAME_MTAS_KWIC_QUERY_TYPE = "query.type";
+
+ /** The Constant NAME_MTAS_KWIC_QUERY_VALUE. */
+ public static final String NAME_MTAS_KWIC_QUERY_VALUE = "query.value";
+
+ /** The Constant NAME_MTAS_KWIC_KEY. */
+ public static final String NAME_MTAS_KWIC_KEY = "key";
+
+ /** The Constant NAME_MTAS_KWIC_PREFIX. */
+ public static final String NAME_MTAS_KWIC_PREFIX = "prefix";
+
+ /** The Constant NAME_MTAS_KWIC_NUMBER. */
+ public static final String NAME_MTAS_KWIC_NUMBER = "number";
+
+ /** The Constant NAME_MTAS_KWIC_START. */
+ public static final String NAME_MTAS_KWIC_START = "start";
+
+ /** The Constant NAME_MTAS_KWIC_LEFT. */
+ public static final String NAME_MTAS_KWIC_LEFT = "left";
+
+ /** The Constant NAME_MTAS_KWIC_RIGHT. */
+ public static final String NAME_MTAS_KWIC_RIGHT = "right";
+
+ /** The Constant NAME_MTAS_KWIC_OUTPUT. */
+ public static final String NAME_MTAS_KWIC_OUTPUT = "output";
+
+ /**
+ * Instantiates a new mtas solr component kwic.
+ *
+ * @param searchComponent
+ * the search component
+ */
+ public MtasSolrComponentKwic(MtasSolrSearchComponent searchComponent) {
+ this.searchComponent = searchComponent;
+ }
+
+ /**
+ * Gets the positive integer.
+ *
+ * @param number
+ * the number
+ * @return the positive integer
+ */
+ private int getPositiveInteger(String number) {
+ try {
+ return Math.max(0, Integer.parseInt(number));
+ } catch (NumberFormatException e) {
+ return 0;
+ }
+ }
+
+ /**
+ * Prepare.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ Set<String> ids = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_KWIC);
+ if (ids.size() > 0) {
+ int tmpCounter = 0;
+ String[] fields = new String[ids.size()];
+ String[] queryTypes = new String[ids.size()];
+ String[] queryValues = new String[ids.size()];
+ String[] keys = new String[ids.size()];
+ String[] prefixes = new String[ids.size()];
+ String[] numbers = new String[ids.size()];
+ String[] starts = new String[ids.size()];
+ String[] lefts = new String[ids.size()];
+ String[] rights = new String[ids.size()];
+ String[] outputs = new String[ids.size()];
+ for (String id : ids) {
+ fields[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_FIELD, null);
+ queryTypes[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_QUERY_TYPE, null);
+ queryValues[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_QUERY_VALUE,
+ null);
+ keys[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_KEY,
+ String.valueOf(tmpCounter))
+ .trim();
+ prefixes[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_PREFIX, null);
+ numbers[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_NUMBER, null);
+ starts[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_START, null);
+ lefts[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_LEFT, null);
+ rights[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_RIGHT, null);
+ starts[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_START, null);
+ outputs[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_OUTPUT, null);
+ tmpCounter++;
+ }
+ String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
+ mtasFields.doKwic = true;
+ rb.setNeedDocList(true);
+ for (String field : fields) {
+ if (field == null || field.isEmpty()) {
+ throw new IOException("no (valid) field in mtas kwic");
+ } else if (!mtasFields.list.containsKey(field)) {
+ mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
+ }
+ }
+ MtasSolrResultUtil.compareAndCheck(keys, fields, NAME_MTAS_KWIC_KEY,
+ NAME_MTAS_KWIC_FIELD, true);
+ MtasSolrResultUtil.compareAndCheck(queryValues, fields,
+ NAME_MTAS_KWIC_QUERY_VALUE, NAME_MTAS_KWIC_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(queryTypes, fields,
+ NAME_MTAS_KWIC_QUERY_TYPE, NAME_MTAS_KWIC_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(prefixes, fields,
+ NAME_MTAS_KWIC_PREFIX, NAME_MTAS_KWIC_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(numbers, fields, NAME_MTAS_KWIC_NUMBER,
+ NAME_MTAS_KWIC_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(starts, fields, NAME_MTAS_KWIC_START,
+ NAME_MTAS_KWIC_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(lefts, fields, NAME_MTAS_KWIC_LEFT,
+ NAME_MTAS_KWIC_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(rights, fields, NAME_MTAS_KWIC_RIGHT,
+ NAME_MTAS_KWIC_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(outputs, fields, NAME_MTAS_KWIC_OUTPUT,
+ NAME_MTAS_KWIC_FIELD, false);
+ for (int i = 0; i < fields.length; i++) {
+ ComponentField cf = mtasFields.list.get(fields[i]);
+ SpanQuery q = MtasSolrResultUtil.constructQuery(queryValues[i],
+ queryTypes[i], fields[i]);
+ // minimize number of queries
+ if (cf.spanQueryList.contains(q)) {
+ q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
+ } else {
+ cf.spanQueryList.add(q);
+ }
+ String key = (keys[i] == null) || (keys[i].isEmpty())
+ ? String.valueOf(i) + ":" + fields[i] + ":" + queryValues[i]
+ : keys[i].trim();
+ String prefix = prefixes[i];
+ Integer number = (numbers[i] != null) ? getPositiveInteger(numbers[i])
+ : null;
+ int start = getPositiveInteger(starts[i]);
+ int left = getPositiveInteger(lefts[i]);
+ int right = getPositiveInteger(rights[i]);
+ String output = outputs[i];
+ mtasFields.list.get(fields[i]).kwicList.add(new ComponentKwic(q, key,
+ prefix, number, start, left, right, output));
+ }
+ }
+ }
+
+ /**
+ * Creates the.
+ *
+ * @param kwic
+ * the kwic
+ * @return the simple ordered map
+ */
+ public SimpleOrderedMap<Object> create(ComponentKwic kwic) {
+ SimpleOrderedMap<Object> mtasKwicResponse = new SimpleOrderedMap<>();
+ mtasKwicResponse.add("key", kwic.key);
+ ArrayList<NamedList<Object>> mtasKwicItemResponses = new ArrayList<NamedList<Object>>();
+ if (kwic.output.equals(ComponentKwic.KWIC_OUTPUT_HIT)) {
+ for (int docId : kwic.hits.keySet()) {
+ NamedList<Object> mtasKwicItemResponse = new SimpleOrderedMap<>();
+ ArrayList<KwicHit> list = kwic.hits.get(docId);
+ ArrayList<NamedList<Object>> mtasKwicItemResponseItems = new ArrayList<NamedList<Object>>();
+ for (KwicHit h : list) {
+ NamedList<Object> mtasKwicItemResponseItem = new SimpleOrderedMap<>();
+ TreeMap<Integer, ArrayList<ArrayList<String>>> hitData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
+ TreeMap<Integer, ArrayList<ArrayList<String>>> leftData = null,
+ rightData = null;
+ if (kwic.left > 0) {
+ leftData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
+ }
+ if (kwic.right > 0) {
+ rightData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
+ }
+ for (int position = Math.max(0,
+ h.startPosition - kwic.left); position <= (h.endPosition
+ + kwic.right); position++) {
+ if (h.hits.containsKey(position)) {
+ ArrayList<ArrayList<String>> hitDataItem = new ArrayList<ArrayList<String>>();
+ for (String term : h.hits.get(position)) {
+ ArrayList<String> hitDataSubItem = new ArrayList<String>();
+ hitDataSubItem.add(CodecUtil.termPrefix(term));
+ hitDataSubItem.add(CodecUtil.termValue(term));
+ hitDataItem.add(hitDataSubItem);
+ }
+ if (position < h.startPosition) {
+ leftData.put(position, hitDataItem);
+ } else if (position > h.endPosition) {
+ rightData.put(position, hitDataItem);
+ } else {
+ hitData.put(position, hitDataItem);
+ }
+ }
+ }
+ if (kwic.left > 0) {
+ mtasKwicItemResponseItem.add("left", leftData);
+ }
+ mtasKwicItemResponseItem.add("hit", hitData);
+ if (kwic.right > 0) {
+ mtasKwicItemResponseItem.add("right", rightData);
+ }
+ mtasKwicItemResponseItems.add(mtasKwicItemResponseItem);
+ }
+ mtasKwicItemResponse.add("documentKey", kwic.uniqueKey.get(docId));
+ mtasKwicItemResponse.add("documentTotal", kwic.subTotal.get(docId));
+ mtasKwicItemResponse.add("documentMinPosition",
+ kwic.minPosition.get(docId));
+ mtasKwicItemResponse.add("documentMaxPosition",
+ kwic.maxPosition.get(docId));
+ mtasKwicItemResponse.add("list", mtasKwicItemResponseItems);
+ mtasKwicItemResponses.add(mtasKwicItemResponse);
+ }
+ } else if (kwic.output.equals(ComponentKwic.KWIC_OUTPUT_TOKEN)) {
+ for (int docId : kwic.tokens.keySet()) {
+ NamedList<Object> mtasKwicItemResponse = new SimpleOrderedMap<>();
+ ArrayList<KwicToken> list = kwic.tokens.get(docId);
+ ArrayList<NamedList<Object>> mtasKwicItemResponseItems = new ArrayList<NamedList<Object>>();
+ for (KwicToken k : list) {
+ NamedList<Object> mtasKwicItemResponseItem = new SimpleOrderedMap<>();
+ mtasKwicItemResponseItem.add("startPosition", k.startPosition);
+ mtasKwicItemResponseItem.add("endPosition", k.endPosition);
+ ArrayList<NamedList<Object>> mtasKwicItemResponseItemTokens = new ArrayList<NamedList<Object>>();
+ for (MtasToken<?> token : k.tokens) {
+ NamedList<Object> mtasKwicItemResponseItemToken = new SimpleOrderedMap<>();
+ if (token.getId() != null) {
+ mtasKwicItemResponseItemToken.add("mtasId", token.getId());
+ }
+ mtasKwicItemResponseItemToken.add("prefix", token.getPrefix());
+ mtasKwicItemResponseItemToken.add("value", token.getPostfix());
+ if (token.getPositionStart() != null) {
+ mtasKwicItemResponseItemToken.add("positionStart",
+ token.getPositionStart());
+ mtasKwicItemResponseItemToken.add("positionEnd",
+ token.getPositionEnd());
+ }
+ if (token.getPositions() != null) {
+ mtasKwicItemResponseItemToken.add("positions",
+ token.getPositions());
+ }
+ if (token.getParentId() != null) {
+ mtasKwicItemResponseItemToken.add("parentMtasId",
+ token.getParentId());
+ }
+ if (token.getPayload() != null) {
+ mtasKwicItemResponseItemToken.add("payload", token.getPayload());
+ }
+ if (token.getOffsetStart() != null) {
+ mtasKwicItemResponseItemToken.add("offsetStart",
+ token.getOffsetStart());
+ mtasKwicItemResponseItemToken.add("offsetEnd",
+ token.getOffsetEnd());
+ }
+ if (token.getRealOffsetStart() != null) {
+ mtasKwicItemResponseItemToken.add("realOffsetStart",
+ token.getRealOffsetStart());
+ mtasKwicItemResponseItemToken.add("realOffsetEnd",
+ token.getRealOffsetEnd());
+ }
+ mtasKwicItemResponseItemTokens.add(mtasKwicItemResponseItemToken);
+ }
+ mtasKwicItemResponseItem.add("tokens",
+ mtasKwicItemResponseItemTokens);
+ mtasKwicItemResponseItems.add(mtasKwicItemResponseItem);
+ }
+ mtasKwicItemResponse.add("documentKey", kwic.uniqueKey.get(docId));
+ mtasKwicItemResponse.add("documentTotal", kwic.subTotal.get(docId));
+ mtasKwicItemResponse.add("documentMinPosition",
+ kwic.minPosition.get(docId));
+ mtasKwicItemResponse.add("documentMaxPosition",
+ kwic.maxPosition.get(docId));
+ mtasKwicItemResponse.add("list", mtasKwicItemResponseItems);
+ mtasKwicItemResponses.add(mtasKwicItemResponse);
+ }
+ }
+ mtasKwicResponse.add("list", mtasKwicItemResponses);
+ return mtasKwicResponse;
+ }
+
+ /**
+ * Modify request.
+ *
+ * @param rb
+ * the rb
+ * @param who
+ * the who
+ * @param sreq
+ * the sreq
+ */
+ public void modifyRequest(ResponseBuilder rb, SearchComponent who,
+ ShardRequest sreq) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (sreq.params.getBool(PARAM_MTAS_KWIC, false)) {
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
+ // do nothing
+ } else {
+ Set<String> keys = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_KWIC);
+ sreq.params.remove(PARAM_MTAS_KWIC);
+ for (String key : keys) {
+ sreq.params.remove(
+ PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_FIELD);
+ sreq.params.remove(
+ PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_QUERY_VALUE);
+ sreq.params
+ .remove(PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_KEY);
+ sreq.params.remove(
+ PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_PREFIX);
+ sreq.params.remove(
+ PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_NUMBER);
+ sreq.params.remove(
+ PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_LEFT);
+ sreq.params.remove(
+ PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_RIGHT);
+ sreq.params.remove(
+ PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_OUTPUT);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Finish stage.
+ *
+ * @param rb
+ * the rb
+ */
+ public void finishStage(ResponseBuilder rb) {
+ if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (rb.stage >= ResponseBuilder.STAGE_EXECUTE_QUERY
+ && rb.stage < ResponseBuilder.STAGE_GET_FIELDS) {
+ for (ShardRequest sreq : rb.finished) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
+ && sreq.params.getBool(PARAM_MTAS_KWIC, false)) {
+ // nothing to do
+ }
+ }
+ }
+ }
+ }
+
+}
diff --git a/src/mtas/solr/handler/component/util/MtasSolrComponentList.java b/src/mtas/solr/handler/component/util/MtasSolrComponentList.java
new file mode 100644
index 0000000..6d6ef50
--- /dev/null
+++ b/src/mtas/solr/handler/component/util/MtasSolrComponentList.java
@@ -0,0 +1,524 @@
+package mtas.solr.handler.component.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.handler.component.ShardResponse;
+
+import mtas.analysis.token.MtasToken;
+import mtas.codec.util.CodecUtil;
+import mtas.codec.util.CodecComponent.ComponentField;
+import mtas.codec.util.CodecComponent.ComponentFields;
+import mtas.codec.util.CodecComponent.ComponentList;
+import mtas.codec.util.CodecComponent.ListHit;
+import mtas.codec.util.CodecComponent.ListToken;
+import mtas.solr.handler.component.MtasSolrSearchComponent;
+
+/**
+ * The Class MtasSolrComponentList.
+ */
+public class MtasSolrComponentList {
+
+ /** The search component. */
+ MtasSolrSearchComponent searchComponent;
+
+ /** The Constant PARAM_MTAS_LIST. */
+ public static final String PARAM_MTAS_LIST = MtasSolrSearchComponent.PARAM_MTAS
+ + ".list";
+
+ /** The Constant NAME_MTAS_LIST_FIELD. */
+ public static final String NAME_MTAS_LIST_FIELD = "field";
+
+ /** The Constant NAME_MTAS_LIST_QUERY_TYPE. */
+ public static final String NAME_MTAS_LIST_QUERY_TYPE = "query.type";
+
+ /** The Constant NAME_MTAS_LIST_QUERY_VALUE. */
+ public static final String NAME_MTAS_LIST_QUERY_VALUE = "query.value";
+
+ /** The Constant NAME_MTAS_LIST_KEY. */
+ public static final String NAME_MTAS_LIST_KEY = "key";
+
+ /** The Constant NAME_MTAS_LIST_PREFIX. */
+ public static final String NAME_MTAS_LIST_PREFIX = "prefix";
+
+ /** The Constant NAME_MTAS_LIST_START. */
+ public static final String NAME_MTAS_LIST_START = "start";
+
+ /** The Constant NAME_MTAS_LIST_NUMBER. */
+ public static final String NAME_MTAS_LIST_NUMBER = "number";
+
+ /** The Constant NAME_MTAS_LIST_LEFT. */
+ public static final String NAME_MTAS_LIST_LEFT = "left";
+
+ /** The Constant NAME_MTAS_LIST_RIGHT. */
+ public static final String NAME_MTAS_LIST_RIGHT = "right";
+
+ /** The Constant NAME_MTAS_LIST_OUTPUT. */
+ public static final String NAME_MTAS_LIST_OUTPUT = "output";
+
+ /**
+ * Instantiates a new mtas solr component list.
+ *
+ * @param searchComponent
+ * the search component
+ */
+ public MtasSolrComponentList(MtasSolrSearchComponent searchComponent) {
+ this.searchComponent = searchComponent;
+ }
+
+ /**
+ * Prepare.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ Set<String> ids = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_LIST);
+ if (ids.size() > 0) {
+ int tmpCounter = 0;
+ String[] fields = new String[ids.size()];
+ String[] queryTypes = new String[ids.size()];
+ String[] queryValues = new String[ids.size()];
+ String[] keys = new String[ids.size()];
+ String[] prefixes = new String[ids.size()];
+ String[] starts = new String[ids.size()];
+ String[] numbers = new String[ids.size()];
+ String[] lefts = new String[ids.size()];
+ String[] rights = new String[ids.size()];
+ String[] outputs = new String[ids.size()];
+ for (String id : ids) {
+ fields[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_FIELD, null);
+ keys[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_KEY,
+ String.valueOf(tmpCounter))
+ .trim();
+ queryTypes[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_QUERY_TYPE, null);
+ queryValues[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_QUERY_VALUE,
+ null);
+ prefixes[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_PREFIX, null);
+ starts[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_START, null);
+ numbers[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_NUMBER, null);
+ lefts[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_LEFT, null);
+ rights[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_RIGHT, null);
+ outputs[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_OUTPUT, null);
+ tmpCounter++;
+ }
+ String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
+ mtasFields.doList = true;
+ rb.setNeedDocSet(true);
+ for (String field : fields) {
+ if (field == null || field.isEmpty()) {
+ throw new IOException("no (valid) field in mtas list");
+ } else if (!mtasFields.list.containsKey(field)) {
+ mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
+ }
+ }
+ MtasSolrResultUtil.compareAndCheck(keys, fields, NAME_MTAS_LIST_KEY,
+ NAME_MTAS_LIST_FIELD, true);
+ MtasSolrResultUtil.compareAndCheck(prefixes, queryValues,
+ NAME_MTAS_LIST_QUERY_VALUE, NAME_MTAS_LIST_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(prefixes, queryTypes,
+ NAME_MTAS_LIST_QUERY_TYPE, NAME_MTAS_LIST_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(prefixes, fields,
+ NAME_MTAS_LIST_PREFIX, NAME_MTAS_LIST_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(starts, fields, NAME_MTAS_LIST_START,
+ NAME_MTAS_LIST_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(numbers, fields, NAME_MTAS_LIST_NUMBER,
+ NAME_MTAS_LIST_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(lefts, fields, NAME_MTAS_LIST_LEFT,
+ NAME_MTAS_LIST_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(rights, fields, NAME_MTAS_LIST_RIGHT,
+ NAME_MTAS_LIST_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(outputs, fields, NAME_MTAS_LIST_OUTPUT,
+ NAME_MTAS_LIST_FIELD, false);
+ for (int i = 0; i < fields.length; i++) {
+ ComponentField cf = mtasFields.list.get(fields[i]);
+ SpanQuery q = MtasSolrResultUtil.constructQuery(queryValues[i],
+ queryTypes[i], fields[i]);
+ // minimize number of queries
+ if (cf.spanQueryList.contains(q)) {
+ q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
+ } else {
+ cf.spanQueryList.add(q);
+ }
+ String key = (keys[i] == null) || (keys[i].isEmpty())
+ ? String.valueOf(i) + ":" + fields[i] + ":" + queryValues[i]
+ : keys[i].trim();
+ String prefix = prefixes[i];
+ int start = (starts[i] == null) || (starts[i].isEmpty()) ? 0
+ : Integer.parseInt(starts[i]);
+ int number = (numbers[i] == null) || (numbers[i].isEmpty()) ? 10
+ : Integer.parseInt(numbers[i]);
+ int left = (lefts[i] == null) || lefts[i].isEmpty() ? 0
+ : Integer.parseInt(lefts[i]);
+ int right = (rights[i] == null) || rights[i].isEmpty() ? 0
+ : Integer.parseInt(rights[i]);
+ String output = outputs[i];
+ mtasFields.list.get(fields[i]).listList
+ .add(new ComponentList(q, fields[i], queryValues[i], queryTypes[i],
+ key, prefix, start, number, left, right, output));
+ }
+ }
+ }
+
+ /**
+ * Modify request.
+ *
+ * @param rb
+ * the rb
+ * @param who
+ * the who
+ * @param sreq
+ * the sreq
+ */
+ public void modifyRequest(ResponseBuilder rb, SearchComponent who,
+ ShardRequest sreq) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (sreq.params.getBool(PARAM_MTAS_LIST, false)) {
+ // compute keys
+ Set<String> keys = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_LIST);
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
+ for (String key : keys) {
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_PREFIX);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_START);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_NUMBER);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_LEFT);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_RIGHT);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_OUTPUT);
+ // don't get data
+ sreq.params.add(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_NUMBER, "0");
+ }
+ } else {
+ sreq.params.remove(PARAM_MTAS_LIST);
+ for (String key : keys) {
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_FIELD);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_QUERY_VALUE);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_QUERY_TYPE);
+ sreq.params
+ .remove(PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_KEY);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_PREFIX);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_START);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_NUMBER);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_LEFT);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_RIGHT);
+ sreq.params.remove(
+ PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_OUTPUT);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Distributed process.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ */
+ @SuppressWarnings("unchecked")
+ public void distributedProcess(ResponseBuilder rb,
+ ComponentFields mtasFields) {
+
+ if (mtasFields.doList) {
+ // compute total from shards
+ HashMap<String, HashMap<String, Integer>> listShardTotals = new HashMap<String, HashMap<String, Integer>>();
+ for (ShardRequest sreq : rb.finished) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
+ && sreq.params.getBool(PARAM_MTAS_LIST, false)) {
+ for (ShardResponse response : sreq.responses) {
+ NamedList<Object> result = response.getSolrResponse().getResponse();
+ try {
+ ArrayList<NamedList<Object>> data = (ArrayList<NamedList<Object>>) result
+ .findRecursive("mtas", "list");
+ if (data != null) {
+ for (NamedList<Object> dataItem : data) {
+ Object key = dataItem.get("key");
+ Object total = dataItem.get("total");
+ if ((key != null) && (key instanceof String)
+ && (total != null) && (total instanceof Integer)) {
+ if (!listShardTotals.containsKey(key)) {
+ listShardTotals.put((String) key,
+ new HashMap<String, Integer>());
+ }
+ HashMap<String, Integer> listShardTotal = listShardTotals
+ .get(key);
+ listShardTotal.put(response.getShard(), (Integer) total);
+ }
+ }
+ }
+ } catch (ClassCastException e) {
+ }
+ }
+ }
+ }
+ // compute shard requests
+ HashMap<String, ModifiableSolrParams> shardRequests = new HashMap<String, ModifiableSolrParams>();
+ int requestId = 0;
+ for (String field : mtasFields.list.keySet()) {
+ for (ComponentList list : mtasFields.list.get(field).listList) {
+ requestId++;
+ if (listShardTotals.containsKey(list.key) && (list.number > 0)) {
+ Integer position = 0;
+ Integer start = list.start;
+ Integer number = list.number;
+ HashMap<String, Integer> totals = listShardTotals.get(list.key);
+ for (int i = 0; i < rb.shards.length; i++) {
+ if (number < 0) {
+ break;
+ }
+ int subTotal = totals.get(rb.shards[i]);
+ // System.out.println(i + " : " + rb.shards[i] + " : "
+ // + totals.get(rb.shards[i]) + " - " + start + " " + number);
+ if ((start >= 0) && (start < subTotal)) {
+ ModifiableSolrParams params;
+ if (!shardRequests.containsKey(rb.shards[i])) {
+ shardRequests.put(rb.shards[i], new ModifiableSolrParams());
+ }
+ params = shardRequests.get(rb.shards[i]);
+ params.add(PARAM_MTAS_LIST + "." + requestId + "."
+ + NAME_MTAS_LIST_FIELD, list.field);
+ params.add(PARAM_MTAS_LIST + "." + requestId + "."
+ + NAME_MTAS_LIST_QUERY_VALUE, list.queryValue);
+ params.add(PARAM_MTAS_LIST + "." + requestId + "."
+ + NAME_MTAS_LIST_QUERY_TYPE, list.queryType);
+ params.add(PARAM_MTAS_LIST + "." + requestId + "."
+ + NAME_MTAS_LIST_KEY, list.key);
+ params.add(PARAM_MTAS_LIST + "." + requestId + "."
+ + NAME_MTAS_LIST_PREFIX, list.prefix);
+ params.add(PARAM_MTAS_LIST + "." + requestId + "."
+ + NAME_MTAS_LIST_START, Integer.toString(start));
+ params.add(
+ PARAM_MTAS_LIST + "." + requestId + "."
+ + NAME_MTAS_LIST_NUMBER,
+ Integer.toString(Math.min(number, (subTotal - start))));
+ params.add(PARAM_MTAS_LIST + "." + requestId + "."
+ + NAME_MTAS_LIST_LEFT, Integer.toString(list.left));
+ params.add(PARAM_MTAS_LIST + "." + requestId + "."
+ + NAME_MTAS_LIST_RIGHT, Integer.toString(list.right));
+ params.add(PARAM_MTAS_LIST + "." + requestId + "."
+ + NAME_MTAS_LIST_OUTPUT, list.output);
+ number -= (subTotal - start);
+ start = 0;
+ } else {
+ start -= subTotal;
+ }
+ position += subTotal;
+ }
+ }
+ }
+ }
+
+ for (String shardName : shardRequests.keySet()) {
+ ShardRequest sreq = new ShardRequest();
+ sreq.shards = new String[] { shardName };
+ sreq.purpose = ShardRequest.PURPOSE_PRIVATE;
+ sreq.params = new ModifiableSolrParams();
+ sreq.params.add("fq", rb.req.getParams().getParams("fq"));
+ sreq.params.add("q", rb.req.getParams().getParams("q"));
+ sreq.params.add("cache", rb.req.getParams().getParams("cache"));
+ sreq.params.add("rows", "0");
+ sreq.params.add(MtasSolrSearchComponent.PARAM_MTAS, rb.req
+ .getOriginalParams().getParams(MtasSolrSearchComponent.PARAM_MTAS));
+ sreq.params.add(PARAM_MTAS_LIST,
+ rb.req.getOriginalParams().getParams(PARAM_MTAS_LIST));
+ sreq.params.add(shardRequests.get(shardName));
+ rb.addRequest(searchComponent, sreq);
+ }
+ }
+ }
+
+ /**
+ * Creates the.
+ *
+ * @param list
+ * the list
+ * @return the simple ordered map
+ */
+ public SimpleOrderedMap<Object> create(ComponentList list) {
+ SimpleOrderedMap<Object> mtasListResponse = new SimpleOrderedMap<>();
+ mtasListResponse.add("key", list.key);
+ mtasListResponse.add("total", list.total);
+ if (list.output != null) {
+ ArrayList<NamedList<Object>> mtasListItemResponses = new ArrayList<NamedList<Object>>();
+ if (list.output.equals(ComponentList.LIST_OUTPUT_HIT)) {
+ mtasListResponse.add("number", list.hits.size());
+ for (ListHit hit : list.hits) {
+ NamedList<Object> mtasListItemResponse = new SimpleOrderedMap<>();
+ mtasListItemResponse.add("documentKey",
+ list.uniqueKey.get(hit.docId));
+ mtasListItemResponse.add("documentHitPosition", hit.docPosition);
+ mtasListItemResponse.add("documentHitTotal",
+ list.subTotal.get(hit.docId));
+ mtasListItemResponse.add("documentMinPosition",
+ list.minPosition.get(hit.docId));
+ mtasListItemResponse.add("documentMaxPosition",
+ list.maxPosition.get(hit.docId));
+ mtasListItemResponse.add("startPosition", hit.startPosition);
+ mtasListItemResponse.add("endPosition", hit.endPosition);
+
+ TreeMap<Integer, ArrayList<ArrayList<String>>> hitData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
+ TreeMap<Integer, ArrayList<ArrayList<String>>> leftData = null,
+ rightData = null;
+ if (list.left > 0) {
+ leftData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
+ }
+ if (list.right > 0) {
+ rightData = new TreeMap<Integer, ArrayList<ArrayList<String>>>();
+ }
+ for (int position = Math.max(0,
+ hit.startPosition - list.left); position <= (hit.endPosition
+ + list.right); position++) {
+ ArrayList<ArrayList<String>> hitDataItem = new ArrayList<ArrayList<String>>();
+ if (hit.hits.containsKey(position)) {
+ for (String term : hit.hits.get(position)) {
+ ArrayList<String> hitDataSubItem = new ArrayList<String>();
+ hitDataSubItem.add(CodecUtil.termPrefix(term));
+ hitDataSubItem.add(CodecUtil.termValue(term));
+ hitDataItem.add(hitDataSubItem);
+ }
+ }
+ if (position < hit.startPosition) {
+ leftData.put(position, hitDataItem);
+ } else if (position > hit.endPosition) {
+ rightData.put(position, hitDataItem);
+ } else {
+ hitData.put(position, hitDataItem);
+ }
+ }
+ if (list.left > 0) {
+ mtasListItemResponse.add("left", leftData);
+ }
+ mtasListItemResponse.add("hit", hitData);
+ if (list.right > 0) {
+ mtasListItemResponse.add("right", rightData);
+ }
+ mtasListItemResponses.add(mtasListItemResponse);
+ }
+ } else if (list.output.equals(ComponentList.LIST_OUTPUT_TOKEN)) {
+ mtasListResponse.add("number", list.tokens.size());
+ for (ListToken tokenHit : list.tokens) {
+ NamedList<Object> mtasListItemResponse = new SimpleOrderedMap<>();
+ mtasListItemResponse.add("documentKey",
+ list.uniqueKey.get(tokenHit.docId));
+ mtasListItemResponse.add("documentHitPosition", tokenHit.docPosition);
+ mtasListItemResponse.add("documentHitTotal",
+ list.subTotal.get(tokenHit.docId));
+ mtasListItemResponse.add("documentMinPosition",
+ list.minPosition.get(tokenHit.docId));
+ mtasListItemResponse.add("documentMaxPosition",
+ list.maxPosition.get(tokenHit.docId));
+ mtasListItemResponse.add("startPosition", tokenHit.startPosition);
+ mtasListItemResponse.add("endPosition", tokenHit.endPosition);
+
+ ArrayList<NamedList<Object>> mtasListItemResponseItemTokens = new ArrayList<NamedList<Object>>();
+ for (MtasToken<?> token : tokenHit.tokens) {
+ NamedList<Object> mtasListItemResponseItemToken = new SimpleOrderedMap<>();
+ if (token.getId() != null) {
+ mtasListItemResponseItemToken.add("mtasId", token.getId());
+ }
+ mtasListItemResponseItemToken.add("prefix", token.getPrefix());
+ mtasListItemResponseItemToken.add("value", token.getPostfix());
+ if (token.getPositionStart() != null) {
+ mtasListItemResponseItemToken.add("positionStart",
+ token.getPositionStart());
+ mtasListItemResponseItemToken.add("positionEnd",
+ token.getPositionEnd());
+ }
+ if (token.getPositions() != null) {
+ mtasListItemResponseItemToken.add("positions",
+ token.getPositions());
+ }
+ if (token.getParentId() != null) {
+ mtasListItemResponseItemToken.add("parentMtasId",
+ token.getParentId());
+ }
+ if (token.getPayload() != null) {
+ mtasListItemResponseItemToken.add("payload", token.getPayload());
+ }
+ if (token.getOffsetStart() != null) {
+ mtasListItemResponseItemToken.add("offsetStart",
+ token.getOffsetStart());
+ mtasListItemResponseItemToken.add("offsetEnd",
+ token.getOffsetEnd());
+ }
+ if (token.getRealOffsetStart() != null) {
+ mtasListItemResponseItemToken.add("realOffsetStart",
+ token.getRealOffsetStart());
+ mtasListItemResponseItemToken.add("realOffsetEnd",
+ token.getRealOffsetEnd());
+ }
+ mtasListItemResponseItemTokens.add(mtasListItemResponseItemToken);
+ }
+ mtasListItemResponse.add("tokens", mtasListItemResponseItemTokens);
+ mtasListItemResponses.add(mtasListItemResponse);
+ }
+ }
+ mtasListResponse.add("list", mtasListItemResponses);
+ }
+ return mtasListResponse;
+ }
+
+ /**
+ * Finish stage.
+ *
+ * @param rb
+ * the rb
+ */
+ public void finishStage(ResponseBuilder rb) {
+ if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (rb.stage >= ResponseBuilder.STAGE_EXECUTE_QUERY
+ && rb.stage < ResponseBuilder.STAGE_GET_FIELDS) {
+ for (ShardRequest sreq : rb.finished) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
+ && sreq.params.getBool(PARAM_MTAS_LIST, false)) {
+ // nothing to do
+ }
+ }
+ }
+ }
+ }
+
+}
diff --git a/src/mtas/solr/handler/component/util/MtasSolrComponentPrefix.java b/src/mtas/solr/handler/component/util/MtasSolrComponentPrefix.java
new file mode 100644
index 0000000..9a3e997
--- /dev/null
+++ b/src/mtas/solr/handler/component/util/MtasSolrComponentPrefix.java
@@ -0,0 +1,259 @@
+package mtas.solr.handler.component.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.handler.component.ShardResponse;
+
+import mtas.codec.util.CodecComponent.ComponentField;
+import mtas.codec.util.CodecComponent.ComponentFields;
+import mtas.codec.util.CodecComponent.ComponentPrefix;
+import mtas.solr.handler.component.MtasSolrSearchComponent;
+
+/**
+ * The Class MtasSolrComponentPrefix.
+ */
+public class MtasSolrComponentPrefix {
+
+ /** The search component. */
+ MtasSolrSearchComponent searchComponent;
+
+ /** The Constant PARAM_MTAS_PREFIX. */
+ public static final String PARAM_MTAS_PREFIX = MtasSolrSearchComponent.PARAM_MTAS
+ + ".prefix";
+
+ /** The Constant NAME_MTAS_PREFIX_FIELD. */
+ public static final String NAME_MTAS_PREFIX_FIELD = "field";
+
+ /** The Constant NAME_MTAS_PREFIX_KEY. */
+ public static final String NAME_MTAS_PREFIX_KEY = "key";
+
+ /**
+ * Instantiates a new mtas solr component prefix.
+ *
+ * @param searchComponent
+ * the search component
+ */
+ public MtasSolrComponentPrefix(MtasSolrSearchComponent searchComponent) {
+ this.searchComponent = searchComponent;
+ }
+
+ /**
+ * Prepare.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ Set<String> ids = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_PREFIX);
+ if (ids.size() > 0) {
+ int tmpCounter = 0;
+ String[] fields = new String[ids.size()];
+ String[] keys = new String[ids.size()];
+ for (String id : ids) {
+ fields[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_PREFIX + "." + id + "." + NAME_MTAS_PREFIX_FIELD, null);
+ keys[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_PREFIX + "." + id + "." + NAME_MTAS_PREFIX_KEY,
+ String.valueOf(tmpCounter))
+ .trim();
+ tmpCounter++;
+ }
+ String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
+ mtasFields.doPrefix = true;
+ // init and checks
+ for (String field : fields) {
+ if (field == null || field.isEmpty()) {
+ throw new IOException("no (valid) field in mtas prefix");
+ } else if (!mtasFields.list.containsKey(field)) {
+ mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
+ }
+ }
+ MtasSolrResultUtil.compareAndCheck(keys, fields, NAME_MTAS_PREFIX_KEY,
+ NAME_MTAS_PREFIX_FIELD, true);
+ for (int i = 0; i < fields.length; i++) {
+ String field = fields[i];
+ String key = ((keys == null) || (keys[i] == null)
+ || (keys[i].isEmpty())) ? String.valueOf(i) + ":" + field
+ : keys[i].trim();
+ mtasFields.list.get(field).prefix = new ComponentPrefix(key);
+ }
+ }
+ }
+
+ /**
+ * Modify request.
+ *
+ * @param rb
+ * the rb
+ * @param who
+ * the who
+ * @param sreq
+ * the sreq
+ */
+ public void modifyRequest(ResponseBuilder rb, SearchComponent who,
+ ShardRequest sreq) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (sreq.params.getBool(PARAM_MTAS_PREFIX, false)) {
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
+ // do nothing
+ } else {
+ // remove prefix for other requests
+ Set<String> keys = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_PREFIX);
+ sreq.params.remove(PARAM_MTAS_PREFIX);
+ for (String key : keys) {
+ sreq.params.remove(
+ PARAM_MTAS_PREFIX + "." + key + "." + NAME_MTAS_PREFIX_FIELD);
+ sreq.params.remove(
+ PARAM_MTAS_PREFIX + "." + key + "." + NAME_MTAS_PREFIX_KEY);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Creates the.
+ *
+ * @param prefix
+ * the prefix
+ * @param encode
+ * the encode
+ * @return the simple ordered map
+ */
+ public SimpleOrderedMap<Object> create(ComponentPrefix prefix,
+ Boolean encode) {
+ SimpleOrderedMap<Object> mtasPrefixResponse = new SimpleOrderedMap<Object>();
+ mtasPrefixResponse.add("key", prefix.key);
+ if (encode) {
+ mtasPrefixResponse.add("_encoded_singlePosition",
+ MtasSolrResultUtil.encode(prefix.singlePositionList));
+ mtasPrefixResponse.add("_encoded_multiplePosition",
+ MtasSolrResultUtil.encode(prefix.multiplePositionList));
+ mtasPrefixResponse.add("_encoded_setPosition",
+ MtasSolrResultUtil.encode(prefix.setPositionList));
+ } else {
+ mtasPrefixResponse.add("singlePosition", prefix.singlePositionList);
+ mtasPrefixResponse.add("multiplePosition", prefix.multiplePositionList);
+ mtasPrefixResponse.add("setPosition", prefix.setPositionList);
+ }
+ return mtasPrefixResponse;
+ }
+
+ /**
+ * Finish stage.
+ *
+ * @param rb
+ * the rb
+ */
+ @SuppressWarnings("unchecked")
+ public void finishStage(ResponseBuilder rb) {
+ if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (rb.stage >= ResponseBuilder.STAGE_EXECUTE_QUERY
+ && rb.stage < ResponseBuilder.STAGE_GET_FIELDS) {
+ for (ShardRequest sreq : rb.finished) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
+ && sreq.params.getBool(PARAM_MTAS_PREFIX, false)) {
+ for (ShardResponse shardResponse : sreq.responses) {
+ NamedList<Object> response = shardResponse.getSolrResponse()
+ .getResponse();
+ try {
+ ArrayList<NamedList<Object>> data = (ArrayList<NamedList<Object>>) response
+ .findRecursive("mtas", "prefix");
+ if (data != null) {
+ MtasSolrResultUtil.decode(data);
+ }
+ } catch (ClassCastException e) {
+ // shouldnt happen
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Distributed process.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ public void distributedProcess(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ // rewrite
+ NamedList<Object> mtasResponse = null;
+ try {
+ mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
+ if (mtasResponse != null) {
+ NamedList<Object> mtasResponsePrefix;
+ try {
+ mtasResponsePrefix = (NamedList<Object>) mtasResponse.get("prefix");
+ if (mtasResponsePrefix != null) {
+ repairPrefixItems(mtasResponsePrefix);
+ MtasSolrResultUtil.rewrite(mtasResponsePrefix);
+ }
+ } catch (ClassCastException e) {
+ mtasResponsePrefix = null;
+ }
+ }
+ } catch (ClassCastException e) {
+ mtasResponse = null;
+ }
+ }
+
+ /**
+ * Repair prefix items.
+ *
+ * @param mtasResponse
+ * the mtas response
+ */
+ @SuppressWarnings("unchecked")
+ private void repairPrefixItems(NamedList<Object> mtasResponse) {
+ // repair prefix lists
+ try {
+ ArrayList<NamedList<?>> list = (ArrayList<NamedList<?>>) mtasResponse
+ .findRecursive("prefix");
+ // MtasSolrResultUtil.rewrite(list);
+ if (list != null) {
+ for (NamedList<?> item : list) {
+ TreeSet<String> singlePosition = (TreeSet<String>) item
+ .get("singlePosition");
+ TreeSet<String> multiplePosition = (TreeSet<String>) item
+ .get("multiplePosition");
+ if (singlePosition != null) {
+ if (multiplePosition != null) {
+ for (String prefix : multiplePosition) {
+ if (singlePosition.contains(prefix)) {
+ singlePosition.remove(prefix);
+ }
+ }
+ }
+ }
+ }
+ }
+ } catch (ClassCastException e) {
+
+ }
+ }
+
+}
diff --git a/src/mtas/solr/handler/component/util/MtasSolrComponentStats.java b/src/mtas/solr/handler/component/util/MtasSolrComponentStats.java
new file mode 100644
index 0000000..48a9fb4
--- /dev/null
+++ b/src/mtas/solr/handler/component/util/MtasSolrComponentStats.java
@@ -0,0 +1,641 @@
+package mtas.solr.handler.component.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.handler.component.ShardResponse;
+
+import mtas.codec.util.CodecComponent.ComponentField;
+import mtas.codec.util.CodecComponent.ComponentFields;
+import mtas.codec.util.CodecComponent.ComponentPosition;
+import mtas.codec.util.CodecComponent.ComponentSpan;
+import mtas.codec.util.CodecComponent.ComponentToken;
+import mtas.codec.util.CodecComponent.SubComponentFunction;
+import mtas.codec.util.collector.MtasDataCollector;
+import mtas.parser.function.ParseException;
+import mtas.solr.handler.component.MtasSolrSearchComponent;
+
+/**
+ * The Class MtasSolrComponentStats.
+ */
+public class MtasSolrComponentStats {
+
+ /** The search component. */
+ MtasSolrSearchComponent searchComponent;
+
+ /** The Constant PARAM_MTAS_STATS. */
+ public static final String PARAM_MTAS_STATS = MtasSolrSearchComponent.PARAM_MTAS
+ + ".stats";
+
+ /** The Constant PARAM_MTAS_STATS_POSITIONS. */
+ public static final String PARAM_MTAS_STATS_POSITIONS = PARAM_MTAS_STATS
+ + ".positions";
+
+ /** The Constant NAME_MTAS_STATS_POSITIONS_FIELD. */
+ public static final String NAME_MTAS_STATS_POSITIONS_FIELD = "field";
+
+ /** The Constant NAME_MTAS_STATS_POSITIONS_KEY. */
+ public static final String NAME_MTAS_STATS_POSITIONS_KEY = "key";
+
+ /** The Constant NAME_MTAS_STATS_POSITIONS_TYPE. */
+ public static final String NAME_MTAS_STATS_POSITIONS_TYPE = "type";
+
+ /** The Constant NAME_MTAS_STATS_POSITIONS_MINIMUM. */
+ public static final String NAME_MTAS_STATS_POSITIONS_MINIMUM = "minimum";
+
+ /** The Constant NAME_MTAS_STATS_POSITIONS_MAXIMUM. */
+ public static final String NAME_MTAS_STATS_POSITIONS_MAXIMUM = "maximum";
+
+ /** The Constant PARAM_MTAS_STATS_TOKENS. */
+ public static final String PARAM_MTAS_STATS_TOKENS = PARAM_MTAS_STATS
+ + ".tokens";
+
+ /** The Constant NAME_MTAS_STATS_TOKENS_FIELD. */
+ public static final String NAME_MTAS_STATS_TOKENS_FIELD = "field";
+
+ /** The Constant NAME_MTAS_STATS_TOKENS_KEY. */
+ public static final String NAME_MTAS_STATS_TOKENS_KEY = "key";
+
+ /** The Constant NAME_MTAS_STATS_TOKENS_TYPE. */
+ public static final String NAME_MTAS_STATS_TOKENS_TYPE = "type";
+
+ /** The Constant NAME_MTAS_STATS_TOKENS_MINIMUM. */
+ public static final String NAME_MTAS_STATS_TOKENS_MINIMUM = "minimum";
+
+ /** The Constant NAME_MTAS_STATS_TOKENS_MAXIMUM. */
+ public static final String NAME_MTAS_STATS_TOKENS_MAXIMUM = "maximum";
+
+ /** The Constant PARAM_MTAS_STATS_SPANS. */
+ public static final String PARAM_MTAS_STATS_SPANS = PARAM_MTAS_STATS
+ + ".spans";
+
+ /** The Constant NAME_MTAS_STATS_SPANS_FIELD. */
+ public static final String NAME_MTAS_STATS_SPANS_FIELD = "field";
+
+ /** The Constant NAME_MTAS_STATS_SPANS_QUERY. */
+ public static final String NAME_MTAS_STATS_SPANS_QUERY = "query";
+
+ /** The Constant NAME_MTAS_STATS_SPANS_KEY. */
+ public static final String NAME_MTAS_STATS_SPANS_KEY = "key";
+
+ /** The Constant NAME_MTAS_STATS_SPANS_TYPE. */
+ public static final String NAME_MTAS_STATS_SPANS_TYPE = "type";
+
+ /** The Constant NAME_MTAS_STATS_SPANS_MINIMUM. */
+ public static final String NAME_MTAS_STATS_SPANS_MINIMUM = "minimum";
+
+ /** The Constant NAME_MTAS_STATS_SPANS_MAXIMUM. */
+ public static final String NAME_MTAS_STATS_SPANS_MAXIMUM = "maximum";
+
+ /** The Constant NAME_MTAS_STATS_SPANS_FUNCTION. */
+ public static final String NAME_MTAS_STATS_SPANS_FUNCTION = "function";
+
+ /** The Constant NAME_MTAS_STATS_SPANS_FUNCTION_EXPRESSION. */
+ public static final String NAME_MTAS_STATS_SPANS_FUNCTION_EXPRESSION = "expression";
+
+ /** The Constant NAME_MTAS_STATS_SPANS_FUNCTION_KEY. */
+ public static final String NAME_MTAS_STATS_SPANS_FUNCTION_KEY = "key";
+
+ /** The Constant NAME_MTAS_STATS_SPANS_FUNCTION_TYPE. */
+ public static final String NAME_MTAS_STATS_SPANS_FUNCTION_TYPE = "type";
+
+ /** The Constant SUBNAME_MTAS_STATS_SPANS_QUERY_TYPE. */
+ public static final String SUBNAME_MTAS_STATS_SPANS_QUERY_TYPE = "type";
+
+ /** The Constant SUBNAME_MTAS_STATS_SPANS_QUERY_VALUE. */
+ public static final String SUBNAME_MTAS_STATS_SPANS_QUERY_VALUE = "value";
+
+ /**
+ * Instantiates a new mtas solr component stats.
+ *
+ * @param searchComponent
+ * the search component
+ */
+ public MtasSolrComponentStats(MtasSolrSearchComponent searchComponent) {
+ this.searchComponent = searchComponent;
+ }
+
+ /**
+ * Prepare.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ if (rb.req.getParams().getBool(PARAM_MTAS_STATS_POSITIONS, false)) {
+ preparePositions(rb, mtasFields);
+ }
+ if (rb.req.getParams().getBool(PARAM_MTAS_STATS_TOKENS, false)) {
+ prepareTokens(rb, mtasFields);
+ }
+ if (rb.req.getParams().getBool(PARAM_MTAS_STATS_SPANS, false)) {
+ prepareSpans(rb, mtasFields);
+ }
+ }
+
+ /**
+ * Prepare positions.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ private void preparePositions(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ Set<String> ids = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_STATS_POSITIONS);
+ if (ids.size() > 0) {
+ int tmpCounter = 0;
+ String[] fields = new String[ids.size()];
+ String[] keys = new String[ids.size()];
+ String[] minima = new String[ids.size()];
+ String[] maxima = new String[ids.size()];
+ String[] types = new String[ids.size()];
+ for (String id : ids) {
+ fields[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_POSITIONS
+ + "." + id + "." + NAME_MTAS_STATS_POSITIONS_FIELD, null);
+ keys[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_STATS_POSITIONS + "." + id + "."
+ + NAME_MTAS_STATS_POSITIONS_KEY, String.valueOf(tmpCounter))
+ .trim();
+ minima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_POSITIONS
+ + "." + id + "." + NAME_MTAS_STATS_POSITIONS_MINIMUM, null);
+ maxima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_POSITIONS
+ + "." + id + "." + NAME_MTAS_STATS_POSITIONS_MAXIMUM, null);
+ types[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_POSITIONS
+ + "." + id + "." + NAME_MTAS_STATS_POSITIONS_TYPE, null);
+ tmpCounter++;
+ }
+ String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
+ mtasFields.doStats = true;
+ mtasFields.doStatsPositions = true;
+ rb.setNeedDocSet(true);
+ for (String field : fields) {
+ if (field == null || field.isEmpty()) {
+ throw new IOException("no (valid) field in mtas stats positions");
+ } else if (!mtasFields.list.containsKey(field)) {
+ mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
+ }
+ }
+ MtasSolrResultUtil.compareAndCheck(keys, fields,
+ NAME_MTAS_STATS_POSITIONS_KEY, NAME_MTAS_STATS_POSITIONS_FIELD, true);
+ MtasSolrResultUtil.compareAndCheck(minima, fields,
+ NAME_MTAS_STATS_POSITIONS_MINIMUM, NAME_MTAS_STATS_POSITIONS_FIELD,
+ false);
+ MtasSolrResultUtil.compareAndCheck(maxima, fields,
+ NAME_MTAS_STATS_POSITIONS_MAXIMUM, NAME_MTAS_STATS_POSITIONS_FIELD,
+ false);
+ MtasSolrResultUtil.compareAndCheck(types, fields,
+ NAME_MTAS_STATS_POSITIONS_TYPE, NAME_MTAS_STATS_POSITIONS_FIELD,
+ false);
+ for (int i = 0; i < fields.length; i++) {
+ String field = fields[i];
+ String key = keys[i];
+ String type = (types[i] == null) || (types[i].isEmpty()) ? null
+ : types[i].trim();
+ Double minimum = (minima[i] == null) || (minima[i].isEmpty()) ? null
+ : Double.parseDouble(minima[i]);
+ Double maximum = (maxima[i] == null) || (maxima[i].isEmpty()) ? null
+ : Double.parseDouble(maxima[i]);
+ try {
+ mtasFields.list.get(field).statsPositionList
+ .add(new ComponentPosition(field, key, minimum, maximum, type));
+ } catch (ParseException e) {
+ throw new IOException(e.getMessage());
+ }
+ }
+ }
+ }
+
+ /**
+ * Prepare tokens.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ private void prepareTokens(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ Set<String> ids = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_STATS_TOKENS);
+ if (ids.size() > 0) {
+ int tmpCounter = 0;
+ String[] fields = new String[ids.size()];
+ String[] keys = new String[ids.size()];
+ String[] minima = new String[ids.size()];
+ String[] maxima = new String[ids.size()];
+ String[] types = new String[ids.size()];
+ for (String id : ids) {
+ fields[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_TOKENS
+ + "." + id + "." + NAME_MTAS_STATS_TOKENS_FIELD, null);
+ keys[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_STATS_TOKENS + "." + id + "."
+ + NAME_MTAS_STATS_TOKENS_KEY, String.valueOf(tmpCounter))
+ .trim();
+ minima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_TOKENS
+ + "." + id + "." + NAME_MTAS_STATS_TOKENS_MINIMUM, null);
+ maxima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_TOKENS
+ + "." + id + "." + NAME_MTAS_STATS_TOKENS_MAXIMUM, null);
+ types[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_TOKENS + "."
+ + id + "." + NAME_MTAS_STATS_TOKENS_TYPE, null);
+ tmpCounter++;
+ }
+ String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
+ mtasFields.doStats = true;
+ mtasFields.doStatsTokens = true;
+ rb.setNeedDocSet(true);
+ for (String field : fields) {
+ if (field == null || field.isEmpty()) {
+ throw new IOException("no (valid) field in mtas stats tokens");
+ } else if (!mtasFields.list.containsKey(field)) {
+ mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
+ }
+ }
+ MtasSolrResultUtil.compareAndCheck(keys, fields,
+ NAME_MTAS_STATS_TOKENS_KEY, NAME_MTAS_STATS_TOKENS_FIELD, true);
+ MtasSolrResultUtil.compareAndCheck(minima, fields,
+ NAME_MTAS_STATS_TOKENS_MINIMUM, NAME_MTAS_STATS_TOKENS_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(maxima, fields,
+ NAME_MTAS_STATS_TOKENS_MAXIMUM, NAME_MTAS_STATS_TOKENS_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(types, fields,
+ NAME_MTAS_STATS_TOKENS_TYPE, NAME_MTAS_STATS_TOKENS_FIELD, false);
+ for (int i = 0; i < fields.length; i++) {
+ String field = fields[i];
+ String key = keys[i];
+ String type = (types[i] == null) || (types[i].isEmpty()) ? null
+ : types[i].trim();
+ Double minimum = (minima[i] == null) || (minima[i].isEmpty()) ? null
+ : Double.parseDouble(minima[i]);
+ Double maximum = (maxima[i] == null) || (maxima[i].isEmpty()) ? null
+ : Double.parseDouble(maxima[i]);
+ try {
+ mtasFields.list.get(field).statsTokenList
+ .add(new ComponentToken(field, key, minimum, maximum, type));
+ } catch (ParseException e) {
+ throw new IOException(e.getMessage());
+ }
+ }
+ }
+ }
+
+ /**
+ * Prepare spans.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ private void prepareSpans(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ SortedSet<String> ids = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_STATS_SPANS);
+ if (ids.size() > 0) {
+ int tmpCounter = 0;
+ String[] fields = new String[ids.size()];
+ String[] keys = new String[ids.size()];
+ String[] minima = new String[ids.size()];
+ String[] maxima = new String[ids.size()];
+ String[] types = new String[ids.size()];
+ String[][] functionExpressions = new String[ids.size()][];
+ String[][] functionKeys = new String[ids.size()][];
+ String[][] functionTypes = new String[ids.size()][];
+ String[][] queryTypes = new String[ids.size()][];
+ String[][] queryValues = new String[ids.size()][];
+ for (String id : ids) {
+ fields[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_SPANS + "."
+ + id + "." + NAME_MTAS_STATS_SPANS_FIELD, null);
+ keys[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_STATS_SPANS + "." + id + "." + NAME_MTAS_STATS_SPANS_KEY,
+ String.valueOf(tmpCounter)).trim();
+ minima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_SPANS + "."
+ + id + "." + NAME_MTAS_STATS_SPANS_MINIMUM, null);
+ maxima[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_SPANS + "."
+ + id + "." + NAME_MTAS_STATS_SPANS_MAXIMUM, null);
+ types[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_STATS_SPANS + "."
+ + id + "." + NAME_MTAS_STATS_SPANS_TYPE, null);
+ Set<String> functionIds = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_STATS_SPANS
+ + "." + id + "." + NAME_MTAS_STATS_SPANS_FUNCTION);
+ functionExpressions[tmpCounter] = new String[functionIds.size()];
+ functionKeys[tmpCounter] = new String[functionIds.size()];
+ functionTypes[tmpCounter] = new String[functionIds.size()];
+ int tmpSubCounter = 0;
+ for (String functionId : functionIds) {
+ functionKeys[tmpCounter][tmpSubCounter] = rb.req.getParams()
+ .get(
+ PARAM_MTAS_STATS_SPANS + "." + id + "."
+ + NAME_MTAS_STATS_SPANS_FUNCTION + "." + functionId + "."
+ + NAME_MTAS_STATS_SPANS_FUNCTION_KEY,
+ String.valueOf(tmpSubCounter))
+ .trim();
+ functionExpressions[tmpCounter][tmpSubCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_STATS_SPANS + "." + id + "."
+ + NAME_MTAS_STATS_SPANS_FUNCTION + "." + functionId + "."
+ + NAME_MTAS_STATS_SPANS_FUNCTION_EXPRESSION, null);
+ functionTypes[tmpCounter][tmpSubCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_STATS_SPANS + "." + id + "."
+ + NAME_MTAS_STATS_SPANS_FUNCTION + "." + functionId + "."
+ + NAME_MTAS_STATS_SPANS_FUNCTION_TYPE, null);
+ tmpSubCounter++;
+ }
+
+ Set<String> qIds = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_STATS_SPANS
+ + "." + id + "." + NAME_MTAS_STATS_SPANS_QUERY);
+ if (qIds.size() > 0) {
+ int tmpQCounter = 0;
+ queryTypes[tmpCounter] = new String[qIds.size()];
+ queryValues[tmpCounter] = new String[qIds.size()];
+ for (String qId : qIds) {
+ queryTypes[tmpCounter][tmpQCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_STATS_SPANS + "." + id + "."
+ + NAME_MTAS_STATS_SPANS_QUERY + "." + qId + "."
+ + SUBNAME_MTAS_STATS_SPANS_QUERY_TYPE, null);
+ queryValues[tmpCounter][tmpQCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_STATS_SPANS + "." + id + "."
+ + NAME_MTAS_STATS_SPANS_QUERY + "." + qId + "."
+ + SUBNAME_MTAS_STATS_SPANS_QUERY_VALUE, null);
+ tmpQCounter++;
+ }
+ } else {
+ throw new IOException("no " + NAME_MTAS_STATS_SPANS_QUERY
+ + " for mtas stats span " + id);
+ }
+ tmpCounter++;
+ }
+ String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
+ mtasFields.doStats = true;
+ mtasFields.doStatsSpans = true;
+ rb.setNeedDocSet(true);
+ for (String field : fields) {
+ if (field == null || field.isEmpty()) {
+ throw new IOException("no (valid) field in mtas stats spans");
+ } else if (!mtasFields.list.containsKey(field)) {
+ mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
+ }
+ }
+ MtasSolrResultUtil.compareAndCheck(keys, fields,
+ NAME_MTAS_STATS_SPANS_KEY, NAME_MTAS_STATS_SPANS_FIELD, true);
+ MtasSolrResultUtil.compareAndCheck(minima, fields,
+ NAME_MTAS_STATS_SPANS_MINIMUM, NAME_MTAS_STATS_SPANS_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(maxima, fields,
+ NAME_MTAS_STATS_SPANS_MAXIMUM, NAME_MTAS_STATS_SPANS_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(types, fields,
+ NAME_MTAS_STATS_SPANS_TYPE, NAME_MTAS_STATS_SPANS_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(types, fields,
+ NAME_MTAS_STATS_SPANS_FUNCTION, NAME_MTAS_STATS_SPANS_FIELD, false);
+
+ for (int i = 0; i < fields.length; i++) {
+ ComponentField cf = mtasFields.list.get(fields[i]);
+ int queryNumber = queryValues[i].length;
+ SpanQuery ql[] = new SpanQuery[queryNumber];
+ for (int j = 0; j < queryNumber; j++) {
+ SpanQuery q = MtasSolrResultUtil.constructQuery(queryValues[i][j],
+ queryTypes[i][j], fields[i]);
+ // minimize number of queries
+ if (cf.spanQueryList.contains(q)) {
+ q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
+ } else {
+ cf.spanQueryList.add(q);
+ }
+ ql[j] = q;
+ }
+ Double minimum = (minima[i] == null) || (minima[i].isEmpty()) ? null
+ : Double.parseDouble(minima[i]);
+ Double maximum = (maxima[i] == null) || (maxima[i].isEmpty()) ? null
+ : Double.parseDouble(maxima[i]);
+ String key = (keys[i] == null) || (keys[i].isEmpty())
+ ? String.valueOf(i) + ":" + fields[i] + ":" + queryNumber
+ : keys[i].trim();
+ String type = (types[i] == null) || (types[i].isEmpty()) ? null
+ : types[i].trim();
+ String[] functionKey = functionKeys[i];
+ String[] functionExpression = functionExpressions[i];
+ String[] functionType = functionTypes[i];
+ try {
+ mtasFields.list.get(fields[i]).statsSpanList
+ .add(new ComponentSpan(ql, key, minimum, maximum, type,
+ functionKey, functionExpression, functionType));
+ } catch (ParseException e) {
+ throw new IOException(e.getMessage());
+ }
+ }
+ } else {
+ throw new IOException("missing parameters stats spans");
+ }
+ }
+
+ /**
+ * Modify request.
+ *
+ * @param rb
+ * the rb
+ * @param who
+ * the who
+ * @param sreq
+ * the sreq
+ */
+ public void modifyRequest(ResponseBuilder rb, SearchComponent who,
+ ShardRequest sreq) {
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
+ // do nothing
+ } else {
+ // remove stats for other requests
+ sreq.params.remove(PARAM_MTAS_STATS);
+ sreq.params.remove(PARAM_MTAS_STATS_POSITIONS);
+ sreq.params.remove(PARAM_MTAS_STATS_SPANS);
+ }
+ }
+
+ /**
+ * Creates the position.
+ *
+ * @param position
+ * the position
+ * @param encode
+ * the encode
+ * @return the simple ordered map
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public SimpleOrderedMap<Object> createPosition(ComponentPosition position,
+ Boolean encode) throws IOException {
+ // System.out.println("Create stats position " + position.dataType + " "
+ // + position.statsType + " " + position.statsItems + " --- " + encode);
+ SimpleOrderedMap<Object> mtasPositionResponse = new SimpleOrderedMap<>();
+ mtasPositionResponse.add("key", position.key);
+ MtasSolrResult data = new MtasSolrResult(position.dataCollector,
+ position.dataType, position.statsType, position.statsItems, null);
+ if (encode) {
+ mtasPositionResponse.add("_encoded_data",
+ MtasSolrResultUtil.encode(data));
+ } else {
+ mtasPositionResponse.add(position.dataCollector.getCollectorType(), data);
+ MtasSolrResultUtil.rewrite(mtasPositionResponse);
+ }
+ return mtasPositionResponse;
+ }
+
+ /**
+ * Creates the token.
+ *
+ * @param token
+ * the token
+ * @param encode
+ * the encode
+ * @return the simple ordered map
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public SimpleOrderedMap<Object> createToken(ComponentToken token,
+ Boolean encode) throws IOException {
+ // System.out.println("Create stats position " + position.dataType + " "
+ // + position.statsType + " " + position.statsItems + " --- " + encode);
+ SimpleOrderedMap<Object> mtasTokenResponse = new SimpleOrderedMap<>();
+ mtasTokenResponse.add("key", token.key);
+ MtasSolrResult data = new MtasSolrResult(token.dataCollector,
+ token.dataType, token.statsType, token.statsItems, null);
+ if (encode) {
+ mtasTokenResponse.add("_encoded_data", MtasSolrResultUtil.encode(data));
+ } else {
+ mtasTokenResponse.add(token.dataCollector.getCollectorType(), data);
+ MtasSolrResultUtil.rewrite(mtasTokenResponse);
+ }
+ return mtasTokenResponse;
+ }
+
+ /**
+ * Creates the span.
+ *
+ * @param span
+ * the span
+ * @param encode
+ * the encode
+ * @return the simple ordered map
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ public SimpleOrderedMap<Object> createSpan(ComponentSpan span, Boolean encode)
+ throws IOException {
+ // System.out.println("Create stats span " + span.dataType + " "
+ // + span.statsType + " " + span.statsItems + " --- " + encode);
+ SimpleOrderedMap<Object> mtasSpanResponse = new SimpleOrderedMap<>();
+ mtasSpanResponse.add("key", span.key);
+ HashMap<MtasDataCollector<?, ?>, HashMap<String, MtasSolrResult>> functionData = new HashMap<MtasDataCollector<?, ?>, HashMap<String, MtasSolrResult>>();
+ HashMap<String, MtasSolrResult> functionDataItem = new HashMap<String, MtasSolrResult>();
+ functionData.put(span.dataCollector, functionDataItem);
+ if (span.functions != null) {
+ for (SubComponentFunction function : span.functions) {
+ function.dataCollector.close();
+ functionDataItem.put(function.key,
+ new MtasSolrResult(function.dataCollector,
+ new String[] { function.dataType },
+ new String[] { function.statsType },
+ new TreeSet[] { function.statsItems }, new String[] { null },
+ new String[] { null }, new Integer[] { 0 },
+ new Integer[] { Integer.MAX_VALUE }, null));
+ }
+ }
+ MtasSolrResult data = new MtasSolrResult(span.dataCollector, span.dataType,
+ span.statsType, span.statsItems, functionData);
+ if (encode) {
+ mtasSpanResponse.add("_encoded_data", MtasSolrResultUtil.encode(data));
+ } else {
+ mtasSpanResponse.add(span.dataCollector.getCollectorType(), data);
+ MtasSolrResultUtil.rewrite(mtasSpanResponse);
+ }
+ return mtasSpanResponse;
+ }
+
+ /**
+ * Finish stage.
+ *
+ * @param rb
+ * the rb
+ */
+ @SuppressWarnings("unchecked")
+ public void finishStage(ResponseBuilder rb) {
+ if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (rb.stage >= ResponseBuilder.STAGE_EXECUTE_QUERY
+ && rb.stage < ResponseBuilder.STAGE_GET_FIELDS) {
+ for (ShardRequest sreq : rb.finished) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
+ && sreq.params.getBool(PARAM_MTAS_STATS, false)) {
+ for (ShardResponse shardResponse : sreq.responses) {
+ NamedList<Object> response = shardResponse.getSolrResponse()
+ .getResponse();
+ try {
+ ArrayList<NamedList<Object>> data = (ArrayList<NamedList<Object>>) response
+ .findRecursive("mtas", "stats");
+ if (data != null) {
+ MtasSolrResultUtil.decode(data);
+ }
+ } catch (ClassCastException e) {
+ // shouldnt happen
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Distributed process.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ public void distributedProcess(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ // rewrite
+ NamedList<Object> mtasResponse = null;
+ try {
+ mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
+ if (mtasResponse != null) {
+ NamedList<Object> mtasResponseStats;
+ try {
+ mtasResponseStats = (NamedList<Object>) mtasResponse.get("stats");
+ if (mtasResponseStats != null) {
+ MtasSolrResultUtil.rewrite(mtasResponseStats);
+ }
+ } catch (ClassCastException e) {
+ mtasResponseStats = null;
+ }
+ }
+ } catch (ClassCastException e) {
+ mtasResponse = null;
+ }
+ }
+
+}
diff --git a/src/mtas/solr/handler/component/util/MtasSolrComponentTermvector.java b/src/mtas/solr/handler/component/util/MtasSolrComponentTermvector.java
new file mode 100644
index 0000000..986d5fe
--- /dev/null
+++ b/src/mtas/solr/handler/component/util/MtasSolrComponentTermvector.java
@@ -0,0 +1,953 @@
+package mtas.solr.handler.component.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.Map.Entry;
+
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.handler.component.ShardResponse;
+
+import mtas.codec.util.CodecUtil;
+import mtas.codec.util.CodecComponent.ComponentField;
+import mtas.codec.util.CodecComponent.ComponentFields;
+import mtas.codec.util.CodecComponent.ComponentTermVector;
+import mtas.codec.util.CodecComponent.SubComponentFunction;
+import mtas.codec.util.collector.MtasDataCollector;
+import mtas.codec.util.collector.MtasDataItem.NumberComparator;
+import mtas.parser.function.ParseException;
+import mtas.solr.handler.component.MtasSolrSearchComponent;
+
+/**
+ * The Class MtasSolrComponentTermvector.
+ */
+public class MtasSolrComponentTermvector {
+
+ /** The search component. */
+ MtasSolrSearchComponent searchComponent;
+
+ /** The Constant PARAM_MTAS_TERMVECTOR. */
+ public static final String PARAM_MTAS_TERMVECTOR = MtasSolrSearchComponent.PARAM_MTAS
+ + ".termvector";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_FIELD. */
+ public static final String NAME_MTAS_TERMVECTOR_FIELD = "field";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_KEY. */
+ public static final String NAME_MTAS_TERMVECTOR_KEY = "key";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_PREFIX. */
+ public static final String NAME_MTAS_TERMVECTOR_PREFIX = "prefix";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_REGEXP. */
+ public static final String NAME_MTAS_TERMVECTOR_REGEXP = "regexp";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_TYPE. */
+ public static final String NAME_MTAS_TERMVECTOR_TYPE = "type";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_SORT_TYPE. */
+ public static final String NAME_MTAS_TERMVECTOR_SORT_TYPE = "sort.type";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_SORT_DIRECTION. */
+ public static final String NAME_MTAS_TERMVECTOR_SORT_DIRECTION = "sort.direction";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_NUMBER. */
+ public static final String NAME_MTAS_TERMVECTOR_NUMBER = "number";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_FUNCTION. */
+ public static final String NAME_MTAS_TERMVECTOR_FUNCTION = "function";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_FUNCTION_EXPRESSION. */
+ public static final String NAME_MTAS_TERMVECTOR_FUNCTION_EXPRESSION = "expression";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_FUNCTION_KEY. */
+ public static final String NAME_MTAS_TERMVECTOR_FUNCTION_KEY = "key";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_FUNCTION_TYPE. */
+ public static final String NAME_MTAS_TERMVECTOR_FUNCTION_TYPE = "type";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_BOUNDARY. */
+ public static final String NAME_MTAS_TERMVECTOR_BOUNDARY = "boundary";
+
+ /** The Constant NAME_MTAS_TERMVECTOR_LIST. */
+ public static final String NAME_MTAS_TERMVECTOR_LIST = "list";
+
+ /**
+ * Instantiates a new mtas solr component termvector.
+ *
+ * @param searchComponent
+ * the search component
+ */
+ public MtasSolrComponentTermvector(MtasSolrSearchComponent searchComponent) {
+ this.searchComponent = searchComponent;
+ }
+
+ /**
+ * Prepare.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
+ throws IOException {
+ Set<String> ids = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_TERMVECTOR);
+ if (ids.size() > 0) {
+ int tmpCounter = 0;
+ String[] fields = new String[ids.size()];
+ String[] keys = new String[ids.size()];
+ String[] prefixes = new String[ids.size()];
+ String[] regexps = new String[ids.size()];
+ String[] sortTypes = new String[ids.size()];
+ String[] sortDirections = new String[ids.size()];
+ String[] types = new String[ids.size()];
+ String[] startValues = new String[ids.size()];
+ String[] numbers = new String[ids.size()];
+ String[][] functionExpressions = new String[ids.size()][];
+ String[][] functionKeys = new String[ids.size()][];
+ String[][] functionTypes = new String[ids.size()][];
+ String[] boundaries = new String[ids.size()];
+ String[][] lists = new String[ids.size()][];
+ for (String id : ids) {
+ fields[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_TERMVECTOR + "." + id + "." + NAME_MTAS_TERMVECTOR_FIELD,
+ null);
+ keys[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_TERMVECTOR + "." + id + "." + NAME_MTAS_TERMVECTOR_KEY,
+ String.valueOf(tmpCounter)).trim();
+ prefixes[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_TERMVECTOR
+ + "." + id + "." + NAME_MTAS_TERMVECTOR_PREFIX, null);
+ regexps[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_TERMVECTOR + "."
+ + id + "." + NAME_MTAS_TERMVECTOR_REGEXP, null);
+ sortTypes[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_TERMVECTOR
+ + "." + id + "." + NAME_MTAS_TERMVECTOR_SORT_TYPE, null);
+ sortDirections[tmpCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_TERMVECTOR + "." + id + "."
+ + NAME_MTAS_TERMVECTOR_SORT_DIRECTION, null);
+ numbers[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_TERMVECTOR + "."
+ + id + "." + NAME_MTAS_TERMVECTOR_NUMBER, null);
+ types[tmpCounter] = rb.req.getParams().get(
+ PARAM_MTAS_TERMVECTOR + "." + id + "." + NAME_MTAS_TERMVECTOR_TYPE,
+ null);
+ Set<String> functionIds = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_TERMVECTOR
+ + "." + id + "." + NAME_MTAS_TERMVECTOR_FUNCTION);
+ functionExpressions[tmpCounter] = new String[functionIds.size()];
+ functionKeys[tmpCounter] = new String[functionIds.size()];
+ functionTypes[tmpCounter] = new String[functionIds.size()];
+ int tmpSubCounter = 0;
+ for (String functionId : functionIds) {
+ functionKeys[tmpCounter][tmpSubCounter] = rb.req.getParams()
+ .get(
+ PARAM_MTAS_TERMVECTOR + "." + id + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION + "." + functionId + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION_KEY,
+ String.valueOf(tmpSubCounter))
+ .trim();
+ functionExpressions[tmpCounter][tmpSubCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_TERMVECTOR + "." + id + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION + "." + functionId + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION_EXPRESSION, null);
+ functionTypes[tmpCounter][tmpSubCounter] = rb.req.getParams()
+ .get(PARAM_MTAS_TERMVECTOR + "." + id + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION + "." + functionId + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION_TYPE, null);
+ tmpSubCounter++;
+ }
+ boundaries[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_TERMVECTOR
+ + "." + id + "." + NAME_MTAS_TERMVECTOR_BOUNDARY, null);
+ lists[tmpCounter] = rb.req.getParams().getParams(
+ PARAM_MTAS_TERMVECTOR + "." + id + "." + NAME_MTAS_TERMVECTOR_LIST);
+ tmpCounter++;
+ }
+ String uniqueKeyField = rb.req.getSchema().getUniqueKeyField().getName();
+ mtasFields.doTermVector = true;
+ rb.setNeedDocSet(true);
+ // init and checks
+ for (String field : fields) {
+ if (field == null || field.isEmpty()) {
+ throw new IOException("no (valid) field in mtas termvector");
+ } else if (!mtasFields.list.containsKey(field)) {
+ mtasFields.list.put(field, new ComponentField(field, uniqueKeyField));
+ }
+ }
+ MtasSolrResultUtil.compareAndCheck(keys, fields, NAME_MTAS_TERMVECTOR_KEY,
+ NAME_MTAS_TERMVECTOR_FIELD, true);
+ MtasSolrResultUtil.compareAndCheck(prefixes, fields,
+ NAME_MTAS_TERMVECTOR_PREFIX, NAME_MTAS_TERMVECTOR_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(regexps, fields,
+ NAME_MTAS_TERMVECTOR_REGEXP, NAME_MTAS_TERMVECTOR_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(types, fields,
+ NAME_MTAS_TERMVECTOR_TYPE, NAME_MTAS_TERMVECTOR_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(sortTypes, fields,
+ NAME_MTAS_TERMVECTOR_SORT_TYPE, NAME_MTAS_TERMVECTOR_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(sortDirections, fields,
+ NAME_MTAS_TERMVECTOR_SORT_DIRECTION, NAME_MTAS_TERMVECTOR_FIELD,
+ false);
+ MtasSolrResultUtil.compareAndCheck(numbers, fields,
+ NAME_MTAS_TERMVECTOR_NUMBER, NAME_MTAS_TERMVECTOR_FIELD, false);
+ MtasSolrResultUtil.compareAndCheck(boundaries, fields,
+ NAME_MTAS_TERMVECTOR_BOUNDARY, NAME_MTAS_TERMVECTOR_FIELD, false);
+ for (int i = 0; i < fields.length; i++) {
+ String field = fields[i];
+ String prefix = (prefixes[i] == null) || (prefixes[i].isEmpty()) ? null
+ : prefixes[i].trim();
+ String key = (keys[i] == null) || (keys[i].isEmpty())
+ ? String.valueOf(i) + ":" + field + ":" + prefix : keys[i].trim();
+ String regexp = (regexps[i] == null) || (regexps[i].isEmpty()) ? null
+ : regexps[i].trim();
+ String startValue = (startValues[i] == null)
+ || (startValues[i].isEmpty()) ? null : startValues[i].trim();
+ int number = (numbers[i] == null) || (numbers[i].isEmpty()) ? 10
+ : Integer.parseInt(numbers[i]);
+ String type = (types[i] == null) || (types[i].isEmpty()) ? null
+ : types[i].trim();
+ String sortType = (sortTypes[i] == null) || (sortTypes[i].isEmpty())
+ ? null : sortTypes[i].trim();
+ String sortDirection = (sortDirections[i] == null)
+ || (sortDirections[i].isEmpty()) ? null : sortDirections[i].trim();
+ String[] functionKey = functionKeys[i];
+ String[] functionExpression = functionExpressions[i];
+ String[] functionType = functionTypes[i];
+ String boundary = boundaries[i];
+ String[] list = lists[i];
+ if (prefix == null || prefix.isEmpty()) {
+ throw new IOException("no (valid) prefix in mtas termvector");
+ } else {
+ try {
+ mtasFields.list.get(field).termVectorList
+ .add(new ComponentTermVector(key, prefix, regexp, type,
+ sortType, sortDirection, startValue, number, functionKey,
+ functionExpression, functionType, boundary, list));
+ } catch (ParseException e) {
+ throw new IOException(e.getMessage());
+ }
+ }
+ }
+
+ }
+ }
+
+ /**
+ * Modify request.
+ *
+ * @param rb
+ * the rb
+ * @param who
+ * the who
+ * @param sreq
+ * the sreq
+ */
+ public void modifyRequest(ResponseBuilder rb, SearchComponent who,
+ ShardRequest sreq) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (sreq.params.getBool(PARAM_MTAS_TERMVECTOR, false)) {
+ // compute keys
+ Set<String> keys = MtasSolrResultUtil
+ .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_TERMVECTOR);
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
+ } else {
+ sreq.params.remove(PARAM_MTAS_TERMVECTOR);
+ for (String key : keys) {
+ sreq.params.remove(PARAM_MTAS_TERMVECTOR + "." + key + "."
+ + NAME_MTAS_TERMVECTOR_FIELD);
+ sreq.params.remove(PARAM_MTAS_TERMVECTOR + "." + key + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION_EXPRESSION);
+ sreq.params.remove(PARAM_MTAS_TERMVECTOR + "." + key + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION_TYPE);
+ sreq.params.remove(PARAM_MTAS_TERMVECTOR + "." + key + "."
+ + NAME_MTAS_TERMVECTOR_KEY);
+ sreq.params.remove(PARAM_MTAS_TERMVECTOR + "." + key + "."
+ + NAME_MTAS_TERMVECTOR_NUMBER);
+ sreq.params.remove(PARAM_MTAS_TERMVECTOR + "." + key + "."
+ + NAME_MTAS_TERMVECTOR_PREFIX);
+ sreq.params.remove(PARAM_MTAS_TERMVECTOR + "." + key + "."
+ + NAME_MTAS_TERMVECTOR_REGEXP);
+ sreq.params.remove(PARAM_MTAS_TERMVECTOR + "." + key + "."
+ + NAME_MTAS_TERMVECTOR_SORT_TYPE);
+ sreq.params.remove(PARAM_MTAS_TERMVECTOR + "." + key + "."
+ + NAME_MTAS_TERMVECTOR_SORT_DIRECTION);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Creates the.
+ *
+ * @param termVector
+ * the term vector
+ * @param encode
+ * the encode
+ * @return the simple ordered map
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ public SimpleOrderedMap<Object> create(ComponentTermVector termVector,
+ Boolean encode) throws IOException {
+ SimpleOrderedMap<Object> mtasTermVectorResponse = new SimpleOrderedMap<>();
+ mtasTermVectorResponse.add("key", termVector.key);
+ termVector.subComponentFunction.dataCollector.close();
+ HashMap<MtasDataCollector<?, ?>, HashMap<String, MtasSolrResult>> functionData = new HashMap<MtasDataCollector<?, ?>, HashMap<String, MtasSolrResult>>();
+ HashMap<String, MtasSolrResult> functionDataItem = new HashMap<String, MtasSolrResult>();
+ functionData.put(termVector.subComponentFunction.dataCollector,
+ functionDataItem);
+ if (termVector.functions != null) {
+ for (SubComponentFunction function : termVector.functions) {
+ function.dataCollector.reduceToKeys(
+ termVector.subComponentFunction.dataCollector.getKeyList());
+ function.dataCollector.close();
+ functionDataItem.put(function.key,
+ new MtasSolrResult(function.dataCollector,
+ new String[] { function.dataType },
+ new String[] { function.statsType },
+ new TreeSet[] { function.statsItems }, new String[] { null },
+ new String[] { null }, new Integer[] { 0 },
+ new Integer[] { Integer.MAX_VALUE }, null));
+ }
+ }
+ MtasSolrResult data = new MtasSolrResult(
+ termVector.subComponentFunction.dataCollector,
+ new String[] { termVector.subComponentFunction.dataType },
+ new String[] { termVector.subComponentFunction.statsType },
+ new TreeSet[] { termVector.subComponentFunction.statsItems },
+ new String[] { termVector.subComponentFunction.sortType },
+ new String[] { termVector.subComponentFunction.sortDirection },
+ new Integer[] { 0 }, new Integer[] { termVector.number }, functionData);
+ if (encode) {
+ mtasTermVectorResponse.add("_encoded_list",
+ MtasSolrResultUtil.encode(data));
+ } else {
+ mtasTermVectorResponse.add("list", data);
+ MtasSolrResultUtil.rewrite(mtasTermVectorResponse);
+ }
+ return mtasTermVectorResponse;
+ }
+
+ /**
+ * Finish stage.
+ *
+ * @param rb
+ * the rb
+ */
+ @SuppressWarnings("unchecked")
+ public void finishStage(ResponseBuilder rb) {
+ if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ if (rb.stage >= ResponseBuilder.STAGE_EXECUTE_QUERY
+ && rb.stage < ResponseBuilder.STAGE_GET_FIELDS) {
+ for (ShardRequest sreq : rb.finished) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
+ && sreq.params.getBool(PARAM_MTAS_TERMVECTOR, false)) {
+ for (ShardResponse shardResponse : sreq.responses) {
+ NamedList<Object> response = shardResponse.getSolrResponse()
+ .getResponse();
+ try {
+ ArrayList<NamedList<Object>> data = (ArrayList<NamedList<Object>>) response
+ .findRecursive("mtas", "termvector");
+ if (data != null) {
+ MtasSolrResultUtil.decode(data);
+ }
+ } catch (ClassCastException e) {
+ // shouldnt happen
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Distributed process finish.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ public void distributedProcessFinish(ResponseBuilder rb,
+ ComponentFields mtasFields) throws IOException {
+ // rewrite
+ NamedList<Object> mtasResponse = null;
+ try {
+ mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
+ if (mtasResponse != null) {
+ ArrayList<Object> mtasResponseTermvector;
+ try {
+ mtasResponseTermvector = (ArrayList<Object>) mtasResponse
+ .get("termvector");
+ if (mtasResponseTermvector != null) {
+ MtasSolrResultUtil.rewrite(mtasResponseTermvector);
+ }
+ } catch (ClassCastException e) {
+ mtasResponseTermvector = null;
+ }
+ }
+ } catch (ClassCastException e) {
+ mtasResponse = null;
+ }
+ }
+
+ /**
+ * Distributed process missing top.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public void distributedProcessMissingTop(ResponseBuilder rb,
+ ComponentFields mtasFields) throws IOException {
+ // initialise
+ HashMap<String, TreeMap<String, NumberComparator>> mergedComparatorLists = new HashMap<String, TreeMap<String, NumberComparator>>();
+ HashMap<String, NumberComparator> mergedComparatorBoundaryList = new HashMap<String, NumberComparator>();
+ HashMap<String, NumberComparator> summedComparatorBoundaryList = new HashMap<String, NumberComparator>();
+ HashMap<String, HashMap<String, NumberComparator>> comparatorBoundariesList = new HashMap<String, HashMap<String, NumberComparator>>();
+ // check all termvectors, and initialize
+ for (String field : mtasFields.list.keySet()) {
+ List<ComponentTermVector> tvList = mtasFields.list
+ .get(field).termVectorList;
+ if (tvList != null) {
+ for (ComponentTermVector tv : tvList) {
+ if (tv.subComponentFunction.sortType != CodecUtil.SORT_TERM) {
+ mergedComparatorLists.put(tv.key,
+ new TreeMap<String, NumberComparator>());
+ }
+ }
+ }
+ }
+ // compute for each termvector the mergedComparatorList and the
+ // summedBoundary
+ for (ShardRequest sreq : rb.finished) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
+ && sreq.params.getBool(PARAM_MTAS_TERMVECTOR, false)) {
+ for (ShardResponse shardResponse : sreq.responses) {
+ TreeMap<String, NumberComparator> mergedComparatorList;
+ NamedList<Object> response = shardResponse.getSolrResponse()
+ .getResponse();
+ String key;
+ MtasSolrResult list;
+ NumberComparator comparatorLast;
+ LinkedHashMap<String, NumberComparator> comparatorList;
+ try {
+ ArrayList<NamedList<Object>> data = (ArrayList<NamedList<Object>>) response
+ .findRecursive("mtas", "termvector");
+ if (data != null) {
+ for (int i = 0; i < data.size(); i++) {
+ NamedList<Object> dataItem = data.get(i);
+ try {
+ key = (String) dataItem.get("key");
+ list = (MtasSolrResult) dataItem.get("list");
+ comparatorLast = list.getResult().getLastSortValue();
+ comparatorList = list.getResult().getComparatorList();
+ if (key == null) {
+ dataItem.clear();
+ } else if (comparatorLast == null || comparatorList == null
+ || !mergedComparatorLists.containsKey(key)) {
+ // do nothing
+ } else {
+ mergedComparatorList = mergedComparatorLists.get(key);
+ for (String value : comparatorList.keySet()) {
+ if (mergedComparatorList.containsKey(value)) {
+ mergedComparatorList.get(value)
+ .add(comparatorList.get(value).getValue());
+ } else {
+ mergedComparatorList.put(value,
+ comparatorList.get(value).clone());
+ }
+ }
+ if (!comparatorBoundariesList.containsKey(key)) {
+ comparatorBoundariesList.put(key,
+ new HashMap<String, NumberComparator>());
+ }
+ comparatorBoundariesList.get(key)
+ .put(shardResponse.getShardAddress(), comparatorLast);
+ if (summedComparatorBoundaryList.containsKey(key)) {
+ summedComparatorBoundaryList.get(key)
+ .add(comparatorLast.getValue());
+ } else {
+ summedComparatorBoundaryList.put(key,
+ comparatorLast.clone());
+ }
+ }
+ } catch (ClassCastException e) {
+ dataItem.clear();
+ }
+ }
+ }
+ } catch (ClassCastException e) {
+ // shouldnt happen
+ }
+ shardResponse.getSolrResponse().setResponse(response);
+ }
+ }
+ }
+ // compute for each relevant termvector the mergedComparatorBoundary
+ for (String field : mtasFields.list.keySet()) {
+ List<ComponentTermVector> tvList = mtasFields.list
+ .get(field).termVectorList;
+ if (tvList != null) {
+ for (ComponentTermVector tv : tvList) {
+ TreeMap<String, NumberComparator> mergedComparatorList;
+ if (mergedComparatorLists.containsKey(tv.key)) {
+ mergedComparatorList = mergedComparatorLists.get(tv.key);
+ if (mergedComparatorList.size() < tv.number) {
+ // do nothing
+ } else {
+ SortedSet<Map.Entry<String, NumberComparator>> sortedSet = new TreeSet<Map.Entry<String, NumberComparator>>(
+ new Comparator<Map.Entry<String, NumberComparator>>() {
+ @Override
+ public int compare(Map.Entry<String, NumberComparator> e1,
+ Map.Entry<String, NumberComparator> e2) {
+ int compare = e1.getValue()
+ .compareTo(e2.getValue().getValue());
+ if (compare == 0) {
+ compare = e1.getKey().compareTo(e2.getKey());
+ } else if (tv.subComponentFunction.sortDirection
+ .equals(CodecUtil.SORT_DESC)) {
+ compare *= -1;
+ }
+ return compare;
+ }
+ });
+ sortedSet.addAll(mergedComparatorLists.get(tv.key).entrySet());
+ Optional<Map.Entry<String, NumberComparator>> optionalItem = sortedSet
+ .stream().skip(tv.number - 1).findFirst();
+ if (optionalItem.isPresent()) {
+ mergedComparatorBoundaryList.put(tv.key,
+ optionalItem.get().getValue());
+ }
+ }
+ }
+ }
+ }
+ }
+ // compute which termvectors to recompute
+ HashMap<String, HashMap<String, NumberComparator>> recomputeList = new HashMap<String, HashMap<String, NumberComparator>>();
+ for (String key : mergedComparatorBoundaryList.keySet()) {
+ if (summedComparatorBoundaryList.containsKey(key)) {
+ if (summedComparatorBoundaryList.get(key)
+ .compareTo(mergedComparatorBoundaryList.get(key).getValue()) > 0) {
+ // set termvector to recompute
+ recomputeList.put(key, new HashMap<String, NumberComparator>());
+ // compute difference
+ NumberComparator difference = summedComparatorBoundaryList.get(key)
+ .clone();
+ difference.subtract(mergedComparatorBoundaryList.get(key).getValue());
+ // sort
+ List<Entry<String, NumberComparator>> list = new LinkedList<Entry<String, NumberComparator>>(
+ comparatorBoundariesList.get(key).entrySet());
+ Collections.sort(list,
+ new Comparator<Entry<String, NumberComparator>>() {
+ public int compare(Entry<String, NumberComparator> e1,
+ Entry<String, NumberComparator> e2) {
+ return e1.getValue().compareTo(e2.getValue().getValue());
+ }
+ });
+
+ HashMap<String, NumberComparator> sortedHashMap = new LinkedHashMap<String, NumberComparator>();
+ for (Iterator<Map.Entry<String, NumberComparator>> it = list
+ .iterator(); it.hasNext();) {
+ Map.Entry<String, NumberComparator> entry = it.next();
+ sortedHashMap.put(entry.getKey(), entry.getValue());
+ }
+ NumberComparator sum = null;
+ for (String shardAddress : sortedHashMap.keySet()) {
+ if (sum == null) {
+ sum = sortedHashMap.get(shardAddress).clone();
+ } else {
+ sum.add(sortedHashMap.get(shardAddress).getValue());
+ }
+ // skip shards with low boundaries
+ if (sum.compareTo(
+ mergedComparatorBoundaryList.get(key).getValue()) > 0) {
+ NumberComparator newBoundary = sortedHashMap.get(shardAddress)
+ .clone();
+ newBoundary.subtract(difference.getValue());
+ recomputeList.get(key).put(shardAddress, newBoundary);
+ }
+ }
+ }
+ }
+ }
+
+ // finally, recompute
+ if (recomputeList.size() > 0) {
+ // remove output for termvectors in recompute list and get list of shards
+ HashSet<String> shards = new HashSet<String>();
+ for (ShardRequest sreq : rb.finished) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
+ && sreq.params.getBool(PARAM_MTAS_TERMVECTOR, false)) {
+ for (ShardResponse shardResponse : sreq.responses) {
+ NamedList<Object> response = shardResponse.getSolrResponse()
+ .getResponse();
+ String key;
+ String shardAddress = shardResponse.getShardAddress();
+ try {
+ ArrayList<NamedList<Object>> data = (ArrayList<NamedList<Object>>) response
+ .findRecursive("mtas", "termvector");
+ if (data != null) {
+ for (int i = 0; i < data.size(); i++) {
+ NamedList<Object> dataItem = data.get(i);
+ try {
+ key = (String) dataItem.get("key");
+ if (key != null && recomputeList.containsKey(key)
+ && recomputeList.get(key).containsKey(shardAddress)) {
+ dataItem.clear();
+ dataItem.add("key", key);
+ shards.add(shardAddress);
+ }
+ } catch (ClassCastException e) {
+ dataItem.clear();
+ }
+ }
+ }
+ } catch (ClassCastException e) {
+ // shouldnt happen
+ }
+ shardResponse.getSolrResponse().setResponse(response);
+ }
+ }
+ }
+
+ // parameter
+ HashMap<String, ModifiableSolrParams> requestParamList = new HashMap<String, ModifiableSolrParams>();
+ for (String shardAddress : shards) {
+ ModifiableSolrParams paramsNewRequest = new ModifiableSolrParams();
+ requestParamList.put(shardAddress, paramsNewRequest);
+ int termvectorCounter = 0;
+ for (String field : mtasFields.list.keySet()) {
+ List<ComponentTermVector> tvList = mtasFields.list
+ .get(field).termVectorList;
+ if (tvList != null) {
+ for (ComponentTermVector tv : tvList) {
+ if (recomputeList.containsKey(tv.key)
+ && recomputeList.get(tv.key).containsKey(shardAddress)) {
+ paramsNewRequest.add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter + "."
+ + NAME_MTAS_TERMVECTOR_BOUNDARY,
+ String.valueOf(recomputeList.get(tv.key).get(shardAddress)
+ .getValue()));
+ paramsNewRequest.add(PARAM_MTAS_TERMVECTOR + "."
+ + termvectorCounter + "." + NAME_MTAS_TERMVECTOR_FIELD,
+ field);
+ paramsNewRequest.add(PARAM_MTAS_TERMVECTOR + "."
+ + termvectorCounter + "." + NAME_MTAS_TERMVECTOR_PREFIX,
+ tv.prefix);
+ paramsNewRequest.add(PARAM_MTAS_TERMVECTOR + "."
+ + termvectorCounter + "." + NAME_MTAS_TERMVECTOR_KEY,
+ tv.key);
+ paramsNewRequest.add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter + "."
+ + NAME_MTAS_TERMVECTOR_NUMBER,
+ String.valueOf(tv.number));
+ if (tv.subComponentFunction.sortType != null) {
+ paramsNewRequest.add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter + "."
+ + NAME_MTAS_TERMVECTOR_SORT_TYPE,
+ tv.subComponentFunction.sortType);
+ }
+ if (tv.subComponentFunction.sortDirection != null) {
+ paramsNewRequest.add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter + "."
+ + NAME_MTAS_TERMVECTOR_SORT_DIRECTION,
+ tv.subComponentFunction.sortDirection);
+ }
+ if (tv.subComponentFunction.type != null) {
+ paramsNewRequest.add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter + "."
+ + NAME_MTAS_TERMVECTOR_TYPE,
+ tv.subComponentFunction.type);
+ }
+ if (tv.functions != null) {
+ int functionCounter = 0;
+ for (SubComponentFunction function : tv.functions) {
+ paramsNewRequest.add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION + "."
+ + functionCounter + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION_EXPRESSION,
+ function.expression);
+ paramsNewRequest
+ .add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter
+ + "." + NAME_MTAS_TERMVECTOR_FUNCTION + "."
+ + functionCounter + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION_KEY,
+ function.key);
+ if (function.type != null) {
+ paramsNewRequest.add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION + "."
+ + functionCounter + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION_TYPE,
+ function.type);
+ }
+ functionCounter++;
+ }
+ }
+ if (tv.regexp != null) {
+ paramsNewRequest.add(PARAM_MTAS_TERMVECTOR + "."
+ + termvectorCounter + "." + NAME_MTAS_TERMVECTOR_REGEXP,
+ tv.regexp);
+ }
+ termvectorCounter++;
+ }
+ }
+ }
+ }
+ }
+
+ // new requests
+ for (String shardAddress : requestParamList.keySet()) {
+ ShardRequest sreq = new ShardRequest();
+ sreq.shards = new String[] { shardAddress };
+ sreq.purpose = ShardRequest.PURPOSE_PRIVATE;
+ sreq.params = requestParamList.get(shardAddress);
+ sreq.params.add("fq", rb.req.getParams().getParams("fq"));
+ sreq.params.add("q", rb.req.getParams().getParams("q"));
+ sreq.params.add("cache", rb.req.getParams().getParams("cache"));
+ sreq.params.add("rows", "0");
+ sreq.params.add(MtasSolrSearchComponent.PARAM_MTAS, rb.req
+ .getOriginalParams().getParams(MtasSolrSearchComponent.PARAM_MTAS));
+ sreq.params.add(PARAM_MTAS_TERMVECTOR,
+ rb.req.getOriginalParams().getParams(PARAM_MTAS_TERMVECTOR));
+ rb.addRequest(searchComponent, sreq);
+ }
+ }
+
+ }
+
+ /**
+ * Distributed process missing key.
+ *
+ * @param rb
+ * the rb
+ * @param mtasFields
+ * the mtas fields
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public void distributedProcessMissingKey(ResponseBuilder rb,
+ ComponentFields mtasFields) throws IOException {
+
+ HashMap<String, HashMap<String, HashSet<String>>> missingTermvectorKeys = computeMissingTermvectorItemsPerShard(
+ rb.finished, "mtas", "termvector");
+
+ for (String shardName : missingTermvectorKeys.keySet()) {
+ HashMap<String, HashSet<String>> missingTermvectorKeysShard = missingTermvectorKeys
+ .get(shardName);
+ ModifiableSolrParams paramsNewRequest = new ModifiableSolrParams();
+ int termvectorCounter = 0;
+ for (String field : mtasFields.list.keySet()) {
+ List<ComponentTermVector> tvList = mtasFields.list
+ .get(field).termVectorList;
+ if (tvList != null) {
+ for (ComponentTermVector tv : tvList) {
+ if (missingTermvectorKeysShard.containsKey(tv.key)) {
+ HashSet<String> list = missingTermvectorKeysShard.get(tv.key);
+ if (list.size() > 0) {
+ paramsNewRequest.add(PARAM_MTAS_TERMVECTOR + "."
+ + termvectorCounter + "." + NAME_MTAS_TERMVECTOR_FIELD,
+ field);
+ paramsNewRequest.add(PARAM_MTAS_TERMVECTOR + "."
+ + termvectorCounter + "." + NAME_MTAS_TERMVECTOR_PREFIX,
+ tv.prefix);
+ paramsNewRequest.add(PARAM_MTAS_TERMVECTOR + "."
+ + termvectorCounter + "." + NAME_MTAS_TERMVECTOR_KEY,
+ tv.key);
+ if (tv.subComponentFunction.type != null) {
+ paramsNewRequest.add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter + "."
+ + NAME_MTAS_TERMVECTOR_TYPE,
+ tv.subComponentFunction.type);
+ }
+ if (tv.functions != null) {
+ int functionCounter = 0;
+ for (SubComponentFunction function : tv.functions) {
+ paramsNewRequest.add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION + "."
+ + functionCounter + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION_EXPRESSION,
+ function.expression);
+ paramsNewRequest
+ .add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter
+ + "." + NAME_MTAS_TERMVECTOR_FUNCTION + "."
+ + functionCounter + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION_KEY,
+ function.key);
+ if (function.type != null) {
+ paramsNewRequest.add(
+ PARAM_MTAS_TERMVECTOR + "." + termvectorCounter + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION + "."
+ + functionCounter + "."
+ + NAME_MTAS_TERMVECTOR_FUNCTION_TYPE,
+ function.type);
+ }
+ functionCounter++;
+ }
+ }
+ if (tv.regexp != null) {
+ paramsNewRequest.add(PARAM_MTAS_TERMVECTOR + "."
+ + termvectorCounter + "." + NAME_MTAS_TERMVECTOR_REGEXP,
+ tv.regexp);
+ }
+ // TESTITEM
+ // paramsNewRequest.add(PARAM_MTAS_TERMVECTOR + "."
+ // + termvectorCounter + "." + NAME_MTAS_TERMVECTOR_LIST,
+ // "de");
+ for (String listItem : list) {
+ paramsNewRequest.add(PARAM_MTAS_TERMVECTOR + "."
+ + termvectorCounter + "." + NAME_MTAS_TERMVECTOR_LIST,
+ listItem);
+ }
+ termvectorCounter++;
+ }
+ }
+ }
+
+ if (termvectorCounter > 0) {
+ ShardRequest nsreq = new ShardRequest();
+ nsreq.shards = new String[] { shardName };
+ nsreq.purpose = ShardRequest.PURPOSE_PRIVATE;
+ nsreq.params = new ModifiableSolrParams();
+ nsreq.params.add("fq", rb.req.getParams().getParams("fq"));
+ nsreq.params.add("q", rb.req.getParams().getParams("q"));
+ nsreq.params.add("cache", rb.req.getParams().getParams("cache"));
+ nsreq.params.add("rows", "0");
+ nsreq.params.add(MtasSolrSearchComponent.PARAM_MTAS,
+ rb.req.getOriginalParams()
+ .getParams(MtasSolrSearchComponent.PARAM_MTAS));
+ nsreq.params.add(PARAM_MTAS_TERMVECTOR,
+ rb.req.getOriginalParams().getParams(PARAM_MTAS_TERMVECTOR));
+ nsreq.params.add(paramsNewRequest);
+ rb.addRequest(searchComponent, nsreq);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Compute missing termvector items per shard.
+ *
+ * @param requests
+ * the requests
+ * @param args
+ * the args
+ * @return the hash map
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ private HashMap<String, HashMap<String, HashSet<String>>> computeMissingTermvectorItemsPerShard(
+ List<ShardRequest> requests, String... args) throws IOException {
+ HashMap<String, HashMap<String, HashSet<String>>> result = new HashMap<String, HashMap<String, HashSet<String>>>();
+ HashMap<String, HashMap<String, HashSet<String>>> itemsPerShardSets = new HashMap<String, HashMap<String, HashSet<String>>>();
+ HashMap<String, HashSet<String>> itemSets = new HashMap<String, HashSet<String>>();
+ // loop over responses different shards
+ for (ShardRequest sreq : requests) {
+ if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
+ && sreq.params.getBool(PARAM_MTAS_TERMVECTOR, false)) {
+ for (ShardResponse shardResponse : sreq.responses) {
+ NamedList<Object> response = shardResponse.getSolrResponse()
+ .getResponse();
+ try {
+ ArrayList<NamedList<Object>> data = (ArrayList<NamedList<Object>>) response
+ .findRecursive(args);
+ if (data != null) {
+ for (int i = 0; i < data.size(); i++) {
+ NamedList<Object> dataItem = data.get(i);
+ try {
+ String termvectorKey = (String) dataItem.get("key");
+ MtasSolrResult list = (MtasSolrResult) dataItem.get("list");
+ if (termvectorKey != null && list != null) {
+ Set<String> keyList = list.getKeyList();
+ HashMap<String, HashSet<String>> itemsPerShardSet;
+ HashSet<String> itemSet, tmpItemSet = new HashSet<String>();
+ if (itemsPerShardSets.containsKey(termvectorKey)) {
+ itemsPerShardSet = itemsPerShardSets.get(termvectorKey);
+ itemSet = itemSets.get(termvectorKey);
+ } else {
+ itemsPerShardSet = new HashMap<String, HashSet<String>>();
+ itemSet = new HashSet<String>();
+ itemsPerShardSets.put(termvectorKey, itemsPerShardSet);
+ itemSets.put(termvectorKey, itemSet);
+ }
+ itemsPerShardSet.put(shardResponse.getShardAddress(),
+ tmpItemSet);
+ tmpItemSet.addAll(keyList);
+ itemSet.addAll(keyList);
+ }
+ } catch (ClassCastException e) {
+
+ }
+ }
+
+ }
+ } catch (ClassCastException e) {
+ }
+ }
+ }
+ }
+
+ // construct result
+ for (String key : itemSets.keySet()) {
+ if (itemsPerShardSets.containsKey(key)) {
+ HashMap<String, HashSet<String>> itemsPerShardSet = itemsPerShardSets
+ .get(key);
+ for (String shardAddress : itemsPerShardSet.keySet()) {
+ HashMap<String, HashSet<String>> tmpShardKeySet;
+ if (result.containsKey(shardAddress)) {
+ tmpShardKeySet = result.get(shardAddress);
+ } else {
+ tmpShardKeySet = new HashMap<String, HashSet<String>>();
+ result.put(shardAddress, tmpShardKeySet);
+ }
+ HashSet<String> tmpResult = new HashSet<String>();
+ tmpShardKeySet.put(key, tmpResult);
+ HashSet<String> itemsSet = itemsPerShardSet.get(shardAddress);
+ for (String item : itemSets.get(key)) {
+ if (!itemsSet.contains(item)) {
+ tmpResult.add(item);
+ }
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+}
diff --git a/src/mtas/solr/handler/component/util/MtasSolrResult.java b/src/mtas/solr/handler/component/util/MtasSolrResult.java
new file mode 100644
index 0000000..9f91965
--- /dev/null
+++ b/src/mtas/solr/handler/component/util/MtasSolrResult.java
@@ -0,0 +1,381 @@
+package mtas.solr.handler.component.util;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import mtas.codec.util.DataCollector;
+import mtas.codec.util.collector.MtasDataCollector;
+import mtas.codec.util.collector.MtasDataCollectorResult;
+import mtas.codec.util.collector.MtasDataItem;
+
+/**
+ * The Class MtasSolrResult.
+ */
+public class MtasSolrResult implements Serializable {
+
+ /** The Constant serialVersionUID. */
+ private static final long serialVersionUID = 1L;
+
+ /** The stats type. */
+ public String dataType, statsType;
+
+ /** The stats items. */
+ public TreeSet<String> statsItems;
+
+ /** The sort direction. */
+ public String sortType, sortDirection;
+
+ /** The number. */
+ public Integer start, number;
+
+ /** The data collector. */
+ public MtasDataCollector<?, ?> dataCollector = null;
+
+ /** The function data. */
+ public HashMap<MtasDataCollector<?, ?>, HashMap<String, MtasSolrResult>> functionData;
+
+ /** The sub stats type. */
+ private String[] subDataType, subStatsType;
+
+ /** The sub stats items. */
+ private TreeSet<String>[] subStatsItems;
+
+ /** The sub sort direction. */
+ private String[] subSortType, subSortDirection;
+
+ /** The sub number. */
+ private Integer[] subStart, subNumber;
+
+ /**
+ * Instantiates a new mtas solr result.
+ *
+ * @param dataCollector
+ * the data collector
+ * @param dataType
+ * the data type
+ * @param statsType
+ * the stats type
+ * @param statsItems
+ * the stats items
+ * @param sortType
+ * the sort type
+ * @param sortDirection
+ * the sort direction
+ * @param start
+ * the start
+ * @param number
+ * the number
+ * @param functionData
+ * the function data
+ */
+ @SuppressWarnings("unchecked")
+ public MtasSolrResult(MtasDataCollector<?, ?> dataCollector,
+ String[] dataType, String[] statsType, TreeSet<String>[] statsItems,
+ String[] sortType, String[] sortDirection, Integer[] start,
+ Integer[] number,
+ HashMap<MtasDataCollector<?, ?>, HashMap<String, MtasSolrResult>> functionData) {
+ this.dataCollector = dataCollector;
+ this.functionData = functionData;
+ this.dataType = (dataType == null) ? null : dataType[0];
+ this.statsType = (statsType == null) ? null : statsType[0];
+ this.statsItems = (statsItems == null) ? null : statsItems[0];
+ this.sortType = (sortType == null) ? null : sortType[0];
+ this.sortDirection = (sortDirection == null) ? null : sortDirection[0];
+ this.start = (start == null) ? null : start[0];
+ this.number = (number == null) ? null : number[0];
+ if ((dataType != null) && (dataType.length > 1)) {
+ subDataType = new String[dataType.length - 1];
+ subStatsType = new String[dataType.length - 1];
+ subStatsItems = new TreeSet[dataType.length - 1];
+ subSortType = new String[dataType.length - 1];
+ subSortDirection = new String[dataType.length - 1];
+ System.arraycopy(dataType, 1, subDataType, 0, dataType.length - 1);
+ System.arraycopy(statsType, 1, subStatsType, 0, dataType.length - 1);
+ System.arraycopy(statsItems, 1, subStatsItems, 0, dataType.length - 1);
+ System.arraycopy(sortType, 1, subSortType, 0, dataType.length - 1);
+ System.arraycopy(sortDirection, 1, subSortDirection, 0,
+ dataType.length - 1);
+ } else {
+ subDataType = null;
+ subStatsType = null;
+ subStatsItems = null;
+ subSortType = null;
+ subSortDirection = null;
+ }
+ }
+
+ /**
+ * Instantiates a new mtas solr result.
+ *
+ * @param dataCollector
+ * the data collector
+ * @param dataType
+ * the data type
+ * @param statsType
+ * the stats type
+ * @param statsItems
+ * the stats items
+ * @param functionData
+ * the function data
+ */
+ @SuppressWarnings("unchecked")
+ public MtasSolrResult(MtasDataCollector<?, ?> dataCollector, String dataType,
+ String statsType, TreeSet<String> statsItems,
+ HashMap<MtasDataCollector<?, ?>, HashMap<String, MtasSolrResult>> functionData) {
+ this(dataCollector, new String[] { dataType }, new String[] { statsType },
+ new TreeSet[] { statsItems }, new String[] { null },
+ new String[] { null }, new Integer[] { 0 }, new Integer[] { 1 },
+ functionData);
+ }
+
+ /**
+ * Merge.
+ *
+ * @param newItem
+ * the new item
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ void merge(MtasSolrResult newItem) throws IOException {
+ HashMap<MtasDataCollector<?, ?>, MtasDataCollector<?, ?>> map = new HashMap<MtasDataCollector<?, ?>, MtasDataCollector<?, ?>>();
+ dataCollector.merge(newItem.dataCollector, map, true);
+ if (newItem.functionData != null) {
+ if (functionData == null) {
+ functionData = new HashMap<MtasDataCollector<?, ?>, HashMap<String, MtasSolrResult>>();
+ }
+ for (MtasDataCollector<?, ?> keyCollector : newItem.functionData
+ .keySet()) {
+ if (map.containsKey(keyCollector)) {
+ // compute mapped key
+ MtasDataCollector<?, ?> newKeyCollector = keyCollector;
+ while (map.containsKey(newKeyCollector)) {
+ newKeyCollector = map.get(keyCollector);
+ }
+ if (functionData.containsKey(newKeyCollector)) {
+ HashMap<String, MtasSolrResult> tmpList = functionData
+ .get(newKeyCollector);
+ for (String functionKey : newItem.functionData.get(keyCollector)
+ .keySet()) {
+ if (tmpList.containsKey(functionKey)) {
+ tmpList.get(functionKey).merge(
+ newItem.functionData.get(keyCollector).get(functionKey));
+ } else {
+ tmpList.put(functionKey,
+ newItem.functionData.get(keyCollector).get(functionKey));
+ }
+ }
+ } else {
+ functionData.put(newKeyCollector,
+ newItem.functionData.get(keyCollector));
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Gets the data.
+ *
+ * @param showDebugInfo
+ * the show debug info
+ * @return the data
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ NamedList<Object> getData(boolean showDebugInfo) throws IOException {
+ if (dataCollector.getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ NamedList<Object> mtasResponse = new SimpleOrderedMap<>();
+ // functions
+ Map<String, NamedList<Object>> functionList = new HashMap<String, NamedList<Object>>();
+ if (functionData != null && functionData.containsKey(dataCollector)) {
+ HashMap<String, MtasSolrResult> functionDataItem = functionData
+ .get(dataCollector);
+ for (String functionKey : functionDataItem.keySet()) {
+ MtasSolrResult functionResult = functionDataItem.get(functionKey);
+ if (functionResult.dataCollector.getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ NamedList<Object> functionData = functionResult
+ .getData(showDebugInfo);
+ functionList.put(functionKey, functionData);
+ } else {
+ throw new IOException("unexpected function collectorType "
+ + functionResult.dataCollector.getCollectorType());
+ }
+ }
+ }
+ // main result
+ MtasDataItem<?, ?> dataItem = dataCollector.getResult().getData();
+ if (dataItem != null) {
+ mtasResponse.addAll(dataItem.rewrite(showDebugInfo));
+ if (functionList.size() > 0) {
+ mtasResponse.add("functions", functionList);
+ }
+ }
+ if ((subDataType != null) && (dataItem.getSub() != null)) {
+ MtasSolrResult css = new MtasSolrResult(dataItem.getSub(), subDataType,
+ subStatsType, subStatsItems, subSortType, subSortDirection,
+ subStart, subNumber, functionData);
+ if (dataItem.getSub().getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ mtasResponse.add(dataItem.getSub().getCollectorType(),
+ css.getNamedList(showDebugInfo));
+ } else if (dataItem.getSub().getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ mtasResponse.add(dataItem.getSub().getCollectorType(),
+ css.getData(showDebugInfo));
+ }
+ }
+ return mtasResponse;
+ } else {
+ throw new IOException(
+ "only allowed for " + DataCollector.COLLECTOR_TYPE_DATA);
+ }
+ }
+
+ /**
+ * Gets the key list.
+ *
+ * @return the key list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public Set<String> getKeyList() throws IOException {
+ if (dataCollector.getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ return dataCollector.getResult().getComparatorList().keySet();
+ } else {
+ throw new IOException(
+ "only allowed for " + DataCollector.COLLECTOR_TYPE_LIST);
+ }
+ }
+
+ /**
+ * Gets the full key list.
+ *
+ * @return the full key list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public Set<String> getFullKeyList() throws IOException {
+ if (dataCollector.getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ return dataCollector.getKeyList();
+ } else {
+ throw new IOException(
+ "only allowed for " + DataCollector.COLLECTOR_TYPE_LIST);
+ }
+ }
+
+ /**
+ * Gets the named list.
+ *
+ * @param showDebugInfo
+ * the show debug info
+ * @return the named list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ NamedList<Object> getNamedList(boolean showDebugInfo) throws IOException {
+ if (dataCollector.getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ SimpleOrderedMap<Object> mtasResponseList = new SimpleOrderedMap<>();
+ // functions
+ Map<String, SimpleOrderedMap<Object>> functionList = new HashMap<String, SimpleOrderedMap<Object>>();
+ if (functionData != null && functionData.containsKey(dataCollector)) {
+ HashMap<String, MtasSolrResult> functionDataItem = functionData
+ .get(dataCollector);
+ for (String functionKey : functionDataItem.keySet()) {
+ MtasSolrResult functionResult = functionDataItem.get(functionKey);
+ if (functionResult.dataCollector.getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ NamedList<Object> functionNamedList = functionResult
+ .getNamedList(showDebugInfo);
+ for (int i = 0; i < functionNamedList.size(); i++) {
+ if (functionList.containsKey(functionNamedList.getName(i))) {
+ SimpleOrderedMap<Object> tmpMap = functionList
+ .get(functionNamedList.getName(i));
+ tmpMap.add(functionKey, functionNamedList.getVal(i));
+ } else {
+ SimpleOrderedMap<Object> tmpMap = new SimpleOrderedMap<>();
+ tmpMap.add(functionKey, functionNamedList.getVal(i));
+ functionList.put(functionNamedList.getName(i), tmpMap);
+ }
+ }
+ } else {
+ throw new IOException("unexpected function collectorType "
+ + functionResult.dataCollector.getCollectorType());
+ }
+ }
+ }
+ // main result
+ Map<String, ?> dataList = dataCollector.getResult().getList();
+ for (Entry<String, ?> entry : dataList.entrySet()) {
+ SimpleOrderedMap<Object> mtasResponseListItem = new SimpleOrderedMap<>();
+ MtasDataItem<?, ?> dataItem = (MtasDataItem<?, ?>) entry.getValue();
+ mtasResponseListItem.addAll(dataItem.rewrite(showDebugInfo));
+ if (functionList.containsKey(entry.getKey())) {
+ mtasResponseListItem.add("functions",
+ functionList.get(entry.getKey()));
+ }
+ if ((subDataType != null) && (dataItem.getSub() != null)) {
+ MtasSolrResult css = new MtasSolrResult(dataItem.getSub(),
+ subDataType, subStatsType, subStatsItems, subSortType,
+ subSortDirection, subStart, subNumber, functionData);
+ if (dataItem.getSub().getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ mtasResponseListItem.add(dataItem.getSub().getCollectorType(),
+ css.getNamedList(showDebugInfo));
+ } else if (dataItem.getSub().getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ mtasResponseListItem.add(dataItem.getSub().getCollectorType(),
+ css.getData(showDebugInfo));
+ }
+ }
+ mtasResponseList.add(entry.getKey(), mtasResponseListItem);
+ }
+ return mtasResponseList;
+ } else {
+ throw new IOException(
+ "only allowed for " + DataCollector.COLLECTOR_TYPE_LIST);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ if (dataCollector.getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ return "MtasSolrResult(data-" + hashCode() + ")";
+ }
+ if (dataCollector.getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ return "MtasSolrResult(list(" + dataCollector.getSize() + ")-"
+ + hashCode() + ")";
+ } else {
+ return "MtasSolrResult: unknown";
+ }
+ }
+
+ /**
+ * Gets the result.
+ *
+ * @return the result
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public MtasDataCollectorResult<?, ?> getResult() throws IOException {
+ return dataCollector.getResult();
+ }
+
+}
diff --git a/src/mtas/solr/handler/component/util/MtasSolrResultMerge.java b/src/mtas/solr/handler/component/util/MtasSolrResultMerge.java
new file mode 100644
index 0000000..02c985e
--- /dev/null
+++ b/src/mtas/solr/handler/component/util/MtasSolrResultMerge.java
@@ -0,0 +1,366 @@
+package mtas.solr.handler.component.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.TreeSet;
+import java.util.Map.Entry;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.handler.component.ShardResponse;
+import mtas.solr.handler.component.MtasSolrSearchComponent;
+
+/**
+ * The Class MtasMergeStrategy.
+ */
+public class MtasSolrResultMerge {
+
+ @SuppressWarnings("unchecked")
+ public void merge(ResponseBuilder rb) {
+ if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
+ // mtas response
+ NamedList<Object> mtasResponse = null;
+ try {
+ mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
+ } catch (ClassCastException e) {
+ mtasResponse = null;
+ }
+ if (mtasResponse == null) {
+ mtasResponse = new SimpleOrderedMap<>();
+ rb.rsp.add("mtas", mtasResponse);
+ }
+ for (ShardRequest sreq : rb.finished) {
+ if (rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY) {
+ // merge stats
+ if (rb.req.getParams()
+ .getBool(MtasSolrComponentStats.PARAM_MTAS_STATS, false)) {
+ mergeNamedList(sreq, mtasResponse, "stats", null);
+ }
+ // merge group
+ if (rb.req.getParams()
+ .getBool(MtasSolrComponentGroup.PARAM_MTAS_GROUP, false)) {
+ mergeArrayList(sreq, mtasResponse, "group", null, false);
+ }
+ // merge facet
+ if (rb.req.getParams()
+ .getBool(MtasSolrComponentFacet.PARAM_MTAS_FACET, false)) {
+ mergeArrayList(sreq, mtasResponse, "facet", null, false);
+ }
+ // merge prefix
+ if (rb.req.getParams()
+ .getBool(MtasSolrComponentPrefix.PARAM_MTAS_PREFIX, false)) {
+ mergeArrayList(sreq, mtasResponse, "prefix", null, false);
+ }
+ } else if (rb.stage == MtasSolrSearchComponent.STAGE_TERMVECTOR_MISSING_KEY) {
+ // merge termvector
+ if (rb.req.getParams().getBool(
+ MtasSolrComponentTermvector.PARAM_MTAS_TERMVECTOR, false)) {
+ mergeArrayList(sreq, mtasResponse, "termvector", null, false);
+ }
+ } else if (rb.stage == MtasSolrSearchComponent.STAGE_LIST) {
+ // merge list
+ if (rb.req.getParams().getBool(MtasSolrComponentList.PARAM_MTAS_LIST,
+ false)) {
+ mergeArrayList(sreq, mtasResponse, "list",
+ ShardRequest.PURPOSE_PRIVATE, true);
+ }
+ } else if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
+ // merge kwic
+ if (rb.req.getParams().getBool(MtasSolrComponentKwic.PARAM_MTAS_KWIC,
+ false)) {
+ mergeArrayList(sreq, mtasResponse, "kwic",
+ ShardRequest.PURPOSE_PRIVATE, true);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Merge named list.
+ *
+ * @param sreq
+ * the sreq
+ * @param mtasResponse
+ * the mtas response
+ * @param key
+ * the key
+ * @param preferredPurpose
+ * the preferred purpose
+ */
+ @SuppressWarnings("unchecked")
+ private void mergeNamedList(ShardRequest sreq, NamedList<Object> mtasResponse,
+ String key, Integer preferredPurpose) {
+ // create new response for key
+ NamedList<Object> mtasListResponse;
+ Object o = mtasResponse.get(key);
+ if (o instanceof NamedList) {
+ mtasListResponse = (NamedList<Object>) o;
+ } else {
+ mtasListResponse = new SimpleOrderedMap<>();
+ mtasResponse.removeAll(key);
+ mtasResponse.add(key, mtasListResponse);
+ }
+ // collect responses for each shard
+ HashMap<String, NamedList<Object>> mtasListShardResponses = new HashMap<String, NamedList<Object>>();
+ for (ShardResponse response : sreq.responses) {
+ // only continue if new shard or preferred purpose
+ if (mtasListShardResponses.containsKey(response.getShard())
+ && ((preferredPurpose == null)
+ || (sreq.purpose != preferredPurpose))) {
+ break;
+ }
+ // update
+ try {
+ NamedList<Object> result = response.getSolrResponse().getResponse();
+ NamedList<Object> data = (NamedList<Object>) result
+ .findRecursive("mtas", key);
+ if (data != null) {
+ mtasListShardResponses.put(response.getShard(),
+ MtasSolrResultUtil.decode(data));
+ }
+ } catch (ClassCastException e) {
+
+ }
+ }
+ try {
+ for (NamedList<Object> mtasListShardResponse : mtasListShardResponses
+ .values()) {
+ mergeResponsesNamedList(mtasListResponse, mtasListShardResponse);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Merge array list.
+ *
+ * @param sreq
+ * the sreq
+ * @param mtasResponse
+ * the mtas response
+ * @param key
+ * the key
+ * @param preferredPurpose
+ * the preferred purpose
+ * @param mergeAllShardResponses
+ * the merge all shard responses
+ */
+ @SuppressWarnings("unchecked")
+ private void mergeArrayList(ShardRequest sreq, NamedList<Object> mtasResponse,
+ String key, Integer preferredPurpose, boolean mergeAllShardResponses) {
+ // create new response for key
+ ArrayList<Object> mtasListResponse;
+ Object o = mtasResponse.get(key);
+ if (o instanceof ArrayList) {
+ mtasListResponse = (ArrayList<Object>) o;
+ } else {
+ mtasListResponse = new ArrayList<Object>();
+ mtasResponse.removeAll(key);
+ mtasResponse.add(key, mtasListResponse);
+ }
+ // collect responses for each shard
+ HashMap<String, ArrayList<Object>> mtasListShardResponses = new HashMap<String, ArrayList<Object>>();
+ ArrayList<ArrayList<Object>> mtasListShardResponsesExtra = new ArrayList<ArrayList<Object>>();
+ for (ShardResponse response : sreq.responses) {
+ // only continue if new shard or preferred purpose
+ if (mtasListShardResponses.containsKey(response.getShard())
+ && ((preferredPurpose == null)
+ || (sreq.purpose != preferredPurpose))) {
+ break;
+ }
+ // update
+ try {
+ NamedList<Object> result = response.getSolrResponse().getResponse();
+ ArrayList<Object> data = (ArrayList<Object>) result
+ .findRecursive("mtas", key);
+ if (data != null) {
+ if (mtasListShardResponses.containsKey(response.getShardAddress())) {
+ if (mergeAllShardResponses) {
+ mtasListShardResponsesExtra.add(data);
+ }
+ } else {
+ mtasListShardResponses.put(response.getShardAddress(), data);
+ }
+ }
+ } catch (ClassCastException e) {
+ e.printStackTrace();
+ }
+ }
+
+ try {
+ for (ArrayList<Object> mtasListShardResponse : mtasListShardResponses
+ .values()) {
+ mergeResponsesArrayList(mtasListResponse, mtasListShardResponse);
+ }
+ for (ArrayList<Object> mtasListShardResponse : mtasListShardResponsesExtra) {
+ mergeResponsesArrayList(mtasListResponse, mtasListShardResponse);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Merge responses tree set.
+ *
+ * @param originalList
+ * the original list
+ * @param shardList
+ * the shard list
+ */
+ private void mergeResponsesTreeSet(TreeSet<Object> originalList,
+ TreeSet<Object> shardList) {
+ for (Object item : shardList) {
+ originalList.add(item);
+ }
+ }
+
+ /**
+ * Merge responses array list.
+ *
+ * @param originalList
+ * the original list
+ * @param shardList
+ * the shard list
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ private void mergeResponsesArrayList(ArrayList<Object> originalList,
+ ArrayList<Object> shardList) throws IOException {
+ // get keys from original
+ HashMap<String, Object> originalKeyList = new HashMap<String, Object>();
+ for (Object item : originalList) {
+ if (item instanceof NamedList<?>) {
+ NamedList<Object> itemList = (NamedList<Object>) item;
+ Object key = itemList.get("key");
+ if ((key != null) && (key instanceof String)) {
+ originalKeyList.put((String) key, item);
+ }
+ }
+ }
+ for (Object item : shardList) {
+ if (item instanceof NamedList<?>) {
+ NamedList<Object> itemList = (NamedList<Object>) item;
+ Object key = itemList.get("key");
+ // item with key
+ if ((key != null) && (key instanceof String)) {
+ // merge
+ if (originalKeyList.containsKey(key)) {
+ Object originalItem = originalKeyList.get(key);
+ if (originalItem.getClass().equals(item.getClass())) {
+ mergeResponsesNamedList((NamedList<Object>) originalItem,
+ (NamedList<Object>) item);
+ } else {
+ // ignore?
+ }
+ // add
+ } else {
+ originalList.add(adjustablePartsCloned(item));
+ }
+ } else {
+ originalList.add(item);
+ }
+ } else {
+ originalList.add(item);
+ }
+ }
+ }
+
+ /**
+ * Merge responses named list.
+ *
+ * @param mainResponse
+ * the main response
+ * @param shardResponse
+ * the shard response
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private void mergeResponsesNamedList(NamedList<Object> mainResponse,
+ NamedList<Object> shardResponse) throws IOException {
+ Iterator<Entry<String, Object>> it = shardResponse.iterator();
+ while (it.hasNext()) {
+ Entry<String, Object> entry = it.next();
+ String name = entry.getKey();
+ Object shardValue = entry.getValue();
+ int originalId = mainResponse.indexOf(name, 0);
+ if (originalId < 0) {
+ mainResponse.add(name, adjustablePartsCloned(shardValue));
+ } else {
+ Object original = mainResponse.getVal(originalId);
+ if (original == null) {
+ original = adjustablePartsCloned(shardValue);
+ } else if (shardValue != null
+ && original.getClass().equals(shardValue.getClass())) {
+ // merge ArrayList
+ if (original instanceof ArrayList) {
+ ArrayList originalList = (ArrayList) original;
+ ArrayList shardList = (ArrayList) shardValue;
+ mergeResponsesArrayList(originalList, shardList);
+ // merge Namedlist
+ } else if (original instanceof NamedList<?>) {
+ mergeResponsesNamedList((NamedList<Object>) original,
+ (NamedList<Object>) shardValue);
+ // merge TreeSet
+ } else if (original instanceof TreeSet<?>) {
+ mergeResponsesTreeSet((TreeSet<Object>) original,
+ (TreeSet<Object>) shardValue);
+ } else if (original instanceof MtasSolrResult) {
+ MtasSolrResult originalComponentResult = (MtasSolrResult) original;
+ originalComponentResult.merge((MtasSolrResult) shardValue);
+ } else if (original instanceof String) {
+ // ignore?
+ } else if (original instanceof Integer) {
+ original = (Integer) original + ((Integer) shardValue);
+ } else if (original instanceof Long) {
+ original = (Long) original + ((Long) shardValue);
+ } else {
+ // ignore?
+ }
+ mainResponse.setVal(originalId, original);
+ } else {
+ // ignore?
+ }
+ }
+ }
+ }
+
+ /**
+ * Adjustable parts cloned.
+ *
+ * @param original
+ * the original
+ * @return the object
+ */
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private Object adjustablePartsCloned(Object original) {
+ if (original instanceof NamedList) {
+ NamedList<Object> newObject = new SimpleOrderedMap();
+ NamedList<Object> originalObject = (NamedList<Object>) original;
+ for (int i = 0; i < originalObject.size(); i++) {
+ newObject.add(originalObject.getName(i),
+ adjustablePartsCloned(originalObject.getVal(i)));
+ }
+ return newObject;
+ } else if (original instanceof ArrayList) {
+ ArrayList<Object> newObject = new ArrayList<Object>();
+ ArrayList<Object> originalObject = (ArrayList<Object>) original;
+ for (int i = 0; i < originalObject.size(); i++) {
+ newObject.add(adjustablePartsCloned(originalObject.get(i)));
+ }
+ return newObject;
+ } else if (original instanceof Integer) {
+ Integer originalObject = (Integer) original;
+ return new Integer(originalObject.intValue());
+ }
+ return original;
+ }
+
+}
diff --git a/src/mtas/solr/handler/component/util/MtasSolrResultUtil.java b/src/mtas/solr/handler/component/util/MtasSolrResultUtil.java
new file mode 100644
index 0000000..4eb4030
--- /dev/null
+++ b/src/mtas/solr/handler/component/util/MtasSolrResultUtil.java
@@ -0,0 +1,399 @@
+package mtas.solr.handler.component.util;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Base64;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+
+import mtas.codec.util.DataCollector;
+import mtas.codec.util.collector.MtasDataItem;
+import mtas.parser.cql.MtasCQLParser;
+import mtas.parser.cql.TokenMgrError;
+
+/**
+ * The Class MtasSolrResultUtil.
+ */
+public class MtasSolrResultUtil {
+
+ /** The Constant QUERY_TYPE_CQL. */
+ public static final String QUERY_TYPE_CQL = "cql";
+
+ /**
+ * Rewrite.
+ *
+ * @param al
+ * the al
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ public static void rewrite(ArrayList<?> al) throws IOException {
+ for (int i = 0; i < al.size(); i++) {
+ if (al.get(i) instanceof NamedList) {
+ rewrite((NamedList) al.get(i));
+ } else if (al.get(i) instanceof ArrayList) {
+ rewrite((ArrayList) al.get(i));
+ }
+ }
+ }
+
+ /**
+ * Rewrite.
+ *
+ * @param nl
+ * the nl
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public static void rewrite(NamedList<Object> nl) throws IOException {
+ rewrite(nl, true);
+ }
+
+ /**
+ * Rewrite.
+ *
+ * @param nl
+ * the nl
+ * @param doCollapse
+ * the do collapse
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private static void rewrite(NamedList<Object> nl, boolean doCollapse)
+ throws IOException {
+ boolean showDebugInfo = false;
+ HashMap<String, NamedList<Object>> collapseNamedList = null;
+ int length = nl.size();
+ for (int i = 0; i < length; i++) {
+ if (nl.getVal(i) instanceof NamedList) {
+ NamedList o = (NamedList) nl.getVal(i);
+ rewrite(o, true);
+ nl.setVal(i, o);
+ } else if (nl.getVal(i) instanceof ArrayList) {
+ ArrayList o = (ArrayList) nl.getVal(i);
+ rewrite(o);
+ nl.setVal(i, o);
+ } else if (nl.getVal(i) instanceof MtasDataItem) {
+ MtasDataItem dataItem = (MtasDataItem) nl.getVal(i);
+ nl.setVal(i, dataItem.rewrite(showDebugInfo));
+ } else if (nl.getVal(i) instanceof MtasSolrResult) {
+ MtasSolrResult o = (MtasSolrResult) nl.getVal(i);
+ if (o.dataCollector.getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
+ NamedList<Object> nnl = o.getNamedList(showDebugInfo);
+ for (int j = 0; j < nnl.size(); j++) {
+ if (nnl.getVal(j) != null
+ && nnl.getVal(j) instanceof MtasDataItem) {
+ MtasDataItem mdi = (MtasDataItem) nnl.getVal(j);
+ mdi.rewrite(showDebugInfo);
+ nnl.setVal(j, mdi);
+ }
+ }
+ nl.setVal(i, nnl);
+ } else if (o.dataCollector.getCollectorType()
+ .equals(DataCollector.COLLECTOR_TYPE_DATA)) {
+ NamedList<Object> nnl = o.getData(showDebugInfo);
+ if (nnl.size() > 0) {
+ rewrite(nnl);
+ collapseNamedList = new HashMap<String, NamedList<Object>>();
+ collapseNamedList.put(nl.getName(i), nnl);
+ nl.setVal(i, nnl);
+ } else {
+ nl.setVal(i, null);
+ }
+ }
+ }
+ }
+ // collapse
+ if (doCollapse && collapseNamedList != null) {
+ for (String key : collapseNamedList.keySet()) {
+ nl.remove(key);
+ }
+ for (NamedList<Object> items : collapseNamedList.values()) {
+ nl.addAll(items);
+ }
+ }
+ }
+
+ /**
+ * Rewrite merge list.
+ *
+ * @param key
+ * the key
+ * @param subKey
+ * the sub key
+ * @param snl
+ * the snl
+ * @param tnl
+ * the tnl
+ */
+ @SuppressWarnings({ "unchecked", "unused" })
+ private static void rewriteMergeList(String key, String subKey,
+ NamedList<Object> snl, NamedList<Object> tnl) {
+ for (int i = 0; i < tnl.size(); i++) {
+ Object item = snl.get(tnl.getName(i));
+ if (item != null && tnl.getVal(i) instanceof NamedList) {
+ NamedList<Object> tnnl = (NamedList<Object>) tnl.getVal(i);
+ Object o = tnnl.get(key);
+ NamedList<Object> tnnnl;
+ if (o != null && o instanceof NamedList) {
+ tnnnl = (NamedList<Object>) o;
+ } else {
+ tnnnl = new SimpleOrderedMap<>();
+ tnnl.add(key, tnnnl);
+ }
+ tnnnl.add(subKey, item);
+ }
+ }
+ }
+
+ /**
+ * Rewrite merge data.
+ *
+ * @param key
+ * the key
+ * @param subKey
+ * the sub key
+ * @param snl
+ * the snl
+ * @param tnl
+ * the tnl
+ */
+ @SuppressWarnings({ "unused", "unchecked" })
+ private static void rewriteMergeData(String key, String subKey,
+ NamedList<Object> snl, NamedList<Object> tnl) {
+ if (snl != null) {
+ Object o = tnl.get(key);
+ NamedList<Object> tnnnl;
+ if (o != null && o instanceof NamedList) {
+ tnnnl = (NamedList<Object>) o;
+ } else {
+ tnnnl = new SimpleOrderedMap<>();
+ tnl.add(key, tnnnl);
+ }
+ tnnnl.add(subKey, snl);
+ }
+ }
+
+ /**
+ * Encode.
+ *
+ * @param o
+ * the o
+ * @return the string
+ */
+ public static String encode(Object o) {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ ObjectOutputStream objectOutputStream;
+ try {
+ objectOutputStream = new ObjectOutputStream(byteArrayOutputStream);
+ objectOutputStream.writeObject(o);
+ objectOutputStream.close();
+ return Base64.byteArrayToBase64(byteArrayOutputStream.toByteArray());
+ } catch (IOException e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ /**
+ * Decode.
+ *
+ * @param s
+ * the s
+ * @return the object
+ */
+ static Object decode(String s) {
+ byte[] bytes = Base64.base64ToByteArray(s);
+ ObjectInputStream objectInputStream;
+ try {
+ objectInputStream = new ObjectInputStream(
+ new ByteArrayInputStream(bytes));
+ return objectInputStream.readObject();
+ } catch (IOException | ClassNotFoundException e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ /**
+ * Decode.
+ *
+ * @param l
+ * the l
+ * @return the array list
+ */
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ static ArrayList decode(ArrayList l) {
+ for (int i = 0; i < l.size(); i++) {
+ if (l.get(i) instanceof NamedList) {
+ l.set(i, decode((NamedList) l.get(i)));
+ } else if (l.get(i) instanceof ArrayList) {
+ l.set(i, decode((ArrayList) l.get(i)));
+ }
+ }
+ return l;
+ }
+
+ /**
+ * Decode.
+ *
+ * @param nl
+ * the nl
+ * @return the named list
+ */
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ static NamedList<Object> decode(NamedList<Object> nl) {
+ for (int i = 0; i < nl.size(); i++) {
+ String key = nl.getName(i);
+ Object o = nl.getVal(i);
+ if (key.matches("^_encoded_.*$")) {
+ if (o instanceof String) {
+ Object decodedObject = decode((String) nl.getVal(i));
+ String decodedKey = key.replaceFirst("^_encoded_", "");
+ if (decodedKey.equals("")) {
+ decodedKey = "_" + decodedObject.getClass().getSimpleName() + "_";
+ }
+ nl.setName(i, decodedKey);
+ nl.setVal(i, decodedObject);
+ } else if (o instanceof NamedList) {
+ NamedList nl2 = (NamedList) o;
+ for (int j = 0; j < nl2.size(); j++) {
+ if (nl2.getVal(j) instanceof String) {
+ nl2.setVal(j, decode((String) nl2.getVal(j)));
+ }
+ }
+ } else {
+ // System.out.println("unknown type " +
+ // o.getClass().getCanonicalName());
+ }
+ } else {
+ if (o instanceof NamedList) {
+ nl.setVal(i, decode((NamedList<Object>) o));
+ } else if (o instanceof ArrayList) {
+ nl.setVal(i, decode((ArrayList<Object>) o));
+ }
+ }
+ }
+ return nl;
+ }
+
+ /**
+ * Gets the ids from parameters.
+ *
+ * @param params
+ * the params
+ * @param prefix
+ * the prefix
+ * @return the ids from parameters
+ */
+ public static SortedSet<String> getIdsFromParameters(SolrParams params,
+ String prefix) {
+ SortedSet<String> ids = new TreeSet<String>();
+ Iterator<String> it = params.getParameterNamesIterator();
+ Pattern pattern = Pattern
+ .compile("^" + Pattern.quote(prefix) + "\\.([^\\.]+)(\\..*|$)");
+ while (it.hasNext()) {
+ String item = it.next();
+ Matcher m = pattern.matcher(item);
+ if (m.matches()) {
+ ids.add(m.group(1));
+ }
+ }
+ return ids;
+ }
+
+ /**
+ * Compare and check.
+ *
+ * @param list
+ * the list
+ * @param original
+ * the original
+ * @param nameNew
+ * the name new
+ * @param nameOriginal
+ * the name original
+ * @param unique
+ * the unique
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public static void compareAndCheck(String[] list, String[] original,
+ String nameNew, String nameOriginal, Boolean unique) throws IOException {
+ if (list != null) {
+ if (list.length != original.length) {
+ throw new IOException(
+ "unequal size " + nameNew + " and " + nameOriginal);
+ }
+ if (unique) {
+ Set<String> set = new HashSet<String>();
+ for (int i = 0; i < list.length; i++) {
+ set.add(list[i]);
+ }
+ if (set.size() < list.length) {
+ throw new IOException("duplicate " + nameNew);
+ }
+ }
+ }
+ }
+
+ /**
+ * Construct query.
+ *
+ * @param queryValue
+ * the query value
+ * @param queryType
+ * the query type
+ * @param field
+ * the field
+ * @return the span query
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public static SpanQuery constructQuery(String queryValue, String queryType,
+ String field) throws IOException {
+ if (queryType == null || queryType.isEmpty()) {
+ throw new IOException("no (valid) type for query " + queryValue);
+ } else if (queryValue == null || queryValue.isEmpty()) {
+ throw new IOException("no (valid) value for " + queryType + " query");
+ }
+ Reader reader = new BufferedReader(new StringReader(queryValue));
+ if (queryType.equals(QUERY_TYPE_CQL)) {
+ MtasCQLParser p = new MtasCQLParser(reader);
+ try {
+ return p.parse(field, null);
+ } catch (mtas.parser.cql.ParseException e) {
+ throw new IOException("couldn't parse " + queryType + " query "
+ + queryValue + " (" + e.getMessage() + ")");
+ } catch (TokenMgrError e) {
+ throw new IOException("couldn't parse " + queryType + " query "
+ + queryValue + " (" + e.getMessage() + ")");
+ }
+ } else {
+ throw new IOException(
+ "unknown queryType " + queryType + " for query " + queryValue);
+ }
+ }
+
+}
diff --git a/src/mtas/solr/schema/MtasPreAnalyzedField.java b/src/mtas/solr/schema/MtasPreAnalyzedField.java
index 7eaa888..fd7e57a 100644
--- a/src/mtas/solr/schema/MtasPreAnalyzedField.java
+++ b/src/mtas/solr/schema/MtasPreAnalyzedField.java
@@ -12,57 +12,63 @@ public class MtasPreAnalyzedField extends PreAnalyzedField {
/** The follow index analyzer. */
private static String FOLLOW_INDEX_ANALYZER = "followIndexAnalyzer";
-
+
/** The default configuration. */
private static String DEFAULT_CONFIGURATION = "defaultConfiguration";
-
+
/** The configuration from field. */
private static String CONFIGURATION_FROM_FIELD = "configurationFromField";
-
+
/** The set number of tokens. */
private static String SET_NUMBER_OF_TOKENS = "setNumberOfTokens";
-
+
/** The set number of positions. */
private static String SET_NUMBER_OF_POSITIONS = "setNumberOfPositions";
-
+
/** The set size. */
private static String SET_SIZE = "setSize";
-
+
/** The set error. */
private static String SET_ERROR = "setError";
-
+
/** The follow index analyzer. */
public String followIndexAnalyzer = null;
-
+
/** The default configuration. */
public String defaultConfiguration = null;
-
+
/** The configuration from field. */
public String configurationFromField = null;
-
+
/** The set number of tokens. */
public String setNumberOfTokens = null;
-
+
/** The set number of positions. */
public String setNumberOfPositions = null;
-
+
/** The set size. */
public String setSize = null;
-
+
/** The set error. */
public String setError = null;
-
- /* (non-Javadoc)
- * @see org.apache.solr.schema.PreAnalyzedField#init(org.apache.solr.schema.IndexSchema, java.util.Map)
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.solr.schema.PreAnalyzedField#init(org.apache.solr.schema.
+ * IndexSchema, java.util.Map)
*/
@Override
- public void init(IndexSchema schema, Map<String, String> args) {
+ public void init(IndexSchema schema, Map<String, String> args) {
args.put(PARSER_IMPL, MtasPreAnalyzedParser.class.getName());
super.init(schema, args);
}
- /* (non-Javadoc)
- * @see org.apache.solr.schema.FieldType#setArgs(org.apache.solr.schema.IndexSchema, java.util.Map)
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.solr.schema.FieldType#setArgs(org.apache.solr.schema.
+ * IndexSchema, java.util.Map)
*/
@Override
protected void setArgs(IndexSchema schema, Map<String, String> args) {
@@ -73,9 +79,10 @@ public class MtasPreAnalyzedField extends PreAnalyzedField {
setNumberOfPositions = args.get(SET_NUMBER_OF_POSITIONS);
setSize = args.get(SET_SIZE);
setError = args.get(SET_ERROR);
- if(followIndexAnalyzer==null) {
- throw new RuntimeException("No " + FOLLOW_INDEX_ANALYZER + " for fieldType "+this.getTypeName());
- }
+ if (followIndexAnalyzer == null) {
+ throw new RuntimeException("No " + FOLLOW_INDEX_ANALYZER
+ + " for fieldType " + this.getTypeName());
+ }
args.remove(FOLLOW_INDEX_ANALYZER);
args.remove(DEFAULT_CONFIGURATION);
args.remove(CONFIGURATION_FROM_FIELD);
diff --git a/src/mtas/solr/schema/MtasPreAnalyzedParser.java b/src/mtas/solr/schema/MtasPreAnalyzedParser.java
index c319657..0ed6214 100644
--- a/src/mtas/solr/schema/MtasPreAnalyzedParser.java
+++ b/src/mtas/solr/schema/MtasPreAnalyzedParser.java
@@ -23,28 +23,32 @@ import mtas.solr.update.processor.MtasUpdateRequestProcessorResultReader;
*/
public class MtasPreAnalyzedParser implements PreAnalyzedParser {
- /* (non-Javadoc)
- * @see org.apache.solr.schema.PreAnalyzedField.PreAnalyzedParser#parse(java.io.Reader, org.apache.lucene.util.AttributeSource)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.solr.schema.PreAnalyzedField.PreAnalyzedParser#parse(java.io.
+ * Reader, org.apache.lucene.util.AttributeSource)
*/
@Override
- public ParseResult parse(Reader reader, AttributeSource parent) throws IOException
- {
+ public ParseResult parse(Reader reader, AttributeSource parent)
+ throws IOException {
ParseResult res = new ParseResult();
- //get MtasUpdateRequestProcessorResult
+ // get MtasUpdateRequestProcessorResult
StringBuilder sb = new StringBuilder();
char[] buf = new char[128];
int cnt;
while ((cnt = reader.read(buf)) > 0) {
sb.append(buf, 0, cnt);
}
-
+
MtasUpdateRequestProcessorResultReader result;
Iterator<MtasUpdateRequestProcessorResultItem> iterator;
-
+
try {
result = new MtasUpdateRequestProcessorResultReader(sb.toString());
iterator = result.getIterator();
- if(iterator!=null && iterator.hasNext()) {
+ if (iterator != null && iterator.hasNext()) {
res.str = result.getStoredStringValue();
res.bin = result.getStoredBinValue();
} else {
@@ -53,27 +57,28 @@ public class MtasPreAnalyzedParser implements PreAnalyzedParser {
result.close();
return res;
}
-
- parent.clearAttributes();
- while(iterator.hasNext()) {
+
+ parent.clearAttributes();
+ while (iterator.hasNext()) {
MtasUpdateRequestProcessorResultItem item = iterator.next();
- if(item.tokenTerm!=null) {
+ if (item.tokenTerm != null) {
CharTermAttribute catt = parent.addAttribute(CharTermAttribute.class);
catt.append(item.tokenTerm);
- }
- if(item.tokenFlags!=null) {
+ }
+ if (item.tokenFlags != null) {
FlagsAttribute flags = parent.addAttribute(FlagsAttribute.class);
flags.setFlags(item.tokenFlags);
}
- if(item.tokenPosIncr!=null) {
- PositionIncrementAttribute patt = parent.addAttribute(PositionIncrementAttribute.class);
+ if (item.tokenPosIncr != null) {
+ PositionIncrementAttribute patt = parent
+ .addAttribute(PositionIncrementAttribute.class);
patt.setPositionIncrement(item.tokenPosIncr);
- }
- if(item.tokenPayload!=null) {
+ }
+ if (item.tokenPayload != null) {
PayloadAttribute p = parent.addAttribute(PayloadAttribute.class);
p.setPayload(new BytesRef(item.tokenPayload));
}
- if(item.tokenOffsetStart!=null && item.tokenOffsetEnd!=null) {
+ if (item.tokenOffsetStart != null && item.tokenOffsetEnd != null) {
OffsetAttribute offset = parent.addAttribute(OffsetAttribute.class);
offset.setOffset(item.tokenOffsetStart, item.tokenOffsetEnd);
}
@@ -81,22 +86,26 @@ public class MtasPreAnalyzedParser implements PreAnalyzedParser {
State state = parent.captureState();
res.states.add(state.clone());
// reset for reuse
- parent.clearAttributes();
+ parent.clearAttributes();
}
result.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
- }
+ }
return res;
}
- /* (non-Javadoc)
- * @see org.apache.solr.schema.PreAnalyzedField.PreAnalyzedParser#toFormattedString(org.apache.lucene.document.Field)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.solr.schema.PreAnalyzedField.PreAnalyzedParser#toFormattedString
+ * (org.apache.lucene.document.Field)
*/
@Override
public String toFormattedString(Field f) throws IOException {
- return this.getClass().getName()+" "+f.name();
+ return this.getClass().getName() + " " + f.name();
}
}
diff --git a/src/mtas/solr/search/MtasCQLQParser.java b/src/mtas/solr/search/MtasCQLQParser.java
index f1b8c33..f8e6acc 100644
--- a/src/mtas/solr/search/MtasCQLQParser.java
+++ b/src/mtas/solr/search/MtasCQLQParser.java
@@ -21,13 +21,13 @@ public class MtasCQLQParser extends QParser {
/** The field. */
String field = null;
-
+
/** The cql. */
String cql = null;
-
+
/** The msc. */
MtasSolrSearchComponent msc = null;
-
+
/**
* Instantiates a new mtas cqlq parser.
*
@@ -36,58 +36,62 @@ public class MtasCQLQParser extends QParser {
* @param params the params
* @param req the req
*/
- public MtasCQLQParser(String qstr, SolrParams localParams,
- SolrParams params, SolrQueryRequest req) {
+ public MtasCQLQParser(String qstr, SolrParams localParams, SolrParams params,
+ SolrQueryRequest req) {
super(qstr, localParams, params, req);
-
+
SearchComponent sc = req.getCore().getSearchComponent("mtas");
- if((sc!=null) && (sc instanceof MtasSolrSearchComponent)) {
- msc = (MtasSolrSearchComponent) sc;
- }
- if((localParams.getParams("field")!=null) && (localParams.getParams("field").length==1)) {
- field = localParams.getParams("field")[0];
- }
- if((localParams.getParams("cql")!=null) && (localParams.getParams("cql").length==1)) {
- cql = localParams.getParams("cql")[0];
- }
-
+ if ((sc != null) && (sc instanceof MtasSolrSearchComponent)) {
+ msc = (MtasSolrSearchComponent) sc;
+ }
+ if ((localParams.getParams("field") != null)
+ && (localParams.getParams("field").length == 1)) {
+ field = localParams.getParams("field")[0];
+ }
+ if ((localParams.getParams("cql") != null)
+ && (localParams.getParams("cql").length == 1)) {
+ cql = localParams.getParams("cql")[0];
+ }
+
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
+ *
* @see org.apache.solr.search.QParser#parse()
*/
@Override
public Query parse() throws SyntaxError {
- if(field==null) {
+ if (field == null) {
throw new SyntaxError("no field");
- } else if(cql==null) {
+ } else if (cql == null) {
throw new SyntaxError("no cql");
- } else {
+ } else {
Reader reader = new BufferedReader(new StringReader(cql));
MtasCQLParser p = new MtasCQLParser(reader);
SpanQuery q = null;
-
+
try {
- q = p.parse(field);
+ q = p.parse(field, null);
} catch (mtas.parser.cql.ParseException e) {
throw new SyntaxError(e.getMessage());
- }
-// Map<String,Object> j = req.getJSON();
-// if(j==null) {
-// j = new HashMap<String,Object>();
-// }
-// Set<SpanQuery> l;
-// if(j.containsKey("mtas.stats")) {
-// l = (HashSet<SpanQuery>) j.get("mtas.stats");
-// } else {
-// l = new HashSet<SpanQuery>();
-// }
-// l.add(q);
-// j.put("mtas.stats", l);
- //req.setJSON(j);
+ }
+ // Map<String,Object> j = req.getJSON();
+ // if(j==null) {
+ // j = new HashMap<String,Object>();
+ // }
+ // Set<SpanQuery> l;
+ // if(j.containsKey("mtas.stats")) {
+ // l = (HashSet<SpanQuery>) j.get("mtas.stats");
+ // } else {
+ // l = new HashSet<SpanQuery>();
+ // }
+ // l.add(q);
+ // j.put("mtas.stats", l);
+ // req.setJSON(j);
return q;
- }
+ }
}
}
diff --git a/src/mtas/solr/search/MtasSolrCQLQParserPlugin.java b/src/mtas/solr/search/MtasSolrCQLQParserPlugin.java
index 6b226dc..3d92cb8 100644
--- a/src/mtas/solr/search/MtasSolrCQLQParserPlugin.java
+++ b/src/mtas/solr/search/MtasSolrCQLQParserPlugin.java
@@ -11,15 +11,24 @@ import org.apache.solr.search.QParserPlugin;
*/
public class MtasSolrCQLQParserPlugin extends QParserPlugin {
- /* (non-Javadoc)
- * @see org.apache.solr.search.QParserPlugin#init(org.apache.solr.common.util.NamedList)
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.solr.search.QParserPlugin#init(org.apache.solr.common.util.
+ * NamedList)
*/
+ @SuppressWarnings("rawtypes")
@Override
public void init(NamedList args) {
}
- /* (non-Javadoc)
- * @see org.apache.solr.search.QParserPlugin#createParser(java.lang.String, org.apache.solr.common.params.SolrParams, org.apache.solr.common.params.SolrParams, org.apache.solr.request.SolrQueryRequest)
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.solr.search.QParserPlugin#createParser(java.lang.String,
+ * org.apache.solr.common.params.SolrParams,
+ * org.apache.solr.common.params.SolrParams,
+ * org.apache.solr.request.SolrQueryRequest)
*/
@Override
public QParser createParser(String qstr, SolrParams localParams,
diff --git a/src/mtas/solr/update/processor/MtasUpdateRequestProcessorFactory.java b/src/mtas/solr/update/processor/MtasUpdateRequestProcessorFactory.java
index a9b3742..b2a7ea8 100644
--- a/src/mtas/solr/update/processor/MtasUpdateRequestProcessorFactory.java
+++ b/src/mtas/solr/update/processor/MtasUpdateRequestProcessorFactory.java
@@ -43,8 +43,12 @@ public class MtasUpdateRequestProcessorFactory
/** The config. */
private MtasUpdateRequestProcessorConfig config = null;
- /* (non-Javadoc)
- * @see org.apache.solr.update.processor.UpdateRequestProcessorFactory#init(org.apache.solr.common.util.NamedList)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.solr.update.processor.UpdateRequestProcessorFactory#init(org.
+ * apache.solr.common.util.NamedList)
*/
@Override
@SuppressWarnings("rawtypes")
@@ -55,8 +59,10 @@ public class MtasUpdateRequestProcessorFactory
/**
* Inits the.
*
- * @param req the req
- * @throws IOException Signals that an I/O exception has occurred.
+ * @param req
+ * the req
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
*/
@SuppressWarnings("unchecked")
private void init(SolrQueryRequest req) throws IOException {
@@ -235,8 +241,14 @@ public class MtasUpdateRequestProcessorFactory
}
}
- /* (non-Javadoc)
- * @see org.apache.solr.update.processor.UpdateRequestProcessorFactory#getInstance(org.apache.solr.request.SolrQueryRequest, org.apache.solr.response.SolrQueryResponse, org.apache.solr.update.processor.UpdateRequestProcessor)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.solr.update.processor.UpdateRequestProcessorFactory#getInstance(
+ * org.apache.solr.request.SolrQueryRequest,
+ * org.apache.solr.response.SolrQueryResponse,
+ * org.apache.solr.update.processor.UpdateRequestProcessor)
*/
@Override
public UpdateRequestProcessor getInstance(SolrQueryRequest req,
@@ -262,7 +274,7 @@ class MtasUpdateRequestProcessor extends UpdateRequestProcessor {
}
@Override
- public void processAdd(AddUpdateCommand cmd) throws IOException {
+ public void processAdd(AddUpdateCommand cmd) throws IOException {
if (config != null && config.fieldMapping.size() > 0) {
// get document
SolrInputDocument doc = cmd.getSolrInputDocument();
@@ -310,13 +322,13 @@ class MtasUpdateRequestProcessor extends UpdateRequestProcessor {
}
}
}
-
+
sizeReader = new MtasUpdateRequestProcessorSizeReader(reader);
-
+
// tokenizerFactory
- result = new MtasUpdateRequestProcessorResultWriter(
- storedValue);
- MtasTokenizer tokenizer = tokenizerFactory.create(configuration);
+ result = new MtasUpdateRequestProcessorResultWriter(storedValue);
+ MtasTokenizer<?> tokenizer = tokenizerFactory
+ .create(configuration);
tokenizer.setReader(sizeReader);
tokenizer.reset();
// attributes
@@ -332,8 +344,8 @@ class MtasUpdateRequestProcessor extends UpdateRequestProcessor {
.getAttribute(FlagsAttribute.class);
int numberOfPositions = 0;
- int numberOfTokens = 0;
-
+ int numberOfTokens = 0;
+
while (tokenizer.incrementToken()) {
String term = null;
Integer offsetStart = null, offsetEnd = null, posIncr = null,
@@ -359,17 +371,18 @@ class MtasUpdateRequestProcessor extends UpdateRequestProcessor {
numberOfPositions += posIncr;
result.addItem(term, offsetStart, offsetEnd, posIncr, payload,
flags);
- // System.out.println(term);
- }
+ // System.out.print(term+" ");
+ }
// update field
doc.remove(field);
- if(result.getTokenNumber()>0) {
+ if (result.getTokenNumber() > 0) {
doc.addField(field, result.getFileName());
}
result.close();
// update size
- setFields(doc, config.fieldTypeSizeField.get(fieldType), sizeReader.getTotalReadSize());
+ setFields(doc, config.fieldTypeSizeField.get(fieldType),
+ sizeReader.getTotalReadSize());
// update numberOfPositions
setFields(doc,
config.fieldTypeNumberOfPositionsField.get(fieldType),
@@ -382,17 +395,16 @@ class MtasUpdateRequestProcessor extends UpdateRequestProcessor {
doc.addField(config.fieldTypeErrorField.get(fieldType),
e.getMessage());
// update size
- setFields(doc,
- config.fieldTypeSizeField.get(fieldType), 0);
+ setFields(doc, config.fieldTypeSizeField.get(fieldType), 0);
// update numberOfPositions
setFields(doc,
config.fieldTypeNumberOfPositionsField.get(fieldType), 0);
// update numberOfTokens
setFields(doc, config.fieldTypeNumberOfTokensField.get(fieldType),
0);
- if(result!=null) {
+ if (result != null) {
result.forceCloseAndDelete();
- doc.remove(field);
+ doc.remove(field);
}
}
}
@@ -400,7 +412,7 @@ class MtasUpdateRequestProcessor extends UpdateRequestProcessor {
}
}
- // pass it up the chain
+ // pass it up the chain
super.processAdd(cmd);
}
diff --git a/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultItem.java b/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultItem.java
index 57fb86f..ee825eb 100644
--- a/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultItem.java
+++ b/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultItem.java
@@ -8,28 +8,28 @@ import org.apache.lucene.util.BytesRef;
* The Class MtasUpdateRequestProcessorResultItem.
*/
public class MtasUpdateRequestProcessorResultItem implements Serializable {
-
+
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
/** The token term. */
public String tokenTerm;
-
+
/** The token offset start. */
public Integer tokenOffsetStart;
-
+
/** The token offset end. */
public Integer tokenOffsetEnd;
-
+
/** The token pos incr. */
public Integer tokenPosIncr;
-
+
/** The token payload. */
public byte[] tokenPayload;
-
+
/** The token flags. */
public Integer tokenFlags;
-
+
/**
* Instantiates a new mtas update request processor result item.
*
@@ -40,26 +40,27 @@ public class MtasUpdateRequestProcessorResultItem implements Serializable {
* @param payload the payload
* @param flags the flags
*/
- public MtasUpdateRequestProcessorResultItem(String term, Integer offsetStart, Integer offsetEnd, Integer posIncr, BytesRef payload, Integer flags) {
+ public MtasUpdateRequestProcessorResultItem(String term, Integer offsetStart,
+ Integer offsetEnd, Integer posIncr, BytesRef payload, Integer flags) {
tokenTerm = term;
- if(offsetStart!=null && offsetEnd!=null) {
+ if (offsetStart != null && offsetEnd != null) {
tokenOffsetStart = offsetStart;
tokenOffsetEnd = offsetEnd;
} else {
tokenOffsetStart = null;
tokenOffsetEnd = null;
}
- if(posIncr!=null && posIncr!=1) {
+ if (posIncr != null && posIncr != 1) {
tokenPosIncr = posIncr;
} else {
tokenPosIncr = null;
}
- if(payload!=null) {
+ if (payload != null) {
tokenPayload = payload.bytes;
} else {
tokenPayload = null;
}
tokenFlags = flags;
}
-
+
}
\ No newline at end of file
diff --git a/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultReader.java b/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultReader.java
index d1e0c51..64bef43 100644
--- a/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultReader.java
+++ b/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultReader.java
@@ -15,32 +15,33 @@ public class MtasUpdateRequestProcessorResultReader implements Closeable {
/** The stored string value. */
private String storedStringValue;
+ /** The file input stream. */
private FileInputStream fileInputStream;
/** The object input stream. */
private ObjectInputStream objectInputStream;
+ /** The file. */
private File file;
/** The iterator. */
private Iterator<MtasUpdateRequestProcessorResultItem> iterator;
+ /** The closed. */
private boolean closed;
/**
* Instantiates a new mtas update request processor result reader.
*
- * @param fileName
- * the file name
- * @throws IOException
- * Signals that an I/O exception has occurred.
+ * @param fileName the file name
+ * @throws IOException Signals that an I/O exception has occurred.
*/
public MtasUpdateRequestProcessorResultReader(String fileName)
throws IOException {
file = null;
fileInputStream = null;
objectInputStream = null;
- closed=false;
+ closed = false;
iterator = null;
if (fileName != null) {
file = new File(fileName);
@@ -48,21 +49,22 @@ public class MtasUpdateRequestProcessorResultReader implements Closeable {
objectInputStream = new ObjectInputStream(fileInputStream);
try {
Object o = objectInputStream.readObject();
- if(o instanceof String) {
+ if (o instanceof String) {
storedStringValue = (String) o;
} else {
- throw new IOException("invalid tokenStream");
- }
+ throw new IOException("invalid tokenStream");
+ }
iterator = new Iterator<MtasUpdateRequestProcessorResultItem>() {
MtasUpdateRequestProcessorResultItem next = null;
+
@Override
public boolean hasNext() {
if (!closed) {
- if(next!=null) {
+ if (next != null) {
return true;
} else {
next = getNext();
- if(next!=null) {
+ if (next != null) {
return true;
} else {
return false;
@@ -72,17 +74,18 @@ public class MtasUpdateRequestProcessorResultReader implements Closeable {
return false;
}
}
+
@Override
public MtasUpdateRequestProcessorResultItem next() {
- if(!closed) {
+ if (!closed) {
MtasUpdateRequestProcessorResultItem result;
- if(next!=null) {
+ if (next != null) {
result = next;
next = null;
return result;
} else {
next = getNext();
- if(next!=null) {
+ if (next != null) {
result = next;
next = null;
return result;
@@ -94,9 +97,9 @@ public class MtasUpdateRequestProcessorResultReader implements Closeable {
return null;
}
}
-
+
private MtasUpdateRequestProcessorResultItem getNext() {
- if(!closed) {
+ if (!closed) {
try {
Object o = objectInputStream.readObject();
if (o instanceof MtasUpdateRequestProcessorResultItem) {
@@ -168,7 +171,7 @@ public class MtasUpdateRequestProcessorResultReader implements Closeable {
private void forceClose() {
if (file != null) {
if (file.exists() && file.canWrite()) {
- file.delete();
+ file.delete();
}
file = null;
}
diff --git a/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultWriter.java b/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultWriter.java
index 4e278fc..d4b095d 100644
--- a/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultWriter.java
+++ b/src/mtas/solr/update/processor/MtasUpdateRequestProcessorResultWriter.java
@@ -12,22 +12,25 @@ import org.apache.lucene.util.BytesRef;
*/
public class MtasUpdateRequestProcessorResultWriter implements Closeable {
+ /** The object output stream. */
private ObjectOutputStream objectOutputStream;
+ /** The file output stream. */
private FileOutputStream fileOutputStream;
+ /** The closed. */
private boolean closed;
+ /** The token number. */
private int tokenNumber;
+ /** The file. */
private File file;
/**
* Instantiates a new mtas update request processor result writer.
*
- * @param value
- * the value
- * @throws IOException
+ * @param value the value
*/
public MtasUpdateRequestProcessorResultWriter(String value) {
closed = false;
@@ -48,19 +51,12 @@ public class MtasUpdateRequestProcessorResultWriter implements Closeable {
/**
* Adds the item.
*
- * @param term
- * the term
- * @param offsetStart
- * the offset start
- * @param offsetEnd
- * the offset end
- * @param posIncr
- * the pos incr
- * @param payload
- * the payload
- * @param flags
- * the flags
- * @throws IOException
+ * @param term the term
+ * @param offsetStart the offset start
+ * @param offsetEnd the offset end
+ * @param posIncr the pos incr
+ * @param payload the payload
+ * @param flags the flags
*/
public void addItem(String term, Integer offsetStart, Integer offsetEnd,
Integer posIncr, BytesRef payload, Integer flags) {
@@ -78,16 +74,20 @@ public class MtasUpdateRequestProcessorResultWriter implements Closeable {
}
}
+ /**
+ * Gets the token number.
+ *
+ * @return the token number
+ */
public int getTokenNumber() {
return tokenNumber;
}
/**
- * Creates the file.
+ * Gets the file name.
*
- * @return the string
- * @throws IOException
- * Signals that an I/O exception has occurred.
+ * @return the file name
+ * @throws IOException Signals that an I/O exception has occurred.
*/
public String getFileName() throws IOException {
if (file != null) {
@@ -97,6 +97,11 @@ public class MtasUpdateRequestProcessorResultWriter implements Closeable {
}
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.io.Closeable#close()
+ */
@Override
public void close() throws IOException {
if (!closed) {
@@ -106,6 +111,9 @@ public class MtasUpdateRequestProcessorResultWriter implements Closeable {
}
}
+ /**
+ * Force close and delete.
+ */
public void forceCloseAndDelete() {
try {
if (objectOutputStream != null) {
diff --git a/src/site/markdown/download.md b/src/site/markdown/download.md.vm
index f15bf4d..63843d1 100644
--- a/src/site/markdown/download.md
+++ b/src/site/markdown/download.md.vm
@@ -11,12 +11,12 @@
<th>Description</th>
</tr>
<tr>
- <td>6.1.0</td>
- <td>20160714</td>
- <td><a href='https://github.com/matthijsbrouwer/mtas/releases/download/20160714/mtas-6.1.0.jar'>Binary (jar)</a></td>
- <td><a href='https://github.com/matthijsbrouwer/mtas/archive/20160714.tar.gz'>Source (tgz)</a></td>
- <td><a href='https://github.com/matthijsbrouwer/mtas/archive/20160714.zip'>Source (zip)</a></td>
+ <td>$context.get("currentDevelopmentVersion")</td>
+ <td>$context.get("currentDevelopmentRelease")</td>
+ <td><a href='https://github.com/meertensinstituut/mtas/releases/download/${currentDevelopmentRelease}/mtas-${currentDevelopmentVersion}.jar'>Binary (jar)</a></td>
+ <td><a href='https://github.com/meertensinstituut/mtas/archive/${currentDevelopmentRelease}.tar.gz'>Source (tgz)</a></td>
+ <td><a href='https://github.com/meertensinstituut/mtas/archive/${currentDevelopmentRelease}.zip'>Source (zip)</a></td>
<td>Development version</td>
</tr>
</tbody>
-</table>
\ No newline at end of file
+</table>
diff --git a/src/site/markdown/features.md b/src/site/markdown/features.md
new file mode 100644
index 0000000..b8f52f2
--- /dev/null
+++ b/src/site/markdown/features.md
@@ -0,0 +1,5 @@
+#Features
+
+
+
+
diff --git a/src/site/markdown/index.md b/src/site/markdown/index.md
index f469b6c..59d65f1 100644
--- a/src/site/markdown/index.md
+++ b/src/site/markdown/index.md
@@ -1 +1,5 @@
#Multi Tier Annotation Search
+
+
+
+
diff --git a/src/site/markdown/indexing.md b/src/site/markdown/indexing.md
new file mode 100644
index 0000000..939a57b
--- /dev/null
+++ b/src/site/markdown/indexing.md
@@ -0,0 +1,5 @@
+#Indexing
+
+
+
+
diff --git a/src/site/markdown/indexing_configuration.md b/src/site/markdown/indexing_configuration.md
new file mode 100644
index 0000000..8ffcc0a
--- /dev/null
+++ b/src/site/markdown/indexing_configuration.md
@@ -0,0 +1,5 @@
+#Indexing - configuration
+
+
+
+
diff --git a/src/site/markdown/indexing_formats.md b/src/site/markdown/indexing_formats.md
new file mode 100644
index 0000000..45af625
--- /dev/null
+++ b/src/site/markdown/indexing_formats.md
@@ -0,0 +1,5 @@
+#Indexing - formats
+
+
+
+
diff --git a/src/site/markdown/indexing_formats_crm.md b/src/site/markdown/indexing_formats_crm.md
new file mode 100644
index 0000000..32c96c2
--- /dev/null
+++ b/src/site/markdown/indexing_formats_crm.md
@@ -0,0 +1,5 @@
+#Indexing - formats - CRM
+
+
+
+
diff --git a/src/site/markdown/indexing_formats_folia.md b/src/site/markdown/indexing_formats_folia.md
new file mode 100644
index 0000000..9daa82e
--- /dev/null
+++ b/src/site/markdown/indexing_formats_folia.md
@@ -0,0 +1,5 @@
+#Indexing - formats - FoLiA
+
+
+
+
diff --git a/src/site/markdown/indexing_formats_sketch.md b/src/site/markdown/indexing_formats_sketch.md
new file mode 100644
index 0000000..4e15e30
--- /dev/null
+++ b/src/site/markdown/indexing_formats_sketch.md
@@ -0,0 +1,5 @@
+#Indexing - formats - sketch
+
+
+
+
diff --git a/src/site/markdown/indexing_formats_tei.md b/src/site/markdown/indexing_formats_tei.md
new file mode 100644
index 0000000..b1fad29
--- /dev/null
+++ b/src/site/markdown/indexing_formats_tei.md
@@ -0,0 +1,5 @@
+#Indexing - formats - tei
+
+
+
+
diff --git a/src/site/markdown/installation.md b/src/site/markdown/installation.md
deleted file mode 100644
index e69de29..0000000
--- a/src/site/markdown/installation.md
+++ /dev/null
diff --git a/src/site/markdown/installation.md.vm b/src/site/markdown/installation.md.vm
new file mode 100644
index 0000000..0c18973
--- /dev/null
+++ b/src/site/markdown/installation.md.vm
@@ -0,0 +1,28 @@
+#Getting started
+
+Prebuilt jar libraries are available from the [download](download.html) page, current version is [${currentDevelopmentVersion}, release ${currentDevelopmentRelease}](https://github.com/meertensinstituut/mtas/releases/download/${currentDevelopmentRelease}/mtas-${currentDevelopmentVersion}.jar) (development)
+
+###Build from Source with Maven
+
+Download the source code from GitHub
+
+``` console
+git clone git://github.com/meertensinstituut/mtas.git
+```
+
+Build the library from the created project directory `mtas`
+
+``` console
+mvn package
+```
+
+After a successful build, the directory `target` will contain the new jar library.
+
+###Installation
+
+Mtas can be used as plugin for [Apache Solr](http://lucene.apache.org/solr/) or as library in combination with [Apache Lucene](http://lucene.apache.org/solr/).
+
+- Getting started with [Mtas and Lucene](installation_lucene.html)
+- Getting started with [Mtas and Solr](installation_solr.html)
+
+Versioning follows the required version of both Solr and Lucene, the latest available version is ${currentDevelopmentVersion}.
diff --git a/src/site/markdown/installation_lucene.md b/src/site/markdown/installation_lucene.md
new file mode 100644
index 0000000..ce7f117
--- /dev/null
+++ b/src/site/markdown/installation_lucene.md
@@ -0,0 +1 @@
+#Mtas and Lucene
diff --git a/src/site/markdown/installation_solr.md b/src/site/markdown/installation_solr.md
new file mode 100644
index 0000000..2237b9f
--- /dev/null
+++ b/src/site/markdown/installation_solr.md
@@ -0,0 +1,3 @@
+#Mtas and Solr
+
+
diff --git a/src/site/markdown/search.md b/src/site/markdown/search.md
new file mode 100644
index 0000000..d1a3c5e
--- /dev/null
+++ b/src/site/markdown/search.md
@@ -0,0 +1,5 @@
+# Search
+
+
+
+
diff --git a/src/site/markdown/search_configuration.md b/src/site/markdown/search_configuration.md
new file mode 100644
index 0000000..50af34c
--- /dev/null
+++ b/src/site/markdown/search_configuration.md
@@ -0,0 +1,5 @@
+#Configuration
+
+
+
+
diff --git a/src/site/markdown/search_cql.md b/src/site/markdown/search_cql.md
new file mode 100644
index 0000000..95e5d16
--- /dev/null
+++ b/src/site/markdown/search_cql.md
@@ -0,0 +1,94 @@
+# Corpus Query Language
+
+Within Lucene and Solr, each field containing tokenized text can be considered as a set of tokens, where each token is associated with a position and its value can be seen as a word from the original text. Mtas extends this concept by allowing to associate multiple positions with one token and by associating each token with a prefix and optional postfix instead of the single value. This makes it possible to use multiple tokens on the same position, and distinguish annotations by using a unique prefix for each type, and allows structures like sentences, paragraphs or entities consisting of multiple adjacent or non-adjacent positions.
+
+To describe sets of tokens matching some condition, a query language is needed. Mtas supports CQL based on the Corpus Query Language introduced by the [Corpus WorkBench](http://cwb.sourceforge.net/files/CQP_Tutorial/) and supported by the Lexicom [Sketch Engine](http://www.sketchengine.co.uk/documentation/wiki/SkE/CorpusQuerying).
+
+<a name="prefix"></a>
+
+#### Prefix
+
+For each field containing Mtas tokenized text, every token is associated with a prefix. Within the field, only a limited set of prefixes is used to distinguish between the different types of annotation. By using a [prefix query](search_query_prefix.html) a full list of used prefixes can be produced.
+
+<a name="value"></a>
+
+#### Value
+
+The optional postfix associated with a token can be queried within CQL by providing a *value*. This is a regular expression, the supported syntax is documented in the RegExp class provided by Lucene. By using a [termvector query](search_query_termvector.html), for each [prefix](#prefix) a list of postfix values can be produced.
+
+<a name="#cql"></a>
+
+## CQL
+
+| Syntax | Description | Example |
+|---------------------------------------|----------------------------------|--------------|
+| [token](#token) | Matches a single position token | `[t=\"de\"]` |
+| [multi-position](#multi-position) | Matches a (single or) multi position token | `<s/>` |
+| [sequence](#sequence) | Matches a sequence | `[pos="ADJ"]{2}[pos="N"]` |
+
+| Syntax | Description | Example |
+|---------------------------------------|----------------------------------|--------------|
+| [cql](#cql)**{** \<number\> **}** | Matches provided number of occurrence from [cql](#cql)| `[pos=\"ADJ\"]{2}` |
+| [cql](#cql)**{** \<number\> , \<number\>**}** | Matches each number between provided start and end of occurrence from [cql](#cql)| `[pos=\"ADJ\"]{2,3}` |
+
+
+
+| Syntax | Description | Example |
+|---------------------------------------|-------------------------------------------------|---------|
+| **\(** [cql](#cql) **\) within \(** [cql](#cql) **\)** | Matches cql expression within another cql expression | `([t=\"de\"]) within (<s/>)` |
+| **\(** [cql](#cql) **\) !within \(** [cql](#cql) **\)** | Matches cql expression not within another cql expression | `([t=\"de\"]) !within (<s/>)` |
+| **\(** [cql](#cql) **\) containing \(** [cql](#cql) **\)** | Matches cql expression containing another cql expression | `(<s/>) containing ([t=\"de\"])` |
+| **\(** [cql](#cql) **\) !containing \(** [cql](#cql) **\)** | Matches cql expression not containing another cql expression | `(<s/>) !containing ([t=\"de\"])` |
+
+<a name="#token"></a>
+
+## Token
+
+| Syntax | Description | Example |
+|-------------------------------------|-------------------------------------------------|---------|
+| **\[ \]** | Matches each single position token | `[]` |
+| **\[** [single-position-expression](#single-position-expression) **\]** | Matches single position token with condition defined by an [single-position-expression](#single-position-expression) | `[t="de"]` |
+
+<a name="single-position-expression"></a>
+
+#### Single Position Expression
+
+| Expression | Syntax | Example |
+|-------------|---------------------------------------------|---------|
+| basic | [prefix](#prefix) **= \"**[value](#value)**\"** | `t="de"`
+| not | **\!** [single-position-expression](#single-position-expression) | `!t="de"` |
+| and | **\(** [single-position-expression](#single-position-expression) **\&** [single-position-expression](#single-position-expression) **\&** ... **\)** | `t="de" & pos="LID"`|
+| or | **\(** [single-position-expression](#single-position-expression) **\|** [single-position-expression](#single-position-expression) **\|** ... **\)** | `t="de" | t="het"` |
+| position | **\#** \<position\> | `#100` |
+| range | **\#** \<position\> **-** \<position\> | `#100-110` |
+
+
+<a name="#multi-position"></a>
+
+## Multi-position
+
+| Syntax | Description | Example |
+|-------------------------------------|-------------------------------------------------|---------|
+| **\<** [multi-position-expression](#multi-position-expression) **/\>** | Matches (single and) multi position tokens with condition defined by [multi-position-expression](#multi-position-expression) | `<s/>` |
+| **\<** [multi-position-expression](#multi-position-expression) **\>** | Matches start of (single and) multi position tokens with condition defined by [multi-position-expression](#multi-position-expression) | `<s>` |
+| **\</** [multi-position-expression](#multi-position-expression) **\>** | Matches end of (single and) multi position tokens with condition defined by [multi-position-expression](#multi-position-expression) | `</s>` |
+
+
+<a name="multi-position-expression"></a>
+
+#### Multi Position Expression
+
+| Expression | Syntax |
+|-------------|---------------------------------------------------|
+| prefix | [prefix](#prefix) |
+| basic | [prefix](#prefix) **= \"**[value](#value)**\"** |
+
+
+<a name="#sequence"></a>
+
+## Sequence
+
+| Syntax | Description | Example |
+|---------------------------------------|----------------------------------|--------------|
+| [cql](#cql) [cql](#cql) [cql](#cql)... | A sequence of [cql](#cql) | `[t="de"][pos=\"ADJ\"]{2}[pos="N"]` |
+
diff --git a/src/site/markdown/search_functions.md b/src/site/markdown/search_functions.md
new file mode 100644
index 0000000..1f454f9
--- /dev/null
+++ b/src/site/markdown/search_functions.md
@@ -0,0 +1,2 @@
+#Functions
+
diff --git a/src/site/markdown/search_query.md b/src/site/markdown/search_query.md
new file mode 100644
index 0000000..86d8e65
--- /dev/null
+++ b/src/site/markdown/search_query.md
@@ -0,0 +1,43 @@
+# Query
+
+Mtas can be used in both regular and specific mtas queries.
+
+## Regular queries
+
+By [configuring](search_configuration.html) the mtas query parser in solrconfig.xml, [cql](search_cql.html) can be used in regular queries.
+
+**Example 1**
+
+Search for documents containing the word "de" with a query.
+
+`q={!mtas_cql+field%3D"text"+cql%3D"[t%3D\"de\"]"}&fl=*&start=0&rows=0&wt=json&indent=true`
+
+``` json
+"response":{"numFound":1664241,"start":0,"docs":[]
+ }
+```
+
+**Example 2**
+
+Search for documents containing the word "de" with a filter query.
+
+**Request and response**
+
+`fq={!mtas_cql+field%3D"text"+cql%3D"[t%3D\"de\"]"}&q=*%3A*&fl=*&start=0&rows=0&wt=json&indent=true`
+
+``` json
+"response":{"numFound":1664241,"start":0,"docs":[]
+ }
+```
+
+## Mtas queries
+
+To perform specific mtas queries, the following parameter should be used.
+
+| Parameter | Value | Obligatory |
+|-------------|--------|-------------|
+| mtas | true | yes |
+
+See [statistics](search_query_stats.html),
+[kwic/list](search_query_kwic_and_list.html), [termvector](search_query_termvector.html), [facet](search_query_facet.html), [group](search_query_group.html) and [prefix](search_query_prefix.html) for more details and examples.
+
diff --git a/src/site/markdown/search_query_facet.md b/src/site/markdown/search_query_facet.md
new file mode 100644
index 0000000..771ada3
--- /dev/null
+++ b/src/site/markdown/search_query_facet.md
@@ -0,0 +1,5 @@
+#Query - facet
+
+
+
+
diff --git a/src/site/markdown/search_query_group.md b/src/site/markdown/search_query_group.md
new file mode 100644
index 0000000..324b730
--- /dev/null
+++ b/src/site/markdown/search_query_group.md
@@ -0,0 +1,5 @@
+#Query - group
+
+
+
+
diff --git a/src/site/markdown/search_query_kwic_and_list.md b/src/site/markdown/search_query_kwic_and_list.md
new file mode 100644
index 0000000..e7a9730
--- /dev/null
+++ b/src/site/markdown/search_query_kwic_and_list.md
@@ -0,0 +1,5 @@
+#Query - kwic and lists
+
+
+
+
diff --git a/src/site/markdown/search_query_prefix.md b/src/site/markdown/search_query_prefix.md
new file mode 100644
index 0000000..65b98db
--- /dev/null
+++ b/src/site/markdown/search_query_prefix.md
@@ -0,0 +1,69 @@
+# Prefix
+
+Mtas can produce a list of available prefixes. To get this information, besides the parameter to enable [mtas queries](search_query.html), the following parameter should be provided.
+
+| Parameter | Value | Obligatory |
+|-----------------------|--------|-------------|
+| mtas.stats.prefix | true | yes |
+
+Information for multiple fields can be produced within the same request. To distinguish them, a unique identifier has to be provided for each of the required statistics. The list of available prefixes is independent of any restriction in the document set, and also prefixes of deleted documents can be taken into account when the core hasn't been optimized.
+
+| Parameter | Value | Info | Obligatory |
+|-------------------------------------------------|--------------|--------------------------------|-------------|
+| mtas.stats.prefix.\<identifier\>.key | \<string\> | key used in response | no |
+| mtas.stats.prefix.\<identifier\>.field | \<string\> | mtas field | yes |
+
+The *key* is added to the response and may be used to distinguish between multiple lists, and should therefore be unique. The response will contain three lists: prefixes strictly used for single position tokens, prefixes (also) used for multiple position tokens and prefixes used for multiple non adjacent positions. Notice that the last list will always be a subset of the second list.
+
+## Examples
+1. [Basic](#basic) : list of available prefixes.
+
+<a name="basic"></a>
+
+### Basic
+
+**Example**
+List of avilable prefixes.
+
+**Request and response**
+`q=*%3A*&mtas=true&mtas.prefix=true&mtas.prefix.0.field=text&mtas.prefix.0.key=example+-+basic&rows=0&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "prefix":[{
+ "key":"example - basic",
+ "singlePosition":["feat.buiging",
+ "feat.conjtype",
+ "feat.dial",
+ "feat.genus",
+ "feat.getal",
+ "feat.getal-n",
+ "feat.graad",
+ "feat.head",
+ "feat.lwtype",
+ "feat.naamval",
+ "feat.npagr",
+ "feat.ntype",
+ "feat.numtype",
+ "feat.pdtype",
+ "feat.persoon",
+ "feat.positie",
+ "feat.pvagr",
+ "feat.pvtijd",
+ "feat.spectype",
+ "feat.status",
+ "feat.vwtype",
+ "feat.vztype",
+ "feat.wvorm",
+ "lemma",
+ "morpheme",
+ "pos",
+ "t",
+ "t_lc"],
+ "multiplePosition":["div",
+ "entity",
+ "head",
+ "p",
+ "s"],
+ "setPosition":["entity"]}]}
+```
diff --git a/src/site/markdown/search_query_stats.md b/src/site/markdown/search_query_stats.md
new file mode 100644
index 0000000..74d9e59
--- /dev/null
+++ b/src/site/markdown/search_query_stats.md
@@ -0,0 +1,10 @@
+# Statistics
+
+To produce statistics, besides the parameter to enable [mtas queries](search_query.html), the following parameter should be used.
+
+| Parameter | Value | Obligatory |
+|-------------|--------|-------------|
+| mtas.stats | true | yes |
+
+Using this parameter, it is possible to add statistics on [positions](search_query_stats_positions.html), [tokens](search_query_stats_tokens.html) and [spans](search_query_stats_spans.html) to the response on a request.
+
diff --git a/src/site/markdown/search_query_stats_positions.md b/src/site/markdown/search_query_stats_positions.md
new file mode 100644
index 0000000..4a6dc64
--- /dev/null
+++ b/src/site/markdown/search_query_stats_positions.md
@@ -0,0 +1,136 @@
+# Statistics - positions
+
+To produce statistics on the number of positions within a set of documents, besides the parameter to enable [statistics](search_query_stats.html), the following parameter should be provided.
+
+| Parameter | Value | Obligatory |
+|-----------------------|--------|-------------|
+| mtas.stats.positions | true | yes |
+
+Multiple statistics on positions can be produced within the same request.
+To distinguish them, a unique identifier has to be provided for
+each of the required statistics.
+
+| Parameter | Value | Info | Obligatory |
+|-------------------------------------------------|--------------|--------------------------------|-------------|
+| mtas.stats.positions.\<identifier\>.key | \<string\> | key used in response | no |
+| mtas.stats.positions.\<identifier\>.field | \<string\> | mtas field | yes |
+| mtas.stats.positions.\<identifier\>.type | \<string\> | required [type of statistics](search_stats.html) | no |
+| mtas.stats.positions.\<identifier\>.minimum | \<double\> | minimum number of positions | no |
+| mtas.stats.positions.\<identifier\>.maximum | \<double\> | maximum number of positions | no |
+
+The *key* is added to the response and may be used to distinguish between multiple statistics on positions, and should therefore be unique. The optional *minimum* and *maximum* can be used to focus only on documents satisfying a condition on the number of positions.
+
+---
+
+## Examples
+1. [Basic](#basic) : basic statistics on the number of positions.
+2. [Minimum and maximum](#minimum-and-maximum) : statistics on the number of positions with restrictions on this number.
+3. [Subset](#subset) : statistics on the number of positions within a subset of documents.
+
+---
+
+<a name="basic"></a>
+
+### Basic
+
+**Example**
+Total and average number of positions and the number of documents.
+
+**Request and response**
+`q=*%3A*&rows=0&mtas=true&mtas.stats=true&mtas.stats.positions=true&mtas.stats.positions.0.field=text&mtas.stats.positions.0.key=example - basic&mtas.stats.positions.0.type=sum,mean,n&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "positions":[{
+ "key":"example - basic",
+ "mean":163.32306296759796,
+ "sum":337230767,
+ "n":2064808}]}}
+```
+
+<a name="minimum-and-maximum"></a>
+
+### Minimum and maximum
+
+**Example**
+Full statistics on positions for documents with a minimum of 100 positions, for documents with a maximum of 200 positions, and for documents with between 100 and 200 positions.
+
+**Request and response**
+`q=*%3A*&rows=0&mtas=true&mtas.stats=true&mtas.stats.positions=true&mtas.stats.positions.0.field=text&mtas.stats.positions.0.key=example - minimum&mtas.stats.positions.0.type=all&mtas.stats.positions.0.minimum=100&mtas.stats.positions.1.field=text&mtas.stats.positions.1.key=example - maximum&mtas.stats.positions.1.type=all&mtas.stats.positions.1.maximum=200&mtas.stats.positions.2.field=text&mtas.stats.positions.2.key=example - minimum and maximum&mtas.stats.positions.2.type=all&mtas.stats.positions.2.minimum=100&mtas.stats.positions.2.maximum=200&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "positions":[{
+ "key":"example - minimum",
+ "sumsq":3.217738942941E12,
+ "populationvariance":4429860.598303376,
+ "max":419252.0,
+ "sum":3.03356085E8,
+ "kurtosis":7475.629984106579,
+ "standarddeviation":2104.724912671482,
+ "n":696551,
+ "quadraticmean":2149.309425767974,
+ "min":100.0,
+ "median":232.0,
+ "variance":4429866.9580199765,
+ "mean":435.5116638982477,
+ "geometricmean":269.08829791186866,
+ "sumoflogs":3897230.408057158,
+ "skewness":69.96996634727896},
+ {
+ "key":"example - maximum",
+ "sumsq":8.387258635E9,
+ "populationvariance":2952.9318413516844,
+ "max":200.0,
+ "sum":7.6021711E7,
+ "kurtosis":0.06752103336188053,
+ "standarddeviation":54.34090186412456,
+ "n":1664511,
+ "quadraticmean":70.98501573316724,
+ "min":0.0,
+ "median":25.0,
+ "variance":2952.9336154064163,
+ "mean":45.672098892754136,
+ "geometricmean":0.0,
+ "sumoflogs":"-Infinity",
+ "skewness":1.055453308387049},
+ {
+ "key":"example - minimum and maximum",
+ "sumsq":6.242915333E9,
+ "populationvariance":833.0814348912809,
+ "max":200.0,
+ "sum":4.2147029E7,
+ "kurtosis":-1.089851452153094,
+ "standarddeviation":28.86319883436408,
+ "n":296254,
+ "quadraticmean":145.16489726459713,
+ "min":100.0,
+ "median":139.0,
+ "variance":833.084246952036,
+ "mean":142.2665314223605,
+ "geometricmean":139.38264759454245,
+ "sumoflogs":1462672.066002271,
+ "skewness":0.30992716799338843}]}}
+```
+
+<a name="subset"></a>
+
+### Subset
+
+**Example**
+Total and average number of positions and the number of documents for a subset of documents.
+
+**Request and response**
+`q=text:koe&rows=0&mtas=true&mtas.stats=true&mtas.stats.positions=true&mtas.stats.positions.0.field=text&mtas.stats.positions.0.key=example - subset&mtas.stats.positions.0.type=sum,mean,n&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "positions":[{
+ "key":"example - subset",
+ "mean":3578.5040620384048,
+ "sum":9690589,
+ "n":2708}]}}
+```
diff --git a/src/site/markdown/search_query_stats_spans.md b/src/site/markdown/search_query_stats_spans.md
new file mode 100644
index 0000000..e129201
--- /dev/null
+++ b/src/site/markdown/search_query_stats_spans.md
@@ -0,0 +1,329 @@
+# Statistics - spans
+
+To produce statistics on the occurrence of a span within a set of documents, besides the parameter to enable [statistics](search_query_stats.html), the following parameter should be provided.
+
+| Parameter | Value | Obligatory |
+|-----------------------|--------|-------------|
+| mtas.stats.spans | true | yes |
+
+Multiple statistics on the occurrence of a span can be produced within the same request. To distinguish them, a unique identifier has to be provided for each of the required statistics. Furthermore, statistics for the occurrence of multiple spans can be produced. Spans are described by a query, and to distinguish multiple spans, also a query identifier has to be provided.
+
+| Parameter | Value | Info | Obligatory |
+|-------------------------------------------------|--------------|--------------------------------|-------------|
+| mtas.stats.spans.\<identifier\>.key | \<string\> | key used in response | no |
+| mtas.stats.spans.\<identifier\>.field | \<string\> | mtas field | yes |
+| mtas.stats.spans.\<identifier\>.query.\<identifier query\>.type | \<string\> | query language: [cql](search_cql.html) | yes |
+| mtas.stats.spans.\<identifier\>.query.\<identifier query\>.value | \<string\> | query: [cql](search_cql.html) | yes |
+| mtas.stats.spans.\<identifier\>.type | \<string\> | required [type of statistics](search_stats.html) | no |
+| mtas.stats.spans.\<identifier\>.minimum | \<double\> | minimum number of occurrences span | no |
+| mtas.stats.spans.\<identifier\>.maximum | \<double\> | maximum number of occurrences span | no |
+
+The *key* is added to the response and may be used to distinguish between multiple statistics on the occurrence of spans, and should therefore be unique. The optional *minimum* and *maximum* can be used to focus only on documents satisfying a condition on the number of occurrences of the spans. When multiple queries are provided, the provided boundary will hold on the sum of occurrences of the resulting spans.
+
+## Functions
+
+To compute statistics for values based on the occurrence of one or multiple spans, optionally [functions](search_functions.html) can be added. The parameters for these functions are the number of occurrences *$q0*, *$q1*, ... for each span and the number of positions *$n* in a document. Statistics on the value computed for each document in the set are added to the response.
+
+| Parameter | Value | Info | Obligatory |
+|-------------------------------------------------|--------------|--------------------------------|-------------|
+| mtas.stats.spans.\<identifier\>.function.\<identifier function\>.key | \<string\> | key used in response | no |
+| mtas.stats.spans.\<identifier\>.function.\<identifier function\>.expression | \<string\> | see [functions](search_functions.html) | yes |
+| mtas.stats.spans.\<identifier\>.function.\<identifier function\>.type | \<string\> | required [type of statistics](search_stats.html) | no |
+
+Again, the *key* is added to the response and may be used to distinguish between multiple functions, and should therefore be unique.
+
+---
+
+## Examples
+1. [Basic](#basic) : basic statistics on the occurrence of a word.
+2. [Minimum and Maximum](#minimum-and-maximum) : statistics on the occurrence of a word with restrictions on the number of occurrences.
+3. [Subset](#subset) : statistics on the occurrence of a word within a subset of documents.
+4. [Multiple](#multiple) : statistics on the occurrence of multiple words.
+5. [Functions](#functions) : statistics using functions.
+6. [Multiple and Functions](#multiple-and-functions) : statistics using functions on the occurrence of multiple words.
+
+---
+
+<a name="basic"></a>
+
+### Basic
+
+**Example**
+Total and average number of occurrences of the word "de" and the number of documents.
+
+**CQL**
+`[t="de"]`
+
+**Request and response**
+`q=*%3A*&mtas=true&mtas.stats=true&mtas.stats.spans=true&mtas.stats.spans.0.field=text&mtas.stats.spans.0.query.0.type=cql&mtas.stats.spans.0.query.0.value=%5Bt%3D%22de%22%5D&mtas.stats.spans.0.key=example - basic&mtas.stats.spans.0.type=n%2Csum%2Cmean&rows=0&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "spans":[{
+ "key":"example - basic",
+ "mean":10.484745312881392,
+ "sum":21648986,
+ "n":2064808}]}}
+```
+
+<a name="minimum-and-maximum"></a>
+
+### Minimum and Maximum
+
+**Example**
+Full statistics on the number of occurrences of the word "de" for documents with a minimum of 100 occurrences, for documents with a maximum of 200 occurrences, and for documents with between 100 and 200 occurrences.
+
+**CQL**
+`[t="de"]`
+
+**Request and response**
+`q=*%3A*&mtas=true&mtas.stats=true&mtas.stats.spans=true&mtas.stats.spans.0.field=text&mtas.stats.spans.0.query.0.type=cql&mtas.stats.spans.0.query.0.value=[t%3D"de"]&mtas.stats.spans.0.key=example - minimum&mtas.stats.spans.0.type=all&mtas.stats.spans.0.minimum=100&mtas.stats.spans.1.field=text&mtas.stats.spans.1.query.0.type=cql&mtas.stats.spans.1.query.0.value=[t%3D"de"]&mtas.stats.spans.1.key=example - maximum&mtas.stats.spans.1.type=all&mtas.stats.spans.1.maximum=200&mtas.stats.spans.2.field=text&mtas.stats.spans.2.query.0.type=cql&mtas.stats.spans.2.query.0.value=[t%3D"de"]&mtas.stats.spans.2.key=example - minimum and maximum&mtas.stats.spans.2.type=all&mtas.stats.spans.2.minimum=100&mtas.stats.spans.2.maximum=200&rows=0&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "spans":[{
+ "key":"example - minimum",
+ "sumsq":8.697716821E9,
+ "populationvariance":419246.79216222034,
+ "max":18192.0,
+ "sum":4531791.0,
+ "kurtosis":164.0070630212961,
+ "standarddeviation":647.5106543337658,
+ "n":18029,
+ "quadraticmean":694.5712655602209,
+ "min":100.0,
+ "median":136.0,
+ "variance":419270.04747574165,
+ "mean":251.36119585112766,
+ "geometricmean":160.51059457862394,
+ "sumoflogs":91557.75154264584,
+ "skewness":10.551742594336096},
+ {
+ "key":"example - maximum",
+ "sumsq":7.37120919E8,
+ "populationvariance":271.7584436494812,
+ "max":200.0,
+ "sum":1.9094863E7,
+ "kurtosis":31.747278519903332,
+ "standarddeviation":16.485101621383148,
+ "n":2061622,
+ "quadraticmean":18.90883830898543,
+ "min":0.0,
+ "median":4.0,
+ "variance":271.75857546732925,
+ "mean":9.26205822405864,
+ "geometricmean":0.0,
+ "sumoflogs":"-Infinity",
+ "skewness":4.742017950067824},
+ {
+ "key":"example - minimum and maximum",
+ "sumsq":2.73659206E8,
+ "populationvariance":684.261700791398,
+ "max":200.0,
+ "sum":1977668.0,
+ "kurtosis":-0.4734932618605048,
+ "standarddeviation":26.15927758668186,
+ "n":14843,
+ "quadraticmean":135.78262099542508,
+ "min":100.0,
+ "median":127.0,
+ "variance":684.3078038570759,
+ "mean":133.23910260728962,
+ "geometricmean":130.83058078303992,
+ "sumoflogs":72343.34534205312,
+ "skewness":0.7178184624731176}]}}
+```
+
+<a name="subset"></a>
+
+### Subset
+
+**Example**
+Total and average number of occurrences of the word "de" and the number of documents for a subset of documents.
+
+**CQL**
+`[t="de"]`
+
+**Request and response**
+`q=text:koe&rows=0&mtas=true&mtas.stats=true&mtas.stats.tokens=true&mtas.stats.tokens.0.field=text&mtas.stats.tokens.0.key=example - subset&mtas.stats.tokens.0.type=sum,mean,n&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "tokens":[{
+ "key":"example - subset",
+ "mean":49644.49556868538,
+ "sum":134437294,
+ "n":2708}]}}
+```
+
+<a name="multiple"></a>
+
+### Multiple
+
+**Example**
+Total and average number of occurrences of the word "de" and "het", and the number of documents.
+
+**CQL**
+1. combined cql: `[t="de"|t="het"]`
+2. combined regexp: `[t="(de|het)"]`
+3. two queries: `[t="de"]` `[t="het"]`
+
+**Request and response**
+`q=*%3A*&mtas=true&mtas.stats=true&mtas.stats.spans=true&mtas.stats.spans.0.field=text&mtas.stats.spans.0.query.0.type=cql&mtas.stats.spans.0.query.0.value=[t%3D"de"|t%3D"het"]&mtas.stats.spans.0.key=multiple+-+combined+cql&mtas.stats.spans.0.type=n%2Csum%2Cmean&mtas.stats.spans.1.field=text&mtas.stats.spans.1.query.0.type=cql&mtas.stats.spans.1.query.0.value=[t%3D"(de|het)"]&mtas.stats.spans.1.key=multiple+-+combined+regexp&mtas.stats.spans.1.type=n%2Csum%2Cmean&mtas.stats.spans.2.field=text&mtas.stats.spans.2.query.0.type=cql&mtas.stats.spans.2.query.0.value=[t%3D"de"]&mtas.stats.spans.2.query.1.type=cql&mtas.stats.spans.2.query.1.value=[t%3D"het"]&mtas.stats.spans.2.key=multiple+-+two+queries&mtas.stats.spans.2.type=n%2Csum%2Cmean&rows=0&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "spans":[{
+ "key":"multiple - combined cql",
+ "mean":15.173083405333571,
+ "sum":31329504,
+ "n":2064808},
+ {
+ "key":"multiple - combined regexp",
+ "mean":15.173083405333571,
+ "sum":31329504,
+ "n":2064808},
+ {
+ "key":"multiple - two queries",
+ "mean":15.173083405333571,
+ "sum":31329504,
+ "n":2064808}]}}}
+```
+
+<a name="functions"></a>
+
+### Functions
+
+**Example**
+Statistics for the relative frequency of the word "de" and the total number of words in documents containing this word.
+
+**CQL**
+`[t="de"]`
+
+**Functions**
+`$q0/$n`
+`$n`
+
+**Request and response**
+`q=*%3A*&mtas=true&mtas.stats=true&mtas.stats.spans=true&mtas.stats.spans.0.field=text&mtas.stats.spans.0.query.0.type=cql&mtas.stats.spans.0.query.0.value=[t_lc%3D"de"]&mtas.stats.spans.0.key=functions+-+de&mtas.stats.spans.0.type=n%2Csum%2Cmean&mtas.stats.spans.0.function.0.expression=%24q0%2F%24n&mtas.stats.spans.0.function.0.key=relative+frequency&mtas.stats.spans.0.function.0.type=mean%2Cstandarddeviation%2Cdistribution(start%3D0%2Cend%3D0.1%2Cnumber%3D10)&mtas.stats.spans.0.function.1.expression=%24n&mtas.stats.spans.0.function.1.key=number+of+words&mtas.stats.spans.0.function.1.type=n%2Csum&rows=0&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "spans":[{
+ "key":"functions - de",
+ "mean":12.34790062804871,
+ "sum":25496044,
+ "n":2064808,
+ "functions":{
+ "number of words":{
+ "sum":337230767,
+ "n":2064808},
+ "relative frequency":{
+ "distribution(start=0,end=0.1,number=10)":{
+ "[0.000,0.010)":950500,
+ "[0.010,0.020)":80369,
+ "[0.020,0.030)":115695,
+ "[0.030,0.040)":139752,
+ "[0.040,0.050)":162877,
+ "[0.050,0.060)":168598,
+ "[0.060,0.070)":145493,
+ "[0.070,0.080)":109117,
+ "[0.080,0.090)":77214,
+ "[0.090,0.100)":51243},
+ "mean":0.030196372045937097,
+ "errorList":{"division by zero":691633},
+ "standarddeviation":0.03428066513492476,
+ "errorNumber":691633}}}]}}
+```
+
+<a name="multiple-and-functions"></a>
+
+### Multiple and Functions
+
+**Example**
+Statistics for the absolute and relative frequency of the words "de", "het" and "een", for *part of speech* type "LID" and the total number of words in documents containing this word.
+
+**CQL**
+`[t="de"]`
+`[t="het"]`
+`[t="een"]`
+`[pos="LID"]`
+
+**Functions**
+`$q0/$n`
+`$q1/$n`
+`$q2/$n`
+`$q3/$n`
+`$q0/$q3`
+`$q1/$q3`
+`$q2/$q3`
+`($q0+$q1+$q2)/$q3`
+
+**Request and response**
+`q=*%3A*&mtas=true&mtas.stats=true&mtas.stats.spans=true&mtas.stats.spans.0.field=text&mtas.stats.spans.0.query.0.type=cql&mtas.stats.spans.0.query.0.value=[t_lc%3D"de"]&mtas.stats.spans.0.query.1.type=cql&mtas.stats.spans.0.query.1.value=[t_lc%3D"het"]&mtas.stats.spans.0.query.2.type=cql&mtas.stats.spans.0.query.2.value=[t_lc%3D"een"]&mtas.stats.spans.0.query.3.type=cql&mtas.stats.spans.0.query.3.value=[pos%3D"LID"]&mtas.stats.spans.0.key=multiple+and+functions+-+de%2Bhet%2Been+and+LID&mtas.stats.spans.0.type=n&mtas.stats.spans.0.minimum=1&mtas.stats.spans.0.function.0.expression=%24q0&mtas.stats.spans.0.function.0.key=de+-+absolute&mtas.stats.spans.0.function.0.type=n%2Csum&mtas.stats.spans.0.function.1.expression=%24q1&mtas.stats.spans.0.function.1.key=het+-+absolute&mtas.stats.spans.0.function.1.type=n%2Csum&mtas.stats.spans.0.function.2.expression=%24q2&mtas.stats.spans.0.function.2.key=een+-+absolute&mtas.stats.spans.0.function.2.type=n%2Csum&mtas.stats.spans.0.function.3.expression=%24q3&mtas.stats.spans.0.function.3.key=LID+-+absolute&mtas.stats.spans.0.function.3.type=n%2Csum&mtas.stats.spans.0.function.4.expression=%24q0%2F%24n&mtas.stats.spans.0.function.4.key=de+-+relative+to+positions&mtas.stats.spans.0.function.4.type=n%2Cmean&mtas.stats.spans.0.function.5.expression=%24q1%2F%24n&mtas.stats.spans.0.function.5.key=het+-+relative+to+positions&mtas.stats.spans.0.function.5.type=n%2Cmean&mtas.stats.spans.0.function.6.expression=%24q2%2F%24n&mtas.stats.spans.0.function.6.key=een+-+relative+to+positions&mtas.stats.spans.0.function.6.type=n%2Cmean&mtas.stats.spans.0.function.7.expression=%24q3%2F%24n&mtas.stats.spans.0.function.7.key=LID+-+relative+to+positions&mtas.stats.spans.0.function.7.type=n%2Cmean&mtas.stats.spans.0.function.8.expression=%24q0%2F%24q3&mtas.stats.spans.0.function.8.key=de+-+relative+to+LID&mtas.stats.spans.0.function.8.type=n%2Cmean&mtas.stats.spans.0.function.9.expression=%24q1%2F%24q3&mtas.stats.spans.0.function.9.key=het+-+relative+to+LID&mtas.stats.spans.0.function.9.type=n%2Cmean&mtas.stats.spans.0.function.10.expression=%24q2%2F%24q3&mtas.stats.spans.0.function.10.key=een+-+relative+to+LID&mtas.stats.spans.0.function.10.type=n%2Cmean&mtas.stats.spans.0.function.11.expression=(%24q0%2B%24q1%2B%24q2)%2F%24q3&mtas.stats.spans.0.function.11.key=de%2Bhet%2Been+-+relative+to+LID&mtas.stats.spans.0.function.11.type=n%2Cmean&rows=0&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "spans":[{
+ "key":"multiple and functions - de+het+een and LID",
+ "n":1889646,
+ "functions":{
+ "een - relative to LID":{
+ "mean":0.26176457027078637,
+ "errorList":{"division by zero":24165},
+ "n":1889646,
+ "errorNumber":24165},
+ "LID - absolute":{
+ "sum":44062088,
+ "n":1889646},
+ "de+het+een - relative to LID":{
+ "mean":1.0864042218694616,
+ "errorList":{"division by zero":24165},
+ "n":1889646,
+ "errorNumber":24165},
+ "het - relative to LID":{
+ "mean":0.27408299432800154,
+ "errorList":{"division by zero":24165},
+ "n":1889646,
+ "errorNumber":24165},
+ "een - relative to positions":{
+ "mean":0.014397108127121947,
+ "errorList":{"division by zero":631875},
+ "n":1889646,
+ "errorNumber":631875},
+ "een - absolute":{
+ "sum":10616743,
+ "n":1889646},
+ "het - relative to positions":{
+ "mean":0.014874612933292992,
+ "errorList":{"division by zero":631875},
+ "n":1889646,
+ "errorNumber":631875},
+ "de - absolute":{
+ "sum":25496044,
+ "n":1889646},
+ "het - absolute":{
+ "sum":11527080,
+ "n":1889646},
+ "LID - relative to positions":{
+ "mean":0.05786145893684233,
+ "errorList":{"division by zero":631875},
+ "n":1889646,
+ "errorNumber":631875},
+ "de - relative to LID":{
+ "mean":0.5505566572707496,
+ "errorList":{"division by zero":24165},
+ "n":1889646,
+ "errorNumber":24165},
+ "de - relative to positions":{
+ "mean":0.03299544495178249,
+ "errorList":{"division by zero":631875},
+ "n":1889646,
+ "errorNumber":631875}}}]}}
+```
diff --git a/src/site/markdown/search_query_stats_tokens.md b/src/site/markdown/search_query_stats_tokens.md
new file mode 100644
index 0000000..c1b8d42
--- /dev/null
+++ b/src/site/markdown/search_query_stats_tokens.md
@@ -0,0 +1,136 @@
+# Statistics - tokens
+
+To produce statistics on the number of tokens within a set of documents, besides the parameter to enable [statistics](search_query_stats.html), the following parameter should be provided.
+
+| Parameter | Value | Obligatory |
+|-----------------------|--------|-------------|
+| mtas.stats.tokens | true | yes |
+
+Multiple statistics on tokens can be produced within the same request.
+To distinguish them, a unique identifier has to be provided for
+each of the required statistics.
+
+| Parameter | Value | Info | Obligatory |
+|-------------------------------------------------|--------------|--------------------------------|-------------|
+| mtas.stats.tokens.\<identifier\>.key | \<string\> | key used in response | no |
+| mtas.stats.tokens.\<identifier\>.field | \<string\> | mtas field | yes |
+| mtas.stats.tokens.\<identifier\>.type | \<string\> | required [type of statistics](search_stats.html) | no |
+| mtas.stats.tokens.\<identifier\>.minimum | \<double\> | minimum number of tokens | no |
+| mtas.stats.tokens.\<identifier\>.maximum | \<double\> | maximum number of tokens | no |
+
+The *key* is added to the response and may be used to distinguish between multiple statistics on tokens, and should therefore be unique. The optional *minimum* and *maximum* can be used to focus only on documents satisfying a condition on the number of tokens.
+
+---
+
+## Examples
+1. [Basic](#basic) : basic statistics on the number of tokens.
+2. [Minimum and maximum](#minimum-and-maximum) : statistics on the number of tokens with restrictions on this number.
+3. [Subset](#subset) : statistics on the number of tokens within a subset of documents.
+
+---
+
+<a name="basic"></a>
+
+### Basic
+
+**Example**
+Total and average number of tokens and the number of documents.
+
+**Request and response**
+`q=*%3A*&rows=0&mtas=true&mtas.stats=true&mtas.stats.tokens=true&mtas.stats.tokens.0.field=text&mtas.stats.tokens.0.key=example - basic&mtas.stats.tokens.0.type=sum,mean,n&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "tokens":[{
+ "key":"example - basic",
+ "mean":2208.617131956095,
+ "sum":4560370323,
+ "n":2064808}]}}
+```
+
+<a name="minimum-and-maximum"></a>
+
+### Minimum and maximum
+
+**Example**
+Full statistics on tokens for documents with a minimum of 100 tokens, for documents with a maximum of 200 tokens, and for documents with between 100 and 200 tokens.
+
+**Request and response**
+`q=*%3A*&rows=0&mtas=true&mtas.stats=true&mtas.stats.tokens=true&mtas.stats.tokens.0.field=text&mtas.stats.tokens.0.key=example - minimum&mtas.stats.tokens.0.type=all&mtas.stats.tokens.0.minimum=500&mtas.stats.tokens.1.field=text&mtas.stats.tokens.1.key=example - maximum&mtas.stats.tokens.1.type=all&mtas.stats.tokens.1.maximum=1000&mtas.stats.tokens.2.field=text&mtas.stats.tokens.2.key=example - minimum and maximum&mtas.stats.tokens.2.type=all&mtas.stats.tokens.2.minimum=500&mtas.stats.tokens.2.maximum=1000&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "tokens":[{
+ "key":"example - minimum",
+ "sumsq":6.17630129413745E14,
+ "populationvariance":5.2902986678636354E8,
+ "max":5626223.0,
+ "sum":4.489695699E9,
+ "kurtosis":11397.708386174421,
+ "standarddeviation":23000.659411322642,
+ "n":1133873,
+ "quadraticmean":23338.98849418418,
+ "min":500.0,
+ "median":1788.0,
+ "variance":5.290303333556648E8,
+ "mean":3959.610731536747,
+ "geometricmean":2073.1829663657477,
+ "sumoflogs":8659207.101050543,
+ "skewness":86.80974688421588},
+ {
+ "key":"example - maximum",
+ "sumsq":1.88841931218E11,
+ "populationvariance":100404.89269776225,
+ "max":1000.0,
+ "sum":2.84511144E8,
+ "kurtosis":-0.5052359496974037,
+ "standarddeviation":316.8674407373186,
+ "n":1219963,
+ "quadraticmean":393.4376224488639,
+ "min":0.0,
+ "median":0.0,
+ "variance":100404.97499941812,
+ "mean":233.2129285891795,
+ "geometricmean":0.0,
+ "sumoflogs":"-Infinity",
+ "skewness":0.9900439471758536},
+ {
+ "key":"example - minimum and maximum",
+ "sumsq":1.640713236E11,
+ "populationvariance":20292.151766107923,
+ "max":1000.0,
+ "sum":2.1383652E8,
+ "kurtosis":-1.1536799879589004,
+ "standarddeviation":142.45077035455677,
+ "n":289028,
+ "quadraticmean":753.4360252467811,
+ "min":500.0,
+ "median":734.0,
+ "variance":20292.221974606666,
+ "mean":739.8470736399236,
+ "geometricmean":725.9419502039267,
+ "sumoflogs":1903963.2945452952,
+ "skewness":0.08906106230519327}]}}
+```
+
+<a name="subset"></a>
+
+### Subset
+
+**Example**
+Total and average number of tokens and the number of documents for a subset of documents.
+
+**Request and response**
+`q=text:koe&rows=0&mtas=true&mtas.stats=true&mtas.stats.tokens=true&mtas.stats.tokens.0.field=text&mtas.stats.tokens.0.key=example - subset&mtas.stats.tokens.0.type=sum,mean,n&wt=json&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "tokens":[{
+ "key":"example - subset",
+ "mean":49644.49556868538,
+ "sum":134437294,
+ "n":2708}]}}
+```
diff --git a/src/site/markdown/search_query_termvector.md b/src/site/markdown/search_query_termvector.md
new file mode 100644
index 0000000..a94bc2b
--- /dev/null
+++ b/src/site/markdown/search_query_termvector.md
@@ -0,0 +1,5 @@
+#Query - termvector
+
+
+
+
diff --git a/src/site/markdown/search_sharding.md b/src/site/markdown/search_sharding.md
new file mode 100644
index 0000000..ad8f5fe
--- /dev/null
+++ b/src/site/markdown/search_sharding.md
@@ -0,0 +1,30 @@
+#Sharding
+
+All [mtas queries](search_query.html) support sharding.
+
+**Example**
+
+Query simultaneously for the distribution of the word "de" on two cores
+
+**Request and response**
+
+`q=*%3A*&mtas=true&mtas.stats=true&mtas.stats.spans=true&mtas.stats.spans.0.field=text&mtas.stats.spans.0.query.0.type=cql&mtas.stats.spans.0.query.0.value=[t%3D"de"]&mtas.stats.spans.0.key=example+-+sharding&mtas.stats.spans.0.type=distribution()&rows=0&wt=json&shards=localhost%3A8080%2Fsolr%2Fcore1%2F%2Clocalhost%3A8080%2Fsolr%2Fcore2%2F%2C&indent=true`
+
+``` json
+"mtas":{
+ "stats":{
+ "spans":[{
+ "key":"example - sharding",
+ "distribution()":{
+ "[0,1819]":2064420,
+ "[1820,3639]":224,
+ "[3640,5459]":103,
+ "[5460,7279]":42,
+ "[7280,9099]":10,
+ "[9100,10919]":1,
+ "[10920,12739]":3,
+ "[12740,14559]":1,
+ "[14560,16379]":2,
+ "[16380,18199]":2}}]}}
+```
+
diff --git a/src/site/markdown/search_stats.md b/src/site/markdown/search_stats.md
new file mode 100644
index 0000000..c9bfca8
--- /dev/null
+++ b/src/site/markdown/search_stats.md
@@ -0,0 +1,58 @@
+#Type of statistics
+
+Mtas can produce several type of statistics, e.g. for [positions](search_query_stats_positions.html),
+[tokens](search_query_stats_tokens.html) or [spans](search_query_stats_spans.html).
+
+In general, statistics of type *basic* will require less resources than statistics of type *advanced*, whereas
+statistics of type *advanced* will require less than these of type *full*. If multiple
+statistics are possible and required, the parameters can be seperated with a comma.
+
+
+| Parameter | Description | Type |
+|-----------------------|----------------------|-----------|
+| n | number of documents | basic |
+| sum | total | basic |
+| mean | mean | basic |
+| min | minimum | advanced |
+| max | maximum | advanced |
+| sumsq | sum of squares | advanced |
+| sumoflogs | sum of logs | advanced |
+| geometricmean | geometric mean | advanced |
+| quadraticmean | quadratic mean | advanced |
+| variance | variance | advanced |
+| populationvariance | population variance | advanced |
+| standarddeviation | standard deviation | advanced |
+| skewness | skewness | full |
+| median | median | full |
+| kurtosis | kurtosis | full |
+| all | all of the above | full |
+
+**Examples**
+
+``` console
+n,sum
+mean,standarddeviation
+```
+
+When not obligatory, usually \"*n,sum,mean*\" will be the default.
+
+## Distribution
+
+Besides these parameters, a distribution can requested with (optional) arguments,
+optionally combined with one or more of the parameters listed above.
+
+| Parameter | Type | Arguments | Description | Obligatory |
+|----------------|------|--------------|---------------------|------------|
+| distribution() | full | start | start value | no |
+| | | end | end value | no |
+| | | step | size of intervals | no |
+| | | number | number of intervals | no |
+
+**Examples**
+
+``` console
+distribution()
+distribution(start=0,end=1,number=10)
+mean,standarddeviation,median,distribution(start=0,end=1,number=10)
+```
+
diff --git a/src/site/resources/images/meertens.png b/src/site/resources/images/meertens.png
new file mode 100644
index 0000000..beffe72
--- /dev/null
+++ b/src/site/resources/images/meertens.png
diff --git a/src/site/site.xml b/src/site/site.xml
index 6930bee..387f8ad 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -1,8 +1,58 @@
<project name="Multi Tier Annotation Search">
+ <publishDate position="right" />
+ <version position="left" />
+ <bannerRight>
+ <name>Meertens Instituut</name>
+ <src>images/meertens.png</src>
+ <href>http://www.meertens.knaw.nl/</href>
+ <width>147</width>
+ <height>100</height>
+ <title>Meertens Instituut</title>
+ </bannerRight>
+ <skin>
+ <groupId>org.apache.maven.skins</groupId>
+ <artifactId>maven-fluido-skin</artifactId>
+ <version>1.5</version>
+ </skin>
<body>
+ <breadcrumbs>
+ <item name="Mtas" href="index.html"/>
+ </breadcrumbs>
<menu name="Mtas">
<item name="Introduction" href="index.html"/>
- <item name="Installation" href="installation.html"/>
+ <item name="Features" href="features.html"/>
+ <item name="Getting started" href="installation.html" collapse="true">
+ <item name="Lucene" href="installation_lucene.html"/>
+ <item name="Solr" href="installation_solr.html"/>
+ </item>
+ <item name="Indexing" href="indexing.html" collapse="true">
+ <item name="Configuration" href="indexing_configuration.html"/>
+ <item name="Formats" href="indexing_formats.html">
+ <item name="FoLiA" href="indexing_formats_folia.html" collapse="true"/>
+ <item name="TEI" href="indexing_formats_tei.html" collapse="true"/>
+ <item name="Sketch" href="indexing_formats_sketch.html" collapse="true"/>
+ <item name="CRM" href="indexing_formats_crm.html" collapse="true"/>
+ </item>
+ </item>
+ <item name="Search" href="search.html" collapse="true">
+ <item name="Configuration" href="search_configuration.html"/>
+ <item name="Query" href="search_query.html">
+ <item name="Statistics" href="search_query_stats.html" collapse="true">
+ <item name="Positions" href="search_query_stats_positions.html"/>
+ <item name="Tokens" href="search_query_stats_tokens.html"/>
+ <item name="Spans" href="search_query_stats_spans.html"/>
+ </item>
+ <item name="Kwic/List" href="search_query_kwic_and_list.html"/>
+ <item name="Termvector" href="search_query_termvector.html"/>
+ <item name="Facet" href="search_query_facet.html"/>
+ <item name="Group" href="search_query_group.html"/>
+ <item name="Prefix" href="search_query_prefix.html"/>
+ </item>
+ <item name="Type of statistics" href="search_stats.html"/>
+ <item name="Functions" href="search_functions.html"/>
+ <item name="CQL" href="search_cql.html"/>
+ <item name="Sharding" href="search_sharding.html"/>
+ </item>
<item name="Download" href="download.html"/>
</menu>
<menu ref="reports"/>