Commit ce08df366db74f17ee7502a57acaaf2e88fa9387

Authored by Matthijs Brouwer
1 parent 939c66ba

update

Showing 30 changed files with 2349 additions and 1149 deletions

Too many changes to show.

To preserve performance only 23 of 30 files are displayed.

junit/mtas/parser/MtasCQLParserTestSentence.java
... ... @@ -26,14 +26,19 @@ public class MtasCQLParserTestSentence {
26 26  
27 27 @org.junit.Test
28 28 public void test() {
29   - basicTests();
  29 + try {
  30 + basicTests();
  31 + } catch (ParseException e) {
  32 + // TODO Auto-generated catch block
  33 + e.printStackTrace();
  34 + }
30 35 }
31 36  
32 37 private void testCQLParse(String field, String defaultPrefix, String cql, SpanQuery q) {
33 38 MtasCQLParser p = new MtasCQLParser(new BufferedReader(new StringReader(cql)));
34 39 try {
35 40 System.out.print("CQL parsing:\t"+cql);
36   - assertEquals(p.parse(field, defaultPrefix) ,q);
  41 + assertEquals(p.parse(field, defaultPrefix, null) ,q);
37 42 System.out.print("\n");
38 43 } catch (ParseException e) {
39 44 System.out.println("Error CQL parsing:\t"+cql);
... ... @@ -46,7 +51,7 @@ public class MtasCQLParserTestSentence {
46 51 MtasCQLParser p2 = new MtasCQLParser(new BufferedReader(new StringReader(cql2)));
47 52 try {
48 53 System.out.print("CQL equivalent:\t"+cql1+" and "+cql2);
49   - assertEquals(p1.parse(field, defaultPrefix) ,p2.parse(field, defaultPrefix));
  54 + assertEquals(p1.parse(field, defaultPrefix, null) ,p2.parse(field, defaultPrefix, null));
50 55 System.out.print("\n");
51 56 } catch (ParseException e) {
52 57 System.out.println("Error CQL equivalent:\t"+cql1+" and "+cql2);
... ... @@ -54,7 +59,7 @@ public class MtasCQLParserTestSentence {
54 59 }
55 60 }
56 61  
57   - private void basicTests() {
  62 + private void basicTests() throws ParseException {
58 63 basicTest1();
59 64 basicTest2();
60 65 basicTest3();
... ... @@ -76,11 +81,11 @@ public class MtasCQLParserTestSentence {
76 81 basicTest19();
77 82 }
78 83  
79   - private void basicTest1() {
  84 + private void basicTest1() throws ParseException {
80 85 String field = "testveld";
81 86 String cql = "[pos=\"LID\"] [lemma=\"koe\"]";
82   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID");
83   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe");
  87 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID", null, null);
  88 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe", null, null);
84 89 List<MtasSpanSequenceItem> items = new ArrayList<MtasSpanSequenceItem>();
85 90 items.add(new MtasSpanSequenceItem(q1, false));
86 91 items.add(new MtasSpanSequenceItem(q2, false));
... ... @@ -95,21 +100,21 @@ public class MtasCQLParserTestSentence {
95 100 testCQLEquivalent(field, null, cql1, cql2);
96 101 }
97 102  
98   - private void basicTest3() {
  103 + private void basicTest3() throws ParseException {
99 104 String field = "testveld";
100 105 String cql = "[pos=\"LID\"] | [lemma=\"koe\"]";
101   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID");
102   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe");
  106 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID", null, null);
  107 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe", null, null);
103 108 SpanQuery q = new MtasSpanOrQuery(q1,q2);
104 109 testCQLParse(field, null, cql, q);
105 110 }
106 111  
107   - private void basicTest4() {
  112 + private void basicTest4() throws ParseException {
108 113 String field = "testveld";
109 114 String cql = "[pos=\"LID\"] | ([lemma=\"de\"] [lemma=\"koe\"])";
110   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID");
111   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","de");
112   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","koe");
  115 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID", null, null);
  116 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","de", null, null);
  117 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","koe", null, null);
113 118 List<MtasSpanSequenceItem> items = new ArrayList<MtasSpanSequenceItem>();
114 119 items.add(new MtasSpanSequenceItem(q2, false));
115 120 items.add(new MtasSpanSequenceItem(q3, false));
... ... @@ -139,12 +144,12 @@ public class MtasCQLParserTestSentence {
139 144 testCQLEquivalent(field, null, cql1, cql2);
140 145 }
141 146  
142   - private void basicTest8() {
  147 + private void basicTest8() throws ParseException {
143 148 String field = "testveld";
144 149 String cql = "[lemma=\"koe\"]([pos=\"N\"]|[pos=\"ADJ\"])";
145   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe");
146   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","N");
147   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"pos","ADJ");
  150 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe", null, null);
  151 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","N", null, null);
  152 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"pos","ADJ", null, null);
148 153 SpanQuery q4 = new MtasSpanOrQuery(q2,q3);
149 154 List<MtasSpanSequenceItem> items = new ArrayList<MtasSpanSequenceItem>();
150 155 items.add(new MtasSpanSequenceItem(q1, false));
... ... @@ -153,13 +158,13 @@ public class MtasCQLParserTestSentence {
153 158 testCQLParse(field, null, cql, q);
154 159 }
155 160  
156   - private void basicTest9() {
  161 + private void basicTest9() throws ParseException {
157 162 String field = "testveld";
158 163 String cql = "[lemma=\"koe\"]([pos=\"N\"]|[pos=\"ADJ\"]){2,3}[lemma=\"paard\"]";
159   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe");
160   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","N");
161   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"pos","ADJ");
162   - SpanQuery q4 = new MtasCQLParserWordQuery(field,"lemma","paard");
  164 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe", null, null);
  165 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","N", null, null);
  166 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"pos","ADJ", null, null);
  167 + SpanQuery q4 = new MtasCQLParserWordQuery(field,"lemma","paard",null, null);
163 168 SpanQuery q5 = new MtasSpanOrQuery(new MtasSpanRecurrenceQuery(q2,2,3),new MtasSpanRecurrenceQuery(q3,2,3));
164 169 List<MtasSpanSequenceItem> items = new ArrayList<MtasSpanSequenceItem>();
165 170 items.add(new MtasSpanSequenceItem(q1, false));
... ... @@ -169,12 +174,12 @@ public class MtasCQLParserTestSentence {
169 174 testCQLParse(field, null, cql, q);
170 175 }
171 176  
172   - private void basicTest10() {
  177 + private void basicTest10() throws ParseException {
173 178 String field = "testveld";
174 179 String cql = "[pos=\"LID\"]? [pos=\"ADJ\"]{1,3} [lemma=\"koe\"]";
175   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID");
176   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","ADJ");
177   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","koe");
  180 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID",null, null);
  181 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","ADJ",null, null);
  182 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
178 183 List<MtasSpanSequenceItem> items = new ArrayList<MtasSpanSequenceItem>();
179 184 items.add(new MtasSpanSequenceItem(q1, true));
180 185 items.add(new MtasSpanSequenceItem(new MtasSpanRecurrenceQuery(q2,1,3), false));
... ... @@ -183,29 +188,29 @@ public class MtasCQLParserTestSentence {
183 188 testCQLParse(field, null, cql, q);
184 189 }
185 190  
186   - private void basicTest11() {
  191 + private void basicTest11() throws ParseException {
187 192 String field = "testveld";
188 193 String cql = "<sentence/> containing [lemma=\"koe\"]";
189 194 SpanQuery q1 = new MtasCQLParserGroupQuery(field,"sentence");
190   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe");
  195 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
191 196 SpanQuery q = new SpanContainingQuery(q1, q2);
192 197 testCQLParse(field, null, cql, q);
193 198 }
194 199  
195   - private void basicTest12() {
  200 + private void basicTest12() throws ParseException {
196 201 String field = "testveld";
197 202 String cql = "[lemma=\"koe\"] within <sentence/>";
198   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe");
  203 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
199 204 SpanQuery q2 = new MtasCQLParserGroupQuery(field,"sentence");
200 205 SpanQuery q = new SpanWithinQuery(q2, q1);
201 206 testCQLParse(field, null, cql, q);
202 207 }
203 208  
204   - private void basicTest13() {
  209 + private void basicTest13() throws ParseException {
205 210 String field = "testveld";
206 211 String cql = "[lemma=\"koe\"]([t=\"de\"] within <sentence/>)";
207   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe");
208   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"t","de");
  212 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
  213 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"t","de",null, null);
209 214 SpanQuery q3 = new MtasCQLParserGroupQuery(field,"sentence");
210 215 SpanQuery q4 = new SpanWithinQuery(q3, q2);
211 216 List<MtasSpanSequenceItem> items = new ArrayList<MtasSpanSequenceItem>();
... ... @@ -215,13 +220,13 @@ public class MtasCQLParserTestSentence {
215 220 testCQLParse(field, null, cql, q);
216 221 }
217 222  
218   - private void basicTest14() {
  223 + private void basicTest14() throws ParseException {
219 224 String field = "testveld";
220 225 String cql = "([t=\"de\"] within <sentence/>)[lemma=\"koe\"]";
221   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"t","de");
  226 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"t","de",null, null);
222 227 SpanQuery q2 = new MtasCQLParserGroupQuery(field,"sentence");
223 228 SpanQuery q3 = new SpanWithinQuery(q2, q1);
224   - SpanQuery q4 = new MtasCQLParserWordQuery(field,"lemma","koe");
  229 + SpanQuery q4 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
225 230 List<MtasSpanSequenceItem> items = new ArrayList<MtasSpanSequenceItem>();
226 231 items.add(new MtasSpanSequenceItem(q3, false));
227 232 items.add(new MtasSpanSequenceItem(q4, false));
... ... @@ -229,15 +234,15 @@ public class MtasCQLParserTestSentence {
229 234 testCQLParse(field, null, cql, q);
230 235 }
231 236  
232   - private void basicTest15() {
  237 + private void basicTest15() throws ParseException {
233 238 String field = "testveld";
234 239 String cql = "[lemma=\"koe\"](<sentence/> containing [t=\"de\"]) within <sentence/>[lemma=\"paard\"]";
235   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe");
  240 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
236 241 SpanQuery q2 = new MtasCQLParserGroupQuery(field,"sentence");
237   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"t","de");
  242 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"t","de",null, null);
238 243 SpanQuery q4 = new SpanContainingQuery(q2, q3);
239 244 SpanQuery q5 = new MtasCQLParserGroupQuery(field,"sentence");
240   - SpanQuery q6 = new MtasCQLParserWordQuery(field,"lemma","paard");
  245 + SpanQuery q6 = new MtasCQLParserWordQuery(field,"lemma","paard",null, null);
241 246 List<MtasSpanSequenceItem> items1 = new ArrayList<MtasSpanSequenceItem>();
242 247 items1.add(new MtasSpanSequenceItem(q5, false));
243 248 items1.add(new MtasSpanSequenceItem(q6, false));
... ... @@ -250,12 +255,12 @@ public class MtasCQLParserTestSentence {
250 255 testCQLParse(field, null, cql, q);
251 256 }
252 257  
253   - private void basicTest16() {
  258 + private void basicTest16() throws ParseException {
254 259 String field = "testveld";
255 260 String cql = "(<entity=\"loc\"/> within (<s/> containing [t_lc=\"amsterdam\"])) !containing ([t_lc=\"amsterdam\"])";
256 261 SpanQuery q1 = new MtasCQLParserGroupQuery(field,"entity","loc");
257 262 SpanQuery q2 = new MtasCQLParserGroupQuery(field,"s");
258   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"t_lc","amsterdam");
  263 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"t_lc","amsterdam",null, null);
259 264 SpanQuery q4 = new SpanContainingQuery(q2, q3);
260 265 SpanQuery q5 = new SpanWithinQuery(q4, q1);
261 266 SpanQuery q = new SpanNotQuery(q5,new SpanContainingQuery(q5, q3));
... ... @@ -275,11 +280,11 @@ public class MtasCQLParserTestSentence {
275 280 testCQLParse(field, null, cql, q);
276 281 }
277 282  
278   - private void basicTest18() {
  283 + private void basicTest18() throws ParseException {
279 284 String field = "testveld";
280 285 String cql = "\"de\" [pos=\"N\"]";
281   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"t_lc","de");
282   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","N");
  286 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"t_lc","de",null, null);
  287 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","N",null, null);
283 288 List<MtasSpanSequenceItem> items = new ArrayList<MtasSpanSequenceItem>();
284 289 items.add(new MtasSpanSequenceItem(q1, false));
285 290 items.add(new MtasSpanSequenceItem(q2, false));
... ...
junit/mtas/parser/MtasCQLParserTestWord.java
... ... @@ -19,14 +19,19 @@ public class MtasCQLParserTestWord {
19 19  
20 20 @org.junit.Test
21 21 public void test() {
22   - basicTests();
23   - basicNotTests();
  22 + try {
  23 + basicTests();
  24 + basicNotTests();
  25 + } catch (ParseException e) {
  26 + // TODO Auto-generated catch block
  27 + e.printStackTrace();
  28 + }
24 29 }
25 30  
26 31 private void testCQLParse(String field, String defaultPrefix, String cql, SpanQuery q) {
27 32 MtasCQLParser p = new MtasCQLParser(new BufferedReader(new StringReader(cql)));
28 33 try {
29   - assertEquals(p.parse(field, defaultPrefix) ,q);
  34 + assertEquals(p.parse(field, defaultPrefix, null) ,q);
30 35 System.out.println("Tested CQL parsing:\t"+cql);
31 36 } catch (ParseException e) {
32 37 System.out.println("Error CQL parsing:\t"+cql);
... ... @@ -38,7 +43,7 @@ public class MtasCQLParserTestWord {
38 43 MtasCQLParser p1 = new MtasCQLParser(new BufferedReader(new StringReader(cql1)));
39 44 MtasCQLParser p2 = new MtasCQLParser(new BufferedReader(new StringReader(cql2)));
40 45 try {
41   - assertEquals(p1.parse(field, defaultPrefix) ,p2.parse(field, defaultPrefix));
  46 + assertEquals(p1.parse(field, defaultPrefix,null) ,p2.parse(field, defaultPrefix, null));
42 47 System.out.println("Tested CQL equivalent:\t"+cql1+" and "+cql2);
43 48 } catch (ParseException e) {
44 49 System.out.println("Error CQL equivalent:\t"+cql1+" and "+cql2);
... ... @@ -46,7 +51,7 @@ public class MtasCQLParserTestWord {
46 51 }
47 52 }
48 53  
49   - private void basicNotTests() {
  54 + private void basicNotTests() throws ParseException {
50 55 basicNotTest1();
51 56 basicNotTest2();
52 57 basicNotTest3();
... ... @@ -54,7 +59,7 @@ public class MtasCQLParserTestWord {
54 59 basicNotTest5();
55 60 }
56 61  
57   - private void basicTests() {
  62 + private void basicTests() throws ParseException {
58 63 basicTest1();
59 64 basicTest2();
60 65 basicTest3();
... ... @@ -70,11 +75,11 @@ public class MtasCQLParserTestWord {
70 75 basicTest13();
71 76 }
72 77  
73   - private void basicNotTest1() {
  78 + private void basicNotTest1() throws ParseException {
74 79 String field = "testveld";
75 80 String cql = "[pos=\"LID\" & !lemma=\"de\"]";
76   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID");
77   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","de");
  81 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID",null, null);
  82 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","de",null, null);
78 83 SpanQuery q = new SpanNotQuery(q1,q2);
79 84 testCQLParse(field, null, cql, q);
80 85 }
... ... @@ -86,12 +91,12 @@ public class MtasCQLParserTestWord {
86 91 testCQLEquivalent(field, null, cql1, cql2);
87 92 }
88 93  
89   - private void basicNotTest3() {
  94 + private void basicNotTest3() throws ParseException {
90 95 String field = "testveld";
91 96 String cql = "[pos=\"LID\" & !(lemma=\"de\" | lemma=\"een\")]";
92   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID");
93   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","de");
94   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","een");
  97 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID",null, null);
  98 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","de",null, null);
  99 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","een",null, null);
95 100 SpanQuery q4 = new MtasSpanOrQuery(new SpanQuery[]{q2,q3});
96 101 SpanQuery q = new SpanNotQuery(q1,q4);
97 102 testCQLParse(field, null, cql, q);
... ... @@ -111,27 +116,27 @@ public class MtasCQLParserTestWord {
111 116 testCQLEquivalent(field, null, cql1, cql2);
112 117 }
113 118  
114   - private void basicTest1() {
  119 + private void basicTest1() throws ParseException {
115 120 String field = "testveld";
116 121 String cql = "[lemma=\"koe\"]";
117   - SpanQuery q = new MtasCQLParserWordQuery(field, "lemma", "koe");
  122 + SpanQuery q = new MtasCQLParserWordQuery(field, "lemma", "koe",null, null);
118 123 testCQLParse(field, null, cql, q);
119 124 }
120 125  
121   - private void basicTest2() {
  126 + private void basicTest2() throws ParseException {
122 127 String field = "testveld";
123 128 String cql = "[lemma=\"koe\" & pos=\"N\"]";
124   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe");
125   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","N");
  129 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
  130 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","N",null, null);
126 131 SpanQuery q = new MtasSpanAndQuery(new SpanQuery[]{q1,q2});
127 132 testCQLParse(field, null, cql, q);
128 133 }
129 134  
130   - private void basicTest3() {
  135 + private void basicTest3() throws ParseException {
131 136 String field = "testveld";
132 137 String cql = "[lemma=\"koe\" | lemma=\"paard\"]";
133   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe");
134   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","paard");
  138 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
  139 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","paard",null, null);
135 140 SpanQuery q = new MtasSpanOrQuery(new SpanQuery[]{q1,q2});
136 141 testCQLParse(field, null, cql, q);
137 142 }
... ... @@ -143,60 +148,60 @@ public class MtasCQLParserTestWord {
143 148 testCQLEquivalent(field, null, cql1, cql2);
144 149 }
145 150  
146   - private void basicTest5() {
  151 + private void basicTest5() throws ParseException {
147 152 String field = "testveld";
148 153 String cql = "[(lemma=\"koe\" | lemma=\"paard\") & pos=\"N\"]";
149   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe");
150   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","paard");
  154 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
  155 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","paard",null, null);
151 156 SpanQuery q3 = new MtasSpanOrQuery(new SpanQuery[]{q1,q2});
152   - SpanQuery q4 = new MtasCQLParserWordQuery(field,"pos","N");
  157 + SpanQuery q4 = new MtasCQLParserWordQuery(field,"pos","N",null, null);
153 158 SpanQuery q = new MtasSpanAndQuery(new SpanQuery[]{q3,q4});
154 159 testCQLParse(field, null, cql, q);
155 160 }
156 161  
157   - private void basicTest6() {
  162 + private void basicTest6() throws ParseException {
158 163 String field = "testveld";
159 164 String cql = "[pos=\"N\" & (lemma=\"koe\" | lemma=\"paard\")]";
160   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","N");
161   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe");
162   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","paard");
  165 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","N",null, null);
  166 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
  167 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","paard",null, null);
163 168 SpanQuery q4 = new MtasSpanOrQuery(new SpanQuery[]{q2,q3});
164 169 SpanQuery q = new MtasSpanAndQuery(new SpanQuery[]{q1,q4});
165 170 testCQLParse(field, null, cql, q);
166 171 }
167 172  
168   - private void basicTest7() {
  173 + private void basicTest7() throws ParseException {
169 174 String field = "testveld";
170 175 String cql = "[pos=\"LID\" | (lemma=\"koe\" & pos=\"N\")]";
171   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID");
172   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe");
173   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"pos","N");
  176 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"pos","LID",null, null);
  177 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
  178 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"pos","N",null, null);
174 179 SpanQuery q4 = new MtasSpanAndQuery(new SpanQuery[]{q2,q3});
175 180 SpanQuery q = new MtasSpanOrQuery(new SpanQuery[]{q1,q4});
176 181 testCQLParse(field, null, cql, q);
177 182 }
178 183  
179   - private void basicTest8() {
  184 + private void basicTest8() throws ParseException {
180 185 String field = "testveld";
181 186 String cql = "[(lemma=\"de\" & pos=\"LID\") | (lemma=\"koe\" & pos=\"N\")]";
182   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","de");
183   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","LID");
184   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","koe");
185   - SpanQuery q4 = new MtasCQLParserWordQuery(field,"pos","N");
  187 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","de",null, null);
  188 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"pos","LID",null, null);
  189 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
  190 + SpanQuery q4 = new MtasCQLParserWordQuery(field,"pos","N",null, null);
186 191 SpanQuery q5 = new MtasSpanAndQuery(new SpanQuery[]{q1,q2});
187 192 SpanQuery q6 = new MtasSpanAndQuery(new SpanQuery[]{q3,q4});
188 193 SpanQuery q = new MtasSpanOrQuery(new SpanQuery[]{q5,q6});
189 194 testCQLParse(field, null, cql, q);
190 195 }
191 196  
192   - private void basicTest9() {
  197 + private void basicTest9() throws ParseException {
193 198 String field = "testveld";
194 199 String cql = "[((lemma=\"de\"|lemma=\"het\") & pos=\"LID\") | (lemma=\"koe\" & pos=\"N\")]";
195   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","de");
196   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","het");
197   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"pos","LID");
198   - SpanQuery q4 = new MtasCQLParserWordQuery(field,"lemma","koe");
199   - SpanQuery q5 = new MtasCQLParserWordQuery(field,"pos","N");
  200 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","de",null, null);
  201 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","het",null, null);
  202 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"pos","LID",null, null);
  203 + SpanQuery q4 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
  204 + SpanQuery q5 = new MtasCQLParserWordQuery(field,"pos","N",null, null);
200 205 SpanQuery q6 = new MtasSpanOrQuery(new SpanQuery[]{q1,q2});
201 206 SpanQuery q7 = new MtasSpanAndQuery(new SpanQuery[]{q6,q3});
202 207 SpanQuery q8 = new MtasSpanAndQuery(new SpanQuery[]{q4,q5});
... ... @@ -204,15 +209,15 @@ public class MtasCQLParserTestWord {
204 209 testCQLParse(field, null, cql, q);
205 210 }
206 211  
207   - private void basicTest10() {
  212 + private void basicTest10() throws ParseException {
208 213 String field = "testveld";
209 214 String cql = "[((lemma=\"de\"|lemma=\"het\") & pos=\"LID\") | ((lemma=\"koe\"|lemma=\"paard\") & pos=\"N\")]";
210   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","de");
211   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","het");
212   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"pos","LID");
213   - SpanQuery q4 = new MtasCQLParserWordQuery(field,"lemma","koe");
214   - SpanQuery q5 = new MtasCQLParserWordQuery(field,"lemma","paard");
215   - SpanQuery q6 = new MtasCQLParserWordQuery(field,"pos","N");
  215 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"lemma","de",null, null);
  216 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"lemma","het",null, null);
  217 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"pos","LID",null, null);
  218 + SpanQuery q4 = new MtasCQLParserWordQuery(field,"lemma","koe",null, null);
  219 + SpanQuery q5 = new MtasCQLParserWordQuery(field,"lemma","paard",null, null);
  220 + SpanQuery q6 = new MtasCQLParserWordQuery(field,"pos","N",null, null);
216 221 SpanQuery q7 = new MtasSpanOrQuery(new SpanQuery[]{q1,q2});
217 222 SpanQuery q8 = new MtasSpanAndQuery(new SpanQuery[]{q7,q3});
218 223 SpanQuery q9 = new MtasSpanOrQuery(new SpanQuery[]{q4,q5});
... ... @@ -236,20 +241,20 @@ public class MtasCQLParserTestWord {
236 241 testCQLParse(field, null, cql3, q3);
237 242 }
238 243  
239   - private void basicTest12() {
  244 + private void basicTest12() throws ParseException {
240 245 String field = "testveld";
241 246 String cql = "[(t_lc=\"de\"|t_lc=\"het\"|t_lc=\"paard\")]";
242   - SpanQuery q1 = new MtasCQLParserWordQuery(field,"t_lc","de");
243   - SpanQuery q2 = new MtasCQLParserWordQuery(field,"t_lc","het");
244   - SpanQuery q3 = new MtasCQLParserWordQuery(field,"t_lc","paard");
  247 + SpanQuery q1 = new MtasCQLParserWordQuery(field,"t_lc","de",null, null);
  248 + SpanQuery q2 = new MtasCQLParserWordQuery(field,"t_lc","het",null, null);
  249 + SpanQuery q3 = new MtasCQLParserWordQuery(field,"t_lc","paard",null, null);
245 250 SpanQuery q = new MtasSpanOrQuery(new SpanQuery[]{q1,q2,q3});
246 251 testCQLParse(field, null, cql, q);
247 252 }
248 253  
249   - private void basicTest13() {
  254 + private void basicTest13() throws ParseException {
250 255 String field = "testveld";
251 256 String cql = "\"de\"";
252   - SpanQuery q = new MtasCQLParserWordQuery(field,"t_lc","de");
  257 + SpanQuery q = new MtasCQLParserWordQuery(field,"t_lc","de",null, null);
253 258 testCQLParse(field, "t_lc", cql, q);
254 259 }
255 260 }
... ...
... ... @@ -3,7 +3,7 @@
3 3 <properties>
4 4 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
5 5 <currentDevelopmentVersion>6.2.0</currentDevelopmentVersion>
6   - <currentDevelopmentRelease>20161006</currentDevelopmentRelease>
  6 + <currentDevelopmentRelease>20161024</currentDevelopmentRelease>
7 7 </properties>
8 8 <modelVersion>4.0.0</modelVersion>
9 9 <groupId>dev.meertens.mtas</groupId>
... ...
src/mtas/analysis/token/MtasToken.java
... ... @@ -698,9 +698,15 @@ public abstract class MtasToken&lt;GenericType&gt; {
698 698 * @return the list
699 699 * @throws IOException Signals that an I/O exception has occurred.
700 700 */
701   - public static List<CompiledAutomaton> createAutomata(String prefix,
  701 + public static List<CompiledAutomaton> createAutomata(String prefix, String regexp,
702 702 List<String> valueList) throws IOException {
703 703 List<CompiledAutomaton> list = new ArrayList<CompiledAutomaton>();
  704 + Automaton automatonRegexp = null;
  705 + if(regexp!=null) {
  706 + RegExp re = new RegExp(
  707 + prefix + MtasToken.DELIMITER + regexp + "\u0000*");
  708 + automatonRegexp = re.toAutomaton();
  709 + }
704 710 int step = 500;
705 711 for (int i = 0; i < valueList.size(); i += step) {
706 712 int localStep = step;
... ... @@ -718,7 +724,13 @@ public abstract class MtasToken&lt;GenericType&gt; {
718 724 (new RegExp(prefix + MtasToken.DELIMITER + value + "\u0000*"))
719 725 .toAutomaton());
720 726 }
721   - Automaton automaton = Operations.union(listAutomaton);
  727 + Automaton automatonList = Operations.union(listAutomaton);
  728 + Automaton automaton;
  729 + if(automatonRegexp!=null) {
  730 + automaton = Operations.intersection(automatonList, automatonRegexp);
  731 + } else {
  732 + automaton = automatonList;
  733 + }
722 734 try {
723 735 compiledAutomaton = new CompiledAutomaton(automaton);
724 736 } catch (TooComplexToDeterminizeException e) {
... ...
src/mtas/codec/util/CodecCollector.java
... ... @@ -191,6 +191,7 @@ public class CodecCollector {
191 191 }
192 192 }
193 193 }
  194 +
194 195 }
195 196  
196 197 /**
... ... @@ -617,6 +618,8 @@ public class CodecCollector {
617 618 }
618 619 }
619 620 if (fieldInfo.termVectorList.size() > 0) {
  621 + createTermvectorFull(fieldInfo.termVectorList, positionsData, docSet,
  622 + field, t, r, lrc);
620 623 createTermvectorFirstRound(fieldInfo.termVectorList, positionsData,
621 624 docSet, field, t, r, lrc);
622 625 }
... ... @@ -1474,6 +1477,7 @@ public class CodecCollector {
1474 1477 HashMap<Integer, ArrayList<Match>> matchData;
1475 1478  
1476 1479 for (ComponentGroup group : groupList) {
  1480 + group.dataCollector.setWithTotal();
1477 1481 if (group.prefixes.size() > 0) {
1478 1482 matchData = spansMatchData.get(group.spanQuery);
1479 1483 HashSet<String> knownPrefixes = collectKnownPrefixes(fieldInfo);
... ... @@ -1567,7 +1571,6 @@ public class CodecCollector {
1567 1571 (docId - docBase), group.prefixes, positionsHits);
1568 1572 // administration
1569 1573 for (IntervalTreeNodeData<String> positionHit : positionsHits) {
1570   - // System.out.println(positionHit);
1571 1574 GroupHit hit = new GroupHit(positionHit.list,
1572 1575 positionHit.start, positionHit.end, positionHit.hitStart,
1573 1576 positionHit.hitEnd, group, knownPrefixes);
... ... @@ -2261,6 +2264,7 @@ public class CodecCollector {
2261 2264 // check type
2262 2265 if (dataCollector.getCollectorType()
2263 2266 .equals(DataCollector.COLLECTOR_TYPE_LIST)) {
  2267 + dataCollector.setWithTotal();
2264 2268 // only if documents and facets
2265 2269 if (docSet.length > 0 && list.size() > 0) {
2266 2270 HashMap<String, Integer[]> docLists = new HashMap<String, Integer[]>();
... ... @@ -2539,6 +2543,166 @@ public class CodecCollector {
2539 2543 }
2540 2544 }
2541 2545  
  2546 + private static void createTermvectorFull(
  2547 + List<ComponentTermVector> termVectorList,
  2548 + HashMap<Integer, Integer> positionsData, List<Integer> docSet,
  2549 + String field, Terms t, LeafReader r, LeafReaderContext lrc)
  2550 + throws IOException {
  2551 + if (t != null) {
  2552 + BytesRef term;
  2553 + TermsEnum termsEnum;
  2554 + PostingsEnum postingsEnum = null;
  2555 + String segmentName = "segment" + lrc.ord;
  2556 + int segmentNumber = lrc.parent.leaves().size();
  2557 + // loop over termvectors
  2558 + for (ComponentTermVector termVector : termVectorList) {
  2559 + if (termVector.full || termVector.list != null) {
  2560 + if (termVector.full) {
  2561 + termVector.subComponentFunction.dataCollector.setWithTotal();
  2562 + }
  2563 +
  2564 + List<CompiledAutomaton> listAutomata;
  2565 + if (termVector.list == null) {
  2566 + listAutomata = new ArrayList<CompiledAutomaton>();
  2567 + listAutomata.add(termVector.compiledAutomaton);
  2568 + } else {
  2569 + listAutomata = MtasToken.createAutomata(termVector.prefix,
  2570 + termVector.regexp, new ArrayList<String>(termVector.list));
  2571 + }
  2572 +
  2573 + for (CompiledAutomaton compiledAutomaton : listAutomata) {
  2574 + termsEnum = t.intersect(compiledAutomaton, null);
  2575 + int initSize = Math.min((int) t.size(), 1000);
  2576 + termVector.subComponentFunction.dataCollector.initNewList(initSize,
  2577 + segmentName, segmentNumber, termVector.boundary);
  2578 + boolean doBasic = termVector.subComponentFunction.dataCollector
  2579 + .getStatsType().equals(CodecUtil.STATS_BASIC);
  2580 + if (termVector.functions != null) {
  2581 + for (SubComponentFunction function : termVector.functions) {
  2582 + function.dataCollector.initNewList(initSize);
  2583 + doBasic = doBasic ? (function.parserFunction.sumRule()
  2584 + && !function.parserFunction.needPositions()
  2585 + && function.dataCollector.getStatsType()
  2586 + .equals(CodecUtil.STATS_BASIC))
  2587 + : doBasic;
  2588 + }
  2589 + }
  2590 + // only if documents
  2591 + if (docSet.size() > 0) {
  2592 + int termDocId;
  2593 + // loop over terms
  2594 + while ((term = termsEnum.next()) != null) {
  2595 + termDocId = -1;
  2596 + if (doBasic) {
  2597 + // compute numbers;
  2598 + TermvectorNumberBasic numberBasic = computeTermvectorNumberBasic(
  2599 + docSet, termDocId, termsEnum, r, lrc, postingsEnum);
  2600 + // register
  2601 + if (numberBasic.docNumber > 0) {
  2602 + long valueLong = 0;
  2603 + try {
  2604 + valueLong = termVector.subComponentFunction.parserFunction
  2605 + .getValueLong(numberBasic.valueSum, 1);
  2606 + } catch (IOException e) {
  2607 + termVector.subComponentFunction.dataCollector.error(
  2608 + new String[] { MtasToken.getPostfixFromValue(term) },
  2609 + e.getMessage());
  2610 + }
  2611 + String key = MtasToken.getPostfixFromValue(term);
  2612 + termVector.subComponentFunction.dataCollector.add(
  2613 + new String[] { key }, valueLong, numberBasic.docNumber);
  2614 + if (termVector.functions != null) {
  2615 + for (SubComponentFunction function : termVector.functions) {
  2616 + if (function.dataType
  2617 + .equals(CodecUtil.DATA_TYPE_LONG)) {
  2618 + long valueFunction = function.parserFunction
  2619 + .getValueLong(numberBasic.valueSum, 0);
  2620 + function.dataCollector.add(new String[] { key },
  2621 + valueFunction, numberBasic.docNumber);
  2622 + } else if (function.dataType
  2623 + .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
  2624 + double valueFunction = function.parserFunction
  2625 + .getValueDouble(numberBasic.valueSum, 0);
  2626 + function.dataCollector.add(new String[] { key },
  2627 + valueFunction, numberBasic.docNumber);
  2628 + }
  2629 + }
  2630 + }
  2631 +
  2632 + }
  2633 + } else {
  2634 + TermvectorNumberFull numberFull = computeTermvectorNumberFull(
  2635 + docSet, termDocId, termsEnum, r, lrc, postingsEnum,
  2636 + positionsData);
  2637 + if (numberFull.docNumber > 0) {
  2638 + long[] valuesLong = new long[numberFull.docNumber];
  2639 + String key = MtasToken.getPostfixFromValue(term);
  2640 + for (int i = 0; i < numberFull.docNumber; i++) {
  2641 + try {
  2642 + valuesLong[i] = termVector.subComponentFunction.parserFunction
  2643 + .getValueLong(new long[] { numberFull.args[i] },
  2644 + numberFull.positions[i]);
  2645 + } catch (IOException e) {
  2646 + termVector.subComponentFunction.dataCollector
  2647 + .error(new String[] { key }, e.getMessage());
  2648 + }
  2649 + }
  2650 + termVector.subComponentFunction.dataCollector.add(
  2651 + new String[] { key }, valuesLong, valuesLong.length);
  2652 + if (termVector.functions != null) {
  2653 + for (SubComponentFunction function : termVector.functions) {
  2654 + if (function.dataType
  2655 + .equals(CodecUtil.DATA_TYPE_LONG)) {
  2656 + valuesLong = new long[numberFull.docNumber];
  2657 + for (int i = 0; i < numberFull.docNumber; i++) {
  2658 + try {
  2659 + valuesLong[i] = function.parserFunction
  2660 + .getValueLong(
  2661 + new long[] { numberFull.args[i] },
  2662 + numberFull.positions[i]);
  2663 + } catch (IOException e) {
  2664 + function.dataCollector.error(new String[] { key },
  2665 + e.getMessage());
  2666 + }
  2667 + }
  2668 + function.dataCollector.add(new String[] { key },
  2669 + valuesLong, valuesLong.length);
  2670 + } else if (function.dataType
  2671 + .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
  2672 + double[] valuesDouble = new double[numberFull.docNumber];
  2673 + for (int i = 0; i < numberFull.docNumber; i++) {
  2674 + try {
  2675 + valuesDouble[i] = function.parserFunction
  2676 + .getValueDouble(
  2677 + new long[] { numberFull.args[i] },
  2678 + numberFull.positions[i]);
  2679 + } catch (IOException e) {
  2680 + function.dataCollector.error(new String[] { key },
  2681 + e.getMessage());
  2682 + }
  2683 + }
  2684 + function.dataCollector.add(new String[] { key },
  2685 + valuesDouble, valuesDouble.length);
  2686 + }
  2687 + }
  2688 + }
  2689 + }
  2690 +
  2691 + }
  2692 + }
  2693 + }
  2694 + termVector.subComponentFunction.dataCollector.closeNewList();
  2695 + if (termVector.functions != null) {
  2696 + for (SubComponentFunction function : termVector.functions) {
  2697 + function.dataCollector.closeNewList();
  2698 + }
  2699 + }
  2700 + }
  2701 + }
  2702 + }
  2703 + }
  2704 + }
  2705 +
2542 2706 /**
2543 2707 * Creates the termvector first round.
2544 2708 *
... ... @@ -2572,121 +2736,125 @@ public class CodecCollector {
2572 2736 int segmentNumber = lrc.parent.leaves().size();
2573 2737 // loop over termvectors
2574 2738 for (ComponentTermVector termVector : termVectorList) {
2575   - termsEnum = t.intersect(termVector.compiledAutomaton, null);
2576   - int initSize = Math.min((int) t.size(), 1000);
2577   - termVector.subComponentFunction.dataCollector.initNewList(initSize,
2578   - segmentName, segmentNumber, termVector.boundary);
2579   - if (termVector.functions != null) {
2580   - for (SubComponentFunction function : termVector.functions) {
2581   - function.dataCollector.initNewList(initSize);
2582   - }
2583   - }
2584   - // only if documents
2585   - if (docSet.size() > 0) {
2586   - int termDocId;
2587   - int termNumberMaximum = termVector.number;
2588   - HashMap<BytesRef, RegisterStatus> computeFullList = new HashMap<BytesRef, RegisterStatus>();
2589   - RegisterStatus registerStatus;
2590   - // basic, don't need full values
2591   - if (termVector.subComponentFunction.sortType
2592   - .equals(CodecUtil.SORT_TERM)
2593   - || termVector.subComponentFunction.sortType
2594   - .equals(CodecUtil.STATS_TYPE_SUM)
2595   - || termVector.subComponentFunction.sortType
2596   - .equals(CodecUtil.STATS_TYPE_N)) {
2597   - int termCounter = 0;
2598   -
2599   - boolean continueAfterPreliminaryCheck, preliminaryCheck = false;
2600   - if (r.getLiveDocs() == null && (docSet.size() != r.numDocs())) {
2601   - preliminaryCheck = true;
  2739 + if (!termVector.full && termVector.list == null) {
  2740 + termsEnum = t.intersect(termVector.compiledAutomaton, null);
  2741 + int initSize = Math.min((int) t.size(), 1000);
  2742 + termVector.subComponentFunction.dataCollector.initNewList(initSize,
  2743 + segmentName, segmentNumber, termVector.boundary);
  2744 + if (termVector.functions != null) {
  2745 + for (SubComponentFunction function : termVector.functions) {
  2746 + function.dataCollector.initNewList(initSize);
2602 2747 }
2603   - // loop over terms
2604   - while ((term = termsEnum.next()) != null) {
2605   - termDocId = -1;
2606   - continueAfterPreliminaryCheck = true;
2607   - if (preliminaryCheck) {
2608   - try {
2609   - TermvectorNumberBasic preliminaryNumberBasic = computeTermvectorNumberBasic(
2610   - termsEnum, r);
2611   - if (preliminaryNumberBasic.docNumber > 0) {
2612   - continueAfterPreliminaryCheck = preliminaryRegisterValue(
2613   - term, termVector, preliminaryNumberBasic,
2614   - termNumberMaximum, segmentNumber);
2615   - } else {
2616   - continueAfterPreliminaryCheck = false;
  2748 + }
  2749 + // only if documents
  2750 + if (docSet.size() > 0) {
  2751 + int termDocId;
  2752 + int termNumberMaximum = termVector.number;
  2753 + HashMap<BytesRef, RegisterStatus> computeFullList = new HashMap<BytesRef, RegisterStatus>();
  2754 + RegisterStatus registerStatus;
  2755 + // basic, don't need full values
  2756 + if (termVector.subComponentFunction.sortType
  2757 + .equals(CodecUtil.SORT_TERM)
  2758 + || termVector.subComponentFunction.sortType
  2759 + .equals(CodecUtil.STATS_TYPE_SUM)
  2760 + || termVector.subComponentFunction.sortType
  2761 + .equals(CodecUtil.STATS_TYPE_N)) {
  2762 + int termCounter = 0;
  2763 +
  2764 + boolean continueAfterPreliminaryCheck, preliminaryCheck = false;
  2765 + if (r.getLiveDocs() == null && (docSet.size() != r.numDocs())) {
  2766 + preliminaryCheck = true;
  2767 + }
  2768 + // loop over terms
  2769 + while ((term = termsEnum.next()) != null) {
  2770 + termDocId = -1;
  2771 + continueAfterPreliminaryCheck = true;
  2772 + if (preliminaryCheck) {
  2773 + try {
  2774 + TermvectorNumberBasic preliminaryNumberBasic = computeTermvectorNumberBasic(
  2775 + termsEnum, r);
  2776 + if (preliminaryNumberBasic.docNumber > 0) {
  2777 + continueAfterPreliminaryCheck = preliminaryRegisterValue(
  2778 + term, termVector, preliminaryNumberBasic,
  2779 + termNumberMaximum, segmentNumber);
  2780 + } else {
  2781 + continueAfterPreliminaryCheck = false;
  2782 + }
  2783 + } catch (IOException e) {
  2784 + continueAfterPreliminaryCheck = true;
2617 2785 }
2618   - } catch (IOException e) {
2619   - continueAfterPreliminaryCheck = true;
2620 2786 }
2621   - }
2622   - if (continueAfterPreliminaryCheck) {
2623   - // compute numbers;
2624   - TermvectorNumberBasic numberBasic = computeTermvectorNumberBasic(
2625   - docSet, termDocId, termsEnum, r, lrc, postingsEnum);
2626   - // register
2627   - if (numberBasic.docNumber > 0) {
2628   - termCounter++;
2629   - registerStatus = registerValue(term, termVector, numberBasic,
2630   - termNumberMaximum, segmentNumber, false);
2631   - if (registerStatus != null) {
2632   - computeFullList.put(BytesRef.deepCopyOf(term),
2633   - registerStatus);
  2787 + if (continueAfterPreliminaryCheck) {
  2788 + // compute numbers;
  2789 + TermvectorNumberBasic numberBasic = computeTermvectorNumberBasic(
  2790 + docSet, termDocId, termsEnum, r, lrc, postingsEnum);
  2791 + // register
  2792 + if (numberBasic.docNumber > 0) {
  2793 + termCounter++;
  2794 + registerStatus = registerValue(term, termVector,
  2795 + numberBasic, termNumberMaximum, segmentNumber, false);
  2796 + if (registerStatus != null) {
  2797 + computeFullList.put(BytesRef.deepCopyOf(term),
  2798 + registerStatus);
  2799 + }
2634 2800 }
2635 2801 }
  2802 + // stop after termCounterMaximum
  2803 + if (termVector.subComponentFunction.sortType.equals(
  2804 + CodecUtil.SORT_TERM) && termCounter >= termNumberMaximum) {
  2805 + break;
  2806 + }
2636 2807 }
2637   - // stop after termCounterMaximum
2638   - if (termVector.subComponentFunction.sortType.equals(
2639   - CodecUtil.SORT_TERM) && termCounter >= termNumberMaximum) {
2640   - break;
2641   - }
2642   - }
2643   - // rerun for full
2644   - if (computeFullList.size() > 0) {
2645   - termsEnum = t.intersect(termVector.compiledAutomaton, null);
2646   - while ((term = termsEnum.next()) != null) {
2647   - termDocId = -1;
2648   - // only if (probably) needed
2649   - if (computeFullList.containsKey(term)) {
2650   - registerStatus = computeFullList.get(term);
2651   - if (termVector.subComponentFunction.sortType
2652   - .equals(CodecUtil.SORT_TERM) || termVector.list != null
2653   - || termVector.boundaryRegistration || registerStatus.force
2654   - || termVector.subComponentFunction.dataCollector
2655   - .validateSegmentBoundary(registerStatus.sortValue)) {
2656   - TermvectorNumberFull numberFull = computeTermvectorNumberFull(
2657   - docSet, termDocId, termsEnum, r, lrc, postingsEnum,
2658   - positionsData);
2659   - if (numberFull.docNumber > 0) {
2660   - termCounter++;
2661   - registerValue(term, termVector, numberFull,
2662   - termNumberMaximum, segmentNumber);
  2808 + // rerun for full
  2809 + if (computeFullList.size() > 0) {
  2810 + termsEnum = t.intersect(termVector.compiledAutomaton, null);
  2811 + while ((term = termsEnum.next()) != null) {
  2812 + termDocId = -1;
  2813 + // only if (probably) needed
  2814 + if (computeFullList.containsKey(term)) {
  2815 + registerStatus = computeFullList.get(term);
  2816 + if (termVector.subComponentFunction.sortType
  2817 + .equals(CodecUtil.SORT_TERM) || termVector.list != null
  2818 + || termVector.boundaryRegistration
  2819 + || registerStatus.force
  2820 + || termVector.subComponentFunction.dataCollector
  2821 + .validateSegmentBoundary(
  2822 + registerStatus.sortValue)) {
  2823 + TermvectorNumberFull numberFull = computeTermvectorNumberFull(
  2824 + docSet, termDocId, termsEnum, r, lrc, postingsEnum,
  2825 + positionsData);
  2826 + if (numberFull.docNumber > 0) {
  2827 + termCounter++;
  2828 + registerValue(term, termVector, numberFull,
  2829 + termNumberMaximum, segmentNumber);
  2830 + }
2663 2831 }
2664 2832 }
2665 2833 }
  2834 + computeFullList.clear();
2666 2835 }
2667   - computeFullList.clear();
  2836 + } else {
  2837 + throw new IOException(
  2838 + "sort '" + termVector.subComponentFunction.sortType + " "
  2839 + + termVector.subComponentFunction.sortDirection
  2840 + + "' not supported");
2668 2841 }
2669   - } else {
2670   - throw new IOException(
2671   - "sort '" + termVector.subComponentFunction.sortType + " "
2672   - + termVector.subComponentFunction.sortDirection
2673   - + "' not supported");
2674   - }
2675   - // finish if segments are used
2676   - termVector.subComponentFunction.dataCollector
2677   - .closeSegmentKeyValueRegistration();
  2842 + // finish if segments are used
  2843 + termVector.subComponentFunction.dataCollector
  2844 + .closeSegmentKeyValueRegistration();
  2845 + if (termVector.functions != null) {
  2846 + for (SubComponentFunction function : termVector.functions) {
  2847 + function.dataCollector.closeSegmentKeyValueRegistration();
  2848 + }
  2849 + }
  2850 + }
  2851 + termVector.subComponentFunction.dataCollector.closeNewList();
2678 2852 if (termVector.functions != null) {
2679 2853 for (SubComponentFunction function : termVector.functions) {
2680   - function.dataCollector.closeSegmentKeyValueRegistration();
  2854 + function.dataCollector.closeNewList();
2681 2855 }
2682 2856 }
2683 2857 }
2684   - termVector.subComponentFunction.dataCollector.closeNewList();
2685   - if (termVector.functions != null) {
2686   - for (SubComponentFunction function : termVector.functions) {
2687   - function.dataCollector.closeNewList();
2688   - }
2689   - }
2690 2858 }
2691 2859 }
2692 2860 }
... ... @@ -2723,55 +2891,56 @@ public class CodecCollector {
2723 2891 String segmentName = "segment" + lrc.ord;
2724 2892 int segmentNumber = lrc.parent.leaves().size();
2725 2893 for (ComponentTermVector termVector : termVectorList) {
2726   - if (termVector.subComponentFunction.dataCollector.segmentRecomputeKeyList != null
2727   - && termVector.subComponentFunction.dataCollector.segmentRecomputeKeyList
2728   - .containsKey(segmentName)) {
2729   - HashSet<String> recomputeKeyList = termVector.subComponentFunction.dataCollector.segmentRecomputeKeyList
2730   - .get(segmentName);
2731   - if (recomputeKeyList.size() > 0) {
2732   - List<CompiledAutomaton> listCompiledAutomata = MtasToken
2733   - .createAutomata(termVector.prefix,
2734   - new ArrayList<String>(recomputeKeyList));
2735   - for (CompiledAutomaton compiledAutomaton : listCompiledAutomata) {
2736   - termsEnum = t.intersect(compiledAutomaton, null);
2737   - termVector.subComponentFunction.dataCollector
2738   - .initNewList(
2739   - termVector.subComponentFunction.dataCollector.segmentKeys
2740   - .size(),
2741   - segmentName, segmentNumber, termVector.boundary);
2742   - RegisterStatus registerStatus = null;
2743   - if (termVector.functions != null) {
2744   - for (SubComponentFunction function : termVector.functions) {
2745   - function.dataCollector.initNewList((int) t.size(),
2746   - segmentName, segmentNumber, null);
  2894 + if (!termVector.full && termVector.list == null) {
  2895 + if (termVector.subComponentFunction.dataCollector.segmentRecomputeKeyList != null
  2896 + && termVector.subComponentFunction.dataCollector.segmentRecomputeKeyList
  2897 + .containsKey(segmentName)) {
  2898 + HashSet<String> recomputeKeyList = termVector.subComponentFunction.dataCollector.segmentRecomputeKeyList
  2899 + .get(segmentName);
  2900 + if (recomputeKeyList.size() > 0) {
  2901 + List<CompiledAutomaton> listCompiledAutomata = MtasToken
  2902 + .createAutomata(termVector.prefix, termVector.regexp,
  2903 + new ArrayList<String>(recomputeKeyList));
  2904 + for (CompiledAutomaton compiledAutomaton : listCompiledAutomata) {
  2905 + termsEnum = t.intersect(compiledAutomaton, null);
  2906 + termVector.subComponentFunction.dataCollector.initNewList(
  2907 + termVector.subComponentFunction.dataCollector.segmentKeys
  2908 + .size(),
  2909 + segmentName, segmentNumber, termVector.boundary);
  2910 + RegisterStatus registerStatus = null;
  2911 + if (termVector.functions != null) {
  2912 + for (SubComponentFunction function : termVector.functions) {
  2913 + function.dataCollector.initNewList((int) t.size(),
  2914 + segmentName, segmentNumber, null);
  2915 + }
2747 2916 }
2748   - }
2749   - if (docSet.size() > 0) {
2750   - int termDocId;
2751   - while ((term = termsEnum.next()) != null) {
2752   - termDocId = -1;
2753   - // compute numbers;
2754   - TermvectorNumberBasic numberBasic = computeTermvectorNumberBasic(
2755   - docSet, termDocId, termsEnum, r, lrc, postingsEnum);
2756   - if (numberBasic.docNumber > 0) {
2757   - registerStatus = registerValue(term, termVector,
2758   - numberBasic, 0, segmentNumber, true);
2759   - if (registerStatus != null) {
2760   - TermvectorNumberFull numberFull = computeTermvectorNumberFull(
2761   - docSet, termDocId, termsEnum, r, lrc, postingsEnum,
2762   - positionsData);
2763   - if (numberFull.docNumber > 0) {
2764   - registerValue(term, termVector, numberFull, 0,
2765   - segmentNumber);
  2917 + if (docSet.size() > 0) {
  2918 + int termDocId;
  2919 + while ((term = termsEnum.next()) != null) {
  2920 + termDocId = -1;
  2921 + // compute numbers;
  2922 + TermvectorNumberBasic numberBasic = computeTermvectorNumberBasic(
  2923 + docSet, termDocId, termsEnum, r, lrc, postingsEnum);
  2924 + if (numberBasic.docNumber > 0) {
  2925 + registerStatus = registerValue(term, termVector,
  2926 + numberBasic, 0, segmentNumber, true);
  2927 + if (registerStatus != null) {
  2928 + TermvectorNumberFull numberFull = computeTermvectorNumberFull(
  2929 + docSet, termDocId, termsEnum, r, lrc, postingsEnum,
  2930 + positionsData);
  2931 + if (numberFull.docNumber > 0) {
  2932 + registerValue(term, termVector, numberFull, 0,
  2933 + segmentNumber);
  2934 + }
2766 2935 }
2767 2936 }
2768 2937 }
2769 2938 }
2770   - }
2771   - termVector.subComponentFunction.dataCollector.closeNewList();
2772   - if (termVector.functions != null) {
2773   - for (SubComponentFunction function : termVector.functions) {
2774   - function.dataCollector.closeNewList();
  2939 + termVector.subComponentFunction.dataCollector.closeNewList();
  2940 + if (termVector.functions != null) {
  2941 + for (SubComponentFunction function : termVector.functions) {
  2942 + function.dataCollector.closeNewList();
  2943 + }
2775 2944 }
2776 2945 }
2777 2946 }
... ... @@ -2794,30 +2963,32 @@ public class CodecCollector {
2794 2963 List<ComponentTermVector> termVectorList) throws IOException {
2795 2964 boolean needSecondRound = false;
2796 2965 for (ComponentTermVector termVector : termVectorList) {
2797   - if (termVector.subComponentFunction.dataCollector.segmentRegistration != null
2798   - && (termVector.subComponentFunction.dataCollector.segmentRegistration
2799   - .equals(MtasDataCollector.SEGMENT_SORT_ASC)
2800   - || termVector.subComponentFunction.dataCollector.segmentRegistration
2801   - .equals(MtasDataCollector.SEGMENT_SORT_DESC))
2802   - && termVector.number > 0) {
2803   - termVector.subComponentFunction.dataCollector.recomputeSegmentKeys();
2804   - if (!termVector.subComponentFunction.dataCollector
2805   - .checkExistenceNecessaryKeys()) {
2806   - needSecondRound = true;
2807   - }
2808   - termVector.subComponentFunction.dataCollector.reduceToSegmentKeys();
2809   - } else if (termVector.subComponentFunction.dataCollector.segmentRegistration != null
2810   - && (termVector.subComponentFunction.dataCollector.segmentRegistration
2811   - .equals(MtasDataCollector.SEGMENT_BOUNDARY_ASC)
2812   - || termVector.subComponentFunction.dataCollector.segmentRegistration
2813   - .equals(MtasDataCollector.SEGMENT_BOUNDARY_DESC))
2814   - && termVector.number > 0) {
2815   - termVector.subComponentFunction.dataCollector.recomputeSegmentKeys();
2816   - if (!termVector.subComponentFunction.dataCollector
2817   - .checkExistenceNecessaryKeys()) {
2818   - needSecondRound = true;
2819   - }
2820   - termVector.subComponentFunction.dataCollector.reduceToSegmentKeys();
  2966 + if (!termVector.full && termVector.list == null) {
  2967 + if (termVector.subComponentFunction.dataCollector.segmentRegistration != null
  2968 + && (termVector.subComponentFunction.dataCollector.segmentRegistration
  2969 + .equals(MtasDataCollector.SEGMENT_SORT_ASC)
  2970 + || termVector.subComponentFunction.dataCollector.segmentRegistration
  2971 + .equals(MtasDataCollector.SEGMENT_SORT_DESC))
  2972 + && termVector.number > 0) {
  2973 + termVector.subComponentFunction.dataCollector.recomputeSegmentKeys();
  2974 + if (!termVector.subComponentFunction.dataCollector
  2975 + .checkExistenceNecessaryKeys()) {
  2976 + needSecondRound = true;
  2977 + }
  2978 + termVector.subComponentFunction.dataCollector.reduceToSegmentKeys();
  2979 + } else if (termVector.subComponentFunction.dataCollector.segmentRegistration != null
  2980 + && (termVector.subComponentFunction.dataCollector.segmentRegistration
  2981 + .equals(MtasDataCollector.SEGMENT_BOUNDARY_ASC)
  2982 + || termVector.subComponentFunction.dataCollector.segmentRegistration
  2983 + .equals(MtasDataCollector.SEGMENT_BOUNDARY_DESC))
  2984 + && termVector.number > 0) {
  2985 + termVector.subComponentFunction.dataCollector.recomputeSegmentKeys();
  2986 + if (!termVector.subComponentFunction.dataCollector
  2987 + .checkExistenceNecessaryKeys()) {
  2988 + needSecondRound = true;
  2989 + }
  2990 + termVector.subComponentFunction.dataCollector.reduceToSegmentKeys();
  2991 + }
2821 2992 }
2822 2993 }
2823 2994 return needSecondRound;
... ... @@ -3141,9 +3312,7 @@ public class CodecCollector {
3141 3312 }
3142 3313 }
3143 3314 }
3144   -
3145 3315 }
3146   -
3147 3316 }
3148 3317  
3149 3318 /**
... ...
src/mtas/codec/util/CodecComponent.java
... ... @@ -142,10 +142,8 @@ public class CodecComponent {
142 142 /**
143 143 * Instantiates a new component field.
144 144 *
145   - * @param field
146   - * the field
147   - * @param uniqueKeyField
148   - * the unique key field
  145 + * @param field the field
  146 + * @param uniqueKeyField the unique key field
149 147 */
150 148 public ComponentField(String field, String uniqueKeyField) {
151 149 this.field = field;
... ... @@ -187,8 +185,7 @@ public class CodecComponent {
187 185 /**
188 186 * Instantiates a new component prefix.
189 187 *
190   - * @param key
191   - * the key
  188 + * @param key the key
192 189 */
193 190 public ComponentPrefix(String key) {
194 191 this.key = key;
... ... @@ -201,8 +198,7 @@ public class CodecComponent {
201 198 /**
202 199 * Adds the single position.
203 200 *
204   - * @param prefix
205   - * the prefix
  201 + * @param prefix the prefix
206 202 */
207 203 public void addSinglePosition(String prefix) {
208 204 if (!prefix.trim().equals("")) {
... ... @@ -216,8 +212,7 @@ public class CodecComponent {
216 212 /**
217 213 * Adds the multiple position.
218 214 *
219   - * @param prefix
220   - * the prefix
  215 + * @param prefix the prefix
221 216 */
222 217 public void addMultiplePosition(String prefix) {
223 218 if (!prefix.trim().equals("")) {
... ... @@ -235,8 +230,7 @@ public class CodecComponent {
235 230 /**
236 231 * Adds the set position.
237 232 *
238   - * @param prefix
239   - * the prefix
  233 + * @param prefix the prefix
240 234 */
241 235 public void addSetPosition(String prefix) {
242 236 if (!prefix.trim().equals("")) {
... ... @@ -254,8 +248,7 @@ public class CodecComponent {
254 248 /**
255 249 * Adds the intersecting.
256 250 *
257   - * @param prefix
258   - * the prefix
  251 + * @param prefix the prefix
259 252 */
260 253 public void addIntersecting(String prefix) {
261 254 if (!prefix.trim().equals("")) {
... ... @@ -270,7 +263,7 @@ public class CodecComponent {
270 263 */
271 264 public static class ComponentDistinct {
272 265  
273   - /** The key. */
  266 + /** The regexp. */
274 267 public String key, prefix, regexp;
275 268  
276 269 /** The stats type. */
... ... @@ -282,23 +275,27 @@ public class CodecComponent {
282 275 /** The compiled automaton. */
283 276 public CompiledAutomaton compiledAutomaton;
284 277  
  278 + /** The number. */
285 279 public int number;
286 280  
287 281 /** The unique key. */
288 282 public HashMap<Integer, String> uniqueKey;
289 283  
  284 + /** The stats. */
290 285 public HashMap<Integer, MtasDataCollector> stats;
  286 +
  287 + /** The list. */
291 288 public HashMap<Integer, MtasDataCollector> list;
292 289  
293 290 /**
294 291 * Instantiates a new component distinct.
295 292 *
296   - * @param key
297   - * the key
298   - * @param prefix
299   - * the prefix
300   - * @throws IOException
301   - * Signals that an I/O exception has occurred.
  293 + * @param key the key
  294 + * @param prefix the prefix
  295 + * @param statsType the stats type
  296 + * @param regexp the regexp
  297 + * @param number the number
  298 + * @throws IOException Signals that an I/O exception has occurred.
302 299 */
303 300 public ComponentDistinct(String key, String prefix, String statsType,
304 301 String regexp, int number) throws IOException {
... ... @@ -319,7 +316,7 @@ public class CodecComponent {
319 316 compiledAutomaton = new CompiledAutomaton(re.toAutomaton());
320 317 }
321 318 this.stats = new HashMap<Integer, MtasDataCollector>();
322   - if (this.number > 0) {
  319 + if (this.number > 0) {
323 320 this.list = new HashMap<Integer, MtasDataCollector>();
324 321 } else {
325 322 this.list = null;
... ... @@ -377,24 +374,15 @@ public class CodecComponent {
377 374 /**
378 375 * Instantiates a new component kwic.
379 376 *
380   - * @param query
381   - * the query
382   - * @param key
383   - * the key
384   - * @param prefixes
385   - * the prefixes
386   - * @param number
387   - * the number
388   - * @param start
389   - * the start
390   - * @param left
391   - * the left
392   - * @param right
393   - * the right
394   - * @param output
395   - * the output
396   - * @throws IOException
397   - * Signals that an I/O exception has occurred.
  377 + * @param query the query
  378 + * @param key the key
  379 + * @param prefixes the prefixes
  380 + * @param number the number
  381 + * @param start the start
  382 + * @param left the left
  383 + * @param right the right
  384 + * @param output the output
  385 + * @throws IOException Signals that an I/O exception has occurred.
398 386 */
399 387 public ComponentKwic(SpanQuery query, String key, String prefixes,
400 388 Integer number, int start, int left, int right, String output)
... ... @@ -443,7 +431,9 @@ public class CodecComponent {
443 431 public SpanQuery spanQuery;
444 432  
445 433 /** The key. */
446   - public String field, queryValue, queryType, key;
  434 + public String field, queryValue, queryType, queryPrefix, key;
  435 +
  436 + public HashMap<String, String[]> queryVariables;
447 437  
448 438 /** The tokens. */
449 439 public ArrayList<ListToken> tokens;
... ... @@ -484,38 +474,28 @@ public class CodecComponent {
484 474 /**
485 475 * Instantiates a new component list.
486 476 *
487   - * @param spanQuery
488   - * the span query
489   - * @param field
490   - * the field
491   - * @param queryValue
492   - * the query value
493   - * @param queryType
494   - * the query type
495   - * @param key
496   - * the key
497   - * @param prefix
498   - * the prefix
499   - * @param start
500   - * the start
501   - * @param number
502   - * the number
503   - * @param left
504   - * the left
505   - * @param right
506   - * the right
507   - * @param output
508   - * the output
509   - * @throws IOException
510   - * Signals that an I/O exception has occurred.
  477 + * @param spanQuery the span query
  478 + * @param field the field
  479 + * @param queryValue the query value
  480 + * @param queryType the query type
  481 + * @param key the key
  482 + * @param prefix the prefix
  483 + * @param start the start
  484 + * @param number the number
  485 + * @param left the left
  486 + * @param right the right
  487 + * @param output the output
  488 + * @throws IOException Signals that an I/O exception has occurred.
511 489 */
512 490 public ComponentList(SpanQuery spanQuery, String field, String queryValue,
513   - String queryType, String key, String prefix, int start, int number,
  491 + String queryType, String queryPrefix, HashMap<String, String[]> queryVariables, String key, String prefix, int start, int number,
514 492 int left, int right, String output) throws IOException {
515 493 this.spanQuery = spanQuery;
516 494 this.field = field;
517 495 this.queryValue = queryValue;
518 496 this.queryType = queryType;
  497 + this.queryPrefix = queryPrefix;
  498 + this.queryVariables = queryVariables;
519 499 this.key = key;
520 500 this.left = left;
521 501 this.right = right;
... ... @@ -572,7 +552,7 @@ public class CodecComponent {
572 552 public Integer start, number;
573 553  
574 554 /** The key. */
575   - public String field, queryValue, queryType, key;
  555 + public String field, queryValue, queryType, queryPrefix, key;
576 556  
577 557 /** The data collector. */
578 558 public MtasDataCollector<?, ?> dataCollector;
... ... @@ -590,49 +570,29 @@ public class CodecComponent {
590 570 /**
591 571 * Instantiates a new component group.
592 572 *
593   - * @param spanQuery
594   - * the span query
595   - * @param field
596   - * the field
597   - * @param queryValue
598   - * the query value
599   - * @param queryType
600   - * the query type
601   - * @param key
602   - * the key
603   - * @param number
604   - * the number
605   - * @param groupingHitInsidePrefixes
606   - * the grouping hit inside prefixes
607   - * @param groupingHitInsideLeftPosition
608   - * the grouping hit inside left position
609   - * @param groupingHitInsideLeftPrefixes
610   - * the grouping hit inside left prefixes
611   - * @param groupingHitInsideRightPosition
612   - * the grouping hit inside right position
613   - * @param groupingHitInsideRightPrefixes
614   - * the grouping hit inside right prefixes
615   - * @param groupingHitLeftPosition
616   - * the grouping hit left position
617   - * @param groupingHitLeftPrefixes
618   - * the grouping hit left prefixes
619   - * @param groupingHitRightPosition
620   - * the grouping hit right position
621   - * @param groupingHitRightPrefixes
622   - * the grouping hit right prefixes
623   - * @param groupingLeftPosition
624   - * the grouping left position
625   - * @param groupingLeftPrefixes
626   - * the grouping left prefixes
627   - * @param groupingRightPosition
628   - * the grouping right position
629   - * @param groupingRightPrefixes
630   - * the grouping right prefixes
631   - * @throws IOException
632   - * Signals that an I/O exception has occurred.
  573 + * @param spanQuery the span query
  574 + * @param field the field
  575 + * @param queryValue the query value
  576 + * @param queryType the query type
  577 + * @param key the key
  578 + * @param number the number
  579 + * @param groupingHitInsidePrefixes the grouping hit inside prefixes
  580 + * @param groupingHitInsideLeftPosition the grouping hit inside left position
  581 + * @param groupingHitInsideLeftPrefixes the grouping hit inside left prefixes
  582 + * @param groupingHitInsideRightPosition the grouping hit inside right position
  583 + * @param groupingHitInsideRightPrefixes the grouping hit inside right prefixes
  584 + * @param groupingHitLeftPosition the grouping hit left position
  585 + * @param groupingHitLeftPrefixes the grouping hit left prefixes
  586 + * @param groupingHitRightPosition the grouping hit right position
  587 + * @param groupingHitRightPrefixes the grouping hit right prefixes
  588 + * @param groupingLeftPosition the grouping left position
  589 + * @param groupingLeftPrefixes the grouping left prefixes
  590 + * @param groupingRightPosition the grouping right position
  591 + * @param groupingRightPrefixes the grouping right prefixes
  592 + * @throws IOException Signals that an I/O exception has occurred.
633 593 */
634 594 public ComponentGroup(SpanQuery spanQuery, String field, String queryValue,
635   - String queryType, String key, int number,
  595 + String queryType, String queryPrefix, String key, int number,
636 596 String groupingHitInsidePrefixes,
637 597 String[] groupingHitInsideLeftPosition,
638 598 String[] groupingHitInsideLeftPrefixes,
... ... @@ -647,6 +607,7 @@ public class CodecComponent {
647 607 this.field = field;
648 608 this.queryValue = queryValue;
649 609 this.queryType = queryType;
  610 + this.queryPrefix = queryPrefix;
650 611 this.key = key;
651 612 this.dataType = CodecUtil.DATA_TYPE_LONG;
652 613 this.statsItems = CodecUtil.createStatsItems("n,sum,mean");
... ... @@ -687,7 +648,7 @@ public class CodecComponent {
687 648 dataCollector = DataCollector.getCollector(
688 649 DataCollector.COLLECTOR_TYPE_LIST, this.dataType, this.statsType,
689 650 this.statsItems, this.sortType, this.sortDirection, this.start,
690   - this.number, null, null);
  651 + this.number, null, null);
691 652 }
692 653  
693 654 }
... ... @@ -695,15 +656,11 @@ public class CodecComponent {
695 656 /**
696 657 * Creates the positioned prefixes.
697 658 *
698   - * @param prefixList
699   - * the prefix list
700   - * @param position
701   - * the position
702   - * @param prefixes
703   - * the prefixes
  659 + * @param prefixList the prefix list
  660 + * @param position the position
  661 + * @param prefixes the prefixes
704 662 * @return the hash set[]
705   - * @throws IOException
706   - * Signals that an I/O exception has occurred.
  663 + * @throws IOException Signals that an I/O exception has occurred.
707 664 */
708 665 private static HashSet<String>[] createPositionedPrefixes(
709 666 HashSet<String> prefixList, String[] position, String[] prefixes)
... ... @@ -841,38 +798,22 @@ public class CodecComponent {
841 798 /**
842 799 * Instantiates a new component facet.
843 800 *
844   - * @param spanQueries
845   - * the span queries
846   - * @param field
847   - * the field
848   - * @param key
849   - * the key
850   - * @param baseFields
851   - * the base fields
852   - * @param baseFieldTypes
853   - * the base field types
854   - * @param baseTypes
855   - * the base types
856   - * @param baseSortTypes
857   - * the base sort types
858   - * @param baseSortDirections
859   - * the base sort directions
860   - * @param baseNumbers
861   - * the base numbers
862   - * @param baseMinimumDoubles
863   - * the base minimum doubles
864   - * @param baseMaximumDoubles
865   - * the base maximum doubles
866   - * @param baseFunctionKeys
867   - * the base function keys
868   - * @param baseFunctionExpressions
869   - * the base function expressions
870   - * @param baseFunctionTypes
871   - * the base function types
872   - * @throws IOException
873   - * Signals that an I/O exception has occurred.
874   - * @throws ParseException
875   - * the parse exception
  801 + * @param spanQueries the span queries
  802 + * @param field the field
  803 + * @param key the key
  804 + * @param baseFields the base fields
  805 + * @param baseFieldTypes the base field types
  806 + * @param baseTypes the base types
  807 + * @param baseSortTypes the base sort types
  808 + * @param baseSortDirections the base sort directions
  809 + * @param baseNumbers the base numbers
  810 + * @param baseMinimumDoubles the base minimum doubles
  811 + * @param baseMaximumDoubles the base maximum doubles
  812 + * @param baseFunctionKeys the base function keys
  813 + * @param baseFunctionExpressions the base function expressions
  814 + * @param baseFunctionTypes the base function types
  815 + * @throws IOException Signals that an I/O exception has occurred.
  816 + * @throws ParseException the parse exception
876 817 */
877 818 @SuppressWarnings("unchecked")
878 819 public ComponentFacet(SpanQuery[] spanQueries, String field, String key,
... ... @@ -976,7 +917,7 @@ public class CodecComponent {
976 917 dataCollector = DataCollector.getCollector(this.baseCollectorTypes[0],
977 918 this.baseDataTypes[0], this.baseStatsTypes[0],
978 919 this.baseStatsItems[0], this.baseSortTypes[0],
979   - this.baseSortDirections[0], 0, this.baseNumbers[0], null, null);
  920 + this.baseSortDirections[0], 0, this.baseNumbers[0], null, null);
980 921 } else {
981 922 String[] subBaseCollectorTypes = Arrays
982 923 .copyOfRange(baseCollectorTypes, 1, baseDataTypes.length);
... ... @@ -999,7 +940,7 @@ public class CodecComponent {
999 940 this.baseSortDirections[0], 0, this.baseNumbers[0],
1000 941 subBaseCollectorTypes, subBaseDataTypes, subBaseStatsTypes,
1001 942 subBaseStatsItems, subBaseSortTypes, subBaseSortDirections,
1002   - subStarts, subNumbers, null, null);
  943 + subStarts, subNumbers, null, null);
1003 944 }
1004 945 } else {
1005 946 throw new IOException("no baseFields");
... ... @@ -1080,6 +1021,9 @@ public class CodecComponent {
1080 1021 /** The boundary. */
1081 1022 public String key, prefix, regexp, boundary;
1082 1023  
  1024 + /** The full. */
  1025 + public boolean full;
  1026 +
1083 1027 /** The list. */
1084 1028 public HashSet<String> list;
1085 1029  
... ... @@ -1104,46 +1048,33 @@ public class CodecComponent {
1104 1048 /**
1105 1049 * Instantiates a new component term vector.
1106 1050 *
1107   - * @param key
1108   - * the key
1109   - * @param prefix
1110   - * the prefix
1111   - * @param regexp
1112   - * the regexp
1113   - * @param type
1114   - * the type
1115   - * @param sortType
1116   - * the sort type
1117   - * @param sortDirection
1118   - * the sort direction
1119   - * @param startValue
1120   - * the start value
1121   - * @param number
1122   - * the number
1123   - * @param functionKey
1124   - * the function key
1125   - * @param functionExpression
1126   - * the function expression
1127   - * @param functionType
1128   - * the function type
1129   - * @param boundary
1130   - * the boundary
1131   - * @param list
1132   - * the list
1133   - * @throws IOException
1134   - * Signals that an I/O exception has occurred.
1135   - * @throws ParseException
1136   - * the parse exception
  1051 + * @param key the key
  1052 + * @param prefix the prefix
  1053 + * @param regexp the regexp
  1054 + * @param full the full
  1055 + * @param type the type
  1056 + * @param sortType the sort type
  1057 + * @param sortDirection the sort direction
  1058 + * @param startValue the start value
  1059 + * @param number the number
  1060 + * @param functionKey the function key
  1061 + * @param functionExpression the function expression
  1062 + * @param functionType the function type
  1063 + * @param boundary the boundary
  1064 + * @param list the list
  1065 + * @throws IOException Signals that an I/O exception has occurred.
  1066 + * @throws ParseException the parse exception
1137 1067 */
1138 1068 @SuppressWarnings({ "unchecked", "rawtypes" })
1139 1069 public ComponentTermVector(String key, String prefix, String regexp,
1140   - String type, String sortType, String sortDirection, String startValue,
1141   - int number, String[] functionKey, String[] functionExpression,
1142   - String[] functionType, String boundary, String[] list)
1143   - throws IOException, ParseException {
  1070 + Boolean full, String type, String sortType, String sortDirection,
  1071 + String startValue, int number, String[] functionKey,
  1072 + String[] functionExpression, String[] functionType, String boundary,
  1073 + String[] list) throws IOException, ParseException {
1144 1074 this.key = key;
1145 1075 this.prefix = prefix;
1146 1076 this.regexp = regexp;
  1077 + this.full = (full!=null && full)?true:false;
1147 1078 sortType = sortType == null ? CodecUtil.SORT_TERM : sortType;
1148 1079 sortDirection = sortDirection == null ? (sortType == CodecUtil.SORT_TERM)
1149 1080 ? CodecUtil.SORT_ASC : CodecUtil.SORT_DESC : sortDirection;
... ... @@ -1152,8 +1083,10 @@ public class CodecComponent {
1152 1083 this.boundary = null;
1153 1084 this.number = Integer.MAX_VALUE;
1154 1085 this.startValue = null;
1155   - sortType = CodecUtil.SORT_TERM;
1156   - sortDirection = CodecUtil.SORT_ASC;
  1086 + if(!this.full) {
  1087 + sortType = CodecUtil.SORT_TERM;
  1088 + sortDirection = CodecUtil.SORT_ASC;
  1089 + }
1157 1090 } else {
1158 1091 this.list = null;
1159 1092 this.startValue = startValue;
... ... @@ -1180,11 +1113,11 @@ public class CodecComponent {
1180 1113 if (!sortType.equals(CodecUtil.SORT_TERM)
1181 1114 && !CodecUtil.STATS_TYPES.contains(sortType)) {
1182 1115 throw new IOException("unknown sortType '" + sortType + "'");
1183   - } else if (!sortType.equals(CodecUtil.SORT_TERM)) {
  1116 + } else if (!full && !sortType.equals(CodecUtil.SORT_TERM)) {
1184 1117 if (!(sortType.equals(CodecUtil.STATS_TYPE_SUM)
1185 1118 || sortType.equals(CodecUtil.STATS_TYPE_N))) {
1186 1119 throw new IOException(
1187   - "sortType '" + sortType + "' not supported for termVector");
  1120 + "sortType '" + sortType + "' only supported with full termVector");
1188 1121 }
1189 1122 }
1190 1123 if (!sortDirection.equals(CodecUtil.SORT_ASC)
... ... @@ -1194,7 +1127,10 @@ public class CodecComponent {
1194 1127 }
1195 1128 boundaryRegistration = this.boundary != null;
1196 1129 String segmentRegistration = null;
1197   - if (this.boundary != null) {
  1130 + if(this.full) {
  1131 + this.boundary = null;
  1132 + segmentRegistration = null;
  1133 + } else if (this.boundary != null) {
1198 1134 if (sortDirection.equals(CodecUtil.SORT_ASC)) {
1199 1135 segmentRegistration = MtasDataCollector.SEGMENT_BOUNDARY_ASC;
1200 1136 } else if (sortDirection.equals(CodecUtil.SORT_DESC)) {
... ... @@ -1212,7 +1148,6 @@ public class CodecComponent {
1212 1148 DataCollector.COLLECTOR_TYPE_LIST, key, type,
1213 1149 new MtasFunctionParserFunctionDefault(1), sortType, sortDirection, 0,
1214 1150 this.number, segmentRegistration, boundary);
1215   -
1216 1151 if ((regexp == null) || (regexp.isEmpty())) {
1217 1152 RegExp re = new RegExp(prefix + MtasToken.DELIMITER + ".*");
1218 1153 compiledAutomaton = new CompiledAutomaton(re.toAutomaton());
... ... @@ -1264,7 +1199,7 @@ public class CodecComponent {
1264 1199  
1265 1200 /** The queries. */
1266 1201 public SpanQuery[] queries;
1267   -
  1202 +
1268 1203 /** The key. */
1269 1204 public String key;
1270 1205  
... ... @@ -1295,26 +1230,16 @@ public class CodecComponent {
1295 1230 /**
1296 1231 * Instantiates a new component span.
1297 1232 *
1298   - * @param queries
1299   - * the queries
1300   - * @param key
1301   - * the key
1302   - * @param minimumDouble
1303   - * the minimum double
1304   - * @param maximumDouble
1305   - * the maximum double
1306   - * @param type
1307   - * the type
1308   - * @param functionKey
1309   - * the function key
1310   - * @param functionExpression
1311   - * the function expression
1312   - * @param functionType
1313   - * the function type
1314   - * @throws IOException
1315   - * Signals that an I/O exception has occurred.
1316   - * @throws ParseException
1317   - * the parse exception
  1233 + * @param queries the queries
  1234 + * @param key the key
  1235 + * @param minimumDouble the minimum double
  1236 + * @param maximumDouble the maximum double
  1237 + * @param type the type
  1238 + * @param functionKey the function key
  1239 + * @param functionExpression the function expression
  1240 + * @param functionType the function type
  1241 + * @throws IOException Signals that an I/O exception has occurred.
  1242 + * @throws ParseException the parse exception
1318 1243 */
1319 1244 public ComponentSpan(SpanQuery[] queries, String key, Double minimumDouble,
1320 1245 Double maximumDouble, String type, String[] functionKey,
... ... @@ -1438,20 +1363,13 @@ public class CodecComponent {
1438 1363 /**
1439 1364 * Instantiates a new component position.
1440 1365 *
1441   - * @param field
1442   - * the field
1443   - * @param key
1444   - * the key
1445   - * @param minimumDouble
1446   - * the minimum double
1447   - * @param maximumDouble
1448   - * the maximum double
1449   - * @param statsType
1450   - * the stats type
1451   - * @throws IOException
1452   - * Signals that an I/O exception has occurred.
1453   - * @throws ParseException
1454   - * the parse exception
  1366 + * @param field the field
  1367 + * @param key the key
  1368 + * @param minimumDouble the minimum double
  1369 + * @param maximumDouble the maximum double
  1370 + * @param statsType the stats type
  1371 + * @throws IOException Signals that an I/O exception has occurred.
  1372 + * @throws ParseException the parse exception
1455 1373 */
1456 1374 public ComponentPosition(String field, String key, Double minimumDouble,
1457 1375 Double maximumDouble, String statsType)
... ... @@ -1514,20 +1432,13 @@ public class CodecComponent {
1514 1432 /**
1515 1433 * Instantiates a new component token.
1516 1434 *
1517   - * @param field
1518   - * the field
1519   - * @param key
1520   - * the key
1521   - * @param minimumDouble
1522   - * the minimum double
1523   - * @param maximumDouble
1524   - * the maximum double
1525   - * @param statsType
1526   - * the stats type
1527   - * @throws IOException
1528   - * Signals that an I/O exception has occurred.
1529   - * @throws ParseException
1530   - * the parse exception
  1435 + * @param field the field
  1436 + * @param key the key
  1437 + * @param minimumDouble the minimum double
  1438 + * @param maximumDouble the maximum double
  1439 + * @param statsType the stats type
  1440 + * @throws IOException Signals that an I/O exception has occurred.
  1441 + * @throws ParseException the parse exception
1531 1442 */
1532 1443 public ComponentToken(String field, String key, Double minimumDouble,
1533 1444 Double maximumDouble, String statsType)
... ... @@ -1581,30 +1492,18 @@ public class CodecComponent {
1581 1492 /**
1582 1493 * Instantiates a new sub component function.
1583 1494 *
1584   - * @param collectorType
1585   - * the collector type
1586   - * @param key
1587   - * the key
1588   - * @param type
1589   - * the type
1590   - * @param parserFunction
1591   - * the parser function
1592   - * @param sortType
1593   - * the sort type
1594   - * @param sortDirection
1595   - * the sort direction
1596   - * @param start
1597   - * the start
1598   - * @param number
1599   - * the number
1600   - * @param segmentRegistration
1601   - * the segment registration
1602   - * @param boundary
1603   - * the boundary
1604   - * @throws ParseException
1605   - * the parse exception
1606   - * @throws IOException
1607   - * Signals that an I/O exception has occurred.
  1495 + * @param collectorType the collector type
  1496 + * @param key the key
  1497 + * @param type the type
  1498 + * @param parserFunction the parser function
  1499 + * @param sortType the sort type
  1500 + * @param sortDirection the sort direction
  1501 + * @param start the start
  1502 + * @param number the number
  1503 + * @param segmentRegistration the segment registration
  1504 + * @param boundary the boundary
  1505 + * @throws ParseException the parse exception
  1506 + * @throws IOException Signals that an I/O exception has occurred.
1608 1507 */
1609 1508 public SubComponentFunction(String collectorType, String key, String type,
1610 1509 MtasFunctionParserFunction parserFunction, String sortType,
... ... @@ -1619,7 +1518,7 @@ public class CodecComponent {
1619 1518 this.sortDirection = sortDirection;
1620 1519 this.dataType = parserFunction.getType();
1621 1520 this.statsItems = CodecUtil.createStatsItems(this.type);
1622   - this.statsType = CodecUtil.createStatsType(statsItems, null,
  1521 + this.statsType = CodecUtil.createStatsType(statsItems, sortType,
1623 1522 parserFunction);
1624 1523 if (collectorType.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
1625 1524 dataCollector = DataCollector.getCollector(
... ... @@ -1637,18 +1536,12 @@ public class CodecComponent {
1637 1536 /**
1638 1537 * Instantiates a new sub component function.
1639 1538 *
1640   - * @param collectorType
1641   - * the collector type
1642   - * @param key
1643   - * the key
1644   - * @param expression
1645   - * the expression
1646   - * @param type
1647   - * the type
1648   - * @throws ParseException
1649   - * the parse exception
1650   - * @throws IOException
1651   - * Signals that an I/O exception has occurred.
  1539 + * @param collectorType the collector type
  1540 + * @param key the key
  1541 + * @param expression the expression
  1542 + * @param type the type
  1543 + * @throws ParseException the parse exception
  1544 + * @throws IOException Signals that an I/O exception has occurred.
1652 1545 */
1653 1546 public SubComponentFunction(String collectorType, String key,
1654 1547 String expression, String type) throws ParseException, IOException {
... ... @@ -1691,10 +1584,8 @@ public class CodecComponent {
1691 1584 /**
1692 1585 * Instantiates a new kwic token.
1693 1586 *
1694   - * @param match
1695   - * the match
1696   - * @param tokens
1697   - * the tokens
  1587 + * @param match the match
  1588 + * @param tokens the tokens
1698 1589 */
1699 1590 public KwicToken(Match match, ArrayList<MtasToken<String>> tokens) {
1700 1591 startPosition = match.startPosition;
... ... @@ -1720,10 +1611,8 @@ public class CodecComponent {
1720 1611 /**
1721 1612 * Instantiates a new kwic hit.
1722 1613 *
1723   - * @param match
1724   - * the match
1725   - * @param hits
1726   - * the hits
  1614 + * @param match the match
  1615 + * @param hits the hits
1727 1616 */
1728 1617 public KwicHit(Match match, HashMap<Integer, ArrayList<String>> hits) {
1729 1618 startPosition = match.startPosition;
... ... @@ -1759,8 +1648,7 @@ public class CodecComponent {
1759 1648 /**
1760 1649 * Sort.
1761 1650 *
1762   - * @param data
1763   - * the data
  1651 + * @param data the data
1764 1652 * @return the array list
1765 1653 */
1766 1654 private ArrayList<MtasTreeHit<String>> sort(
... ... @@ -1783,22 +1671,14 @@ public class CodecComponent {
1783 1671 /**
1784 1672 * Instantiates a new group hit.
1785 1673 *
1786   - * @param list
1787   - * the list
1788   - * @param start
1789   - * the start
1790   - * @param end
1791   - * the end
1792   - * @param hitStart
1793   - * the hit start
1794   - * @param hitEnd
1795   - * the hit end
1796   - * @param group
1797   - * the group
1798   - * @param knownPrefixes
1799   - * the known prefixes
1800   - * @throws UnsupportedEncodingException
1801   - * the unsupported encoding exception
  1674 + * @param list the list
  1675 + * @param start the start
  1676 + * @param end the end
  1677 + * @param hitStart the hit start
  1678 + * @param hitEnd the hit end
  1679 + * @param group the group
  1680 + * @param knownPrefixes the known prefixes
  1681 + * @throws UnsupportedEncodingException the unsupported encoding exception
1802 1682 */
1803 1683 @SuppressWarnings("unchecked")
1804 1684 public GroupHit(ArrayList<MtasTreeHit<String>> list, int start, int end,
... ... @@ -2108,10 +1988,8 @@ public class CodecComponent {
2108 1988 /**
2109 1989 * Data equals.
2110 1990 *
2111   - * @param d1
2112   - * the d1
2113   - * @param d2
2114   - * the d2
  1991 + * @param d1 the d1
  1992 + * @param d2 the d2
2115 1993 * @return true, if successful
2116 1994 */
2117 1995 private boolean dataEquals(ArrayList<String>[] d1, ArrayList<String>[] d2) {
... ... @@ -2170,13 +2048,10 @@ public class CodecComponent {
2170 2048 /**
2171 2049 * Data to string.
2172 2050 *
2173   - * @param data
2174   - * the data
2175   - * @param missing
2176   - * the missing
  2051 + * @param data the data
  2052 + * @param missing the missing
2177 2053 * @return the string
2178   - * @throws UnsupportedEncodingException
2179   - * the unsupported encoding exception
  2054 + * @throws UnsupportedEncodingException the unsupported encoding exception
2180 2055 */
2181 2056 private String dataToString(ArrayList<String>[] data,
2182 2057 HashSet<String>[] missing) throws UnsupportedEncodingException {
... ... @@ -2227,10 +2102,8 @@ public class CodecComponent {
2227 2102 /**
2228 2103 * Key to sub sub object.
2229 2104 *
2230   - * @param key
2231   - * the key
2232   - * @param newKey
2233   - * the new key
  2105 + * @param key the key
  2106 + * @param newKey the new key
2234 2107 * @return the hash map[]
2235 2108 */
2236 2109 private static HashMap[] keyToSubSubObject(String key,
... ... @@ -2287,10 +2160,8 @@ public class CodecComponent {
2287 2160 /**
2288 2161 * Key to sub object.
2289 2162 *
2290   - * @param key
2291   - * the key
2292   - * @param newKey
2293   - * the new key
  2163 + * @param key the key
  2164 + * @param newKey the new key
2294 2165 * @return the hash map
2295 2166 */
2296 2167 private static HashMap keyToSubObject(String key, StringBuilder newKey) {
... ... @@ -2313,10 +2184,8 @@ public class CodecComponent {
2313 2184 /**
2314 2185 * Key to object.
2315 2186 *
2316   - * @param key
2317   - * the key
2318   - * @param newKey
2319   - * the new key
  2187 + * @param key the key
  2188 + * @param newKey the new key
2320 2189 * @return the hash map
2321 2190 */
2322 2191 public static HashMap keyToObject(String key, StringBuilder newKey) {
... ... @@ -2374,14 +2243,10 @@ public class CodecComponent {
2374 2243 /**
2375 2244 * Instantiates a new list token.
2376 2245 *
2377   - * @param docId
2378   - * the doc id
2379   - * @param docPosition
2380   - * the doc position
2381   - * @param match
2382   - * the match
2383   - * @param tokens
2384   - * the tokens
  2246 + * @param docId the doc id
  2247 + * @param docPosition the doc position
  2248 + * @param match the match
  2249 + * @param tokens the tokens
2385 2250 */
2386 2251 public ListToken(Integer docId, Integer docPosition, Match match,
2387 2252 ArrayList<MtasToken<String>> tokens) {
... ... @@ -2410,14 +2275,10 @@ public class CodecComponent {
2410 2275 /**
2411 2276 * Instantiates a new list hit.
2412 2277 *
2413   - * @param docId
2414   - * the doc id
2415   - * @param docPosition
2416   - * the doc position
2417   - * @param match
2418   - * the match
2419   - * @param hits
2420   - * the hits
  2278 + * @param docId the doc id
  2279 + * @param docPosition the doc position
  2280 + * @param match the match
  2281 + * @param hits the hits
2421 2282 */
2422 2283 public ListHit(Integer docId, Integer docPosition, Match match,
2423 2284 HashMap<Integer, ArrayList<String>> hits) {
... ... @@ -2443,10 +2304,8 @@ public class CodecComponent {
2443 2304 /**
2444 2305 * Instantiates a new match.
2445 2306 *
2446   - * @param startPosition
2447   - * the start position
2448   - * @param endPosition
2449   - * the end position
  2307 + * @param startPosition the start position
  2308 + * @param endPosition the end position
2450 2309 */
2451 2310 public Match(int startPosition, int endPosition) {
2452 2311 this.startPosition = startPosition;
... ...
src/mtas/codec/util/collector/MtasDataBasic.java
... ... @@ -319,7 +319,7 @@ abstract class MtasDataBasic&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 extends Numb
319 319 || !statsType.equals(newDataCollector.getStatsType())
320 320 || !(newDataCollector instanceof MtasDataBasic)) {
321 321 throw new IOException("cannot merge different dataCollectors");
322   - } else {
  322 + } else {
323 323 segmentRegistration = null;
324 324 @SuppressWarnings("unchecked")
325 325 MtasDataBasic<T1, T2> newMtasDataBasic = (MtasDataBasic<T1, T2>) newDataCollector;
... ...
src/mtas/codec/util/collector/MtasDataCollector.java
... ... @@ -91,6 +91,9 @@ public abstract class MtasDataCollector&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 e
91 91 /** The source number list. */
92 92 protected int[] sourceNumberList;
93 93  
  94 + /** The did segment registration. */
  95 + private boolean withTotal;
  96 +
94 97 /** The segment registration. */
95 98 public transient String segmentRegistration;
96 99  
... ... @@ -240,6 +243,7 @@ public abstract class MtasDataCollector&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 e
240 243 this.start = start;
241 244 this.number = number;
242 245 this.segmentRegistration = segmentRegistration;
  246 + this.withTotal = false;
243 247 if (segmentRegistration != null) {
244 248 segmentKeys = new HashSet<String>();
245 249 segmentKeyValueList = new LinkedHashMap<String, HashMap<String, T1>>();
... ... @@ -909,6 +913,7 @@ public abstract class MtasDataCollector&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 e
909 913 }
910 914 return segmentKeys.contains(key) ? SEGMENT_KEY : SEGMENT_NEW;
911 915 } else if (compareWithBoundary(value, tmpSegmentValueBoundary)) {
  916 + //System.out.println(key+" "+value+" "+tmpSegmentValueBoundary);
912 917 if (!test) {
913 918 segmentKeyValueList.get(segmentName).put(key, value);
914 919 if (compareWithBoundary(value, tmpSegmentValueMaxListMin)) {
... ... @@ -1407,5 +1412,21 @@ public abstract class MtasDataCollector&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 e
1407 1412 public int getSize() {
1408 1413 return size;
1409 1414 }
  1415 +
  1416 + public boolean withTotal() {
  1417 + return withTotal;
  1418 + }
  1419 +
  1420 + public void setWithTotal() throws IOException {
  1421 + if(collectorType.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
  1422 + if(segmentName!=null) {
  1423 + throw new IOException("can't get total with segmentRegistration");
  1424 + } else {
  1425 + withTotal = true;
  1426 + }
  1427 + } else {
  1428 + throw new IOException("can't get total for dataCollector of type "+collectorType);
  1429 + }
  1430 + }
1410 1431  
1411 1432 }
1412 1433 \ No newline at end of file
... ...
src/mtas/parser/cql/MtasCQLParser.java
... ... @@ -20,13 +20,25 @@ import org.apache.lucene.index.Term;
20 20 import mtas.search.spans.MtasSpanSequenceItem;
21 21 import mtas.search.spans.MtasSpanSequenceQuery;
22 22 import java.util.ArrayList;
  23 +import java.util.HashMap;
  24 +import java.util.HashSet;
23 25 import java.util.regex.Matcher;
24 26 import java.util.regex.Pattern;
25 27  
26 28 public class MtasCQLParser implements MtasCQLParserConstants {
27   - public SpanQuery parse(String field, String defaultPrefix) throws ParseException
  29 + public SpanQuery parse(String field, String defaultPrefix, HashMap<String, String[] > variables) throws ParseException
28 30 {
29   - return cql(field, defaultPrefix);
  31 + HashSet<String> usedVariables = new HashSet<String>();
  32 + SpanQuery query = cql(field, defaultPrefix, variables, usedVariables);
  33 + if(variables!=null && variables.size() > usedVariables.size()) {
  34 + for(String key : variables.keySet()) {
  35 + if(!usedVariables.contains(key)) {
  36 + throw new ParseException("variable $"+key+" not used");
  37 + }
  38 + }
  39 + throw new ParseException("unused variables defined");
  40 + }
  41 + return query;
30 42 }
31 43  
32 44 private String unquoteString(String unfiltered)
... ... @@ -39,10 +51,19 @@ public class MtasCQLParser implements MtasCQLParserConstants {
39 51 return unfiltered;
40 52 }
41 53  
42   - final private SpanQuery cql(String field, String defaultPrefix) throws ParseException, ParseException {
  54 + private String variableString(String variable)
  55 + {
  56 + if (variable.startsWith("$"))
  57 + {
  58 + variable = variable.substring(1, variable.length());
  59 + }
  60 + return variable;
  61 + }
  62 +
  63 + final private SpanQuery cql(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException, ParseException {
43 64 SpanQuery q;
44 65 ArrayList < MtasSpanSequenceItem > itemList = new ArrayList < MtasSpanSequenceItem > ();
45   - q = cqlBlock(field, defaultPrefix);
  66 + q = cqlBlock(field, defaultPrefix, variables, usedVariables);
46 67 itemList.add(new MtasSpanSequenceItem(q, false));
47 68 label_1:
48 69 while (true) {
... ... @@ -51,7 +72,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
51 72 } else {
52 73 break label_1;
53 74 }
54   - q = cqlBlock(field, defaultPrefix);
  75 + q = cqlBlock(field, defaultPrefix, variables, usedVariables);
55 76 itemList.add(new MtasSpanSequenceItem(q, false));
56 77 }
57 78 jj_consume_token(0);
... ... @@ -66,7 +87,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
66 87 throw new Error("Missing return statement in function");
67 88 }
68 89  
69   - final private SpanQuery cqlBlock(String field, String defaultPrefix) throws ParseException, ParseException {
  90 + final private SpanQuery cqlBlock(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException, ParseException {
70 91 MtasCQLParserSentenceCondition s1 = null, s2 = null;
71 92 SpanQuery q1 = null, q2 = null;
72 93 Token end = null;
... ... @@ -76,10 +97,10 @@ public class MtasCQLParser implements MtasCQLParserConstants {
76 97 String OPERATOR_WITHIN = "within";
77 98 String OPERATOR_NOT_WITHIN = "not_within";
78 99 if (jj_2_2(1000)) {
79   - s1 = sentence(field, defaultPrefix);
  100 + s1 = sentence(field, defaultPrefix, variables, usedVariables);
80 101 } else if (jj_2_3(1000)) {
81 102 jj_consume_token(BRACKET_START);
82   - q1 = cqlBlock(field, defaultPrefix);
  103 + q1 = cqlBlock(field, defaultPrefix, variables, usedVariables);
83 104 jj_consume_token(BRACKET_END);
84 105 } else {
85 106 jj_consume_token(-1);
... ... @@ -103,10 +124,10 @@ public class MtasCQLParser implements MtasCQLParserConstants {
103 124 throw new ParseException();
104 125 }
105 126 if (jj_2_8(1000)) {
106   - s2 = sentence(field, defaultPrefix);
  127 + s2 = sentence(field, defaultPrefix, variables, usedVariables);
107 128 } else if (jj_2_9(1000)) {
108 129 jj_consume_token(BRACKET_START);
109   - q2 = cqlBlock(field, defaultPrefix);
  130 + q2 = cqlBlock(field, defaultPrefix, variables, usedVariables);
110 131 jj_consume_token(BRACKET_END);
111 132 } else {
112 133 jj_consume_token(-1);
... ... @@ -153,7 +174,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
153 174 throw new Error("Missing return statement in function");
154 175 }
155 176  
156   - final private MtasCQLParserSentenceCondition sentence(String field, String defaultPrefix) throws ParseException, ParseException {
  177 + final private MtasCQLParserSentenceCondition sentence(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException, ParseException {
157 178 MtasCQLParserSentenceCondition sentenceCondition;
158 179 MtasCQLParserSentencePartCondition condition;
159 180 Token questionMark = null;
... ... @@ -161,13 +182,13 @@ public class MtasCQLParser implements MtasCQLParserConstants {
161 182 Token maxValue = null;
162 183 int minimumOccurence = 1;
163 184 int maximumOccurence = 1;
164   - condition = sentencePart(field, defaultPrefix);
  185 + condition = sentencePart(field, defaultPrefix, variables, usedVariables);
165 186 sentenceCondition = condition.createFullSentence();
166 187 {if (true) return sentenceCondition;}
167 188 throw new Error("Missing return statement in function");
168 189 }
169 190  
170   - final private MtasCQLParserSentencePartCondition sentencePart(String field, String defaultPrefix) throws ParseException, ParseException {
  191 + final private MtasCQLParserSentencePartCondition sentencePart(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException, ParseException {
171 192 Token operator;
172 193 MtasCQLParserSentencePartCondition condition, sentencePart;
173 194 MtasCQLParserBasicSentenceCondition basicSentence;
... ... @@ -177,11 +198,11 @@ public class MtasCQLParser implements MtasCQLParserConstants {
177 198 int minimumOccurence = 1;
178 199 int maximumOccurence = 1;
179 200 if (jj_2_15(1000)) {
180   - basicSentence = basicSentence(field, defaultPrefix);
  201 + basicSentence = basicSentence(field, defaultPrefix, variables, usedVariables);
181 202 condition = new MtasCQLParserSentencePartCondition(basicSentence);
182 203 } else if (jj_2_16(1000)) {
183 204 jj_consume_token(BRACKET_START);
184   - sentencePart = sentencePart(field, defaultPrefix);
  205 + sentencePart = sentencePart(field, defaultPrefix, variables, usedVariables);
185 206 jj_consume_token(BRACKET_END);
186 207 if (jj_2_14(1000)) {
187 208 questionMark = null;
... ... @@ -234,7 +255,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
234 255 } else {
235 256 ;
236 257 }
237   - sentencePart = sentencePart(field, defaultPrefix);
  258 + sentencePart = sentencePart(field, defaultPrefix, variables, usedVariables);
238 259 if (operator == null)
239 260 {
240 261 condition.setOr(false);
... ... @@ -251,12 +272,12 @@ public class MtasCQLParser implements MtasCQLParserConstants {
251 272 throw new Error("Missing return statement in function");
252 273 }
253 274  
254   - final private MtasCQLParserBasicSentenceCondition basicSentence(String field, String defaultPrefix) throws ParseException, ParseException {
  275 + final private MtasCQLParserBasicSentenceCondition basicSentence(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException, ParseException {
255 276 MtasCQLParserWordFullCondition subWordCondition;
256 277 MtasCQLParserGroupFullCondition subGroupCondition;
257 278 MtasCQLParserBasicSentenceCondition condition = new MtasCQLParserBasicSentenceCondition();
258 279 if (jj_2_19(1000)) {
259   - subWordCondition = word(field, defaultPrefix);
  280 + subWordCondition = word(field, defaultPrefix, variables, usedVariables);
260 281 condition.addWord(subWordCondition);
261 282 } else if (jj_2_20(1000)) {
262 283 subGroupCondition = group(field);
... ... @@ -273,7 +294,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
273 294 break label_2;
274 295 }
275 296 if (jj_2_22(1000)) {
276   - subWordCondition = word(field, defaultPrefix);
  297 + subWordCondition = word(field, defaultPrefix, variables, usedVariables);
277 298 condition.addWord(subWordCondition);
278 299 } else if (jj_2_23(1000)) {
279 300 subGroupCondition = group(field);
... ... @@ -408,7 +429,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
408 429 throw new Error("Missing return statement in function");
409 430 }
410 431  
411   - final private MtasCQLParserWordFullCondition word(String field, String defaultPrefix) throws ParseException, ParseException {
  432 + final private MtasCQLParserWordFullCondition word(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException, ParseException {
412 433 Token questionMark = null;
413 434 Token value;
414 435 MtasCQLParserWordFullCondition wordCondition;
... ... @@ -419,16 +440,16 @@ public class MtasCQLParser implements MtasCQLParserConstants {
419 440 int maximumOccurence = 1;
420 441 if (jj_2_39(1000)) {
421 442 value = jj_consume_token(QUOTED_VALUE);
422   - condition = new MtasCQLParserDefaultPrefixCondition(field, defaultPrefix, unquoteString(value.image));
  443 + condition = new MtasCQLParserDefaultPrefixCondition(field, defaultPrefix, unquoteString(value.image), variables, usedVariables);
423 444 } else if (jj_2_40(1000)) {
424 445 jj_consume_token(WORD_START);
425 446 if (jj_2_37(1000)) {
426   - subCondition = wordCondition(field);
  447 + subCondition = wordCondition(field, variables, usedVariables);
427 448 if (jj_2_35(1000)) {
428 449 jj_consume_token(AND);
429 450 condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_AND);
430 451 condition.addCondition(subCondition);
431   - subCondition = wordCondition(field);
  452 + subCondition = wordCondition(field, variables, usedVariables);
432 453 condition.addCondition(subCondition);
433 454 label_3:
434 455 while (true) {
... ... @@ -438,14 +459,14 @@ public class MtasCQLParser implements MtasCQLParserConstants {
438 459 break label_3;
439 460 }
440 461 jj_consume_token(AND);
441   - subCondition = wordCondition(field);
  462 + subCondition = wordCondition(field, variables, usedVariables);
442 463 condition.addCondition(subCondition);
443 464 }
444 465 } else if (jj_2_36(1000)) {
445 466 jj_consume_token(OR);
446 467 condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_OR);
447 468 condition.addCondition(subCondition);
448   - subCondition = wordCondition(field);
  469 + subCondition = wordCondition(field, variables, usedVariables);
449 470 condition.addCondition(subCondition);
450 471 label_4:
451 472 while (true) {
... ... @@ -455,7 +476,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
455 476 break label_4;
456 477 }
457 478 jj_consume_token(OR);
458   - subCondition = wordCondition(field);
  479 + subCondition = wordCondition(field, variables, usedVariables);
459 480 condition.addCondition(subCondition);
460 481 }
461 482 } else {
... ... @@ -463,7 +484,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
463 484 throw new ParseException();
464 485 }
465 486 } else if (jj_2_38(1000)) {
466   - condition = wordCondition(field);
  487 + condition = wordCondition(field, variables, usedVariables);
467 488 } else {
468 489 jj_consume_token(-1);
469 490 throw new ParseException();
... ... @@ -523,7 +544,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
523 544 throw new Error("Missing return statement in function");
524 545 }
525 546  
526   - final private MtasCQLParserWordCondition wordCondition(String field) throws ParseException, ParseException {
  547 + final private MtasCQLParserWordCondition wordCondition(String field, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException, ParseException {
527 548 Token negation = null;
528 549 MtasCQLParserWordCondition condition, subCondition;
529 550 if (jj_2_66(1000)) {
... ... @@ -536,9 +557,9 @@ public class MtasCQLParser implements MtasCQLParserConstants {
536 557 if (jj_2_63(1000)) {
537 558 condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_AND);
538 559 if (jj_2_47(1000)) {
539   - subCondition = wordAtomCondition(field);
  560 + subCondition = wordAtomCondition(field, variables, usedVariables);
540 561 } else if (jj_2_48(1000)) {
541   - subCondition = wordCondition(field);
  562 + subCondition = wordCondition(field, variables, usedVariables);
542 563 } else {
543 564 jj_consume_token(-1);
544 565 throw new ParseException();
... ... @@ -546,9 +567,9 @@ public class MtasCQLParser implements MtasCQLParserConstants {
546 567 condition.addCondition(subCondition);
547 568 jj_consume_token(AND);
548 569 if (jj_2_49(1000)) {
549   - subCondition = wordAtomCondition(field);
  570 + subCondition = wordAtomCondition(field, variables, usedVariables);
550 571 } else if (jj_2_50(1000)) {
551   - subCondition = wordCondition(field);
  572 + subCondition = wordCondition(field, variables, usedVariables);
552 573 } else {
553 574 jj_consume_token(-1);
554 575 throw new ParseException();
... ... @@ -563,9 +584,9 @@ public class MtasCQLParser implements MtasCQLParserConstants {
563 584 }
564 585 jj_consume_token(AND);
565 586 if (jj_2_52(1000)) {
566   - subCondition = wordAtomCondition(field);
  587 + subCondition = wordAtomCondition(field, variables, usedVariables);
567 588 } else if (jj_2_53(1000)) {
568   - subCondition = wordCondition(field);
  589 + subCondition = wordCondition(field, variables, usedVariables);
569 590 } else {
570 591 jj_consume_token(-1);
571 592 throw new ParseException();
... ... @@ -575,9 +596,9 @@ public class MtasCQLParser implements MtasCQLParserConstants {
575 596 } else if (jj_2_64(1000)) {
576 597 condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_OR);
577 598 if (jj_2_54(1000)) {
578   - subCondition = wordAtomCondition(field);
  599 + subCondition = wordAtomCondition(field, variables, usedVariables);
579 600 } else if (jj_2_55(1000)) {
580   - subCondition = wordCondition(field);
  601 + subCondition = wordCondition(field, variables, usedVariables);
581 602 } else {
582 603 jj_consume_token(-1);
583 604 throw new ParseException();
... ... @@ -585,9 +606,9 @@ public class MtasCQLParser implements MtasCQLParserConstants {
585 606 condition.addCondition(subCondition);
586 607 jj_consume_token(OR);
587 608 if (jj_2_56(1000)) {
588   - subCondition = wordAtomCondition(field);
  609 + subCondition = wordAtomCondition(field, variables, usedVariables);
589 610 } else if (jj_2_57(1000)) {
590   - subCondition = wordCondition(field);
  611 + subCondition = wordCondition(field, variables, usedVariables);
591 612 } else {
592 613 jj_consume_token(-1);
593 614 throw new ParseException();
... ... @@ -602,9 +623,9 @@ public class MtasCQLParser implements MtasCQLParserConstants {
602 623 }
603 624 jj_consume_token(OR);
604 625 if (jj_2_59(1000)) {
605   - subCondition = wordAtomCondition(field);
  626 + subCondition = wordAtomCondition(field, variables, usedVariables);
606 627 } else if (jj_2_60(1000)) {
607   - subCondition = wordCondition(field);
  628 + subCondition = wordCondition(field, variables, usedVariables);
608 629 } else {
609 630 jj_consume_token(-1);
610 631 throw new ParseException();
... ... @@ -613,9 +634,9 @@ public class MtasCQLParser implements MtasCQLParserConstants {
613 634 }
614 635 } else if (jj_2_65(1000)) {
615 636 if (jj_2_61(1000)) {
616   - condition = wordAtomCondition(field);
  637 + condition = wordAtomCondition(field, variables, usedVariables);
617 638 } else if (jj_2_62(1000)) {
618   - condition = wordCondition(field);
  639 + condition = wordCondition(field, variables, usedVariables);
619 640 } else {
620 641 jj_consume_token(-1);
621 642 throw new ParseException();
... ... @@ -633,7 +654,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
633 654 {if (true) return condition;}
634 655 } else if (jj_2_67(1000)) {
635 656 //plain atom is a valid condition
636   - subCondition = wordAtomCondition(field);
  657 + subCondition = wordAtomCondition(field, variables, usedVariables);
637 658 //System.out.println("=== wordCondition ===\n" + subCondition + "\n");
638 659 {if (true) return subCondition;}
639 660 } else {
... ... @@ -645,7 +666,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
645 666 throw new Error("Missing return statement in function");
646 667 }
647 668  
648   - final private MtasCQLParserWordCondition wordAtomCondition(String field) throws ParseException, ParseException {
  669 + final private MtasCQLParserWordCondition wordAtomCondition(String field, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException, ParseException {
649 670 Token negation = null;
650 671 Token nequals = null;
651 672 Token prefix;
... ... @@ -655,7 +676,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
655 676 } else {
656 677 ;
657 678 }
658   - if (jj_2_73(1000)) {
  679 + if (jj_2_75(1000)) {
659 680 jj_consume_token(OCTOTHORPE);
660 681 if (jj_2_69(1000)) {
661 682 value = jj_consume_token(NUMBER);
... ... @@ -695,7 +716,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
695 716 {if (true) throw new ParseException("invalid range");}
696 717 }
697 718 {if (true) return condition;}
698   - } else if (jj_2_74(1000)) {
  719 + } else if (jj_2_76(1000)) {
699 720 prefix = jj_consume_token(UNQUOTED_VALUE);
700 721 if (jj_2_71(1000)) {
701 722 nequals = jj_consume_token(TOKEN_NOTEQUALS);
... ... @@ -705,6 +726,35 @@ public class MtasCQLParser implements MtasCQLParserConstants {
705 726 jj_consume_token(-1);
706 727 throw new ParseException();
707 728 }
  729 + value = jj_consume_token(VARIABLE);
  730 + if (nequals != null)
  731 + {
  732 + //use RegexpQuery combined with SpanMultiTermQueryWrapper
  733 + {if (true) throw new ParseException("TODO: not implemented '" + prefix.image + nequals.image + value.image + "'");}
  734 + }
  735 + else
  736 + {
  737 + MtasCQLParserWordCondition condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_OR);
  738 + Term term = new Term(field, prefix.image + MtasToken.DELIMITER + value.image);
  739 + SpanQuery q = new MtasCQLParserWordQuery(field, prefix.image, variableString(value.image), MtasCQLParserWordQuery.MTAS_CQL_VARIABLE_QUERY, variables, usedVariables);
  740 + if (negation != null)
  741 + {
  742 + condition.swapNot();
  743 + }
  744 + condition.addPositiveQuery(q);
  745 + //System.out.println("=== wordAtomCondition ===\n" + condition + "\n");
  746 + {if (true) return condition;}
  747 + }
  748 + } else if (jj_2_77(1000)) {
  749 + prefix = jj_consume_token(UNQUOTED_VALUE);
  750 + if (jj_2_73(1000)) {
  751 + nequals = jj_consume_token(TOKEN_NOTEQUALS);
  752 + } else if (jj_2_74(1000)) {
  753 + jj_consume_token(TOKEN_EQUALS);
  754 + } else {
  755 + jj_consume_token(-1);
  756 + throw new ParseException();
  757 + }
708 758 value = jj_consume_token(QUOTED_VALUE);
709 759 if (nequals != null)
710 760 {
... ... @@ -715,7 +765,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
715 765 {
716 766 MtasCQLParserWordCondition condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_AND);
717 767 Term term = new Term(field, prefix.image + MtasToken.DELIMITER + unquoteString(value.image));
718   - SpanQuery q = new MtasCQLParserWordQuery(field, prefix.image, unquoteString(value.image), MtasCQLParserWordQuery.MTAS_CQL_REGEXP_QUERY);
  768 + SpanQuery q = new MtasCQLParserWordQuery(field, prefix.image, unquoteString(value.image), MtasCQLParserWordQuery.MTAS_CQL_REGEXP_QUERY, variables, usedVariables);
719 769 if (negation != null)
720 770 {
721 771 condition.swapNot();
... ... @@ -1249,107 +1299,25 @@ public class MtasCQLParser implements MtasCQLParserConstants {
1249 1299 finally { jj_save(73, xla); }
1250 1300 }
1251 1301  
1252   - private boolean jj_3R_11() {
1253   - Token xsp;
1254   - xsp = jj_scanpos;
1255   - if (jj_3_39()) {
1256   - jj_scanpos = xsp;
1257   - if (jj_3_40()) {
1258   - jj_scanpos = xsp;
1259   - if (jj_3_41()) return true;
1260   - }
1261   - }
1262   - xsp = jj_scanpos;
1263   - if (jj_3_45()) jj_scanpos = xsp;
1264   - return false;
1265   - }
1266   -
1267   - private boolean jj_3_68() {
1268   - if (jj_scan_token(NEGATION)) return true;
1269   - return false;
1270   - }
1271   -
1272   - private boolean jj_3R_15() {
1273   - Token xsp;
1274   - xsp = jj_scanpos;
1275   - if (jj_3_68()) jj_scanpos = xsp;
1276   - xsp = jj_scanpos;
1277   - if (jj_3_73()) {
1278   - jj_scanpos = xsp;
1279   - if (jj_3_74()) return true;
1280   - }
1281   - return false;
1282   - }
1283   -
1284   - private boolean jj_3_13() {
1285   - if (jj_scan_token(QUESTION_MARK)) return true;
1286   - return false;
1287   - }
1288   -
1289   - private boolean jj_3_12() {
1290   - if (jj_scan_token(CURLY_BRACKET_START)) return true;
1291   - if (jj_scan_token(NUMBER)) return true;
1292   - if (jj_scan_token(CURLY_BRACKET_END)) return true;
1293   - return false;
1294   - }
1295   -
1296   - private boolean jj_3_11() {
1297   - if (jj_scan_token(CURLY_BRACKET_START)) return true;
1298   - if (jj_scan_token(NUMBER)) return true;
1299   - if (jj_scan_token(KOMMA)) return true;
1300   - if (jj_scan_token(NUMBER)) return true;
1301   - if (jj_scan_token(CURLY_BRACKET_END)) return true;
1302   - return false;
1303   - }
1304   -
1305   - private boolean jj_3_14() {
1306   - Token xsp;
1307   - xsp = jj_scanpos;
1308   - if (jj_3_11()) {
1309   - jj_scanpos = xsp;
1310   - if (jj_3_12()) {
1311   - jj_scanpos = xsp;
1312   - if (jj_3_13()) return true;
1313   - }
1314   - }
1315   - return false;
1316   - }
1317   -
1318   - private boolean jj_3_67() {
1319   - if (jj_3R_15()) return true;
1320   - return false;
1321   - }
1322   -
1323   - private boolean jj_3_16() {
1324   - if (jj_scan_token(BRACKET_START)) return true;
1325   - if (jj_3R_10()) return true;
1326   - if (jj_scan_token(BRACKET_END)) return true;
1327   - Token xsp;
1328   - xsp = jj_scanpos;
1329   - if (jj_3_14()) jj_scanpos = xsp;
1330   - return false;
1331   - }
1332   -
1333   - private boolean jj_3_15() {
1334   - if (jj_3R_9()) return true;
1335   - return false;
  1302 + private boolean jj_2_75(int xla) {
  1303 + jj_la = xla; jj_lastpos = jj_scanpos = token;
  1304 + try { return !jj_3_75(); }
  1305 + catch(LookaheadSuccess ls) { return true; }
  1306 + finally { jj_save(74, xla); }
1336 1307 }
1337 1308  
1338   - private boolean jj_3R_10() {
1339   - Token xsp;
1340   - xsp = jj_scanpos;
1341   - if (jj_3_15()) {
1342   - jj_scanpos = xsp;
1343   - if (jj_3_16()) return true;
1344   - }
1345   - xsp = jj_scanpos;
1346   - if (jj_3_18()) jj_scanpos = xsp;
1347   - return false;
  1309 + private boolean jj_2_76(int xla) {
  1310 + jj_la = xla; jj_lastpos = jj_scanpos = token;
  1311 + try { return !jj_3_76(); }
  1312 + catch(LookaheadSuccess ls) { return true; }
  1313 + finally { jj_save(75, xla); }
1348 1314 }
1349 1315  
1350   - private boolean jj_3_62() {
1351   - if (jj_3R_14()) return true;
1352   - return false;
  1316 + private boolean jj_2_77(int xla) {
  1317 + jj_la = xla; jj_lastpos = jj_scanpos = token;
  1318 + try { return !jj_3_77(); }
  1319 + catch(LookaheadSuccess ls) { return true; }
  1320 + finally { jj_save(76, xla); }
1353 1321 }
1354 1322  
1355 1323 private boolean jj_3_61() {
... ... @@ -1637,6 +1605,16 @@ public class MtasCQLParser implements MtasCQLParserConstants {
1637 1605 return false;
1638 1606 }
1639 1607  
  1608 + private boolean jj_3_74() {
  1609 + if (jj_scan_token(TOKEN_EQUALS)) return true;
  1610 + return false;
  1611 + }
  1612 +
  1613 + private boolean jj_3_73() {
  1614 + if (jj_scan_token(TOKEN_NOTEQUALS)) return true;
  1615 + return false;
  1616 + }
  1617 +
1640 1618 private boolean jj_3_5() {
1641 1619 if (jj_scan_token(NOT_CONTAINING)) return true;
1642 1620 return false;
... ... @@ -1647,6 +1625,18 @@ public class MtasCQLParser implements MtasCQLParserConstants {
1647 1625 return false;
1648 1626 }
1649 1627  
  1628 + private boolean jj_3_77() {
  1629 + if (jj_scan_token(UNQUOTED_VALUE)) return true;
  1630 + Token xsp;
  1631 + xsp = jj_scanpos;
  1632 + if (jj_3_73()) {
  1633 + jj_scanpos = xsp;
  1634 + if (jj_3_74()) return true;
  1635 + }
  1636 + if (jj_scan_token(QUOTED_VALUE)) return true;
  1637 + return false;
  1638 + }
  1639 +
1650 1640 private boolean jj_3_10() {
1651 1641 Token xsp;
1652 1642 xsp = jj_scanpos;
... ... @@ -1791,7 +1781,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
1791 1781 return false;
1792 1782 }
1793 1783  
1794   - private boolean jj_3_74() {
  1784 + private boolean jj_3_76() {
1795 1785 if (jj_scan_token(UNQUOTED_VALUE)) return true;
1796 1786 Token xsp;
1797 1787 xsp = jj_scanpos;
... ... @@ -1799,7 +1789,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
1799 1789 jj_scanpos = xsp;
1800 1790 if (jj_3_72()) return true;
1801 1791 }
1802   - if (jj_scan_token(QUOTED_VALUE)) return true;
  1792 + if (jj_scan_token(VARIABLE)) return true;
1803 1793 return false;
1804 1794 }
1805 1795  
... ... @@ -1893,7 +1883,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
1893 1883 return false;
1894 1884 }
1895 1885  
1896   - private boolean jj_3_73() {
  1886 + private boolean jj_3_75() {
1897 1887 if (jj_scan_token(OCTOTHORPE)) return true;
1898 1888 Token xsp;
1899 1889 xsp = jj_scanpos;
... ... @@ -1904,6 +1894,112 @@ public class MtasCQLParser implements MtasCQLParserConstants {
1904 1894 return false;
1905 1895 }
1906 1896  
  1897 + private boolean jj_3R_11() {
  1898 + Token xsp;
  1899 + xsp = jj_scanpos;
  1900 + if (jj_3_39()) {
  1901 + jj_scanpos = xsp;
  1902 + if (jj_3_40()) {
  1903 + jj_scanpos = xsp;
  1904 + if (jj_3_41()) return true;
  1905 + }
  1906 + }
  1907 + xsp = jj_scanpos;
  1908 + if (jj_3_45()) jj_scanpos = xsp;
  1909 + return false;
  1910 + }
  1911 +
  1912 + private boolean jj_3_68() {
  1913 + if (jj_scan_token(NEGATION)) return true;
  1914 + return false;
  1915 + }
  1916 +
  1917 + private boolean jj_3R_15() {
  1918 + Token xsp;
  1919 + xsp = jj_scanpos;
  1920 + if (jj_3_68()) jj_scanpos = xsp;
  1921 + xsp = jj_scanpos;
  1922 + if (jj_3_75()) {
  1923 + jj_scanpos = xsp;
  1924 + if (jj_3_76()) {
  1925 + jj_scanpos = xsp;
  1926 + if (jj_3_77()) return true;
  1927 + }
  1928 + }
  1929 + return false;
  1930 + }
  1931 +
  1932 + private boolean jj_3_13() {
  1933 + if (jj_scan_token(QUESTION_MARK)) return true;
  1934 + return false;
  1935 + }
  1936 +
  1937 + private boolean jj_3_12() {
  1938 + if (jj_scan_token(CURLY_BRACKET_START)) return true;
  1939 + if (jj_scan_token(NUMBER)) return true;
  1940 + if (jj_scan_token(CURLY_BRACKET_END)) return true;
  1941 + return false;
  1942 + }
  1943 +
  1944 + private boolean jj_3_11() {
  1945 + if (jj_scan_token(CURLY_BRACKET_START)) return true;
  1946 + if (jj_scan_token(NUMBER)) return true;
  1947 + if (jj_scan_token(KOMMA)) return true;
  1948 + if (jj_scan_token(NUMBER)) return true;
  1949 + if (jj_scan_token(CURLY_BRACKET_END)) return true;
  1950 + return false;
  1951 + }
  1952 +
  1953 + private boolean jj_3_14() {
  1954 + Token xsp;
  1955 + xsp = jj_scanpos;
  1956 + if (jj_3_11()) {
  1957 + jj_scanpos = xsp;
  1958 + if (jj_3_12()) {
  1959 + jj_scanpos = xsp;
  1960 + if (jj_3_13()) return true;
  1961 + }
  1962 + }
  1963 + return false;
  1964 + }
  1965 +
  1966 + private boolean jj_3_67() {
  1967 + if (jj_3R_15()) return true;
  1968 + return false;
  1969 + }
  1970 +
  1971 + private boolean jj_3_16() {
  1972 + if (jj_scan_token(BRACKET_START)) return true;
  1973 + if (jj_3R_10()) return true;
  1974 + if (jj_scan_token(BRACKET_END)) return true;
  1975 + Token xsp;
  1976 + xsp = jj_scanpos;
  1977 + if (jj_3_14()) jj_scanpos = xsp;
  1978 + return false;
  1979 + }
  1980 +
  1981 + private boolean jj_3_15() {
  1982 + if (jj_3R_9()) return true;
  1983 + return false;
  1984 + }
  1985 +
  1986 + private boolean jj_3R_10() {
  1987 + Token xsp;
  1988 + xsp = jj_scanpos;
  1989 + if (jj_3_15()) {
  1990 + jj_scanpos = xsp;
  1991 + if (jj_3_16()) return true;
  1992 + }
  1993 + xsp = jj_scanpos;
  1994 + if (jj_3_18()) jj_scanpos = xsp;
  1995 + return false;
  1996 + }
  1997 +
  1998 + private boolean jj_3_62() {
  1999 + if (jj_3R_14()) return true;
  2000 + return false;
  2001 + }
  2002 +
1907 2003 /** Generated Token Manager. */
1908 2004 public MtasCQLParserTokenManager token_source;
1909 2005 SimpleCharStream jj_input_stream;
... ... @@ -1923,7 +2019,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
1923 2019 private static void jj_la1_init_0() {
1924 2020 jj_la1_0 = new int[] {};
1925 2021 }
1926   - final private JJCalls[] jj_2_rtns = new JJCalls[74];
  2022 + final private JJCalls[] jj_2_rtns = new JJCalls[77];
1927 2023 private boolean jj_rescan = false;
1928 2024 private int jj_gc = 0;
1929 2025  
... ... @@ -2107,7 +2203,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
2107 2203 /** Generate ParseException. */
2108 2204 public ParseException generateParseException() {
2109 2205 jj_expentries.clear();
2110   - boolean[] la1tokens = new boolean[31];
  2206 + boolean[] la1tokens = new boolean[32];
2111 2207 if (jj_kind >= 0) {
2112 2208 la1tokens[jj_kind] = true;
2113 2209 jj_kind = -1;
... ... @@ -2121,7 +2217,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
2121 2217 }
2122 2218 }
2123 2219 }
2124   - for (int i = 0; i < 31; i++) {
  2220 + for (int i = 0; i < 32; i++) {
2125 2221 if (la1tokens[i]) {
2126 2222 jj_expentry = new int[1];
2127 2223 jj_expentry[0] = i;
... ... @@ -2148,7 +2244,7 @@ public class MtasCQLParser implements MtasCQLParserConstants {
2148 2244  
2149 2245 private void jj_rescan_token() {
2150 2246 jj_rescan = true;
2151   - for (int i = 0; i < 74; i++) {
  2247 + for (int i = 0; i < 77; i++) {
2152 2248 try {
2153 2249 JJCalls p = jj_2_rtns[i];
2154 2250 do {
... ... @@ -2229,6 +2325,9 @@ public class MtasCQLParser implements MtasCQLParserConstants {
2229 2325 case 71: jj_3_72(); break;
2230 2326 case 72: jj_3_73(); break;
2231 2327 case 73: jj_3_74(); break;
  2328 + case 74: jj_3_75(); break;
  2329 + case 75: jj_3_76(); break;
  2330 + case 76: jj_3_77(); break;
2232 2331 }
2233 2332 }
2234 2333 p = p.next;
... ...
src/mtas/parser/cql/MtasCQLParser.jj
... ... @@ -6,10 +6,9 @@ options
6 6 DEBUG_PARSER = false;
7 7 DEBUG_LOOKAHEAD = false;
8 8 DEBUG_TOKEN_MANAGER = false;
9   -
10 9 LOOKAHEAD= 1000;
11   -}
12   -
  10 +}
  11 +
13 12 PARSER_BEGIN(MtasCQLParser)
14 13 package mtas.parser.cql;
15 14 import mtas.analysis.token.MtasToken;
... ... @@ -32,14 +31,26 @@ import org.apache.lucene.index.Term;
32 31 import mtas.search.spans.MtasSpanSequenceItem;
33 32 import mtas.search.spans.MtasSpanSequenceQuery;
34 33 import java.util.ArrayList;
  34 +import java.util.HashMap;
  35 +import java.util.HashSet;
35 36 import java.util.regex.Matcher;
36 37 import java.util.regex.Pattern;
37 38  
38 39 public class MtasCQLParser
39 40 {
40   - public SpanQuery parse(String field, String defaultPrefix) throws ParseException
  41 + public SpanQuery parse(String field, String defaultPrefix, HashMap<String, String[] > variables) throws ParseException
41 42 {
42   - return cql(field, defaultPrefix);
  43 + HashSet<String> usedVariables = new HashSet<String>();
  44 + SpanQuery query = cql(field, defaultPrefix, variables, usedVariables);
  45 + if(variables!=null && variables.size() > usedVariables.size()) {
  46 + for(String key : variables.keySet()) {
  47 + if(!usedVariables.contains(key)) {
  48 + throw new ParseException("variable $"+key+" not used");
  49 + }
  50 + }
  51 + throw new ParseException("unused variables defined");
  52 + }
  53 + return query;
43 54 }
44 55  
45 56 private String unquoteString(String unfiltered)
... ... @@ -51,6 +62,15 @@ public class MtasCQLParser
51 62 }
52 63 return unfiltered;
53 64 }
  65 +
  66 + private String variableString(String variable)
  67 + {
  68 + if (variable.startsWith("$"))
  69 + {
  70 + variable = variable.substring(1, variable.length());
  71 + }
  72 + return variable;
  73 + }
54 74 }
55 75  
56 76 PARSER_END(MtasCQLParser)
... ... @@ -170,6 +190,17 @@ TOKEN :
170 190  
171 191 TOKEN :
172 192 {
  193 + < VARIABLE :
  194 + "$"
  195 + (
  196 + < ALLOWED_UNQUOTED_CHARACTER >
  197 + | < DIGIT >
  198 + )+
  199 + >
  200 +}
  201 +
  202 +TOKEN :
  203 +{
173 204 < QUOTED_VALUE :
174 205 "\""
175 206 (
... ... @@ -203,18 +234,18 @@ TOKEN :
203 234 >
204 235 }
205 236  
206   -private SpanQuery cql(String field, String defaultPrefix) throws ParseException :
  237 +private SpanQuery cql(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException :
207 238 {
208 239 SpanQuery q;
209 240 ArrayList < MtasSpanSequenceItem > itemList = new ArrayList < MtasSpanSequenceItem > ();
210 241 }
211 242 {
212   - q = cqlBlock(field, defaultPrefix)
  243 + q = cqlBlock(field, defaultPrefix, variables, usedVariables)
213 244 {
214 245 itemList.add(new MtasSpanSequenceItem(q, false));
215 246 }
216 247 (
217   - q = cqlBlock(field, defaultPrefix)
  248 + q = cqlBlock(field, defaultPrefix, variables, usedVariables)
218 249 {
219 250 itemList.add(new MtasSpanSequenceItem(q, false));
220 251 }
... ... @@ -232,7 +263,7 @@ private SpanQuery cql(String field, String defaultPrefix) throws ParseException
232 263 }
233 264 }
234 265  
235   -private SpanQuery cqlBlock(String field, String defaultPrefix) throws ParseException :
  266 +private SpanQuery cqlBlock(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException :
236 267 {
237 268 MtasCQLParserSentenceCondition s1 = null, s2 = null;
238 269 SpanQuery q1 = null, q2 = null;
... ... @@ -245,9 +276,9 @@ private SpanQuery cqlBlock(String field, String defaultPrefix) throws ParseExcep
245 276 }
246 277 {
247 278 (
248   - s1 = sentence(field, defaultPrefix)
  279 + s1 = sentence(field, defaultPrefix, variables, usedVariables)
249 280 |
250   - < BRACKET_START > q1 = cqlBlock(field, defaultPrefix) < BRACKET_END >
  281 + < BRACKET_START > q1 = cqlBlock(field, defaultPrefix, variables, usedVariables) < BRACKET_END >
251 282 )
252 283 [
253 284 (
... ... @@ -269,8 +300,8 @@ private SpanQuery cqlBlock(String field, String defaultPrefix) throws ParseExcep
269 300 }
270 301 )
271 302 (
272   - s2 = sentence(field, defaultPrefix)
273   - | < BRACKET_START > q2 = cqlBlock(field, defaultPrefix) < BRACKET_END >
  303 + s2 = sentence(field, defaultPrefix, variables, usedVariables)
  304 + | < BRACKET_START > q2 = cqlBlock(field, defaultPrefix, variables, usedVariables) < BRACKET_END >
274 305 )
275 306 ]
276 307 {
... ... @@ -312,7 +343,7 @@ private SpanQuery cqlBlock(String field, String defaultPrefix) throws ParseExcep
312 343 }
313 344 }
314 345  
315   -private MtasCQLParserSentenceCondition sentence(String field, String defaultPrefix) throws ParseException :
  346 +private MtasCQLParserSentenceCondition sentence(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException :
316 347 {
317 348 MtasCQLParserSentenceCondition sentenceCondition;
318 349 MtasCQLParserSentencePartCondition condition;
... ... @@ -323,14 +354,14 @@ private MtasCQLParserSentenceCondition sentence(String field, String defaultPref
323 354 int maximumOccurence = 1;
324 355 }
325 356 {
326   - condition = sentencePart(field, defaultPrefix)
  357 + condition = sentencePart(field, defaultPrefix, variables, usedVariables)
327 358 {
328 359 sentenceCondition = condition.createFullSentence();
329 360 return sentenceCondition;
330 361 }
331 362 }
332 363  
333   -private MtasCQLParserSentencePartCondition sentencePart(String field, String defaultPrefix) throws ParseException :
  364 +private MtasCQLParserSentencePartCondition sentencePart(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException :
334 365 {
335 366 Token operator;
336 367 MtasCQLParserSentencePartCondition condition, sentencePart;
... ... @@ -343,13 +374,13 @@ private MtasCQLParserSentencePartCondition sentencePart(String field, String def
343 374 }
344 375 {
345 376 (
346   - basicSentence = basicSentence(field, defaultPrefix)
  377 + basicSentence = basicSentence(field, defaultPrefix, variables, usedVariables)
347 378 {
348 379 condition = new MtasCQLParserSentencePartCondition(basicSentence);
349 380 }
350 381 |
351 382 (
352   - < BRACKET_START > sentencePart = sentencePart(field, defaultPrefix) < BRACKET_END >
  383 + < BRACKET_START > sentencePart = sentencePart(field, defaultPrefix, variables, usedVariables) < BRACKET_END >
353 384 [
354 385 {
355 386 questionMark = null;
... ... @@ -389,7 +420,7 @@ private MtasCQLParserSentencePartCondition sentencePart(String field, String def
389 420 }
390 421 [ operator = < OR > ]
391 422 (
392   - sentencePart = sentencePart(field, defaultPrefix)
  423 + sentencePart = sentencePart(field, defaultPrefix, variables, usedVariables)
393 424 {
394 425 if (operator == null)
395 426 {
... ... @@ -408,15 +439,15 @@ private MtasCQLParserSentencePartCondition sentencePart(String field, String def
408 439 }
409 440 }
410 441  
411   -private MtasCQLParserBasicSentenceCondition basicSentence(String field, String defaultPrefix) throws ParseException :
  442 +private MtasCQLParserBasicSentenceCondition basicSentence(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException :
412 443 {
413 444 MtasCQLParserWordFullCondition subWordCondition;
414 445 MtasCQLParserGroupFullCondition subGroupCondition;
415 446 MtasCQLParserBasicSentenceCondition condition = new MtasCQLParserBasicSentenceCondition();
416 447 }
417   -{
  448 +{
418 449 (
419   - subWordCondition = word(field, defaultPrefix)
  450 + subWordCondition = word(field, defaultPrefix, variables, usedVariables)
420 451 {
421 452 condition.addWord(subWordCondition);
422 453 }
... ... @@ -427,7 +458,7 @@ private MtasCQLParserBasicSentenceCondition basicSentence(String field, String d
427 458 )
428 459 (
429 460 (
430   - subWordCondition = word(field, defaultPrefix)
  461 + subWordCondition = word(field, defaultPrefix, variables, usedVariables)
431 462 {
432 463 condition.addWord(subWordCondition);
433 464 }
... ... @@ -550,7 +581,7 @@ private MtasCQLParserGroupCondition groupCondition(String field) throws ParseExc
550 581 }
551 582 }
552 583  
553   -private MtasCQLParserWordFullCondition word(String field, String defaultPrefix) throws ParseException :
  584 +private MtasCQLParserWordFullCondition word(String field, String defaultPrefix, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException :
554 585 {
555 586 Token questionMark = null;
556 587 Token value;
... ... @@ -565,24 +596,24 @@ private MtasCQLParserWordFullCondition word(String field, String defaultPrefix)
565 596 (
566 597 value = < QUOTED_VALUE >
567 598 {
568   - condition = new MtasCQLParserDefaultPrefixCondition(field, defaultPrefix, unquoteString(value.image));
  599 + condition = new MtasCQLParserDefaultPrefixCondition(field, defaultPrefix, unquoteString(value.image), variables, usedVariables);
569 600 }
570 601 |
571 602 < WORD_START >
572 603 (
573   - subCondition = wordCondition(field)
  604 + subCondition = wordCondition(field, variables, usedVariables)
574 605 (
575 606 < AND >
576 607 {
577 608 condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_AND);
578 609 condition.addCondition(subCondition);
579 610 }
580   - subCondition = wordCondition(field)
  611 + subCondition = wordCondition(field, variables, usedVariables)
581 612 {
582 613 condition.addCondition(subCondition);
583 614 }
584 615 (
585   - < AND > subCondition = wordCondition(field)
  616 + < AND > subCondition = wordCondition(field, variables, usedVariables)
586 617 {
587 618 condition.addCondition(subCondition);
588 619 }
... ... @@ -592,18 +623,18 @@ private MtasCQLParserWordFullCondition word(String field, String defaultPrefix)
592 623 condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_OR);
593 624 condition.addCondition(subCondition);
594 625 }
595   - subCondition = wordCondition(field)
  626 + subCondition = wordCondition(field, variables, usedVariables)
596 627 {
597 628 condition.addCondition(subCondition);
598 629 }
599 630 (
600   - < OR > subCondition = wordCondition(field)
  631 + < OR > subCondition = wordCondition(field, variables, usedVariables)
601 632 {
602 633 condition.addCondition(subCondition);
603 634 }
604 635 )*
605 636 )
606   - | condition = wordCondition(field)
  637 + | condition = wordCondition(field, variables, usedVariables)
607 638 )
608 639 < WORD_END >
609 640 |
... ... @@ -647,7 +678,7 @@ private MtasCQLParserWordFullCondition word(String field, String defaultPrefix)
647 678 }
648 679 }
649 680  
650   -private MtasCQLParserWordCondition wordCondition(String field) throws ParseException :
  681 +private MtasCQLParserWordCondition wordCondition(String field, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException :
651 682 {
652 683 Token negation = null;
653 684 MtasCQLParserWordCondition condition, subCondition;
... ... @@ -660,16 +691,16 @@ private MtasCQLParserWordCondition wordCondition(String field) throws ParseExcep
660 691 condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_AND);
661 692 }
662 693 (
663   - subCondition = wordAtomCondition(field)
664   - | subCondition = wordCondition(field)
  694 + subCondition = wordAtomCondition(field, variables, usedVariables)
  695 + | subCondition = wordCondition(field, variables, usedVariables)
665 696 )
666 697 {
667 698 condition.addCondition(subCondition);
668 699 }
669 700 < AND >
670 701 (
671   - subCondition = wordAtomCondition(field)
672   - | subCondition = wordCondition(field)
  702 + subCondition = wordAtomCondition(field, variables, usedVariables)
  703 + | subCondition = wordCondition(field, variables, usedVariables)
673 704 )
674 705 {
675 706 condition.addCondition(subCondition);
... ... @@ -677,8 +708,8 @@ private MtasCQLParserWordCondition wordCondition(String field) throws ParseExcep
677 708 (
678 709 < AND >
679 710 (
680   - subCondition = wordAtomCondition(field)
681   - | subCondition = wordCondition(field)
  711 + subCondition = wordAtomCondition(field, variables, usedVariables)
  712 + | subCondition = wordCondition(field, variables, usedVariables)
682 713 )
683 714 {
684 715 condition.addCondition(subCondition);
... ... @@ -689,16 +720,16 @@ private MtasCQLParserWordCondition wordCondition(String field) throws ParseExcep
689 720 condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_OR);
690 721 }
691 722 (
692   - subCondition = wordAtomCondition(field)
693   - | subCondition = wordCondition(field)
  723 + subCondition = wordAtomCondition(field, variables, usedVariables)
  724 + | subCondition = wordCondition(field, variables, usedVariables)
694 725 )
695 726 {
696 727 condition.addCondition(subCondition);
697 728 }
698 729 < OR >
699 730 (
700   - subCondition = wordAtomCondition(field)
701   - | subCondition = wordCondition(field)
  731 + subCondition = wordAtomCondition(field, variables, usedVariables)
  732 + | subCondition = wordCondition(field, variables, usedVariables)
702 733 )
703 734 {
704 735 condition.addCondition(subCondition);
... ... @@ -706,8 +737,8 @@ private MtasCQLParserWordCondition wordCondition(String field) throws ParseExcep
706 737 (
707 738 < OR >
708 739 (
709   - subCondition = wordAtomCondition(field)
710   - | subCondition = wordCondition(field)
  740 + subCondition = wordAtomCondition(field, variables, usedVariables)
  741 + | subCondition = wordCondition(field, variables, usedVariables)
711 742 )
712 743 {
713 744 condition.addCondition(subCondition);
... ... @@ -715,8 +746,8 @@ private MtasCQLParserWordCondition wordCondition(String field) throws ParseExcep
715 746 )*
716 747 |
717 748 (
718   - condition = wordAtomCondition(field)
719   - | condition = wordCondition(field)
  749 + condition = wordAtomCondition(field, variables, usedVariables)
  750 + | condition = wordCondition(field, variables, usedVariables)
720 751 )
721 752 )
722 753 < BRACKET_END >
... ... @@ -727,7 +758,7 @@ private MtasCQLParserWordCondition wordCondition(String field) throws ParseExcep
727 758 }
728 759 //System.out.println("=== wordCondition ===\n" + condition + "\n"); return condition;
729 760 }
730   - | //plain atom is a valid condition subCondition = wordAtomCondition(field)
  761 + | //plain atom is a valid condition subCondition = wordAtomCondition(field, variables, usedVariables)
731 762 {
732 763 //System.out.println("=== wordCondition ===\n" + subCondition + "\n"); return subCondition;
733 764 }
... ... @@ -737,7 +768,7 @@ private MtasCQLParserWordCondition wordCondition(String field) throws ParseExcep
737 768 }
738 769 }
739 770  
740   -private MtasCQLParserWordCondition wordAtomCondition(String field) throws ParseException :
  771 +private MtasCQLParserWordCondition wordAtomCondition(String field, HashMap<String, String[] > variables, HashSet<String > usedVariables) throws ParseException :
741 772 {
742 773 Token negation = null;
743 774 Token nequals = null;
... ... @@ -794,6 +825,35 @@ private MtasCQLParserWordCondition wordAtomCondition(String field) throws ParseE
794 825 nequals = < TOKEN_NOTEQUALS >
795 826 | < TOKEN_EQUALS >
796 827 )
  828 + (value = < VARIABLE >)
  829 + {
  830 + if (nequals != null)
  831 + {
  832 + //use RegexpQuery combined with SpanMultiTermQueryWrapper
  833 + throw new ParseException("TODO: not implemented '" + prefix.image + nequals.image + value.image + "'");
  834 + }
  835 + else
  836 + {
  837 + MtasCQLParserWordCondition condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_OR);
  838 + Term term = new Term(field, prefix.image + MtasToken.DELIMITER + value.image);
  839 + SpanQuery q = new MtasCQLParserWordQuery(field, prefix.image, variableString(value.image), MtasCQLParserWordQuery.MTAS_CQL_VARIABLE_QUERY, variables, usedVariables);
  840 + if (negation != null)
  841 + {
  842 + condition.swapNot();
  843 + }
  844 + condition.addPositiveQuery(q);
  845 + //System.out.println("=== wordAtomCondition ===\n" + condition + "\n");
  846 + return condition;
  847 + }
  848 + }
  849 + )
  850 + |
  851 + (
  852 + (prefix = < UNQUOTED_VALUE >)
  853 + (
  854 + nequals = < TOKEN_NOTEQUALS >
  855 + | < TOKEN_EQUALS >
  856 + )
797 857 (value = < QUOTED_VALUE >)
798 858 {
799 859 if (nequals != null)
... ... @@ -804,7 +864,7 @@ private MtasCQLParserWordCondition wordAtomCondition(String field) throws ParseE
804 864 {
805 865 MtasCQLParserWordCondition condition = new MtasCQLParserWordCondition(field, MtasCQLParserWordCondition.TYPE_AND);
806 866 Term term = new Term(field, prefix.image + MtasToken.DELIMITER + unquoteString(value.image));
807   - SpanQuery q = new MtasCQLParserWordQuery(field, prefix.image, unquoteString(value.image), MtasCQLParserWordQuery.MTAS_CQL_REGEXP_QUERY);
  867 + SpanQuery q = new MtasCQLParserWordQuery(field, prefix.image, unquoteString(value.image), MtasCQLParserWordQuery.MTAS_CQL_REGEXP_QUERY, variables, usedVariables);
808 868 if (negation != null)
809 869 {
810 870 condition.swapNot();
... ...
src/mtas/parser/cql/MtasCQLParserConstants.java
... ... @@ -53,15 +53,17 @@ public interface MtasCQLParserConstants {
53 53 /** RegularExpression Id. */
54 54 int NUMBER = 25;
55 55 /** RegularExpression Id. */
56   - int QUOTED_VALUE = 26;
  56 + int VARIABLE = 26;
57 57 /** RegularExpression Id. */
58   - int UNQUOTED_VALUE = 27;
  58 + int QUOTED_VALUE = 27;
59 59 /** RegularExpression Id. */
60   - int OCTOTHORPE = 28;
  60 + int UNQUOTED_VALUE = 28;
61 61 /** RegularExpression Id. */
62   - int DIGIT = 29;
  62 + int OCTOTHORPE = 29;
63 63 /** RegularExpression Id. */
64   - int ALLOWED_UNQUOTED_CHARACTER = 30;
  64 + int DIGIT = 30;
  65 + /** RegularExpression Id. */
  66 + int ALLOWED_UNQUOTED_CHARACTER = 31;
65 67  
66 68 /** Lexical state. */
67 69 int DEFAULT = 0;
... ... @@ -94,6 +96,7 @@ public interface MtasCQLParserConstants {
94 96 "\"!=\"",
95 97 "\"=\"",
96 98 "<NUMBER>",
  99 + "<VARIABLE>",
97 100 "<QUOTED_VALUE>",
98 101 "<UNQUOTED_VALUE>",
99 102 "<OCTOTHORPE>",
... ...
src/mtas/parser/cql/MtasCQLParserTokenManager.java
... ... @@ -20,6 +20,8 @@ import org.apache.lucene.index.Term;
20 20 import mtas.search.spans.MtasSpanSequenceItem;
21 21 import mtas.search.spans.MtasSpanSequenceQuery;
22 22 import java.util.ArrayList;
  23 +import java.util.HashMap;
  24 +import java.util.HashSet;
23 25 import java.util.regex.Matcher;
24 26 import java.util.regex.Pattern;
25 27  
... ... @@ -38,79 +40,79 @@ private final int jjStopStringLiteralDfa_0(int pos, long active0)
38 40 case 0:
39 41 if ((active0 & 0xa0L) != 0L)
40 42 {
41   - jjmatchedKind = 27;
42   - return 6;
  43 + jjmatchedKind = 28;
  44 + return 8;
43 45 }
44 46 return -1;
45 47 case 1:
46 48 if ((active0 & 0xa0L) != 0L)
47 49 {
48   - jjmatchedKind = 27;
  50 + jjmatchedKind = 28;
49 51 jjmatchedPos = 1;
50   - return 6;
  52 + return 8;
51 53 }
52 54 return -1;
53 55 case 2:
54 56 if ((active0 & 0xa0L) != 0L)
55 57 {
56   - jjmatchedKind = 27;
  58 + jjmatchedKind = 28;
57 59 jjmatchedPos = 2;
58   - return 6;
  60 + return 8;
59 61 }
60 62 return -1;
61 63 case 3:
62 64 if ((active0 & 0xa0L) != 0L)
63 65 {
64   - jjmatchedKind = 27;
  66 + jjmatchedKind = 28;
65 67 jjmatchedPos = 3;
66   - return 6;
  68 + return 8;
67 69 }
68 70 return -1;
69 71 case 4:
70 72 if ((active0 & 0xa0L) != 0L)
71 73 {
72   - jjmatchedKind = 27;
  74 + jjmatchedKind = 28;
73 75 jjmatchedPos = 4;
74   - return 6;
  76 + return 8;
75 77 }
76 78 return -1;
77 79 case 5:
78   - if ((active0 & 0x80L) != 0L)
79   - return 6;
80 80 if ((active0 & 0x20L) != 0L)
81 81 {
82   - jjmatchedKind = 27;
  82 + jjmatchedKind = 28;
83 83 jjmatchedPos = 5;
84   - return 6;
  84 + return 8;
85 85 }
  86 + if ((active0 & 0x80L) != 0L)
  87 + return 8;
86 88 return -1;
87 89 case 6:
88 90 if ((active0 & 0x20L) != 0L)
89 91 {
90   - jjmatchedKind = 27;
  92 + jjmatchedKind = 28;
91 93 jjmatchedPos = 6;
92   - return 6;
  94 + return 8;
93 95 }
94 96 return -1;
95 97 case 7:
96 98 if ((active0 & 0x20L) != 0L)
97 99 {
98   - jjmatchedKind = 27;
  100 + jjmatchedKind = 28;
99 101 jjmatchedPos = 7;
100   - return 6;
  102 + return 8;
101 103 }
102 104 return -1;
103 105 case 8:
104 106 if ((active0 & 0x20L) != 0L)
105 107 {
106   - jjmatchedKind = 27;
  108 + jjmatchedKind = 28;
107 109 jjmatchedPos = 8;
108   - return 6;
  110 + return 8;
109 111 }
110 112 return -1;
111 113 case 9:
112 114 if ((active0 & 0x20L) != 0L)
113   - return 6;
  115 + return 8;
114 116 return -1;
115 117 default :
116 118 return -1;
... ... @@ -282,7 +284,7 @@ private int jjMoveStringLiteralDfa5_0(long old0, long active0)
282 284 return jjMoveStringLiteralDfa6_0(active0, 0x120L);
283 285 case 110:
284 286 if ((active0 & 0x80L) != 0L)
285   - return jjStartNfaWithStates_0(5, 7, 6);
  287 + return jjStartNfaWithStates_0(5, 7, 8);
286 288 break;
287 289 default :
288 290 break;
... ... @@ -364,7 +366,7 @@ private int jjMoveStringLiteralDfa9_0(long old0, long active0)
364 366 {
365 367 case 103:
366 368 if ((active0 & 0x20L) != 0L)
367   - return jjStartNfaWithStates_0(9, 5, 6);
  369 + return jjStartNfaWithStates_0(9, 5, 8);
368 370 break;
369 371 case 110:
370 372 return jjMoveStringLiteralDfa10_0(active0, 0x40L);
... ... @@ -410,7 +412,7 @@ static final long[] jjbitVec2 = {
410 412 private int jjMoveNfa_0(int startState, int curPos)
411 413 {
412 414 int startsAt = 0;
413   - jjnewStateCnt = 8;
  415 + jjnewStateCnt = 10;
414 416 int i = 1;
415 417 jjstateSet[0] = startState;
416 418 int kind = 0x7fffffff;
... ... @@ -428,17 +430,19 @@ private int jjMoveNfa_0(int startState, int curPos)
428 430 case 1:
429 431 if ((0x3ff600000000000L & l) != 0L)
430 432 {
431   - if (kind > 27)
432   - kind = 27;
433   - jjCheckNAdd(6);
  433 + if (kind > 28)
  434 + kind = 28;
  435 + jjCheckNAdd(8);
434 436 }
435 437 else if (curChar == 35)
436 438 {
437   - if (kind > 28)
438   - kind = 28;
  439 + if (kind > 29)
  440 + kind = 29;
439 441 }
440 442 else if (curChar == 34)
441 443 jjCheckNAddStates(0, 2);
  444 + else if (curChar == 36)
  445 + jjCheckNAdd(2);
442 446 if ((0x3ff000000000000L & l) != 0L)
443 447 {
444 448 if (kind > 25)
... ... @@ -453,27 +457,38 @@ private int jjMoveNfa_0(int startState, int curPos)
453 457 kind = 25;
454 458 jjCheckNAdd(0);
455 459 break;
  460 + case 2:
  461 + if ((0x3ff600000000000L & l) == 0L)
  462 + break;
  463 + if (kind > 26)
  464 + kind = 26;
  465 + jjCheckNAdd(2);
  466 + break;
456 467 case 3:
  468 + if (curChar == 34)
  469 + jjCheckNAddStates(0, 2);
  470 + break;
  471 + case 5:
457 472 jjCheckNAddStates(0, 2);
458 473 break;
459   - case 4:
  474 + case 6:
460 475 if ((0xfffffffbffffffffL & l) != 0L)
461 476 jjCheckNAddStates(0, 2);
462 477 break;
463   - case 5:
464   - if (curChar == 34 && kind > 26)
465   - kind = 26;
  478 + case 7:
  479 + if (curChar == 34 && kind > 27)
  480 + kind = 27;
466 481 break;
467   - case 6:
  482 + case 8:
468 483 if ((0x3ff600000000000L & l) == 0L)
469 484 break;
470   - if (kind > 27)
471   - kind = 27;
472   - jjCheckNAdd(6);
473   - break;
474   - case 7:
475   - if (curChar == 35 && kind > 28)
  485 + if (kind > 28)
476 486 kind = 28;
  487 + jjCheckNAdd(8);
  488 + break;
  489 + case 9:
  490 + if (curChar == 35 && kind > 29)
  491 + kind = 29;
477 492 break;
478 493 default : break;
479 494 }
... ... @@ -487,21 +502,28 @@ private int jjMoveNfa_0(int startState, int curPos)
487 502 switch(jjstateSet[--i])
488 503 {
489 504 case 1:
490   - case 6:
  505 + case 8:
491 506 if ((0x7fffffe87fffffeL & l) == 0L)
492 507 break;
493   - if (kind > 27)
494   - kind = 27;
495   - jjCheckNAdd(6);
  508 + if (kind > 28)
  509 + kind = 28;
  510 + jjCheckNAdd(8);
496 511 break;
497 512 case 2:
  513 + if ((0x7fffffe87fffffeL & l) == 0L)
  514 + break;
  515 + if (kind > 26)
  516 + kind = 26;
  517 + jjstateSet[jjnewStateCnt++] = 2;
  518 + break;
  519 + case 4:
498 520 if (curChar == 92)
499   - jjstateSet[jjnewStateCnt++] = 3;
  521 + jjstateSet[jjnewStateCnt++] = 5;
500 522 break;
501   - case 3:
  523 + case 5:
502 524 jjCheckNAddStates(0, 2);
503 525 break;
504   - case 4:
  526 + case 6:
505 527 if ((0xffffffffefffffffL & l) != 0L)
506 528 jjCheckNAddStates(0, 2);
507 529 break;
... ... @@ -520,8 +542,8 @@ private int jjMoveNfa_0(int startState, int curPos)
520 542 {
521 543 switch(jjstateSet[--i])
522 544 {
523   - case 3:
524   - case 4:
  545 + case 5:
  546 + case 6:
525 547 if (jjCanMove_0(hiByte, i1, i2, l1, l2))
526 548 jjCheckNAddStates(0, 2);
527 549 break;
... ... @@ -536,14 +558,14 @@ private int jjMoveNfa_0(int startState, int curPos)
536 558 kind = 0x7fffffff;
537 559 }
538 560 ++curPos;
539   - if ((i = jjnewStateCnt) == (startsAt = 8 - (jjnewStateCnt = startsAt)))
  561 + if ((i = jjnewStateCnt) == (startsAt = 10 - (jjnewStateCnt = startsAt)))
540 562 return curPos;
541 563 try { curChar = input_stream.readChar(); }
542 564 catch(java.io.IOException e) { return curPos; }
543 565 }
544 566 }
545 567 static final int[] jjnextStates = {
546   - 2, 4, 5,
  568 + 4, 6, 7,
547 569 };
548 570 private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
549 571 {
... ... @@ -563,21 +585,21 @@ public static final String[] jjstrLiteralImages = {
563 585 "", null, null, null, null, "\143\157\156\164\141\151\156\151\156\147",
564 586 "\41\143\157\156\164\141\151\156\151\156\147", "\167\151\164\150\151\156", "\41\167\151\164\150\151\156", "\74", "\76",
565 587 "\133", "\135", "\50", "\51", "\173", "\175", "\57", "\77", "\41", "\46", "\174",
566   -"\54", "\41\75", "\75", null, null, null, null, null, null, };
  588 +"\54", "\41\75", "\75", null, null, null, null, null, null, null, };
567 589  
568 590 /** Lexer state names. */
569 591 public static final String[] lexStateNames = {
570 592 "DEFAULT",
571 593 };
572 594 static final long[] jjtoToken = {
573   - 0x1fffffe1L,
  595 + 0x3fffffe1L,
574 596 };
575 597 static final long[] jjtoSkip = {
576 598 0x1eL,
577 599 };
578 600 protected SimpleCharStream input_stream;
579   -private final int[] jjrounds = new int[8];
580   -private final int[] jjstateSet = new int[16];
  601 +private final int[] jjrounds = new int[10];
  602 +private final int[] jjstateSet = new int[20];
581 603 protected char curChar;
582 604 /** Constructor. */
583 605 public MtasCQLParserTokenManager(SimpleCharStream stream){
... ... @@ -604,7 +626,7 @@ private void ReInitRounds()
604 626 {
605 627 int i;
606 628 jjround = 0x80000001;
607   - for (i = 8; i-- > 0;)
  629 + for (i = 10; i-- > 0;)
608 630 jjrounds[i] = 0x80000000;
609 631 }
610 632  
... ...
src/mtas/parser/cql/util/MtasCQLParserDefaultPrefixCondition.java
1 1 package mtas.parser.cql.util;
2 2  
  3 +import java.util.HashMap;
  4 +import java.util.HashSet;
  5 +
3 6 import mtas.parser.cql.ParseException;
4 7  
5 8 /**
... ... @@ -21,12 +24,12 @@ public class MtasCQLParserDefaultPrefixCondition
21 24 * the parse exception
22 25 */
23 26 public MtasCQLParserDefaultPrefixCondition(String field, String prefix,
24   - String value) throws ParseException {
  27 + String value, HashMap<String, String[] > variables, HashSet<String> usedVariables) throws ParseException {
25 28 super(field, TYPE_AND);
26 29 if (prefix == null) {
27 30 throw new ParseException("no default prefix defined");
28 31 } else {
29   - addPositiveQuery(new MtasCQLParserWordQuery(field, prefix, value));
  32 + addPositiveQuery(new MtasCQLParserWordQuery(field, prefix, value, variables, usedVariables));
30 33 }
31 34 }
32 35  
... ...
src/mtas/parser/cql/util/MtasCQLParserWordQuery.java
1 1 package mtas.parser.cql.util;
2 2  
3 3 import java.io.IOException;
  4 +import java.util.HashMap;
  5 +import java.util.HashSet;
  6 +import java.util.regex.Pattern;
4 7  
5 8 import mtas.analysis.token.MtasToken;
  9 +import mtas.parser.cql.ParseException;
  10 +import mtas.search.spans.MtasSpanMatchNoneQuery;
  11 +import mtas.search.spans.MtasSpanOrQuery;
6 12 import mtas.search.spans.MtasSpanPrefixQuery;
7 13 import mtas.search.spans.MtasSpanRegexpQuery;
8 14 import mtas.search.spans.MtasSpanTermQuery;
... ... @@ -13,6 +19,7 @@ import org.apache.lucene.index.Term;
13 19 import org.apache.lucene.search.IndexSearcher;
14 20 import org.apache.lucene.search.Query;
15 21 import org.apache.lucene.search.spans.SpanQuery;
  22 +import org.apache.lucene.search.spans.SpanTermQuery;
16 23 import org.apache.lucene.search.spans.SpanWeight;
17 24  
18 25 /**
... ... @@ -37,6 +44,8 @@ public class MtasCQLParserWordQuery extends SpanQuery {
37 44  
38 45 /** The Constant MTAS_CQL_WILDCARD_QUERY. */
39 46 public static final String MTAS_CQL_WILDCARD_QUERY = "wildcard";
  47 +
  48 + public static final String MTAS_CQL_VARIABLE_QUERY = "variable";
40 49  
41 50 /**
42 51 * Instantiates a new mtas cql parser word query.
... ... @@ -46,7 +55,7 @@ public class MtasCQLParserWordQuery extends SpanQuery {
46 55 * @param prefix
47 56 * the prefix
48 57 */
49   - public MtasCQLParserWordQuery(String field, String prefix) {
  58 + public MtasCQLParserWordQuery(String field, String prefix, HashMap<String, String[] > variables) {
50 59 term = new Term(field, prefix + MtasToken.DELIMITER);
51 60 query = new MtasSpanPrefixQuery(term, true);
52 61 }
... ... @@ -60,9 +69,10 @@ public class MtasCQLParserWordQuery extends SpanQuery {
60 69 * the prefix
61 70 * @param value
62 71 * the value
  72 + * @throws IOException
63 73 */
64   - public MtasCQLParserWordQuery(String field, String prefix, String value) {
65   - this(field, prefix, value, MTAS_CQL_REGEXP_QUERY);
  74 + public MtasCQLParserWordQuery(String field, String prefix, String value, HashMap<String, String[] > variables, HashSet<String> usedVariables) throws ParseException {
  75 + this(field, prefix, value, MTAS_CQL_REGEXP_QUERY, variables, usedVariables);
66 76 }
67 77  
68 78 /**
... ... @@ -76,18 +86,45 @@ public class MtasCQLParserWordQuery extends SpanQuery {
76 86 * the value
77 87 * @param type
78 88 * the type
  89 + * @throws IOException
79 90 */
80   - public MtasCQLParserWordQuery(String field, String prefix, String value,
81   - String type) {
  91 + public MtasCQLParserWordQuery(String field, String prefix, String value,
  92 + String type, HashMap<String, String[] > variables, HashSet<String> usedVariables) throws ParseException {
  93 + String termBase = prefix + MtasToken.DELIMITER + value;
82 94 if (type.equals(MTAS_CQL_REGEXP_QUERY)) {
83   - term = new Term(field, prefix + MtasToken.DELIMITER + value + "\u0000*");
  95 + term = new Term(field, termBase + "\u0000*");
84 96 query = new MtasSpanRegexpQuery(term, true);
85 97 } else if (type.equals(MTAS_CQL_WILDCARD_QUERY)) {
86   - term = new Term(field, prefix + MtasToken.DELIMITER + value);
  98 + term = new Term(field, termBase );
87 99 query = new MtasSpanWildcardQuery(term, true);
88   - } else if (type.equals(MTAS_CQL_TERM_QUERY)) {
89   - term = new Term(field, prefix + MtasToken.DELIMITER + value);
90   - query = new MtasSpanTermQuery(term, true);
  100 + } else if (type.equals(MTAS_CQL_TERM_QUERY)) {
  101 + term = new Term(field, "\""+termBase+"\"\u0000*");
  102 + query = new MtasSpanRegexpQuery(term, true);
  103 + } else if (type.equals(MTAS_CQL_VARIABLE_QUERY)) {
  104 + if(value!=null && variables!=null && variables.containsKey(value) && variables.get(value)!=null) {
  105 + if(usedVariables.contains(value)) {
  106 + throw new ParseException("variable $"+value+" should be used exactly one time");
  107 + } else {
  108 + usedVariables.add(value);
  109 + }
  110 + String[] list = variables.get(value);
  111 + SpanQuery[] queries = new SpanQuery[list.length];
  112 + term = new Term(field, prefix + MtasToken.DELIMITER);
  113 + for(int i=0; i<list.length; i++) {
  114 + termBase = prefix + MtasToken.DELIMITER + list[i];
  115 + term = new Term(field, "\""+termBase+"\"\u0000*");
  116 + queries[i] = new MtasSpanRegexpQuery(term, true);
  117 + }
  118 + if(queries.length==0) {
  119 + query = new MtasSpanMatchNoneQuery(field);
  120 + } else if(queries.length>1) {
  121 + query = new MtasSpanOrQuery(queries);
  122 + } else {
  123 + query = queries[0];
  124 + }
  125 + } else {
  126 + throw new ParseException("variable $"+value+" not defined");
  127 + }
91 128 } else {
92 129 term = new Term(field, prefix + MtasToken.DELIMITER + value);
93 130 query = new MtasSpanTermQuery(term, true);
... ...
src/mtas/search/spans/MtasSpanMatchNoneQuery.java 0 โ†’ 100644
  1 +package mtas.search.spans;
  2 +
  3 +import java.io.IOException;
  4 +import java.lang.reflect.Method;
  5 +import java.util.Map;
  6 +import java.util.Set;
  7 +import mtas.search.similarities.MtasSimScorer;
  8 +import org.apache.lucene.codecs.FieldsProducer;
  9 +import org.apache.lucene.index.LeafReader;
  10 +import org.apache.lucene.index.LeafReaderContext;
  11 +import org.apache.lucene.index.Term;
  12 +import org.apache.lucene.index.TermContext;
  13 +import org.apache.lucene.search.IndexSearcher;
  14 +import org.apache.lucene.search.similarities.Similarity.SimScorer;
  15 +import org.apache.lucene.search.spans.SpanQuery;
  16 +import org.apache.lucene.search.spans.SpanWeight;
  17 +import org.apache.lucene.search.spans.Spans;
  18 +
  19 +/**
  20 + * The Class MtasSpanMatchNoneQuery.
  21 + */
  22 +public class MtasSpanMatchNoneQuery extends SpanQuery {
  23 +
  24 + /** The field. */
  25 + private String field;
  26 +
  27 + /** The query name. */
  28 + private static String QUERY_NAME = "mtasSpanMatchNoneQuery";
  29 +
  30 + /**
  31 + * Instantiates a new mtas span match none query.
  32 + *
  33 + * @param field
  34 + * the field
  35 + */
  36 + public MtasSpanMatchNoneQuery(String field) {
  37 + this.field = field;
  38 + }
  39 +
  40 + /*
  41 + * (non-Javadoc)
  42 + *
  43 + * @see org.apache.lucene.search.spans.SpanQuery#getField()
  44 + */
  45 + @Override
  46 + public String getField() {
  47 + return field;
  48 + }
  49 +
  50 + /*
  51 + * (non-Javadoc)
  52 + *
  53 + * @see
  54 + * org.apache.lucene.search.spans.SpanQuery#createWeight(org.apache.lucene.
  55 + * search.IndexSearcher, boolean)
  56 + */
  57 + @Override
  58 + public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores)
  59 + throws IOException {
  60 + return new SpanNoneWeight(searcher, null);
  61 + }
  62 +
  63 + /**
  64 + * The Class SpanNoneWeight.
  65 + */
  66 + public class SpanNoneWeight extends SpanWeight {
  67 +
  68 + /**
  69 + * Instantiates a new span none weight.
  70 + *
  71 + * @param searcher
  72 + * the searcher
  73 + * @param termContexts
  74 + * the term contexts
  75 + * @throws IOException
  76 + * Signals that an I/O exception has occurred.
  77 + */
  78 + public SpanNoneWeight(IndexSearcher searcher,
  79 + Map<Term, TermContext> termContexts) throws IOException {
  80 + super(MtasSpanMatchNoneQuery.this, searcher, termContexts);
  81 + }
  82 +
  83 + /*
  84 + * (non-Javadoc)
  85 + *
  86 + * @see
  87 + * org.apache.lucene.search.spans.SpanWeight#extractTermContexts(java.util.
  88 + * Map)
  89 + */
  90 + @Override
  91 + public void extractTermContexts(Map<Term, TermContext> contexts) {
  92 + }
  93 +
  94 + /*
  95 + * (non-Javadoc)
  96 + *
  97 + * @see
  98 + * org.apache.lucene.search.spans.SpanWeight#getSpans(org.apache.lucene.
  99 + * index.LeafReaderContext,
  100 + * org.apache.lucene.search.spans.SpanWeight.Postings)
  101 + */
  102 + @Override
  103 + public Spans getSpans(LeafReaderContext context, Postings requiredPostings)
  104 + throws IOException {
  105 + try {
  106 + // get leafreader
  107 + LeafReader r = context.reader();
  108 + // get delegate
  109 + Boolean hasMethod = true;
  110 + while (hasMethod) {
  111 + hasMethod = false;
  112 + Method[] methods = r.getClass().getMethods();
  113 + for (Method m : methods) {
  114 + if (m.getName().equals("getDelegate")) {
  115 + hasMethod = true;
  116 + r = (LeafReader) m.invoke(r, (Object[]) null);
  117 + break;
  118 + }
  119 + }
  120 + }
  121 + // get fieldsproducer
  122 + Method fpm = r.getClass().getMethod("getPostingsReader",
  123 + (Class<?>[]) null);
  124 + FieldsProducer fp = (FieldsProducer) fpm.invoke(r, (Object[]) null);
  125 + // get MtasFieldsProducer using terms
  126 + return new MtasSpanMatchNone(field);
  127 + } catch (Exception e) {
  128 + throw new IOException("Can't get reader");
  129 + }
  130 +
  131 + }
  132 +
  133 + /*
  134 + * (non-Javadoc)
  135 + *
  136 + * @see org.apache.lucene.search.Weight#extractTerms(java.util.Set)
  137 + */
  138 + @Override
  139 + public void extractTerms(Set<Term> terms) {
  140 + }
  141 +
  142 + /*
  143 + * (non-Javadoc)
  144 + *
  145 + * @see
  146 + * org.apache.lucene.search.spans.SpanWeight#getSimScorer(org.apache.lucene.
  147 + * index.LeafReaderContext)
  148 + */
  149 + @Override
  150 + public SimScorer getSimScorer(LeafReaderContext context) {
  151 + return new MtasSimScorer();
  152 + }
  153 +
  154 + }
  155 +
  156 + /*
  157 + * (non-Javadoc)
  158 + *
  159 + * @see
  160 + * org.apache.lucene.search.spans.SpanTermQuery#toString(java.lang.String)
  161 + */
  162 + @Override
  163 + public String toString(String field) {
  164 + StringBuilder buffer = new StringBuilder();
  165 + buffer.append(QUERY_NAME + "([])");
  166 + return buffer.toString();
  167 + }
  168 +
  169 + /*
  170 + * (non-Javadoc)
  171 + *
  172 + * @see org.apache.lucene.search.Query#equals(java.lang.Object)
  173 + */
  174 + @Override
  175 + public boolean equals(Object obj) {
  176 + if (this == obj)
  177 + return true;
  178 + if (obj == null)
  179 + return false;
  180 + if (getClass() != obj.getClass())
  181 + return false;
  182 + final MtasSpanMatchNoneQuery that = (MtasSpanMatchNoneQuery) obj;
  183 + return field.equals(that.field);
  184 + }
  185 +
  186 + /*
  187 + * (non-Javadoc)
  188 + *
  189 + * @see org.apache.lucene.search.Query#hashCode()
  190 + */
  191 + @Override
  192 + public int hashCode() {
  193 + int h = QUERY_NAME.hashCode();
  194 + h = (h * 7) ^ field.hashCode();
  195 + return h;
  196 + }
  197 +
  198 +}
... ...
src/mtas/search/spans/MtasSpanRegexQuery.java
... ... @@ -21,7 +21,7 @@ import org.apache.lucene.search.spans.SpanWeight;
21 21 public class MtasSpanRegexQuery extends SpanQuery {
22 22  
23 23 /** The Constant MTAS_REGEX_EXPAND_BOUNDARY. */
24   - private static final int MTAS_REGEX_EXPAND_BOUNDARY = 1000;
  24 + private static final int MTAS_REGEX_EXPAND_BOUNDARY = 1000000;
25 25  
26 26 /** The prefix. */
27 27 private String prefix;
... ... @@ -54,7 +54,7 @@ public class MtasSpanRegexQuery extends SpanQuery {
54 54 SpanQuery[] clauses = ((SpanOrQuery) q).getClauses();
55 55 if (clauses.length > MTAS_REGEX_EXPAND_BOUNDARY) {
56 56 // TODO : forward index solution
57   - throw new IOException("JAN-ODIJK-EXCEPTION: Regex \""
  57 + throw new IOException("Regex \""
58 58 + CodecUtil.termValue(term.text()) + "\" expands to "
59 59 + clauses.length + " terms, too many (boundary "
60 60 + MTAS_REGEX_EXPAND_BOUNDARY + ")!");
... ...
src/mtas/search/spans/MtasSpanRegexpQuery.java
... ... @@ -22,7 +22,7 @@ import org.apache.lucene.search.spans.SpanWeight;
22 22 public class MtasSpanRegexpQuery extends SpanQuery {
23 23  
24 24 /** The Constant MTAS_REGEXP_EXPAND_BOUNDARY. */
25   - private static final int MTAS_REGEXP_EXPAND_BOUNDARY = 1000;
  25 + private static final int MTAS_REGEXP_EXPAND_BOUNDARY = 1000000;
26 26  
27 27 /** The prefix. */
28 28 private String prefix;
... ... @@ -62,7 +62,7 @@ public class MtasSpanRegexpQuery extends SpanQuery {
62 62 */
63 63 public MtasSpanRegexpQuery(Term term, boolean singlePosition) {
64 64 super();
65   - RegexpQuery req = new RegexpQuery(term);
  65 + RegexpQuery req = new RegexpQuery(term);
66 66 query = new SpanMultiTermQueryWrapper<RegexpQuery>(req);
67 67 this.term = term;
68 68 this.singlePosition = singlePosition;
... ... @@ -90,7 +90,7 @@ public class MtasSpanRegexpQuery extends SpanQuery {
90 90 SpanQuery[] clauses = ((SpanOrQuery) q).getClauses();
91 91 if (clauses.length > MTAS_REGEXP_EXPAND_BOUNDARY) {
92 92 // TODO : forward index solution
93   - throw new IOException("JAN-ODIJK-EXCEPTION: Regexp \""
  93 + throw new IOException("Regexp \""
94 94 + CodecUtil.termValue(term.text()) + "\" expands to "
95 95 + clauses.length + " terms, too many (boundary "
96 96 + MTAS_REGEXP_EXPAND_BOUNDARY + ")!");
... ...
src/mtas/search/spans/MtasSpanWildcardQuery.java
... ... @@ -22,7 +22,7 @@ import org.apache.lucene.search.spans.SpanWeight;
22 22 public class MtasSpanWildcardQuery extends SpanQuery {
23 23  
24 24 /** The Constant MTAS_WILDCARD_EXPAND_BOUNDARY. */
25   - private static final int MTAS_WILDCARD_EXPAND_BOUNDARY = 1000;
  25 + private static final int MTAS_WILDCARD_EXPAND_BOUNDARY = 1000000;
26 26  
27 27 /** The query name. */
28 28 private static String QUERY_NAME = "mtasSpanWildcardQuery";
... ... @@ -90,7 +90,7 @@ public class MtasSpanWildcardQuery extends SpanQuery {
90 90 SpanQuery[] clauses = ((SpanOrQuery) q).getClauses();
91 91 if (clauses.length > MTAS_WILDCARD_EXPAND_BOUNDARY) {
92 92 // TODO : forward index solution
93   - throw new IOException("JAN-ODIJK-EXCEPTION: Wildcard expression \""
  93 + throw new IOException("Wildcard expression \""
94 94 + CodecUtil.termValue(term.text()) + "\" expands to "
95 95 + clauses.length + " terms, too many (boundary "
96 96 + MTAS_WILDCARD_EXPAND_BOUNDARY + ")!");
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentDistinct.java
... ... @@ -163,7 +163,6 @@ public class MtasSolrComponentDistinct {
163 163 }
164 164 mtasDistinctResponse.add("list", mtasDistinctItemResponses);
165 165 MtasSolrResultUtil.rewrite(mtasDistinctResponse);
166   - System.out.println(mtasDistinctResponse.get("list"));
167 166 return mtasDistinctResponse;
168 167 }
169 168  
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentFacet.java
... ... @@ -56,6 +56,18 @@ public class MtasSolrComponentFacet {
56 56 /** The Constant SUBNAME_MTAS_FACET_QUERY_VALUE. */
57 57 public static final String SUBNAME_MTAS_FACET_QUERY_VALUE = "value";
58 58  
  59 + /** The Constant SUBNAME_MTAS_FACET_QUERY_PREFIX. */
  60 + public static final String SUBNAME_MTAS_FACET_QUERY_PREFIX = "prefix";
  61 +
  62 + /** The Constant SUBNAME_MTAS_FACET_QUERY_VARIABLE. */
  63 + public static final String SUBNAME_MTAS_FACET_QUERY_VARIABLE = "variable";
  64 +
  65 + /** The Constant SUBNAME_MTAS_FACET_QUERY_VARIABLE_NAME. */
  66 + public static final String SUBNAME_MTAS_FACET_QUERY_VARIABLE_NAME = "name";
  67 +
  68 + /** The Constant SUBNAME_MTAS_FACET_QUERY_VARIABLE_VALUE. */
  69 + public static final String SUBNAME_MTAS_FACET_QUERY_VARIABLE_VALUE = "value";
  70 +
59 71 /** The Constant SUBNAME_MTAS_FACET_BASE_FIELD. */
60 72 public static final String SUBNAME_MTAS_FACET_BASE_FIELD = "field";
61 73  
... ... @@ -120,6 +132,8 @@ public class MtasSolrComponentFacet {
120 132 String[] keys = new String[ids.size()];
121 133 String[][] queryTypes = new String[ids.size()][];
122 134 String[][] queryValues = new String[ids.size()][];
  135 + String[][] queryPrefixes = new String[ids.size()][];
  136 + HashMap<String, String[]>[][] queryVariables = new HashMap[ids.size()][];
123 137 String[][] baseFields = new String[ids.size()][];
124 138 String[][] baseFieldTypes = new String[ids.size()][];
125 139 String[][] baseTypes = new String[ids.size()][];
... ... @@ -145,6 +159,8 @@ public class MtasSolrComponentFacet {
145 159 int tmpQCounter = 0;
146 160 queryTypes[tmpCounter] = new String[qIds.size()];
147 161 queryValues[tmpCounter] = new String[qIds.size()];
  162 + queryPrefixes[tmpCounter] = new String[qIds.size()];
  163 + queryVariables[tmpCounter] = new HashMap[qIds.size()];
148 164 for (String qId : qIds) {
149 165 queryTypes[tmpCounter][tmpQCounter] = rb.req.getParams()
150 166 .get(
... ... @@ -156,6 +172,47 @@ public class MtasSolrComponentFacet {
156 172 PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_QUERY
157 173 + "." + qId + "." + SUBNAME_MTAS_FACET_QUERY_VALUE,
158 174 null);
  175 + queryPrefixes[tmpCounter][tmpQCounter] = rb.req.getParams()
  176 + .get(
  177 + PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_QUERY
  178 + + "." + qId + "." + SUBNAME_MTAS_FACET_QUERY_PREFIX,
  179 + null);
  180 + Set<String> vIds = MtasSolrResultUtil.getIdsFromParameters(
  181 + rb.req.getParams(),
  182 + PARAM_MTAS_FACET + "." + id + "." + NAME_MTAS_FACET_QUERY + "."
  183 + + qId + "." + SUBNAME_MTAS_FACET_QUERY_VARIABLE);
  184 + queryVariables[tmpCounter][tmpQCounter] = new HashMap<String, String[]>();
  185 + if (vIds.size() > 0) {
  186 + HashMap<String, ArrayList<String>> tmpVariables = new HashMap<String, ArrayList<String>>();
  187 + for (String vId : vIds) {
  188 + String name = rb.req.getParams().get(PARAM_MTAS_FACET + "." + id
  189 + + "." + NAME_MTAS_FACET_QUERY + "." + qId + "."
  190 + + SUBNAME_MTAS_FACET_QUERY_VARIABLE + "." + vId + "."
  191 + + SUBNAME_MTAS_FACET_QUERY_VARIABLE_NAME, null);
  192 + if (name != null) {
  193 + if (!tmpVariables.containsKey(name)) {
  194 + tmpVariables.put(name, new ArrayList<String>());
  195 + }
  196 + String value = rb.req.getParams().get(PARAM_MTAS_FACET + "."
  197 + + id + "." + NAME_MTAS_FACET_QUERY + "." + qId + "."
  198 + + SUBNAME_MTAS_FACET_QUERY_VARIABLE + "." + vId + "."
  199 + + SUBNAME_MTAS_FACET_QUERY_VARIABLE_VALUE, null);
  200 + if (value != null) {
  201 + ArrayList<String> list = new ArrayList<String>();
  202 + String[] subList = value.split("(?<!\\\\),");
  203 + for(int i=0; i<subList.length; i++) {
  204 + list.add(subList[i].replace("\\,", ",").replace("\\\\", "\\"));
  205 + }
  206 + tmpVariables.get(name).addAll(list);
  207 + }
  208 + }
  209 + }
  210 + for (String name : tmpVariables.keySet()) {
  211 + queryVariables[tmpCounter][tmpQCounter].put(name,
  212 + tmpVariables.get(name)
  213 + .toArray(new String[tmpVariables.get(name).size()]));
  214 + }
  215 + }
159 216 tmpQCounter++;
160 217 }
161 218 } else {
... ... @@ -278,7 +335,8 @@ public class MtasSolrComponentFacet {
278 335 SpanQuery ql[] = new SpanQuery[queryNumber];
279 336 for (int j = 0; j < queryNumber; j++) {
280 337 SpanQuery q = MtasSolrResultUtil.constructQuery(queryValues[i][j],
281   - queryTypes[i][j], fields[i]);
  338 + queryTypes[i][j], queryPrefixes[i][j], queryVariables[i][j],
  339 + fields[i]);
282 340 // minimize number of queries
283 341 if (cf.spanQueryList.contains(q)) {
284 342 q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
... ... @@ -328,10 +386,71 @@ public class MtasSolrComponentFacet {
328 386 PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_FIELD);
329 387 sreq.params.remove(
330 388 PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_KEY);
331   - sreq.params.remove(
  389 + Set<String> subKeys = MtasSolrResultUtil.getIdsFromParameters(
  390 + rb.req.getParams(),
332 391 PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_QUERY);
333   - sreq.params.remove(
  392 + for (String subKey : subKeys) {
  393 + sreq.params.remove(
  394 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_QUERY
  395 + + "." + subKey + "." + SUBNAME_MTAS_FACET_QUERY_TYPE);
  396 + sreq.params.remove(
  397 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_QUERY
  398 + + "." + subKey + "." + SUBNAME_MTAS_FACET_QUERY_VALUE);
  399 + sreq.params.remove(
  400 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_QUERY
  401 + + "." + subKey + "." + SUBNAME_MTAS_FACET_QUERY_PREFIX);
  402 + Set<String> subSubKeys = MtasSolrResultUtil.getIdsFromParameters(
  403 + rb.req.getParams(),
  404 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_QUERY+ "." + subKey + "." + SUBNAME_MTAS_FACET_QUERY_VARIABLE);
  405 + for (String subSubKey : subSubKeys) {
  406 + sreq.params.remove(
  407 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_QUERY
  408 + + "." + subKey + "." + SUBNAME_MTAS_FACET_QUERY_VARIABLE+"."+subSubKey+"."+SUBNAME_MTAS_FACET_QUERY_VARIABLE_NAME);
  409 + sreq.params.remove(
  410 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_QUERY
  411 + + "." + subKey + "." + SUBNAME_MTAS_FACET_QUERY_VARIABLE+"."+subSubKey+"."+SUBNAME_MTAS_FACET_QUERY_VARIABLE_VALUE);
  412 + }
  413 + }
  414 + subKeys = MtasSolrResultUtil.getIdsFromParameters(
  415 + rb.req.getParams(),
334 416 PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE);
  417 + for (String subKey : subKeys) {
  418 + sreq.params.remove(
  419 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE
  420 + + "." + subKey + "." + SUBNAME_MTAS_FACET_BASE_FIELD);
  421 + sreq.params.remove(
  422 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE
  423 + + "." + subKey + "." + SUBNAME_MTAS_FACET_BASE_TYPE);
  424 + sreq.params.remove(
  425 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE
  426 + + "." + subKey + "." + SUBNAME_MTAS_FACET_BASE_MAXIMUM);
  427 + sreq.params.remove(
  428 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE
  429 + + "." + subKey + "." + SUBNAME_MTAS_FACET_BASE_MINIMUM);
  430 + sreq.params.remove(
  431 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE
  432 + + "." + subKey + "." + SUBNAME_MTAS_FACET_BASE_NUMBER);
  433 + sreq.params.remove(
  434 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE
  435 + + "." + subKey + "." + SUBNAME_MTAS_FACET_BASE_SORT_DIRECTION);
  436 + sreq.params.remove(
  437 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE
  438 + + "." + subKey + "." + SUBNAME_MTAS_FACET_BASE_SORT_TYPE);
  439 + Set<String> subSubKeys = MtasSolrResultUtil.getIdsFromParameters(
  440 + rb.req.getParams(),
  441 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE+ "." + subKey + "." + SUBNAME_MTAS_FACET_BASE_FUNCTION);
  442 + for (String subSubKey : subSubKeys) {
  443 + sreq.params.remove(
  444 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE
  445 + + "." + subKey + "." + SUBNAME_MTAS_FACET_BASE_FUNCTION+"."+subSubKey+"."+SUBNAME_MTAS_FACET_BASE_FUNCTION_EXPRESSION);
  446 + sreq.params.remove(
  447 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE
  448 + + "." + subKey + "." + SUBNAME_MTAS_FACET_BASE_FUNCTION+"."+subSubKey+"."+SUBNAME_MTAS_FACET_BASE_FUNCTION_KEY);
  449 + sreq.params.remove(
  450 + PARAM_MTAS_FACET + "." + key + "." + NAME_MTAS_FACET_BASE
  451 + + "." + subKey + "." + SUBNAME_MTAS_FACET_BASE_FUNCTION+"."+subSubKey+"."+SUBNAME_MTAS_FACET_BASE_FUNCTION_TYPE);
  452 + }
  453 + }
335 454 }
336 455 }
337 456 }
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentGroup.java
... ... @@ -2,6 +2,7 @@ package mtas.solr.handler.component.util;
2 2  
3 3 import java.io.IOException;
4 4 import java.util.ArrayList;
  5 +import java.util.HashMap;
5 6 import java.util.Set;
6 7 import java.util.SortedSet;
7 8 import java.util.TreeSet;
... ... @@ -41,6 +42,18 @@ public class MtasSolrComponentGroup {
41 42 /** The Constant NAME_MTAS_GROUP_QUERY_VALUE. */
42 43 public static final String NAME_MTAS_GROUP_QUERY_VALUE = "query.value";
43 44  
  45 + /** The Constant NAME_MTAS_GROUP_QUERY_PREFIX. */
  46 + public static final String NAME_MTAS_GROUP_QUERY_PREFIX = "query.prefix";
  47 +
  48 + /** The Constant NAME_MTAS_KWIC_QUERY_VARIABLE. */
  49 + public static final String NAME_MTAS_GROUP_QUERY_VARIABLE = "query.variable";
  50 +
  51 + /** The Constant NAME_MTAS_KWIC_QUERY_VARIABLE_NAME. */
  52 + public static final String SUBNAME_MTAS_GROUP_QUERY_VARIABLE_NAME = "name";
  53 +
  54 + /** The Constant NAME_MTAS_KWIC_QUERY_VARIABLE_VALUE. */
  55 + public static final String SUBNAME_MTAS_GROUP_QUERY_VARIABLE_VALUE = "value";
  56 +
44 57 /** The Constant NAME_MTAS_GROUP_KEY. */
45 58 public static final String NAME_MTAS_GROUP_KEY = "key";
46 59  
... ... @@ -102,6 +115,8 @@ public class MtasSolrComponentGroup {
102 115 String[] fields = new String[ids.size()];
103 116 String[] queryTypes = new String[ids.size()];
104 117 String[] queryValues = new String[ids.size()];
  118 + String[] queryPrefixes = new String[ids.size()];
  119 + HashMap<String, String[]>[] queryVariables = new HashMap[ids.size()];
105 120 String[] keys = new String[ids.size()];
106 121 String[] numbers = new String[ids.size()];
107 122 String[][] groupingLeftPosition = new String[ids.size()][];
... ... @@ -132,6 +147,47 @@ public class MtasSolrComponentGroup {
132 147 queryValues[tmpCounter] = rb.req.getParams().get(
133 148 PARAM_MTAS_GROUP + "." + id + "." + NAME_MTAS_GROUP_QUERY_VALUE,
134 149 null);
  150 + queryPrefixes[tmpCounter] = rb.req.getParams().get(
  151 + PARAM_MTAS_GROUP + "." + id + "." + NAME_MTAS_GROUP_QUERY_PREFIX,
  152 + null);
  153 + Set<String> vIds = MtasSolrResultUtil.getIdsFromParameters(
  154 + rb.req.getParams(),
  155 + PARAM_MTAS_GROUP + "." + id + "."
  156 + + NAME_MTAS_GROUP_QUERY_VARIABLE);
  157 + queryVariables[tmpCounter] = new HashMap<String, String[]>();
  158 + if (vIds.size() > 0) {
  159 + HashMap<String, ArrayList<String>> tmpVariables = new HashMap<String, ArrayList<String>>();
  160 + for (String vId : vIds) {
  161 + String name = rb.req.getParams()
  162 + .get(PARAM_MTAS_GROUP + "." + id + "."
  163 + + NAME_MTAS_GROUP_QUERY_VARIABLE + "." + vId
  164 + + "." + SUBNAME_MTAS_GROUP_QUERY_VARIABLE_NAME,
  165 + null);
  166 + if (name != null) {
  167 + if (!tmpVariables.containsKey(name)) {
  168 + tmpVariables.put(name, new ArrayList<String>());
  169 + }
  170 + String value = rb.req.getParams()
  171 + .get(PARAM_MTAS_GROUP + "." + id + "."
  172 + + NAME_MTAS_GROUP_QUERY_VARIABLE + "." + vId
  173 + + "." + SUBNAME_MTAS_GROUP_QUERY_VARIABLE_VALUE,
  174 + null);
  175 + if (value != null) {
  176 + ArrayList<String> list = new ArrayList<String>();
  177 + String[] subList = value.split("(?<!\\\\),");
  178 + for(int i=0; i<subList.length; i++) {
  179 + list.add(subList[i].replace("\\,", ",").replace("\\\\", "\\"));
  180 + }
  181 + tmpVariables.get(name).addAll(list);
  182 + }
  183 + }
  184 + }
  185 + for (String name : tmpVariables.keySet()) {
  186 + queryVariables[tmpCounter].put(name,
  187 + tmpVariables.get(name)
  188 + .toArray(new String[tmpVariables.get(name).size()]));
  189 + }
  190 + }
135 191 groupingHitInsidePrefixes[tmpCounter] = null;
136 192 // collect
137 193 SortedSet<String> gids;
... ... @@ -222,7 +278,7 @@ public class MtasSolrComponentGroup {
222 278 for (int i = 0; i < fields.length; i++) {
223 279 ComponentField cf = mtasFields.list.get(fields[i]);
224 280 SpanQuery q = MtasSolrResultUtil.constructQuery(queryValues[i],
225   - queryTypes[i], fields[i]);
  281 + queryTypes[i], queryPrefixes[i], queryVariables[i], fields[i]);
226 282 // minimize number of queries
227 283 if (cf.spanQueryList.contains(q)) {
228 284 q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
... ... @@ -235,7 +291,7 @@ public class MtasSolrComponentGroup {
235 291 int number = (numbers[i] == null) || (numbers[i].isEmpty())
236 292 ? DEFAULT_NUMBER : Integer.parseInt(numbers[i]);
237 293 mtasFields.list.get(fields[i]).groupList.add(new ComponentGroup(q,
238   - fields[i], queryValues[i], queryTypes[i], key, number,
  294 + fields[i], queryValues[i], queryTypes[i], queryPrefixes[i], key, number,
239 295 groupingHitInsidePrefixes[i], groupingHitInsideLeftPosition[i],
240 296 groupingHitInsideLeftPrefixes[i], groupingHitInsideRightPosition[i],
241 297 groupingHitInsideRightPrefixes[i], groupingHitLeftPosition[i],
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentKwic.java
... ... @@ -2,6 +2,7 @@ package mtas.solr.handler.component.util;
2 2  
3 3 import java.io.IOException;
4 4 import java.util.ArrayList;
  5 +import java.util.HashMap;
5 6 import java.util.Set;
6 7 import java.util.TreeMap;
7 8  
... ... @@ -42,6 +43,18 @@ public class MtasSolrComponentKwic {
42 43 /** The Constant NAME_MTAS_KWIC_QUERY_VALUE. */
43 44 public static final String NAME_MTAS_KWIC_QUERY_VALUE = "query.value";
44 45  
  46 + /** The Constant NAME_MTAS_KWIC_QUERY_PREFIX. */
  47 + public static final String NAME_MTAS_KWIC_QUERY_PREFIX = "query.prefix";
  48 +
  49 + /** The Constant NAME_MTAS_KWIC_QUERY_VARIABLE. */
  50 + public static final String NAME_MTAS_KWIC_QUERY_VARIABLE = "query.variable";
  51 +
  52 + /** The Constant NAME_MTAS_KWIC_QUERY_VARIABLE_NAME. */
  53 + public static final String SUBNAME_MTAS_KWIC_QUERY_VARIABLE_NAME = "name";
  54 +
  55 + /** The Constant NAME_MTAS_KWIC_QUERY_VARIABLE_VALUE. */
  56 + public static final String SUBNAME_MTAS_KWIC_QUERY_VARIABLE_VALUE = "value";
  57 +
45 58 /** The Constant NAME_MTAS_KWIC_KEY. */
46 59 public static final String NAME_MTAS_KWIC_KEY = "key";
47 60  
... ... @@ -66,8 +79,7 @@ public class MtasSolrComponentKwic {
66 79 /**
67 80 * Instantiates a new mtas solr component kwic.
68 81 *
69   - * @param searchComponent
70   - * the search component
  82 + * @param searchComponent the search component
71 83 */
72 84 public MtasSolrComponentKwic(MtasSolrSearchComponent searchComponent) {
73 85 this.searchComponent = searchComponent;
... ... @@ -76,8 +88,7 @@ public class MtasSolrComponentKwic {
76 88 /**
77 89 * Gets the positive integer.
78 90 *
79   - * @param number
80   - * the number
  91 + * @param number the number
81 92 * @return the positive integer
82 93 */
83 94 private int getPositiveInteger(String number) {
... ... @@ -91,12 +102,9 @@ public class MtasSolrComponentKwic {
91 102 /**
92 103 * Prepare.
93 104 *
94   - * @param rb
95   - * the rb
96   - * @param mtasFields
97   - * the mtas fields
98   - * @throws IOException
99   - * Signals that an I/O exception has occurred.
  105 + * @param rb the rb
  106 + * @param mtasFields the mtas fields
  107 + * @throws IOException Signals that an I/O exception has occurred.
100 108 */
101 109 public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
102 110 throws IOException {
... ... @@ -107,6 +115,8 @@ public class MtasSolrComponentKwic {
107 115 String[] fields = new String[ids.size()];
108 116 String[] queryTypes = new String[ids.size()];
109 117 String[] queryValues = new String[ids.size()];
  118 + String[] queryPrefixes = new String[ids.size()];
  119 + HashMap<String, String[]>[] queryVariables = new HashMap[ids.size()];
110 120 String[] keys = new String[ids.size()];
111 121 String[] prefixes = new String[ids.size()];
112 122 String[] numbers = new String[ids.size()];
... ... @@ -122,6 +132,47 @@ public class MtasSolrComponentKwic {
122 132 queryValues[tmpCounter] = rb.req.getParams().get(
123 133 PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_QUERY_VALUE,
124 134 null);
  135 + queryPrefixes[tmpCounter] = rb.req.getParams().get(
  136 + PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_QUERY_PREFIX,
  137 + null);
  138 + Set<String> vIds = MtasSolrResultUtil.getIdsFromParameters(
  139 + rb.req.getParams(),
  140 + PARAM_MTAS_KWIC + "." + id + "."
  141 + + NAME_MTAS_KWIC_QUERY_VARIABLE);
  142 + queryVariables[tmpCounter] = new HashMap<String, String[]>();
  143 + if (vIds.size() > 0) {
  144 + HashMap<String, ArrayList<String>> tmpVariables = new HashMap<String, ArrayList<String>>();
  145 + for (String vId : vIds) {
  146 + String name = rb.req.getParams()
  147 + .get(PARAM_MTAS_KWIC + "." + id + "."
  148 + + NAME_MTAS_KWIC_QUERY_VARIABLE + "." + vId
  149 + + "." + SUBNAME_MTAS_KWIC_QUERY_VARIABLE_NAME,
  150 + null);
  151 + if (name != null) {
  152 + if (!tmpVariables.containsKey(name)) {
  153 + tmpVariables.put(name, new ArrayList<String>());
  154 + }
  155 + String value = rb.req.getParams()
  156 + .get(PARAM_MTAS_KWIC + "." + id + "."
  157 + + NAME_MTAS_KWIC_QUERY_VARIABLE + "." + vId
  158 + + "." + SUBNAME_MTAS_KWIC_QUERY_VARIABLE_VALUE,
  159 + null);
  160 + if (value != null) {
  161 + ArrayList<String> list = new ArrayList<String>();
  162 + String[] subList = value.split("(?<!\\\\),");
  163 + for(int i=0; i<subList.length; i++) {
  164 + list.add(subList[i].replace("\\,", ",").replace("\\\\", "\\"));
  165 + }
  166 + tmpVariables.get(name).addAll(list);
  167 + }
  168 + }
  169 + }
  170 + for (String name : tmpVariables.keySet()) {
  171 + queryVariables[tmpCounter].put(name,
  172 + tmpVariables.get(name)
  173 + .toArray(new String[tmpVariables.get(name).size()]));
  174 + }
  175 + }
125 176 keys[tmpCounter] = rb.req.getParams()
126 177 .get(PARAM_MTAS_KWIC + "." + id + "." + NAME_MTAS_KWIC_KEY,
127 178 String.valueOf(tmpCounter))
... ... @@ -158,6 +209,8 @@ public class MtasSolrComponentKwic {
158 209 NAME_MTAS_KWIC_QUERY_VALUE, NAME_MTAS_KWIC_FIELD, false);
159 210 MtasSolrResultUtil.compareAndCheck(queryTypes, fields,
160 211 NAME_MTAS_KWIC_QUERY_TYPE, NAME_MTAS_KWIC_FIELD, false);
  212 + MtasSolrResultUtil.compareAndCheck(queryPrefixes, fields,
  213 + NAME_MTAS_KWIC_QUERY_PREFIX, NAME_MTAS_KWIC_FIELD, false);
161 214 MtasSolrResultUtil.compareAndCheck(prefixes, fields,
162 215 NAME_MTAS_KWIC_PREFIX, NAME_MTAS_KWIC_FIELD, false);
163 216 MtasSolrResultUtil.compareAndCheck(numbers, fields, NAME_MTAS_KWIC_NUMBER,
... ... @@ -173,7 +226,7 @@ public class MtasSolrComponentKwic {
173 226 for (int i = 0; i < fields.length; i++) {
174 227 ComponentField cf = mtasFields.list.get(fields[i]);
175 228 SpanQuery q = MtasSolrResultUtil.constructQuery(queryValues[i],
176   - queryTypes[i], fields[i]);
  229 + queryTypes[i], queryPrefixes[i], queryVariables[i], fields[i]);
177 230 // minimize number of queries
178 231 if (cf.spanQueryList.contains(q)) {
179 232 q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
... ... @@ -199,8 +252,7 @@ public class MtasSolrComponentKwic {
199 252 /**
200 253 * Creates the.
201 254 *
202   - * @param kwic
203   - * the kwic
  255 + * @param kwic the kwic
204 256 * @return the simple ordered map
205 257 */
206 258 public SimpleOrderedMap<Object> create(ComponentKwic kwic) {
... ... @@ -330,12 +382,9 @@ public class MtasSolrComponentKwic {
330 382 /**
331 383 * Modify request.
332 384 *
333   - * @param rb
334   - * the rb
335   - * @param who
336   - * the who
337   - * @param sreq
338   - * the sreq
  385 + * @param rb the rb
  386 + * @param who the who
  387 + * @param sreq the sreq
339 388 */
340 389 public void modifyRequest(ResponseBuilder rb, SearchComponent who,
341 390 ShardRequest sreq) {
... ... @@ -351,7 +400,11 @@ public class MtasSolrComponentKwic {
351 400 sreq.params.remove(
352 401 PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_FIELD);
353 402 sreq.params.remove(
  403 + PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_QUERY_TYPE);
  404 + sreq.params.remove(
354 405 PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_QUERY_VALUE);
  406 + sreq.params.remove(
  407 + PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_QUERY_PREFIX);
355 408 sreq.params
356 409 .remove(PARAM_MTAS_KWIC + "." + key + "." + NAME_MTAS_KWIC_KEY);
357 410 sreq.params.remove(
... ... @@ -373,8 +426,7 @@ public class MtasSolrComponentKwic {
373 426 /**
374 427 * Finish stage.
375 428 *
376   - * @param rb
377   - * the rb
  429 + * @param rb the rb
378 430 */
379 431 public void finishStage(ResponseBuilder rb) {
380 432 if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentList.java
... ... @@ -45,6 +45,18 @@ public class MtasSolrComponentList {
45 45 /** The Constant NAME_MTAS_LIST_QUERY_VALUE. */
46 46 public static final String NAME_MTAS_LIST_QUERY_VALUE = "query.value";
47 47  
  48 + /** The Constant NAME_MTAS_LIST_QUERY_PREFIX. */
  49 + public static final String NAME_MTAS_LIST_QUERY_PREFIX = "query.prefix";
  50 +
  51 + /** The Constant NAME_MTAS_LIST_QUERY_VARIABLE. */
  52 + public static final String NAME_MTAS_LIST_QUERY_VARIABLE = "query.variable";
  53 +
  54 + /** The Constant SUBNAME_MTAS_LIST_QUERY_VARIABLE_NAME. */
  55 + public static final String SUBNAME_MTAS_LIST_QUERY_VARIABLE_NAME = "name";
  56 +
  57 + /** The Constant SUBNAME_MTAS_LIST_QUERY_VARIABLE_VALUE. */
  58 + public static final String SUBNAME_MTAS_LIST_QUERY_VARIABLE_VALUE = "value";
  59 +
48 60 /** The Constant NAME_MTAS_LIST_KEY. */
49 61 public static final String NAME_MTAS_LIST_KEY = "key";
50 62  
... ... @@ -69,8 +81,7 @@ public class MtasSolrComponentList {
69 81 /**
70 82 * Instantiates a new mtas solr component list.
71 83 *
72   - * @param searchComponent
73   - * the search component
  84 + * @param searchComponent the search component
74 85 */
75 86 public MtasSolrComponentList(MtasSolrSearchComponent searchComponent) {
76 87 this.searchComponent = searchComponent;
... ... @@ -79,12 +90,9 @@ public class MtasSolrComponentList {
79 90 /**
80 91 * Prepare.
81 92 *
82   - * @param rb
83   - * the rb
84   - * @param mtasFields
85   - * the mtas fields
86   - * @throws IOException
87   - * Signals that an I/O exception has occurred.
  93 + * @param rb the rb
  94 + * @param mtasFields the mtas fields
  95 + * @throws IOException Signals that an I/O exception has occurred.
88 96 */
89 97 public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
90 98 throws IOException {
... ... @@ -95,6 +103,8 @@ public class MtasSolrComponentList {
95 103 String[] fields = new String[ids.size()];
96 104 String[] queryTypes = new String[ids.size()];
97 105 String[] queryValues = new String[ids.size()];
  106 + String[] queryPrefixes = new String[ids.size()];
  107 + HashMap<String, String[]>[] queryVariables = new HashMap[ids.size()];
98 108 String[] keys = new String[ids.size()];
99 109 String[] prefixes = new String[ids.size()];
100 110 String[] starts = new String[ids.size()];
... ... @@ -114,6 +124,47 @@ public class MtasSolrComponentList {
114 124 queryValues[tmpCounter] = rb.req.getParams().get(
115 125 PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_QUERY_VALUE,
116 126 null);
  127 + queryPrefixes[tmpCounter] = rb.req.getParams().get(
  128 + PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_QUERY_PREFIX,
  129 + null);
  130 + Set<String> vIds = MtasSolrResultUtil.getIdsFromParameters(
  131 + rb.req.getParams(),
  132 + PARAM_MTAS_LIST + "." + id + "."
  133 + + NAME_MTAS_LIST_QUERY_VARIABLE);
  134 + queryVariables[tmpCounter] = new HashMap<String, String[]>();
  135 + if (vIds.size() > 0) {
  136 + HashMap<String, ArrayList<String>> tmpVariables = new HashMap<String, ArrayList<String>>();
  137 + for (String vId : vIds) {
  138 + String name = rb.req.getParams()
  139 + .get(PARAM_MTAS_LIST + "." + id + "."
  140 + + NAME_MTAS_LIST_QUERY_VARIABLE + "." + vId
  141 + + "." + SUBNAME_MTAS_LIST_QUERY_VARIABLE_NAME,
  142 + null);
  143 + if (name != null) {
  144 + if (!tmpVariables.containsKey(name)) {
  145 + tmpVariables.put(name, new ArrayList<String>());
  146 + }
  147 + String value = rb.req.getParams()
  148 + .get(PARAM_MTAS_LIST + "." + id + "."
  149 + + NAME_MTAS_LIST_QUERY_VARIABLE + "." + vId
  150 + + "." + SUBNAME_MTAS_LIST_QUERY_VARIABLE_VALUE,
  151 + null);
  152 + if (value != null) {
  153 + ArrayList<String> list = new ArrayList<String>();
  154 + String[] subList = value.split("(?<!\\\\),");
  155 + for(int i=0; i<subList.length; i++) {
  156 + list.add(subList[i].replace("\\,", ",").replace("\\\\", "\\"));
  157 + }
  158 + tmpVariables.get(name).addAll(list);
  159 + }
  160 + }
  161 + }
  162 + for (String name : tmpVariables.keySet()) {
  163 + queryVariables[tmpCounter].put(name,
  164 + tmpVariables.get(name)
  165 + .toArray(new String[tmpVariables.get(name).size()]));
  166 + }
  167 + }
117 168 prefixes[tmpCounter] = rb.req.getParams().get(
118 169 PARAM_MTAS_LIST + "." + id + "." + NAME_MTAS_LIST_PREFIX, null);
119 170 starts[tmpCounter] = rb.req.getParams()
... ... @@ -144,6 +195,8 @@ public class MtasSolrComponentList {
144 195 NAME_MTAS_LIST_QUERY_VALUE, NAME_MTAS_LIST_FIELD, false);
145 196 MtasSolrResultUtil.compareAndCheck(prefixes, queryTypes,
146 197 NAME_MTAS_LIST_QUERY_TYPE, NAME_MTAS_LIST_FIELD, false);
  198 + MtasSolrResultUtil.compareAndCheck(prefixes, queryPrefixes,
  199 + NAME_MTAS_LIST_QUERY_PREFIX, NAME_MTAS_LIST_FIELD, false);
147 200 MtasSolrResultUtil.compareAndCheck(prefixes, fields,
148 201 NAME_MTAS_LIST_PREFIX, NAME_MTAS_LIST_FIELD, false);
149 202 MtasSolrResultUtil.compareAndCheck(starts, fields, NAME_MTAS_LIST_START,
... ... @@ -159,7 +212,7 @@ public class MtasSolrComponentList {
159 212 for (int i = 0; i < fields.length; i++) {
160 213 ComponentField cf = mtasFields.list.get(fields[i]);
161 214 SpanQuery q = MtasSolrResultUtil.constructQuery(queryValues[i],
162   - queryTypes[i], fields[i]);
  215 + queryTypes[i], queryPrefixes[i], queryVariables[i], fields[i]);
163 216 // minimize number of queries
164 217 if (cf.spanQueryList.contains(q)) {
165 218 q = cf.spanQueryList.get(cf.spanQueryList.indexOf(q));
... ... @@ -167,7 +220,8 @@ public class MtasSolrComponentList {
167 220 cf.spanQueryList.add(q);
168 221 }
169 222 String key = (keys[i] == null) || (keys[i].isEmpty())
170   - ? String.valueOf(i) + ":" + fields[i] + ":" + queryValues[i]
  223 + ? String.valueOf(i) + ":" + fields[i] + ":" + queryValues[i] + ":"
  224 + + queryPrefixes[i]
171 225 : keys[i].trim();
172 226 String prefix = prefixes[i];
173 227 int start = (starts[i] == null) || (starts[i].isEmpty()) ? 0
... ... @@ -179,9 +233,9 @@ public class MtasSolrComponentList {
179 233 int right = (rights[i] == null) || rights[i].isEmpty() ? 0
180 234 : Integer.parseInt(rights[i]);
181 235 String output = outputs[i];
182   - mtasFields.list.get(fields[i]).listList
183   - .add(new ComponentList(q, fields[i], queryValues[i], queryTypes[i],
184   - key, prefix, start, number, left, right, output));
  236 + mtasFields.list.get(fields[i]).listList.add(new ComponentList(q,
  237 + fields[i], queryValues[i], queryTypes[i], queryPrefixes[i], queryVariables[i], key,
  238 + prefix, start, number, left, right, output));
185 239 }
186 240 }
187 241 }
... ... @@ -189,12 +243,9 @@ public class MtasSolrComponentList {
189 243 /**
190 244 * Modify request.
191 245 *
192   - * @param rb
193   - * the rb
194   - * @param who
195   - * the who
196   - * @param sreq
197   - * the sreq
  246 + * @param rb the rb
  247 + * @param who the who
  248 + * @param sreq the sreq
198 249 */
199 250 public void modifyRequest(ResponseBuilder rb, SearchComponent who,
200 251 ShardRequest sreq) {
... ... @@ -230,6 +281,16 @@ public class MtasSolrComponentList {
230 281 PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_QUERY_VALUE);
231 282 sreq.params.remove(
232 283 PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_QUERY_TYPE);
  284 + sreq.params.remove(
  285 + PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_QUERY_PREFIX);
  286 + Set<String> subKeys = MtasSolrResultUtil
  287 + .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_QUERY_VARIABLE);
  288 + for (String subKey : subKeys) {
  289 + sreq.params.remove(
  290 + PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_QUERY_VARIABLE+"."+subKey+"."+SUBNAME_MTAS_LIST_QUERY_VARIABLE_NAME);
  291 + sreq.params.remove(
  292 + PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_QUERY_VARIABLE+"."+subKey+"."+SUBNAME_MTAS_LIST_QUERY_VARIABLE_VALUE);
  293 + }
233 294 sreq.params
234 295 .remove(PARAM_MTAS_LIST + "." + key + "." + NAME_MTAS_LIST_KEY);
235 296 sreq.params.remove(
... ... @@ -253,10 +314,8 @@ public class MtasSolrComponentList {
253 314 /**
254 315 * Distributed process.
255 316 *
256   - * @param rb
257   - * the rb
258   - * @param mtasFields
259   - * the mtas fields
  317 + * @param rb the rb
  318 + * @param mtasFields the mtas fields
260 319 */
261 320 @SuppressWarnings("unchecked")
262 321 public void distributedProcess(ResponseBuilder rb,
... ... @@ -325,6 +384,24 @@ public class MtasSolrComponentList {
325 384 params.add(PARAM_MTAS_LIST + "." + requestId + "."
326 385 + NAME_MTAS_LIST_QUERY_TYPE, list.queryType);
327 386 params.add(PARAM_MTAS_LIST + "." + requestId + "."
  387 + + NAME_MTAS_LIST_QUERY_PREFIX, list.queryPrefix);
  388 + int subRequestId=0;
  389 + for(String name : list.queryVariables.keySet()) {
  390 + if(list.queryVariables.get(name)==null || list.queryVariables.get(name).length==0) {
  391 + params.add(PARAM_MTAS_LIST + "." + requestId + "."
  392 + + NAME_MTAS_LIST_QUERY_VARIABLE+ "."+subRequestId+"."+SUBNAME_MTAS_LIST_QUERY_VARIABLE_NAME, name);
  393 + subRequestId++;
  394 + } else {
  395 + for(String value : list.queryVariables.get(name)) {
  396 + params.add(PARAM_MTAS_LIST + "." + requestId + "."
  397 + + NAME_MTAS_LIST_QUERY_VARIABLE+ "."+subRequestId+"."+SUBNAME_MTAS_LIST_QUERY_VARIABLE_NAME, name);
  398 + params.add(PARAM_MTAS_LIST + "." + requestId + "."
  399 + + NAME_MTAS_LIST_QUERY_VARIABLE+ "."+subRequestId+"."+SUBNAME_MTAS_LIST_QUERY_VARIABLE_VALUE, value);
  400 + subRequestId++;
  401 + }
  402 + }
  403 + }
  404 + params.add(PARAM_MTAS_LIST + "." + requestId + "."
328 405 + NAME_MTAS_LIST_KEY, list.key);
329 406 params.add(PARAM_MTAS_LIST + "." + requestId + "."
330 407 + NAME_MTAS_LIST_PREFIX, list.prefix);
... ... @@ -373,8 +450,7 @@ public class MtasSolrComponentList {
373 450 /**
374 451 * Creates the.
375 452 *
376   - * @param list
377   - * the list
  453 + * @param list the list
378 454 * @return the simple ordered map
379 455 */
380 456 public SimpleOrderedMap<Object> create(ComponentList list) {
... ... @@ -504,8 +580,7 @@ public class MtasSolrComponentList {
504 580 /**
505 581 * Finish stage.
506 582 *
507   - * @param rb
508   - * the rb
  583 + * @param rb the rb
509 584 */
510 585 public void finishStage(ResponseBuilder rb) {
511 586 if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
... ...