Commit 7179e4b769f7bea15982a6bc97cdd9f6e2a4adf0

Authored by Matthijs Brouwer
1 parent 347732b5

documentation, bugfixes

Showing 65 changed files with 4601 additions and 1323 deletions

Too many changes to show.

To preserve performance only 30 of 65 files are displayed.

conf/parser/mtas/chat_test.xml
... ... @@ -103,15 +103,6 @@
103 103 <item type="text" />
104 104 </post>
105 105 </token>
106   - <token type="string" offset="false">
107   - <pre>
108   - <item type="name" />
109   - <item type="string" value="_lc" />
110   - </pre>
111   - <post>
112   - <item type="text" filter="ascii,lowercase" />
113   - </post>
114   - </token>
115 106 </mapping>
116 107 <mapping type="word" name="t">
117 108 <token type="string" offset="false">
... ...
conf/parser/mtas/folia_pm.xml
... ... @@ -25,7 +25,7 @@
25 25 <!-- START REFERENCES -->
26 26 <references>
27 27 <reference name="wref" ref="id" />
28   - </references>
  28 + </references>
29 29 <!-- END REFERENCES -->
30 30  
31 31 <!-- START MAPPINGS -->
... ... @@ -284,6 +284,7 @@
284 284 <token type="string" offset="false" realoffset="false" parent="false">
285 285 <pre>
286 286 <item type="name" />
  287 + <item type="ancestorGroupAttribute" name="class" prefix="." />
287 288 <item type="attribute" name="subset" prefix="." />
288 289 </pre>
289 290 <post>
... ...
conf/solr/solrconfig.xml
... ... @@ -1171,7 +1171,8 @@
1171 1171  
1172 1172 http://wiki.apache.org/solr/TermVectorComponent
1173 1173 -->
1174   - <searchComponent name="tvComponent" class="solr.TermVectorComponent"/>
  1174 + <!-- <searchComponent name="tvComponent" class="solr.TermVectorComponent"/> -->
  1175 + <searchComponent name="tvComponent" class="org.apache.solr.handler.component.TermVectorComponent"/>
1175 1176  
1176 1177 <!-- A request handler for demonstrating the term vector component
1177 1178  
... ... @@ -1347,10 +1348,10 @@
1347 1348 </searchComponent>
1348 1349  
1349 1350 <searchComponent name="mtas" class="mtas.solr.handler.component.MtasSolrSearchComponent">
1350   - <!-- <str name="joinCacheDirectory">${solr.core.instanceDir}/cache/join</str>
1351   - <long name="joinLifetime">86400</long>
1352   - <int name="joinMaximumNumber">1000</int>
1353   - <int name="joinMaximumOverflow">10</int> -->
  1351 + <str name="collectionCacheDirectory">${solr.core.instanceDir}/cache/collection</str>
  1352 + <long name="collectionLifetime">86400</long>
  1353 + <int name="collectionMaximumNumber">1000</int>
  1354 + <int name="collectionMaximumOverflow">10</int>
1354 1355 </searchComponent>
1355 1356  
1356 1357 <!-- Update Processors
... ...
docker/Dockerfile
1 1 # Automatically generated Dockerfile
2   -# - Build 2017-07-13 14:19
  2 +# - Build 2017-09-06 06:47
3 3 # - Lucene/Solr version 6.6.0
4 4 # - Mtas release 20170713
5 5 #
... ... @@ -74,7 +74,7 @@ RUN service apache2 stop &amp;&amp; \
74 74 chmod -R 755 /var/www/html && \
75 75 printf "echo\n" >> /start.sh && \
76 76 printf "echo \"================ Mtas -- Multi Tier Annotation Search =================\"\n" >> /start.sh && \
77   - printf "echo \" Timestamp 2017-07-13 14:19\"\n" >> /start.sh && \
  77 + printf "echo \" Timestamp 2017-09-06 06:47\"\n" >> /start.sh && \
78 78 printf "echo \" Lucene/Solr version 6.6.0\"\n" >> /start.sh && \
79 79 printf "echo \" Mtas release 20170713\"\n" >> /start.sh && \
80 80 printf "echo \" See https://meertensinstituut.github.io/mtas/ for more information\"\n" >> /start.sh && \
... ...
docker/solrconfig.xml
... ... @@ -1246,10 +1246,10 @@
1246 1246  
1247 1247 <!-- MTAS: searchComponent -->
1248 1248 <searchComponent name="mtas" class="mtas.solr.handler.component.MtasSolrSearchComponent">
1249   - <!-- <str name="joinCacheDirectory">${solr.core.instanceDir}/cache/join</str>
1250   - <long name="joinLifetime">86400</long>
1251   - <int name="joinMaximumNumber">1000</int>
1252   - <int name="joinMaximumOverflow">10</int> -->
  1249 + <str name="collectionCacheDirectory">${solr.core.instanceDir}/cache/collection</str>
  1250 + <long name="collectionLifetime">86400</long>
  1251 + <int name="collectionMaximumNumber">1000</int>
  1252 + <int name="collectionMaximumOverflow">10</int>
1253 1253 </searchComponent>
1254 1254  
1255 1255  
... ...
junit/data/conf/solrconfig.xml
... ... @@ -1347,10 +1347,10 @@
1347 1347 </searchComponent>
1348 1348  
1349 1349 <searchComponent name="mtas" class="mtas.solr.handler.component.MtasSolrSearchComponent">
1350   - <!-- <str name="joinCacheDirectory">${solr.core.instanceDir}/cache/join</str>
1351   - <long name="joinLifetime">86400</long>
1352   - <int name="joinMaximumNumber">1000</int>
1353   - <int name="joinMaximumOverflow">10</int> -->
  1350 + <str name="collectionCacheDirectory">${solr.core.instanceDir}/cache/collection</str>
  1351 + <long name="collectionLifetime">86400</long>
  1352 + <int name="collectionMaximumNumber">1000</int>
  1353 + <int name="collectionMaximumOverflow">10</int>
1354 1354 </searchComponent>
1355 1355  
1356 1356 <!-- Update Processors
... ...
junit/mtas/solr/MtasSolrBase.java
... ... @@ -14,6 +14,7 @@ import java.util.Map.Entry;
14 14  
15 15 import org.apache.commons.logging.Log;
16 16 import org.apache.commons.logging.LogFactory;
  17 +import org.apache.solr.common.SolrDocumentList;
17 18 import org.apache.solr.common.SolrInputDocument;
18 19 import org.apache.solr.common.util.NamedList;
19 20  
... ... @@ -22,6 +23,24 @@ import org.apache.solr.common.util.NamedList;
22 23 */
23 24 public class MtasSolrBase {
24 25  
  26 + /** The field id. */
  27 + public static String FIELD_ID = "id";
  28 +
  29 + /** The field title. */
  30 + public static String FIELD_TITLE = "title";
  31 +
  32 + /** The field text. */
  33 + public static String FIELD_TEXT = "text";
  34 +
  35 + /** The field mtas. */
  36 + public static String FIELD_MTAS = "mtas";
  37 +
  38 + /** The field mtas advanced. */
  39 + public static String FIELD_MTAS_ADVANCED = "mtasAdvanced";
  40 +
  41 + /** The field source. */
  42 + public static String FIELD_SOURCE = "source";
  43 +
25 44 /**
26 45 * Instantiates a new mtas solr base.
27 46 */
... ... @@ -33,6 +52,28 @@ public class MtasSolrBase {
33 52 private static Log log = LogFactory.getLog(MtasSolrBase.class);
34 53  
35 54 /**
  55 + * Gets the num found.
  56 + *
  57 + * @param response the response
  58 + * @return the num found
  59 + */
  60 + public static long getNumFound(NamedList<Object> response) {
  61 + if (response == null) {
  62 + log.error("no (valid); response");
  63 + } else {
  64 + Object mtasResponseRaw = response.get("response");
  65 + if (mtasResponseRaw != null
  66 + && mtasResponseRaw instanceof SolrDocumentList) {
  67 + SolrDocumentList mtasResponse = (SolrDocumentList) mtasResponseRaw;
  68 + return mtasResponse.getNumFound();
  69 + } else {
  70 + log.error("unexpected " + mtasResponseRaw);
  71 + }
  72 + }
  73 + return 0;
  74 + }
  75 +
  76 + /**
36 77 * Gets the from stats.
37 78 *
38 79 * @param response the response
... ... @@ -52,11 +93,11 @@ public class MtasSolrBase {
52 93 Object mtasStatsFieldsRaw = mtasStats.get("stats_fields");
53 94 if (mtasStatsFieldsRaw != null
54 95 && mtasStatsFieldsRaw instanceof NamedList) {
55   - NamedList<Object> mtasStatsFields = (NamedList) mtasStatsFieldsRaw;
  96 + NamedList<Object> mtasStatsFields = (NamedList<Object>) mtasStatsFieldsRaw;
56 97 Object mtasStatsFieldsFieldRaw = mtasStatsFields.get(field);
57 98 if (mtasStatsFieldsFieldRaw != null
58 99 && mtasStatsFieldsFieldRaw instanceof NamedList) {
59   - NamedList<Object> mtasStatsFieldsField = (NamedList) mtasStatsFieldsFieldRaw;
  100 + NamedList<Object> mtasStatsFieldsField = (NamedList<Object>) mtasStatsFieldsFieldRaw;
60 101 Object mtasStatsFieldsFieldNameRaw = mtasStatsFieldsField.get(name);
61 102 if (mtasStatsFieldsFieldNameRaw != null
62 103 && mtasStatsFieldsFieldNameRaw instanceof Number) {
... ... @@ -148,18 +189,19 @@ public class MtasSolrBase {
148 189 * @param key the key
149 190 * @return the from mtas termvector
150 191 */
151   - public static List<NamedList> getFromMtasTermvector(
  192 + public static List<NamedList<Object>> getFromMtasTermvector(
152 193 NamedList<Object> response, String key) {
153 194 if (response == null) {
154 195 log.error("no (valid); response");
155 196 } else {
156 197 Object mtasResponseRaw = response.get("mtas");
157 198 if (mtasResponseRaw != null && mtasResponseRaw instanceof NamedList) {
158   - NamedList<Object> mtasResponse = (NamedList) response.get("mtas");
  199 + NamedList<Object> mtasResponse = (NamedList<Object>) response
  200 + .get("mtas");
159 201 Object mtasTermvectorResponseRaw = mtasResponse.get("termvector");
160 202 if (mtasTermvectorResponseRaw != null
161 203 && mtasTermvectorResponseRaw instanceof List) {
162   - List<NamedList> mtasTermvectorResponse = (List) mtasTermvectorResponseRaw;
  204 + List<NamedList<Object>> mtasTermvectorResponse = (List) mtasTermvectorResponseRaw;
163 205 if (mtasTermvectorResponse.isEmpty()) {
164 206 log.error("no (valid) mtas termvector response");
165 207 } else {
... ... @@ -173,9 +215,9 @@ public class MtasSolrBase {
173 215 }
174 216 }
175 217 assertFalse("no item with key " + key, item == null);
176   - if (item.get("list") != null
  218 + if (item != null && item.get("list") != null
177 219 && (item.get("list") instanceof List)) {
178   - return (List<NamedList>) item.get("list");
  220 + return (List<NamedList<Object>>) item.get("list");
179 221 }
180 222 }
181 223 } else {
... ... @@ -221,14 +263,16 @@ public class MtasSolrBase {
221 263 }
222 264 assertFalse("no item with key " + key, item == null);
223 265 Map<String, List<String>> result = new HashMap<>();
224   - Iterator<Entry<String, Object>> it = item.iterator();
225   - Entry<String, Object> entry;
226   - while (it.hasNext()) {
227   - entry = it.next();
228   - if (!entry.getKey().equals("key")) {
229   - assertTrue("invalid entry prefix",
230   - entry.getValue() instanceof List);
231   - result.put(entry.getKey(), (List) entry.getValue());
  266 + if (item != null) {
  267 + Iterator<Entry<String, Object>> it = item.iterator();
  268 + Entry<String, Object> entry;
  269 + while (it.hasNext()) {
  270 + entry = it.next();
  271 + if (!entry.getKey().equals("key")) {
  272 + assertTrue("invalid entry prefix",
  273 + entry.getValue() instanceof List);
  274 + result.put(entry.getKey(), (List) entry.getValue());
  275 + }
232 276 }
233 277 }
234 278 return result;
... ... @@ -240,6 +284,78 @@ public class MtasSolrBase {
240 284 }
241 285  
242 286 /**
  287 + * Gets the from mtas collection.
  288 + *
  289 + * @param response the response
  290 + * @param key the key
  291 + * @return the from mtas collection
  292 + */
  293 + public static NamedList<Object> getFromMtasCollection(
  294 + NamedList<Object> response, String key) {
  295 + if (response == null) {
  296 + log.error("no (valid); response");
  297 + } else {
  298 + Object mtasResponseRaw = response.get("mtas");
  299 + if (mtasResponseRaw != null && mtasResponseRaw instanceof NamedList) {
  300 + NamedList<Object> mtasResponse = (NamedList<Object>) response
  301 + .get("mtas");
  302 + Object mtasCollectionResponseRaw = mtasResponse.get("collection");
  303 + if (mtasCollectionResponseRaw != null
  304 + && mtasCollectionResponseRaw instanceof List) {
  305 + List<NamedList<Object>> mtasCollectionResponse = (List<NamedList<Object>>) mtasCollectionResponseRaw;
  306 + if (mtasCollectionResponse.isEmpty()) {
  307 + log.error("no (valid) mtas join response");
  308 + } else {
  309 + for (NamedList<Object> mtasCollectionResponseItem : mtasCollectionResponse) {
  310 + if (mtasCollectionResponseItem.get("key") != null
  311 + && (mtasCollectionResponseItem.get("key") instanceof String)
  312 + && mtasCollectionResponseItem.get("key").equals(key)) {
  313 + return mtasCollectionResponseItem;
  314 + }
  315 + }
  316 + }
  317 + } else {
  318 + log.error("unexpected " + mtasCollectionResponseRaw);
  319 + }
  320 + } else {
  321 + log.error("unexpected " + mtasResponseRaw);
  322 + }
  323 + }
  324 + return null;
  325 + }
  326 +
  327 + /**
  328 + * Gets the from mtas collection list.
  329 + *
  330 + * @param response the response
  331 + * @param key the key
  332 + * @param id the id
  333 + * @return the from mtas collection list
  334 + */
  335 + public static NamedList<Object> getFromMtasCollectionList(
  336 + NamedList<Object> response, String key, String id) {
  337 + NamedList<Object> collectionResponse = getFromMtasCollection(response, key);
  338 + if (collectionResponse != null) {
  339 + Object collectionResponseListRaw = collectionResponse.get("list");
  340 + if (collectionResponseListRaw != null && collectionResponseListRaw instanceof List) {
  341 + List<NamedList<Object>> collectionResponseList = (List<NamedList<Object>>) collectionResponseListRaw;
  342 + for (NamedList<Object> item : collectionResponseList) {
  343 + if (item.get("id") != null && item.get("id") instanceof String) {
  344 + if (id.equals((String) item.get("id"))) {
  345 + return item;
  346 + }
  347 + }
  348 + }
  349 + } else {
  350 + log.error("unexpected " + collectionResponseListRaw + " (searching list)");
  351 + }
  352 + } else {
  353 + log.error("no collectionResponse (searching key " + key + ")");
  354 + }
  355 + return null;
  356 + }
  357 +
  358 + /**
243 359 * Delete directory.
244 360 *
245 361 * @param directory the directory
... ... @@ -273,38 +389,38 @@ public class MtasSolrBase {
273 389 Path dataPath = Paths.get("junit").resolve("data");
274 390 // data
275 391 SolrInputDocument newDoc1 = new SolrInputDocument();
276   - newDoc1.addField("id", "1");
277   - newDoc1.addField("title", "Een onaangenaam mens in de Haarlemmerhout");
278   - newDoc1.addField("text", "Een onaangenaam mens in de Haarlemmerhout");
279   - newDoc1.addField("mtas", dataPath.resolve("resources")
  392 + newDoc1.addField(FIELD_ID, "1");
  393 + newDoc1.addField(FIELD_TITLE, "Een onaangenaam mens in de Haarlemmerhout");
  394 + newDoc1.addField(FIELD_TEXT, "Een onaangenaam mens in de Haarlemmerhout");
  395 + newDoc1.addField(FIELD_MTAS, dataPath.resolve("resources")
280 396 .resolve("beets1.xml.gz").toFile().getAbsolutePath());
281 397 if (includeAdvanced) {
282   - newDoc1.addField("source", "source1");
283   - newDoc1.addField("mtasAdvanced", dataPath.resolve("resources")
  398 + newDoc1.addField(FIELD_SOURCE, "source1");
  399 + newDoc1.addField(FIELD_MTAS_ADVANCED, dataPath.resolve("resources")
284 400 .resolve("beets1").toFile().getAbsolutePath());
285 401 }
286 402 solrDocuments.put(1, newDoc1);
287 403 SolrInputDocument newDoc2 = new SolrInputDocument();
288   - newDoc2.addField("id", "2");
289   - newDoc2.addField("title", "Een oude kennis");
290   - newDoc2.addField("text", "Een oude kennis");
291   - newDoc2.addField("mtas", dataPath.resolve("resources")
  404 + newDoc2.addField(FIELD_ID, "2");
  405 + newDoc2.addField(FIELD_TITLE, "Een oude kennis");
  406 + newDoc2.addField(FIELD_TEXT, "Een oude kennis");
  407 + newDoc2.addField(FIELD_MTAS, dataPath.resolve("resources")
292 408 .resolve("beets2.xml.gz").toFile().getAbsolutePath());
293 409 if (includeAdvanced) {
294   - newDoc2.addField("source", "source2");
295   - newDoc2.addField("mtasAdvanced", dataPath.resolve("resources")
  410 + newDoc2.addField(FIELD_SOURCE, "source2");
  411 + newDoc2.addField(FIELD_MTAS_ADVANCED, dataPath.resolve("resources")
296 412 .resolve("beets2.xml").toFile().getAbsolutePath());
297 413 }
298 414 SolrInputDocument newDoc3 = new SolrInputDocument();
299 415 solrDocuments.put(2, newDoc2);
300   - newDoc3.addField("id", "3");
301   - newDoc3.addField("title", "Varen en Rijden");
302   - newDoc3.addField("text", "Varen en Rijden");
303   - newDoc3.addField("mtas", dataPath.resolve("resources")
  416 + newDoc3.addField(FIELD_ID, "3");
  417 + newDoc3.addField(FIELD_TITLE, "Varen en Rijden");
  418 + newDoc3.addField(FIELD_TEXT, "Varen en Rijden");
  419 + newDoc3.addField(FIELD_MTAS, dataPath.resolve("resources")
304 420 .resolve("beets3.xml.gz").toFile().getAbsolutePath());
305 421 if (includeAdvanced) {
306   - newDoc3.addField("source", "source3");
307   - newDoc3.addField("mtasAdvanced", dataPath.resolve("resources")
  422 + newDoc3.addField(FIELD_SOURCE, "source3");
  423 + newDoc3.addField(FIELD_MTAS_ADVANCED, dataPath.resolve("resources")
308 424 .resolve("beets3.xml.gz").toFile().getAbsolutePath());
309 425 }
310 426 solrDocuments.put(3, newDoc3);
... ...
junit/mtas/solr/MtasSolrTestDistributedSearchConsistency.java
... ... @@ -225,7 +225,7 @@ public class MtasSolrTestDistributedSearchConsistency {
225 225 list.get(COLLECTION_DISTRIBUTED).getResponse(), "tv",
226 226 new String[] { "n", "sum" });
227 227 for (Entry<String, QueryResponse> entry : list.entrySet()) {
228   - List<NamedList> tv = MtasSolrBase
  228 + List<NamedList<Object>> tv = MtasSolrBase
229 229 .getFromMtasTermvector(entry.getValue().getResponse(), "tv");
230 230 for (NamedList<Object> item : tv) {
231 231 String key = item.get("key").toString();
... ... @@ -311,6 +311,310 @@ public class MtasSolrTestDistributedSearchConsistency {
311 311 }
312 312  
313 313 /**
  314 + * Mtas request handler collection 1.
  315 + *
  316 + * @throws IOException Signals that an I/O exception has occurred.
  317 + */
  318 + @org.junit.Test
  319 + public void mtasRequestHandlerCollection1() throws IOException {
  320 + String[] collections = new String[] { COLLECTION_ALL_OPTIMIZED,
  321 + COLLECTION_ALL_MULTIPLE_SEGMENTS, COLLECTION_DISTRIBUTED };
  322 + String[] collectionsParts = new String[] { COLLECTION_PART1_OPTIMIZED,
  323 + COLLECTION_PART2_MULTIPLE_SEGMENTS };
  324 + Map<String, String> listCreateVersion = new HashMap<>();
  325 + Map<String, Number> listCreateSize = new HashMap<>();
  326 + Map<String, String> listPostVersion = new HashMap<>();
  327 + Map<String, Number> listPostSize = new HashMap<>();
  328 + // create
  329 + ModifiableSolrParams paramsCreate = new ModifiableSolrParams();
  330 + paramsCreate.set("q", "*:*");
  331 + paramsCreate.set("rows", "0");
  332 + paramsCreate.set("mtas", "true");
  333 + paramsCreate.set("mtas.collection", "true");
  334 + paramsCreate.set("mtas.collection.0.key", "create");
  335 + paramsCreate.set("mtas.collection.0.action", "create");
  336 + paramsCreate.set("mtas.collection.0.id", "idCreate");
  337 + paramsCreate.set("mtas.collection.0.field", MtasSolrBase.FIELD_ID);
  338 + Map<String, QueryResponse> listCreate = createResults(paramsCreate,
  339 + Arrays.asList(collections));
  340 + for (Entry<String, QueryResponse> entry : listCreate.entrySet()) {
  341 + long size = MtasSolrBase.getNumFound(entry.getValue().getResponse());
  342 + NamedList<Object> create = MtasSolrBase
  343 + .getFromMtasCollection(entry.getValue().getResponse(), "create");
  344 + createCollectionAssertions(create, entry.getKey(), "idCreate", null, size,
  345 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  346 + listCreateVersion.put(entry.getKey(), (String) create.get("version"));
  347 + listCreateSize.put(entry.getKey(), (Number) create.get("size"));
  348 + }
  349 + // post
  350 + ModifiableSolrParams paramsPost = new ModifiableSolrParams();
  351 + paramsPost.set("q", "*:*");
  352 + paramsPost.set("rows", "0");
  353 + paramsPost.set("mtas", "true");
  354 + paramsPost.set("mtas.collection", "true");
  355 + paramsPost.set("mtas.collection.0.key", "post");
  356 + paramsPost.set("mtas.collection.0.action", "post");
  357 + paramsPost.set("mtas.collection.0.id", "idPost");
  358 + paramsPost.set("mtas.collection.0.post", "[1,2,3,4]");
  359 + Map<String, QueryResponse> listPost = createResults(paramsPost,
  360 + Arrays.asList(collections));
  361 + for (Entry<String, QueryResponse> entry : listPost.entrySet()) {
  362 + long size = 4;
  363 + NamedList<Object> post = MtasSolrBase
  364 + .getFromMtasCollection(entry.getValue().getResponse(), "post");
  365 + createCollectionAssertions(post, entry.getKey(), "idPost", null, size,
  366 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  367 + listPostVersion.put(entry.getKey(), (String) post.get("version"));
  368 + listPostSize.put(entry.getKey(), (Number) post.get("size"));
  369 + }
  370 + // list
  371 + ModifiableSolrParams paramsList = new ModifiableSolrParams();
  372 + paramsList.set("q", "*:*");
  373 + paramsList.set("rows", "0");
  374 + paramsList.set("mtas", "true");
  375 + paramsList.set("mtas.collection", "true");
  376 + paramsList.set("mtas.collection.0.key", "list");
  377 + paramsList.set("mtas.collection.0.action", "list");
  378 + Map<String, QueryResponse> listList = createResults(paramsList,
  379 + Arrays.asList(collections));
  380 + for (Entry<String, QueryResponse> entry : listList.entrySet()) {
  381 + // check create
  382 + NamedList<Object> listCreateItem1 = MtasSolrBase
  383 + .getFromMtasCollectionList(entry.getValue().getResponse(), "list",
  384 + "idCreate");
  385 + createCollectionAssertions(listCreateItem1, entry.getKey(), "idCreate",
  386 + listCreateVersion.get(entry.getKey()),
  387 + listCreateSize.get(entry.getKey()),
  388 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  389 + // check post
  390 + NamedList<Object> listPostItem1 = MtasSolrBase.getFromMtasCollectionList(
  391 + entry.getValue().getResponse(), "list", "idPost");
  392 + createCollectionAssertions(listPostItem1, entry.getKey(), "idPost",
  393 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  394 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  395 + }
  396 + // check
  397 + ModifiableSolrParams paramsCheck = new ModifiableSolrParams();
  398 + paramsCheck.set("q", "*:*");
  399 + paramsCheck.set("rows", "0");
  400 + paramsCheck.set("mtas", "true");
  401 + paramsCheck.set("mtas.collection", "true");
  402 + paramsCheck.set("mtas.collection.0.key", "check1");
  403 + paramsCheck.set("mtas.collection.0.action", "check");
  404 + paramsCheck.set("mtas.collection.0.id", "idCreate");
  405 + paramsCheck.set("mtas.collection.1.key", "check2");
  406 + paramsCheck.set("mtas.collection.1.action", "check");
  407 + paramsCheck.set("mtas.collection.1.id", "idPost");
  408 + // check on all
  409 + Map<String, QueryResponse> listCheck = createResults(paramsCheck,
  410 + Arrays.asList(collections));
  411 + for (Entry<String, QueryResponse> entry : listCheck.entrySet()) {
  412 + NamedList<Object> listItemCheck1 = MtasSolrBase
  413 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  414 + createCollectionAssertions(listItemCheck1, entry.getKey(), "idCreate",
  415 + listCreateVersion.get(entry.getKey()),
  416 + listCreateSize.get(entry.getKey()),
  417 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  418 + NamedList<Object> listItemCheck2 = MtasSolrBase
  419 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  420 + createCollectionAssertions(listItemCheck2, entry.getKey(), "idPost",
  421 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  422 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  423 + }
  424 + // check on parts
  425 + createResults(paramsCheck, Arrays.asList(collectionsParts));
  426 + for (Entry<String, QueryResponse> entry : listCheck.entrySet()) {
  427 + NamedList<Object> listItemCheck1 = MtasSolrBase
  428 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  429 + createCollectionAssertions(listItemCheck1, entry.getKey(), "idCreate",
  430 + listCreateVersion.get(entry.getKey()),
  431 + listCreateSize.get(entry.getKey()),
  432 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  433 + NamedList<Object> listItemCheck2 = MtasSolrBase
  434 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  435 + createCollectionAssertions(listItemCheck2, entry.getKey(), "idPost",
  436 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  437 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  438 + }
  439 + // delete
  440 + ModifiableSolrParams paramsDelete = new ModifiableSolrParams();
  441 + paramsDelete.set("q", "*:*");
  442 + paramsDelete.set("rows", "0");
  443 + paramsDelete.set("mtas", "true");
  444 + paramsDelete.set("mtas.collection", "true");
  445 + paramsDelete.set("mtas.collection.0.key", "delete1");
  446 + paramsDelete.set("mtas.collection.0.action", "delete");
  447 + paramsDelete.set("mtas.collection.0.id", "idCreate");
  448 + paramsDelete.set("mtas.collection.1.key", "delete2");
  449 + paramsDelete.set("mtas.collection.1.action", "delete");
  450 + paramsDelete.set("mtas.collection.1.id", "idPost");
  451 + // delete on parts
  452 + createResults(paramsDelete, Arrays.asList(collectionsParts));
  453 + // recheck on parts
  454 + Map<String, QueryResponse> listCheckParts = createResults(paramsCheck,
  455 + Arrays.asList(collectionsParts));
  456 + for (Entry<String, QueryResponse> entry : listCheckParts.entrySet()) {
  457 + NamedList<Object> listItemCheck1 = MtasSolrBase
  458 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  459 + assertTrue(
  460 + entry.getKey() + " - create - should be removed: " + listItemCheck1,
  461 + listItemCheck1 != null && listItemCheck1.get("id") == null);
  462 + NamedList<Object> listItemCheck2 = MtasSolrBase
  463 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  464 + assertTrue(
  465 + entry.getKey() + " - post - should be removed: " + listItemCheck2,
  466 + listItemCheck2 != null && listItemCheck2.get("id") == null);
  467 + }
  468 + // list with empty parts
  469 + listList = createResults(paramsList, Arrays.asList(collections));
  470 + for (Entry<String, QueryResponse> entry : listList.entrySet()) {
  471 + // check create
  472 + NamedList<Object> listCreateItem1 = MtasSolrBase
  473 + .getFromMtasCollectionList(entry.getValue().getResponse(), "list",
  474 + "idCreate");
  475 + createCollectionAssertions(listCreateItem1, entry.getKey(), "idCreate",
  476 + listCreateVersion.get(entry.getKey()),
  477 + listCreateSize.get(entry.getKey()), 0);
  478 + // check post
  479 + NamedList<Object> listPostItem1 = MtasSolrBase.getFromMtasCollectionList(
  480 + entry.getValue().getResponse(), "list", "idPost");
  481 + createCollectionAssertions(listPostItem1, entry.getKey(), "idPost",
  482 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  483 + 0);
  484 + }
  485 + // recheck on all, assuming empty parts, autofix
  486 + listCheck = createResults(paramsCheck, Arrays.asList(collections));
  487 + for (Entry<String, QueryResponse> entry : listCheck.entrySet()) {
  488 + NamedList<Object> listItemCheck1 = MtasSolrBase
  489 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  490 + createCollectionAssertions(listItemCheck1, entry.getKey(), "idCreate",
  491 + listCreateVersion.get(entry.getKey()),
  492 + listCreateSize.get(entry.getKey()),
  493 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  494 + NamedList<Object> listItemCheck2 = MtasSolrBase
  495 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  496 + createCollectionAssertions(listItemCheck2, entry.getKey(), "idPost",
  497 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  498 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  499 + }
  500 + // recheck on parts
  501 + listCheckParts = createResults(paramsCheck,
  502 + Arrays.asList(collectionsParts));
  503 + for (Entry<String, QueryResponse> entry : listCheck.entrySet()) {
  504 + NamedList<Object> listItemCheck1 = MtasSolrBase
  505 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  506 + createCollectionAssertions(listItemCheck1, entry.getKey(), "idCreate",
  507 + listCreateVersion.get(entry.getKey()),
  508 + listCreateSize.get(entry.getKey()),
  509 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  510 + NamedList<Object> listItemCheck2 = MtasSolrBase
  511 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  512 + createCollectionAssertions(listItemCheck2, entry.getKey(), "idPost",
  513 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  514 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  515 + }
  516 + // full delete
  517 + createResults(paramsDelete, Arrays.asList(collections));
  518 + // final check
  519 + listCheck = createResults(paramsCheck, Arrays.asList(collections));
  520 + for (Entry<String, QueryResponse> entry : listCheck.entrySet()) {
  521 + NamedList<Object> listItemCheck1 = MtasSolrBase
  522 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  523 + assertTrue(
  524 + entry.getKey() + " - create - should be removed: " + listItemCheck1,
  525 + listItemCheck1 != null && listItemCheck1.get("id") == null);
  526 + NamedList<Object> listItemCheck2 = MtasSolrBase
  527 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  528 + assertTrue(
  529 + entry.getKey() + " - post - should be removed: " + listItemCheck2,
  530 + listItemCheck2 != null && listItemCheck2.get("id") == null);
  531 + }
  532 + }
  533 +
  534 + /**
  535 + * Mtas request handler collection 2.
  536 + *
  537 + * @throws IOException Signals that an I/O exception has occurred.
  538 + */
  539 + @org.junit.Test
  540 + public void mtasRequestHandlerCollection2() throws IOException {
  541 + String[] collections = new String[] { COLLECTION_ALL_OPTIMIZED,
  542 + COLLECTION_ALL_MULTIPLE_SEGMENTS, COLLECTION_DISTRIBUTED };
  543 + // post
  544 + ModifiableSolrParams paramsPost = new ModifiableSolrParams();
  545 + paramsPost.set("q", "*:*");
  546 + paramsPost.set("mtas", "true");
  547 + paramsPost.set("mtas.collection", "true");
  548 + paramsPost.set("mtas.collection.0.key", "postKey1");
  549 + paramsPost.set("mtas.collection.0.action", "post");
  550 + paramsPost.set("mtas.collection.0.id", "postSet1");
  551 + paramsPost.set("mtas.collection.0.post", "[1,3,4]");
  552 + paramsPost.set("mtas.collection.1.key", "postKey2");
  553 + paramsPost.set("mtas.collection.1.action", "post");
  554 + paramsPost.set("mtas.collection.1.id", "postSet2");
  555 + paramsPost.set("mtas.collection.1.post", "[2]");
  556 + paramsPost.set("mtas.collection.2.key", "createKey1");
  557 + paramsPost.set("mtas.collection.2.action", "create");
  558 + paramsPost.set("mtas.collection.2.id", "createSet1");
  559 + paramsPost.set("mtas.collection.2.field", MtasSolrBase.FIELD_ID);
  560 + createResults(paramsPost, Arrays.asList(collections));
  561 + // query set1
  562 + ModifiableSolrParams paramsSelect1 = new ModifiableSolrParams();
  563 + paramsSelect1.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  564 + + "\" collection=\"postSet1\"}");
  565 + paramsSelect1.set("rows", "0");
  566 + Map<String, QueryResponse> listPost1 = createResults(paramsSelect1,
  567 + Arrays.asList(collections));
  568 + for (Entry<String, QueryResponse> entry : listPost1.entrySet()) {
  569 + long n = MtasSolrBase.getNumFound(entry.getValue().getResponse());
  570 + assertTrue(
  571 + entry.getKey() + " - incorrect number of matching documents : " + n,
  572 + n == 2);
  573 + }
  574 + // query set2
  575 + ModifiableSolrParams paramsSelect2 = new ModifiableSolrParams();
  576 + paramsSelect2.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  577 + + "\" collection=\"postSet2\"}");
  578 + paramsSelect2.set("rows", "0");
  579 + Map<String, QueryResponse> listPost2 = createResults(paramsSelect2,
  580 + Arrays.asList(collections));
  581 + for (Entry<String, QueryResponse> entry : listPost2.entrySet()) {
  582 + long n = MtasSolrBase.getNumFound(entry.getValue().getResponse());
  583 + assertTrue(
  584 + entry.getKey() + " - incorrect number of matching documents : " + n,
  585 + n == 1);
  586 + }
  587 + // query set3
  588 + ModifiableSolrParams paramsSelect3 = new ModifiableSolrParams();
  589 + paramsSelect3.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  590 + + "\" collection=\"createSet1\"}");
  591 + paramsSelect3.set("rows", "0");
  592 + Map<String, QueryResponse> listPost3 = createResults(paramsSelect3,
  593 + Arrays.asList(collections));
  594 + for (Entry<String, QueryResponse> entry : listPost3.entrySet()) {
  595 + long n = MtasSolrBase.getNumFound(entry.getValue().getResponse());
  596 + assertTrue(
  597 + entry.getKey() + " - incorrect number of matching documents : " + n,
  598 + n == 3);
  599 + }
  600 + // query set1 or set2
  601 + ModifiableSolrParams paramsSelect4 = new ModifiableSolrParams();
  602 + paramsSelect4.set("q",
  603 + "({!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  604 + + "\" collection=\"postSet1\"}) OR ({!mtas_join field=\""
  605 + + MtasSolrBase.FIELD_ID + "\" collection=\"postSet2\"})");
  606 + paramsSelect4.set("rows", "0");
  607 + Map<String, QueryResponse> listPost4 = createResults(paramsSelect4,
  608 + Arrays.asList(collections));
  609 + for (Entry<String, QueryResponse> entry : listPost4.entrySet()) {
  610 + long n = MtasSolrBase.getNumFound(entry.getValue().getResponse());
  611 + assertTrue(
  612 + entry.getKey() + " - incorrect number of matching documents : " + n,
  613 + n == 3);
  614 + }
  615 + }
  616 +
  617 + /**
314 618 * Mtas request handler prefix.
315 619 *
316 620 * @throws IOException Signals that an I/O exception has occurred.
... ... @@ -336,12 +640,16 @@ public class MtasSolrTestDistributedSearchConsistency {
336 640 /**
337 641 * Creates the results.
338 642 *
339   - * @param params the params
  643 + * @param initialParams the initial params
340 644 * @param collections the collections
341 645 * @return the hash map
342 646 */
343 647 private static HashMap<String, QueryResponse> createResults(
344   - final ModifiableSolrParams params, List<String> collections) {
  648 + final ModifiableSolrParams initialParams, List<String> collections) {
  649 + // use initial params
  650 + ModifiableSolrParams params = new ModifiableSolrParams();
  651 + params.add(initialParams);
  652 + // continue
345 653 HashMap<String, QueryResponse> list = new HashMap<>();
346 654 CloudSolrClient client = cloudCluster.getSolrClient();
347 655 try {
... ... @@ -450,8 +758,10 @@ public class MtasSolrTestDistributedSearchConsistency {
450 758 */
451 759 private static void createTermvectorAssertions(NamedList<Object> response1,
452 760 NamedList<Object> response2, String key, String[] names) {
453   - List<NamedList> list1 = MtasSolrBase.getFromMtasTermvector(response1, key);
454   - List<NamedList> list2 = MtasSolrBase.getFromMtasTermvector(response2, key);
  761 + List<NamedList<Object>> list1 = MtasSolrBase
  762 + .getFromMtasTermvector(response1, key);
  763 + List<NamedList<Object>> list2 = MtasSolrBase
  764 + .getFromMtasTermvector(response2, key);
455 765 assertFalse("list should be defined", list1 == null || list2 == null);
456 766 if (list1 != null && list2 != null) {
457 767 assertEquals("lists should have equal size", list1.size(), list2.size());
... ... @@ -479,6 +789,57 @@ public class MtasSolrTestDistributedSearchConsistency {
479 789 }
480 790  
481 791 /**
  792 + * Creates the collection assertions.
  793 + *
  794 + * @param create the create
  795 + * @param collection the collection
  796 + * @param id the id
  797 + * @param version the version
  798 + * @param size the size
  799 + * @param shards the shards
  800 + */
  801 + private static void createCollectionAssertions(NamedList<Object> create,
  802 + String collection, String id, String version, Number size, int shards) {
  803 + assertFalse(collection + ": create - not found", create == null);
  804 + assertTrue(collection + ": create - no valid version",
  805 + create.get("id") != null && create.get("id") instanceof String);
  806 + assertTrue(collection + ": create - id incorrect, '" + id
  807 + + "' not equal to '" + create.get("id") + "'",
  808 + ((String) create.get("id")).equals(id));
  809 + assertTrue(
  810 + collection + ": create - no valid version, '" + version
  811 + + "' not equal to '" + create.get("version") + "'",
  812 + create.get("version") != null
  813 + && create.get("version") instanceof String);
  814 + if (version != null) {
  815 + assertTrue(collection + ": create - version incorrect",
  816 + ((String) create.get("version")).equals(version));
  817 + }
  818 + assertTrue(collection + ": create - no valid size",
  819 + create.get("size") != null && create.get("size") instanceof Number);
  820 + Number createSize = (Number) create.get("size");
  821 + assertEquals(collection + ": number of values", size.longValue(),
  822 + createSize.longValue());
  823 + if (shards > 0) {
  824 + assertTrue("no (valid) shards",
  825 + create.get("shards") != null && create.get("shards") instanceof List
  826 + && ((List) create.get("shards")).size() == shards);
  827 + for (Object shardItem : (List<Object>) create.get("shards")) {
  828 + assertTrue(collection + ": invalid shardItem",
  829 + shardItem instanceof NamedList);
  830 + Object sizeRaw = ((NamedList<Object>) shardItem).get("size");
  831 + assertTrue(collection + ": incorrect size",
  832 + sizeRaw != null && sizeRaw instanceof Number
  833 + && ((Number) sizeRaw).longValue() == createSize.longValue());
  834 + }
  835 + } else {
  836 + assertFalse(collection + ": shards found : " + create.get("shards"),
  837 + create.get("shards") != null && create.get("shards") instanceof List
  838 + && !((List) create.get("shards")).isEmpty());
  839 + }
  840 + }
  841 +
  842 + /**
482 843 * Creates the cloud.
483 844 */
484 845 private static void createCloud() {
... ...
junit/mtas/solr/MtasSolrTestSearchConsistency.java
... ... @@ -221,13 +221,13 @@ public class MtasSolrTestSearchConsistency {
221 221 params.set("mtas", "true");
222 222 params.set("mtas.stats", "true");
223 223 params.set("mtas.stats.spans", "true");
224   - params.set("mtas.stats.spans.0.field", "mtas");
  224 + params.set("mtas.stats.spans.0.field", MtasSolrBase.FIELD_MTAS);
225 225 params.set("mtas.stats.spans.0.key", "statsKey");
226 226 params.set("mtas.stats.spans.0.query.0.type", "cql");
227 227 params.set("mtas.stats.spans.0.query.0.value", "[]");
228 228 params.set("mtas.stats.spans.0.type", "n,sum,mean");
229 229 params.set("mtas.stats.positions", "true");
230   - params.set("mtas.stats.positions.0.field", "mtas");
  230 + params.set("mtas.stats.positions.0.field", MtasSolrBase.FIELD_MTAS);
231 231 params.set("mtas.stats.positions.0.key", "statsKey");
232 232 params.set("mtas.stats.positions.0.type", "n,sum,mean");
233 233 params.set("rows", "0");
... ... @@ -264,7 +264,7 @@ public class MtasSolrTestSearchConsistency {
264 264 params.set("mtas", "true");
265 265 params.set("mtas.stats", "true");
266 266 params.set("mtas.stats.tokens", "true");
267   - params.set("mtas.stats.tokens.0.field", "mtas");
  267 + params.set("mtas.stats.tokens.0.field", MtasSolrBase.FIELD_MTAS);
268 268 params.set("mtas.stats.tokens.0.key", "statsKey");
269 269 params.set("mtas.stats.tokens.0.type", String.join(",", types));
270 270 params.set("mtas.stats.tokens.0.minimum", 1);
... ... @@ -293,7 +293,7 @@ public class MtasSolrTestSearchConsistency {
293 293 params.set("rows", 0);
294 294 params.set("mtas", "true");
295 295 params.set("mtas.termvector", "true");
296   - params.set("mtas.termvector.0.field", "mtas");
  296 + params.set("mtas.termvector.0.field", MtasSolrBase.FIELD_MTAS);
297 297 params.set("mtas.termvector.0.prefix", "t_lc");
298 298 params.set("mtas.termvector.0.key", "tv");
299 299 params.set("mtas.termvector.0.sort.type", "sum");
... ... @@ -342,7 +342,7 @@ public class MtasSolrTestSearchConsistency {
342 342 params.set("rows", 0);
343 343 params.set("mtas", "true");
344 344 params.set("mtas.termvector", "true");
345   - params.set("mtas.termvector.0.field", "mtas");
  345 + params.set("mtas.termvector.0.field", MtasSolrBase.FIELD_MTAS);
346 346 params.set("mtas.termvector.0.prefix", "t_lc");
347 347 params.set("mtas.termvector.0.key", "tv");
348 348 params.set("mtas.termvector.0.type", String.join(",", types));
... ... @@ -355,7 +355,8 @@ public class MtasSolrTestSearchConsistency {
355 355 } catch (SolrServerException e) {
356 356 throw new IOException(e);
357 357 }
358   - List<NamedList> tv = MtasSolrBase.getFromMtasTermvector(response, "tv");
  358 + List<NamedList<Object>> tv = MtasSolrBase.getFromMtasTermvector(response,
  359 + "tv");
359 360 for (String key : list) {
360 361 params.clear();
361 362 params.set("q", "*:*");
... ... @@ -363,13 +364,13 @@ public class MtasSolrTestSearchConsistency {
363 364 params.set("mtas", "true");
364 365 params.set("mtas.stats", "true");
365 366 params.set("mtas.stats.spans", "true");
366   - params.set("mtas.stats.spans.0.field", "mtas");
  367 + params.set("mtas.stats.spans.0.field", MtasSolrBase.FIELD_MTAS);
367 368 params.set("mtas.stats.spans.0.key", "statsKey0");
368 369 params.set("mtas.stats.spans.0.minimum", 1);
369 370 params.set("mtas.stats.spans.0.query.0.type", "cql");
370 371 params.set("mtas.stats.spans.0.query.0.value", "[t_lc=\"" + key + "\"]");
371 372 params.set("mtas.stats.spans.0.type", String.join(",", types));
372   - params.set("mtas.stats.spans.1.field", "mtas");
  373 + params.set("mtas.stats.spans.1.field", MtasSolrBase.FIELD_MTAS);
373 374 params.set("mtas.stats.spans.1.key", "statsKey1");
374 375 params.set("mtas.stats.spans.1.minimum", 0);
375 376 params.set("mtas.stats.spans.1.query.0.type", "cql");
... ... @@ -419,7 +420,7 @@ public class MtasSolrTestSearchConsistency {
419 420 params.set("rows", 0);
420 421 params.set("mtas", "true");
421 422 params.set("mtas.termvector", "true");
422   - params.set("mtas.termvector.0.field", "mtas");
  423 + params.set("mtas.termvector.0.field", MtasSolrBase.FIELD_MTAS);
423 424 params.set("mtas.termvector.0.prefix", "t_lc");
424 425 params.set("mtas.termvector.0.key", "tv");
425 426 params.set("mtas.termvector.0.regexp", "een[a-z]*");
... ... @@ -435,7 +436,8 @@ public class MtasSolrTestSearchConsistency {
435 436 } catch (SolrServerException e) {
436 437 throw new IOException(e);
437 438 }
438   - List<NamedList> tv = MtasSolrBase.getFromMtasTermvector(response, "tv");
  439 + List<NamedList<Object>> tv = MtasSolrBase.getFromMtasTermvector(response,
  440 + "tv");
439 441 Set<String> keys = new HashSet<>();
440 442 for (NamedList<Object> item : tv) {
441 443 if (item != null && item.get("key") != null
... ... @@ -455,6 +457,350 @@ public class MtasSolrTestSearchConsistency {
455 457 }
456 458  
457 459 /**
  460 + * Mtas request handler collection 1.
  461 + *
  462 + * @throws IOException Signals that an I/O exception has occurred.
  463 + */
  464 + @org.junit.Test
  465 + public void mtasRequestHandlerCollection1() throws IOException {
  466 + // create
  467 + ModifiableSolrParams paramsCreate = new ModifiableSolrParams();
  468 + paramsCreate.set("q", "*:*");
  469 + paramsCreate.set("mtas", "true");
  470 + paramsCreate.set("mtas.collection", "true");
  471 + paramsCreate.set("mtas.collection.0.key", "create");
  472 + paramsCreate.set("mtas.collection.0.action", "create");
  473 + paramsCreate.set("mtas.collection.0.id", "idCreate");
  474 + paramsCreate.set("mtas.collection.0.field", "id");
  475 + SolrRequest<?> requestCreate = new QueryRequest(paramsCreate, METHOD.POST);
  476 + NamedList<Object> responseCreate;
  477 + try {
  478 + responseCreate = server.request(requestCreate, "collection1");
  479 + } catch (SolrServerException e) {
  480 + throw new IOException(e);
  481 + }
  482 + long n = MtasSolrBase.getNumFound(responseCreate);
  483 + NamedList<Object> create = MtasSolrBase
  484 + .getFromMtasCollection(responseCreate, "create");
  485 + assertFalse("create - id not found", create == null);
  486 + assertTrue("create - no valid version", create.get("version") != null
  487 + && create.get("version") instanceof String);
  488 + assertTrue("create - no valid size",
  489 + create.get("size") != null && create.get("size") instanceof Number);
  490 + String createVersion = (String) create.get("version");
  491 + Number createSize = (Number) create.get("size");
  492 + assertEquals("number of values", n, createSize.longValue());
  493 + // post
  494 + ModifiableSolrParams paramsPost = new ModifiableSolrParams();
  495 + paramsPost.set("q", "*:*");
  496 + paramsPost.set("mtas", "true");
  497 + paramsPost.set("mtas.collection", "true");
  498 + paramsPost.set("mtas.collection.0.key", "post");
  499 + paramsPost.set("mtas.collection.0.action", "post");
  500 + paramsPost.set("mtas.collection.0.id", "idPost");
  501 + paramsPost.set("mtas.collection.0.post", "[1,2,3,4]");
  502 + SolrRequest<?> requestPost = new QueryRequest(paramsPost, METHOD.POST);
  503 + NamedList<Object> responsePost;
  504 + try {
  505 + responsePost = server.request(requestPost, "collection1");
  506 + } catch (SolrServerException e) {
  507 + throw new IOException(e);
  508 + }
  509 + NamedList<Object> post = MtasSolrBase.getFromMtasCollection(responsePost,
  510 + "post");
  511 + assertFalse("post - id not found", post == null);
  512 + assertTrue("post - no valid version",
  513 + post.get("version") != null && post.get("version") instanceof String);
  514 + assertTrue("post - no valid size",
  515 + post.get("size") != null && post.get("size") instanceof Number);
  516 + String postVersion = (String) post.get("version");
  517 + Number postSize = (Number) post.get("size");
  518 + assertTrue("post - incorrect size", postSize.longValue() == 4);
  519 + // list
  520 + ModifiableSolrParams paramsList = new ModifiableSolrParams();
  521 + paramsList.set("q", "*:*");
  522 + paramsList.set("mtas", "true");
  523 + paramsList.set("mtas.collection", "true");
  524 + paramsList.set("mtas.collection.0.key", "list");
  525 + paramsList.set("mtas.collection.0.action", "list");
  526 + SolrRequest<?> requestList1 = new QueryRequest(paramsList, METHOD.POST);
  527 + NamedList<Object> responseList1;
  528 + try {
  529 + responseList1 = server.request(requestList1, "collection1");
  530 + } catch (SolrServerException e) {
  531 + throw new IOException(e);
  532 + }
  533 + // check create
  534 + NamedList<Object> listCreateItem1 = MtasSolrBase
  535 + .getFromMtasCollectionList(responseList1, "list", "idCreate");
  536 + assertFalse("list - create - id not found", listCreateItem1 == null);
  537 + assertTrue("list - create - incorrect version",
  538 + listCreateItem1.get("version") != null
  539 + && listCreateItem1.get("version") instanceof String
  540 + && listCreateItem1.get("version").equals(createVersion));
  541 + assertTrue("list - create - incorrect size",
  542 + listCreateItem1.get("size") != null
  543 + && listCreateItem1.get("size") instanceof Number
  544 + && ((Number) listCreateItem1.get("size")).longValue() == createSize
  545 + .longValue());
  546 + // check post
  547 + NamedList<Object> listPostItem1 = MtasSolrBase
  548 + .getFromMtasCollectionList(responseList1, "list", "idPost");
  549 + assertFalse("list - post - id not found", listPostItem1 == null);
  550 + assertTrue("list - post - incorrect version",
  551 + listPostItem1.get("version") != null
  552 + && listPostItem1.get("version") instanceof String
  553 + && listPostItem1.get("version").equals(postVersion));
  554 + assertTrue("list - post - incorrect size",
  555 + listPostItem1.get("size") != null
  556 + && listPostItem1.get("size") instanceof Number
  557 + && ((Number) listPostItem1.get("size")).longValue() == postSize
  558 + .longValue());
  559 + // check
  560 + ModifiableSolrParams paramsCheck = new ModifiableSolrParams();
  561 + paramsCheck.set("q", "*:*");
  562 + paramsCheck.set("mtas", "true");
  563 + paramsCheck.set("mtas.collection", "true");
  564 + paramsCheck.set("mtas.collection.0.key", "check1");
  565 + paramsCheck.set("mtas.collection.0.action", "check");
  566 + paramsCheck.set("mtas.collection.0.id", "idCreate");
  567 + paramsCheck.set("mtas.collection.1.key", "check2");
  568 + paramsCheck.set("mtas.collection.1.action", "check");
  569 + paramsCheck.set("mtas.collection.1.id", "idPost");
  570 + SolrRequest<?> requestCheck = new QueryRequest(paramsCheck, METHOD.POST);
  571 + NamedList<Object> responseCheck;
  572 + try {
  573 + responseCheck = server.request(requestCheck, "collection1");
  574 + } catch (SolrServerException e) {
  575 + throw new IOException(e);
  576 + }
  577 + // check create
  578 + NamedList<Object> check1 = MtasSolrBase.getFromMtasCollection(responseCheck,
  579 + "check1");
  580 + assertFalse("check - create - no response", check1 == null);
  581 + assertTrue("check - create - no valid version",
  582 + check1.get("version") != null
  583 + && check1.get("version") instanceof String);
  584 + assertTrue("check - create - no valid size",
  585 + check1.get("size") != null && check1.get("size") instanceof Number);
  586 + String check1Version = (String) check1.get("version");
  587 + Number check1Size = (Number) check1.get("size");
  588 + assertEquals("check - create - version", check1Version, createVersion);
  589 + assertEquals("check - create - number of values", check1Size.longValue(),
  590 + createSize.longValue());
  591 + // check post
  592 + NamedList<Object> check2 = MtasSolrBase.getFromMtasCollection(responseCheck,
  593 + "check2");
  594 + assertFalse("check - post - no response", check2 == null);
  595 + assertTrue("check - post - no valid version", check2.get("version") != null
  596 + && check2.get("version") instanceof String);
  597 + assertTrue("check - post - no valid size",
  598 + check2.get("size") != null && check2.get("size") instanceof Number);
  599 + String check2Version = (String) check2.get("version");
  600 + Number check2Size = (Number) check2.get("size");
  601 + assertEquals("check - post - version", check2Version, postVersion);
  602 + assertEquals("check - post - number of values", check2Size.longValue(), 4);
  603 + // delete
  604 + ModifiableSolrParams paramsDelete = new ModifiableSolrParams();
  605 + paramsDelete.set("q", "*:*");
  606 + paramsDelete.set("mtas", "true");
  607 + paramsDelete.set("mtas.collection", "true");
  608 + paramsDelete.set("mtas.collection.0.key", "delete1");
  609 + paramsDelete.set("mtas.collection.0.action", "delete");
  610 + paramsDelete.set("mtas.collection.0.id", "idCreate");
  611 + paramsDelete.set("mtas.collection.1.key", "delete2");
  612 + paramsDelete.set("mtas.collection.1.action", "delete");
  613 + paramsDelete.set("mtas.collection.1.id", "idPost");
  614 + SolrRequest<?> requestDelete = new QueryRequest(paramsDelete, METHOD.POST);
  615 + NamedList<Object> responseDelete;
  616 + try {
  617 + responseDelete = server.request(requestDelete, "collection1");
  618 + } catch (SolrServerException e) {
  619 + throw new IOException(e);
  620 + }
  621 + // check create
  622 + NamedList<Object> delete1 = MtasSolrBase
  623 + .getFromMtasCollection(responseDelete, "delete1");
  624 + assertFalse("delete - create - no response", delete1 == null);
  625 + // check post
  626 + NamedList<Object> delete2 = MtasSolrBase
  627 + .getFromMtasCollection(responseDelete, "delete2");
  628 + assertFalse("delete - post - no response", delete2 == null);
  629 + // list (again)
  630 + SolrRequest<?> requestList2 = new QueryRequest(paramsList, METHOD.POST);
  631 + NamedList<Object> responseList2;
  632 + try {
  633 + responseList2 = server.request(requestList2, "collection1");
  634 + } catch (SolrServerException e) {
  635 + throw new IOException(e);
  636 + }
  637 + // check create
  638 + NamedList<Object> listCreateItem2 = MtasSolrBase
  639 + .getFromMtasCollectionList(responseList2, "list", "idCreate");
  640 + assertTrue("list - create - id found", listCreateItem2 == null);
  641 + // check post
  642 + NamedList<Object> listPostItem2 = MtasSolrBase
  643 + .getFromMtasCollectionList(responseList2, "list", "idPost");
  644 + assertTrue("list - post - id found", listPostItem2 == null);
  645 + }
  646 +
  647 + /**
  648 + * Mtas request handler collection 2.
  649 + *
  650 + * @throws IOException Signals that an I/O exception has occurred.
  651 + */
  652 + @org.junit.Test
  653 + public void mtasRequestHandlerCollection2() throws IOException {
  654 + // post
  655 + ModifiableSolrParams paramsPost = new ModifiableSolrParams();
  656 + paramsPost.set("q", "*:*");
  657 + paramsPost.set("mtas", "true");
  658 + paramsPost.set("mtas.collection", "true");
  659 + paramsPost.set("mtas.collection.0.key", "postKey1");
  660 + paramsPost.set("mtas.collection.0.action", "post");
  661 + paramsPost.set("mtas.collection.0.id", "postSet1");
  662 + paramsPost.set("mtas.collection.0.post", "[1,3,4]");
  663 + paramsPost.set("mtas.collection.1.key", "postKey2");
  664 + paramsPost.set("mtas.collection.1.action", "post");
  665 + paramsPost.set("mtas.collection.1.id", "postSet2");
  666 + paramsPost.set("mtas.collection.1.post", "[2]");
  667 + paramsPost.set("mtas.collection.2.key", "createKey1");
  668 + paramsPost.set("mtas.collection.2.action", "create");
  669 + paramsPost.set("mtas.collection.2.id", "createSet1");
  670 + paramsPost.set("mtas.collection.2.field", MtasSolrBase.FIELD_ID);
  671 + SolrRequest<?> requestPost = new QueryRequest(paramsPost, METHOD.POST);
  672 + NamedList<Object> responsePost;
  673 + try {
  674 + responsePost = server.request(requestPost, "collection1");
  675 + } catch (SolrServerException e) {
  676 + throw new IOException(e);
  677 + }
  678 + MtasSolrBase.getFromMtasCollection(responsePost, "post");
  679 + // query set1
  680 + ModifiableSolrParams paramsSelect1 = new ModifiableSolrParams();
  681 + paramsSelect1.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  682 + + "\" collection=\"postSet1\"}");
  683 + paramsSelect1.set("rows", "0");
  684 + SolrRequest<?> request1 = new QueryRequest(paramsSelect1, METHOD.POST);
  685 + NamedList<Object> response1;
  686 + try {
  687 + response1 = server.request(request1, "collection1");
  688 + } catch (SolrServerException e) {
  689 + throw new IOException(e);
  690 + }
  691 + long n1 = MtasSolrBase.getNumFound(response1);
  692 + assertTrue("incorrect number of matching documents : " + n1, n1 == 2);
  693 + // query set2
  694 + ModifiableSolrParams paramsSelect2 = new ModifiableSolrParams();
  695 + paramsSelect2.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  696 + + "\" collection=\"postSet2\"}");
  697 + paramsSelect2.set("rows", "0");
  698 + SolrRequest<?> request2 = new QueryRequest(paramsSelect2, METHOD.POST);
  699 + NamedList<Object> response2;
  700 + try {
  701 + response2 = server.request(request2, "collection1");
  702 + } catch (SolrServerException e) {
  703 + throw new IOException(e);
  704 + }
  705 + long n2 = MtasSolrBase.getNumFound(response2);
  706 + assertTrue("incorrect number of matching documents : " + n2, n2 == 1);
  707 + // query set3
  708 + ModifiableSolrParams paramsSelect3 = new ModifiableSolrParams();
  709 + paramsSelect3.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  710 + + "\" collection=\"createSet1\"}");
  711 + paramsSelect3.set("rows", "0");
  712 + SolrRequest<?> request3 = new QueryRequest(paramsSelect3, METHOD.POST);
  713 + NamedList<Object> response3;
  714 + try {
  715 + response3 = server.request(request3, "collection1");
  716 + } catch (SolrServerException e) {
  717 + throw new IOException(e);
  718 + }
  719 + long n3 = MtasSolrBase.getNumFound(response3);
  720 + assertTrue("incorrect number of matching documents : " + n3, n3 == 3);
  721 + // query set1 or set2
  722 + ModifiableSolrParams paramsSelect4 = new ModifiableSolrParams();
  723 + paramsSelect4.set("q",
  724 + "({!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  725 + + "\" collection=\"postSet1\"}) OR ({!mtas_join field=\""
  726 + + MtasSolrBase.FIELD_ID + "\" collection=\"postSet2\"})");
  727 + paramsSelect4.set("rows", "0");
  728 + SolrRequest<?> request4 = new QueryRequest(paramsSelect4, METHOD.POST);
  729 + NamedList<Object> response4;
  730 + try {
  731 + response4 = server.request(request4, "collection1");
  732 + } catch (SolrServerException e) {
  733 + throw new IOException(e);
  734 + }
  735 + long n4 = MtasSolrBase.getNumFound(response4);
  736 + assertTrue("incorrect number of matching documents : " + n4, n4 == 3);
  737 + }
  738 +
  739 + @org.junit.Test
  740 + public void mtasRequestHandlerCollection3() throws IOException {
  741 + // post
  742 + ModifiableSolrParams paramsPost = new ModifiableSolrParams();
  743 + paramsPost.set("q", "*:*");
  744 + paramsPost.set("mtas", "true");
  745 + paramsPost.set("mtas.collection", "true");
  746 + paramsPost.set("mtas.collection.0.key", "setCreatedByPost");
  747 + paramsPost.set("mtas.collection.0.action", "post");
  748 + paramsPost.set("mtas.collection.0.id", "setCreatedByPost");
  749 + paramsPost.set("mtas.collection.0.post", "[1,3,4]");
  750 + SolrRequest<?> requestPost = new QueryRequest(paramsPost, METHOD.POST);
  751 + try {
  752 + server.request(requestPost, "collection1");
  753 + } catch (SolrServerException e) {
  754 + throw new IOException(e);
  755 + }
  756 + // import
  757 + ModifiableSolrParams paramsImport = new ModifiableSolrParams();
  758 + paramsImport.set("q", "*:*");
  759 + paramsImport.set("mtas", "true");
  760 + paramsImport.set("mtas.collection", "true");
  761 + paramsImport.set("mtas.collection.0.key", "setCreatedByImport");
  762 + paramsImport.set("mtas.collection.0.action", "post");
  763 + paramsImport.set("mtas.collection.0.id", "setCreatedByImport");
  764 + paramsImport.set("mtas.collection.0.post", "[1,3,4]");
  765 + SolrRequest<?> requestImport = new QueryRequest(paramsImport, METHOD.POST);
  766 + try {
  767 + server.request(requestImport, "collection1");
  768 + } catch (SolrServerException e) {
  769 + throw new IOException(e);
  770 + }
  771 + // query post
  772 + ModifiableSolrParams paramsSelect1 = new ModifiableSolrParams();
  773 + paramsSelect1.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  774 + + "\" collection=\"setCreatedByPost\"}");
  775 + paramsSelect1.set("rows", "0");
  776 + SolrRequest<?> request1 = new QueryRequest(paramsSelect1, METHOD.POST);
  777 + NamedList<Object> response1;
  778 + try {
  779 + response1 = server.request(request1, "collection1");
  780 + } catch (SolrServerException e) {
  781 + throw new IOException(e);
  782 + }
  783 + long n1 = MtasSolrBase.getNumFound(response1);
  784 + assertTrue ("no matching documents for posted set: " + n1, n1>0);
  785 + // query import
  786 + ModifiableSolrParams paramsSelect2 = new ModifiableSolrParams();
  787 + paramsSelect2.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  788 + + "\" collection=\"setCreatedByImport\"}");
  789 + paramsSelect2.set("rows", "0");
  790 + SolrRequest<?> request2 = new QueryRequest(paramsSelect2, METHOD.POST);
  791 + NamedList<Object> response2;
  792 + try {
  793 + response2 = server.request(request2, "collection1");
  794 + } catch (SolrServerException e) {
  795 + throw new IOException(e);
  796 + }
  797 + long n2 = MtasSolrBase.getNumFound(response2);
  798 + assertTrue ("no matching documents for imported set: " + n2, n2>0);
  799 + //compare
  800 + assertTrue("posted set and imported set give different results : "+n1+" and "+n2, n1==n2);
  801 + }
  802 +
  803 + /**
458 804 * Mtas solr schema pre analyzed parser and field.
459 805 *
460 806 * @throws IOException Signals that an I/O exception has occurred.
... ... @@ -468,17 +814,17 @@ public class MtasSolrTestSearchConsistency {
468 814 params.set("mtas", "true");
469 815 params.set("mtas.stats", "true");
470 816 params.set("mtas.stats.spans", "true");
471   - params.set("mtas.stats.spans.0.field", "mtas");
  817 + params.set("mtas.stats.spans.0.field", MtasSolrBase.FIELD_MTAS);
472 818 params.set("mtas.stats.spans.0.key", "statsKey");
473 819 params.set("mtas.stats.spans.0.query.0.type", "cql");
474 820 params.set("mtas.stats.spans.0.query.0.value", "[]");
475 821 params.set("mtas.stats.spans.0.type", "n,sum,sumsq");
476 822 params.set("mtas.stats.positions", "true");
477   - params.set("mtas.stats.positions.0.field", "mtas");
  823 + params.set("mtas.stats.positions.0.field", MtasSolrBase.FIELD_MTAS);
478 824 params.set("mtas.stats.positions.0.key", "statsKey");
479 825 params.set("mtas.stats.positions.0.type", "n,sum,sumsq");
480 826 params.set("mtas.stats.tokens", "true");
481   - params.set("mtas.stats.tokens.0.field", "mtas");
  827 + params.set("mtas.stats.tokens.0.field", MtasSolrBase.FIELD_MTAS);
482 828 params.set("mtas.stats.tokens.0.key", "statsKey");
483 829 params.set("mtas.stats.tokens.0.type", "n,sum,sumsq");
484 830 params.set("rows", "0");
... ... @@ -493,9 +839,10 @@ public class MtasSolrTestSearchConsistency {
493 839 params.remove("mtas.stats.spans.0.field");
494 840 params.remove("mtas.stats.positions.0.field");
495 841 params.remove("mtas.stats.tokens.0.field");
496   - params.set("mtas.stats.spans.0.field", "mtasAdvanced");
497   - params.set("mtas.stats.positions.0.field", "mtasAdvanced");
498   - params.set("mtas.stats.tokens.0.field", "mtasAdvanced");
  842 + params.set("mtas.stats.spans.0.field", MtasSolrBase.FIELD_MTAS_ADVANCED);
  843 + params.set("mtas.stats.positions.0.field",
  844 + MtasSolrBase.FIELD_MTAS_ADVANCED);
  845 + params.set("mtas.stats.tokens.0.field", MtasSolrBase.FIELD_MTAS_ADVANCED);
499 846 try {
500 847 response2 = server.request(request, "collection1");
501 848 } catch (SolrServerException e) {
... ... @@ -523,8 +870,10 @@ public class MtasSolrTestSearchConsistency {
523 870 */
524 871 private static void createTermvectorAssertions(NamedList<Object> response1,
525 872 NamedList<Object> response2, String key, String[] names) {
526   - List<NamedList> list1 = MtasSolrBase.getFromMtasTermvector(response1, key);
527   - List<NamedList> list2 = MtasSolrBase.getFromMtasTermvector(response2, key);
  873 + List<NamedList<Object>> list1 = MtasSolrBase
  874 + .getFromMtasTermvector(response1, key);
  875 + List<NamedList<Object>> list2 = MtasSolrBase
  876 + .getFromMtasTermvector(response2, key);
528 877 assertFalse("list should be defined", list1 == null || list2 == null);
529 878 if (list1 != null && list2 != null) {
530 879 assertFalse("first list should not be longer",
... ...
... ... @@ -144,7 +144,8 @@
144 144 <artifactId>maven-site-plugin</artifactId>
145 145 <version>3.6</version>
146 146 <configuration>
147   - <outputDirectory>${project.basedir}/gh-pages/</outputDirectory>
  147 + <siteDirectory>${project.basedir}/src/site/</siteDirectory>
  148 + <outputDirectory>${project.basedir}/gh-pages/</outputDirectory>
148 149 </configuration>
149 150 </plugin>
150 151 <plugin>
... ... @@ -306,7 +307,7 @@
306 307 <!-- <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>versions-maven-plugin</artifactId>
307 308 <version>2.2</version> <reportSets> <reportSet> <reports> <report>dependency-updates-report</report>
308 309 <report>plugin-updates-report</report> <report>property-updates-report</report>
309   - </reports> </reportSet> </reportSets> </plugin> -->
  310 + </reports> </reportSet> </reportSets> </plugin> -->
310 311 </plugins>
311 312 </reporting>
312 313 <repositories>
... ...
src/mtas/analysis/parser/MtasXMLParser.java
... ... @@ -126,7 +126,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
126 126 super(config);
127 127 try {
128 128 initParser();
129   - // System.out.print(printConfig());
  129 + //System.out.print(printConfig());
130 130 } catch (MtasConfigException e) {
131 131 log.error(e);
132 132 }
... ...
src/mtas/analysis/util/MtasCharFilterFactory.java
... ... @@ -66,8 +66,10 @@ public class MtasCharFilterFactory extends CharFilterFactory
66 66 /**
67 67 * Instantiates a new mtas char filter factory.
68 68 *
69   - * @param args the args
70   - * @throws IOException Signals that an I/O exception has occurred.
  69 + * @param args
  70 + * the args
  71 + * @throws IOException
  72 + * Signals that an I/O exception has occurred.
71 73 */
72 74 public MtasCharFilterFactory(Map<String, String> args) throws IOException {
73 75 this(args, null);
... ... @@ -76,12 +78,15 @@ public class MtasCharFilterFactory extends CharFilterFactory
76 78 /**
77 79 * Instantiates a new mtas char filter factory.
78 80 *
79   - * @param args the args
80   - * @param resourceLoader the resource loader
81   - * @throws IOException Signals that an I/O exception has occurred.
  81 + * @param args
  82 + * the args
  83 + * @param resourceLoader
  84 + * the resource loader
  85 + * @throws IOException
  86 + * Signals that an I/O exception has occurred.
82 87 */
83 88 public MtasCharFilterFactory(Map<String, String> args,
84   - SolrResourceLoader resourceLoader) throws IOException {
  89 + ResourceLoader resourceLoader) throws IOException {
85 90 super(args);
86 91 typeArgument = get(args, ARGUMENT_TYPE);
87 92 prefixArgument = get(args, ARGUMENT_PREFIX);
... ... @@ -110,8 +115,10 @@ public class MtasCharFilterFactory extends CharFilterFactory
110 115 /**
111 116 * Inits the.
112 117 *
113   - * @param resourceLoader the resource loader
114   - * @throws IOException Signals that an I/O exception has occurred.
  118 + * @param resourceLoader
  119 + * the resource loader
  120 + * @throws IOException
  121 + * Signals that an I/O exception has occurred.
115 122 */
116 123 private void init(ResourceLoader resourceLoader) throws IOException {
117 124 if (config == null && configs == null) {
... ... @@ -164,10 +171,13 @@ public class MtasCharFilterFactory extends CharFilterFactory
164 171 /**
165 172 * Creates the.
166 173 *
167   - * @param input the input
168   - * @param configuration the configuration
  174 + * @param input
  175 + * the input
  176 + * @param configuration
  177 + * the configuration
169 178 * @return the reader
170   - * @throws IOException Signals that an I/O exception has occurred.
  179 + * @throws IOException
  180 + * Signals that an I/O exception has occurred.
171 181 */
172 182 public Reader create(Reader input, String configuration) throws IOException {
173 183 if (configs != null && configs.size() > 0) {
... ... @@ -210,10 +220,13 @@ public class MtasCharFilterFactory extends CharFilterFactory
210 220 /**
211 221 * Creates the.
212 222 *
213   - * @param input the input
214   - * @param config the config
  223 + * @param input
  224 + * the input
  225 + * @param config
  226 + * the config
215 227 * @return the reader
216   - * @throws IOException Signals that an I/O exception has occurred.
  228 + * @throws IOException
  229 + * Signals that an I/O exception has occurred.
217 230 */
218 231 public Reader create(Reader input, MtasConfiguration config)
219 232 throws IOException {
... ...
src/mtas/analysis/util/MtasTokenizerFactory.java
... ... @@ -9,8 +9,6 @@ import org.apache.lucene.analysis.util.ResourceLoader;
9 9 import org.apache.lucene.analysis.util.ResourceLoaderAware;
10 10 import org.apache.lucene.analysis.util.TokenizerFactory;
11 11 import org.apache.lucene.util.AttributeFactory;
12   -import org.apache.solr.core.SolrResourceLoader;
13   -
14 12 import java.io.IOException;
15 13 import java.util.HashMap;
16 14 import java.util.Map;
... ... @@ -66,7 +64,7 @@ public class MtasTokenizerFactory extends TokenizerFactory
66 64 * @throws IOException Signals that an I/O exception has occurred.
67 65 */
68 66 public MtasTokenizerFactory(Map<String, String> args,
69   - SolrResourceLoader resourceLoader) throws IOException {
  67 + ResourceLoader resourceLoader) throws IOException {
70 68 super(args);
71 69 configFileArgument = get(args, ARGUMENT_CONFIGFILE);
72 70 configArgument = get(args, ARGUMENT_CONFIG);
... ...
src/mtas/codec/MtasFieldsProducer.java
1 1 package mtas.codec;
2 2  
  3 +import java.io.EOFException;
  4 +import java.io.FileNotFoundException;
3 5 import java.io.IOException;
  6 +import java.nio.file.NoSuchFileException;
4 7 import java.util.ArrayList;
5 8 import java.util.Collection;
6 9 import java.util.Collections;
... ... @@ -52,8 +55,8 @@ public class MtasFieldsProducer extends FieldsProducer {
52 55 public MtasFieldsProducer(SegmentReadState state, String name)
53 56 throws IOException {
54 57 String postingsFormatName = null;
55   - indexInputList = new HashMap<String, IndexInput>();
56   - indexInputOffsetList = new HashMap<String, Long>();
  58 + indexInputList = new HashMap<>();
  59 + indexInputOffsetList = new HashMap<>();
57 60 version = MtasCodecPostingsFormat.VERSION_CURRENT;
58 61  
59 62 postingsFormatName = addIndexInputToList("object", openMtasFile(state, name,
... ... @@ -109,20 +112,25 @@ public class MtasFieldsProducer extends FieldsProducer {
109 112 * @throws IOException Signals that an I/O exception has occurred.
110 113 */
111 114 private String addIndexInputToList(String name, IndexInput in,
112   - String postingsFormatName) throws IOException {
  115 + String postingsFormatName) throws IOException {
113 116 if (indexInputList.get(name) != null) {
114 117 indexInputList.get(name).close();
115 118 }
116   - String localPostingsFormatName = postingsFormatName;
117   - if (localPostingsFormatName == null) {
118   - localPostingsFormatName = in.readString();
119   - } else if (!in.readString().equals(localPostingsFormatName)) {
120   - throw new IOException("delegate codec " + name + " doesn't equal "
121   - + localPostingsFormatName);
  119 + if(in!=null) {
  120 + String localPostingsFormatName = postingsFormatName;
  121 + if (localPostingsFormatName == null) {
  122 + localPostingsFormatName = in.readString();
  123 + } else if (!in.readString().equals(localPostingsFormatName)) {
  124 + throw new IOException("delegate codec " + name + " doesn't equal "
  125 + + localPostingsFormatName);
  126 + }
  127 + indexInputList.put(name, in);
  128 + indexInputOffsetList.put(name, in.getFilePointer());
  129 + return localPostingsFormatName;
  130 + } else {
  131 + log.debug("no "+name+" registered");
  132 + return null;
122 133 }
123   - indexInputList.put(name, in);
124   - indexInputOffsetList.put(name, in.getFilePointer());
125   - return localPostingsFormatName;
126 134 }
127 135  
128 136 /*
... ... @@ -232,7 +240,13 @@ public class MtasFieldsProducer extends FieldsProducer {
232 240 String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
233 241 state.segmentSuffix, extension);
234 242 IndexInput object;
235   - object = state.directory.openInput(fileName, state.context);
  243 + try {
  244 + object = state.directory.openInput(fileName, state.context);
  245 + } catch (FileNotFoundException | NoSuchFileException e) {
  246 + log.debug(e);
  247 + //throw new NoSuchFileException(e.getMessage());
  248 + return null;
  249 + }
236 250 int minVersion = (minimum == null) ? MtasCodecPostingsFormat.VERSION_START
237 251 : minimum.intValue();
238 252 int maxVersion = (maximum == null) ? MtasCodecPostingsFormat.VERSION_CURRENT
... ... @@ -245,7 +259,12 @@ public class MtasFieldsProducer extends FieldsProducer {
245 259 log.debug(e);
246 260 throw new IndexFormatTooOldException(e.getMessage(), e.getVersion(),
247 261 e.getMinVersion(), e.getMaxVersion());
248   - }
  262 + } catch (EOFException e) {
  263 + object.close();
  264 + log.debug(e);
  265 + //throw new EOFException(e.getMessage());
  266 + return null;
  267 + }
249 268 return object;
250 269 }
251 270  
... ...
src/mtas/codec/util/CodecCollector.java
... ... @@ -27,7 +27,7 @@ import mtas.codec.util.CodecComponent.ComponentDocument;
27 27 import mtas.codec.util.CodecComponent.ComponentFacet;
28 28 import mtas.codec.util.CodecComponent.ComponentField;
29 29 import mtas.codec.util.CodecComponent.ComponentGroup;
30   -import mtas.codec.util.CodecComponent.ComponentJoin;
  30 +import mtas.codec.util.CodecComponent.ComponentCollection;
31 31 import mtas.codec.util.CodecComponent.ComponentKwic;
32 32 import mtas.codec.util.CodecComponent.ComponentList;
33 33 import mtas.codec.util.CodecComponent.ComponentPosition;
... ... @@ -203,40 +203,52 @@ public class CodecCollector {
203 203 }
204 204  
205 205 /**
206   - * Collect join.
  206 + * Collect collection.
207 207 *
208 208 * @param reader the reader
209 209 * @param docSet the doc set
210   - * @param joinInfo the join info
  210 + * @param collectionInfo the collection info
211 211 * @throws IOException Signals that an I/O exception has occurred.
212 212 */
213   - public static void collectJoin(IndexReader reader, List<Integer> docSet,
214   - ComponentJoin joinInfo) throws IOException {
215   - BytesRef term = null;
216   - PostingsEnum postingsEnum = null;
217   - Integer docId;
218   - Integer termDocId = -1;
219   - Terms terms;
220   - LeafReaderContext lrc;
221   - LeafReader r;
222   - ListIterator<LeafReaderContext> iterator = reader.leaves().listIterator();
223   - while (iterator.hasNext()) {
224   - lrc = iterator.next();
225   - r = lrc.reader();
226   - for (String field : joinInfo.fields()) {
227   - if ((terms = r.fields().terms(field)) != null) {
228   - TermsEnum termsEnum = terms.iterator();
229   - termDocId = -1;
230   - while ((term = termsEnum.next()) != null) {
231   - Iterator<Integer> docIterator = docSet.iterator();
232   - postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE);
233   - while (docIterator.hasNext()) {
234   - docId = docIterator.next() - lrc.docBase;
235   - if ((docId >= termDocId) && ((docId.equals(termDocId))
236   - || ((termDocId = postingsEnum.advance(docId))
237   - .equals(docId)))) {
238   - joinInfo.add(term.utf8ToString());
239   - break;
  213 + public static void collectCollection(IndexReader reader, List<Integer> docSet,
  214 + ComponentCollection collectionInfo) throws IOException {
  215 + if (collectionInfo.action().equals(ComponentCollection.ACTION_CHECK)) {
  216 + // can't do anything in lucene for check
  217 + } else if (collectionInfo.action()
  218 + .equals(ComponentCollection.ACTION_LIST)) {
  219 + // can't do anything in lucene for list
  220 + } else if (collectionInfo.action()
  221 + .equals(ComponentCollection.ACTION_CREATE)) {
  222 + BytesRef term = null;
  223 + PostingsEnum postingsEnum = null;
  224 + Integer docId;
  225 + Integer termDocId = -1;
  226 + Terms terms;
  227 + LeafReaderContext lrc;
  228 + LeafReader r;
  229 + ListIterator<LeafReaderContext> iterator = reader.leaves().listIterator();
  230 + while (iterator.hasNext()) {
  231 + lrc = iterator.next();
  232 + r = lrc.reader();
  233 + for (String field : collectionInfo.fields()) {
  234 + if ((terms = r.fields().terms(field)) != null) {
  235 + TermsEnum termsEnum = terms.iterator();
  236 + while ((term = termsEnum.next()) != null) {
  237 + Iterator<Integer> docIterator = docSet.iterator();
  238 + postingsEnum = termsEnum.postings(postingsEnum,
  239 + PostingsEnum.NONE);
  240 + termDocId = -1;
  241 + while (docIterator.hasNext()) {
  242 + docId = docIterator.next() - lrc.docBase;
  243 + if ((docId >= termDocId) && ((docId.equals(termDocId))
  244 + || ((termDocId = postingsEnum.advance(docId))
  245 + .equals(docId)))) {
  246 + collectionInfo.addValue(term.utf8ToString());
  247 + break;
  248 + }
  249 + if (termDocId.equals(PostingsEnum.NO_MORE_DOCS)) {
  250 + break;
  251 + }
240 252 }
241 253 }
242 254 }
... ... @@ -270,7 +282,7 @@ public class CodecCollector {
270 282 boolean needSpans = false;
271 283 boolean needPositions = false;
272 284 boolean needTokens = false;
273   -
  285 +
274 286 // results
275 287 Map<Integer, Integer> positionsData = null;
276 288 Map<Integer, Integer> tokensData = null;
... ... @@ -1517,7 +1529,6 @@ public class CodecCollector {
1517 1529 }
1518 1530 }
1519 1531 }
1520   -
1521 1532 } else {
1522 1533 int maximumNumberOfDocuments = 0;
1523 1534 int boundaryMinimumNumberOfDocuments = 1;
... ... @@ -1671,9 +1682,9 @@ public class CodecCollector {
1671 1682 m.endPosition - 1);
1672 1683 }
1673 1684 if (group.hitRight != null) {
1674   - start = Math.min(m.endPosition - group.hitRight.length + 1,
  1685 + start = Math.min(m.endPosition - group.hitRight.length,
1675 1686 m.startPosition);
1676   - end = end == null ? m.endPosition : Math.max(end, m.endPosition);
  1687 + end = end == null ? (m.endPosition - 1) : Math.max(end, (m.endPosition - 1));
1677 1688 }
1678 1689 if (group.left != null) {
1679 1690 start = start == null ? m.startPosition - group.left.length
... ... @@ -1683,8 +1694,8 @@ public class CodecCollector {
1683 1694 }
1684 1695 if (group.right != null) {
1685 1696 start = start == null ? m.endPosition : Math.min(m.endPosition, start);
1686   - end = end == null ? m.endPosition + group.right.length
1687   - : Math.max(m.endPosition + group.right.length, end);
  1697 + end = end == null ? m.endPosition + group.right.length - 1
  1698 + : Math.max(m.endPosition + group.right.length - 1, end);
1688 1699 }
1689 1700 return new IntervalTreeNodeData<>(start, end, m.startPosition,
1690 1701 m.endPosition - 1);
... ...
src/mtas/codec/util/CodecComponent.java
... ... @@ -2,8 +2,13 @@ package mtas.codec.util;
2 2  
3 3 import java.io.BufferedReader;
4 4 import java.io.IOException;
  5 +import java.io.InputStream;
  6 +import java.io.InputStreamReader;
5 7 import java.io.StringReader;
6 8 import java.io.UnsupportedEncodingException;
  9 +import java.net.HttpURLConnection;
  10 +import java.net.URL;
  11 +import java.net.URLEncoder;
7 12 import java.nio.charset.StandardCharsets;
8 13 import java.util.ArrayList;
9 14 import java.util.Arrays;
... ... @@ -31,8 +36,12 @@ import mtas.parser.function.util.MtasFunctionParserFunction;
31 36 import mtas.parser.function.util.MtasFunctionParserFunctionDefault;
32 37 import mtas.search.spans.util.MtasSpanQuery;
33 38  
  39 +import org.apache.commons.io.IOUtils;
34 40 import org.apache.commons.lang.ArrayUtils;
35 41 import org.apache.lucene.util.BytesRef;
  42 +import org.noggit.JSONParser;
  43 +import org.noggit.ObjectBuilder;
  44 +
36 45  
37 46 /**
38 47 * The Class CodecComponent.
... ... @@ -53,8 +62,8 @@ public class CodecComponent {
53 62 /** The list. */
54 63 public Map<String, ComponentField> list;
55 64  
56   - /** The join. */
57   - public ComponentJoin join;
  65 + /** The collection. */
  66 + public List<ComponentCollection> collection;
58 67  
59 68 /** The do document. */
60 69 public boolean doDocument;
... ... @@ -89,15 +98,15 @@ public class CodecComponent {
89 98 /** The do facet. */
90 99 public boolean doFacet;
91 100  
92   - /** The do join. */
93   - public boolean doJoin;
  101 + /** The do collection. */
  102 + public boolean doCollection;
94 103  
95 104 /**
96 105 * Instantiates a new component fields.
97 106 */
98 107 public ComponentFields() {
99 108 list = new HashMap<>();
100   - join = null;
  109 + collection = new ArrayList<>();
101 110 doDocument = false;
102 111 doKwic = false;
103 112 doList = false;
... ... @@ -109,7 +118,7 @@ public class CodecComponent {
109 118 doStatsTokens = false;
110 119 doPrefix = false;
111 120 doFacet = false;
112   - doJoin = false;
  121 + doCollection = false;
113 122 }
114 123 }
115 124  
... ... @@ -163,7 +172,8 @@ public class CodecComponent {
163 172 /**
164 173 * Instantiates a new component field.
165 174 *
166   - * @param uniqueKeyField the unique key field
  175 + * @param uniqueKeyField
  176 + * the unique key field
167 177 */
168 178 public ComponentField(String uniqueKeyField) {
169 179 this.uniqueKeyField = uniqueKeyField;
... ... @@ -205,7 +215,8 @@ public class CodecComponent {
205 215 /**
206 216 * Instantiates a new component prefix.
207 217 *
208   - * @param key the key
  218 + * @param key
  219 + * the key
209 220 */
210 221 public ComponentPrefix(String key) {
211 222 this.key = key;
... ... @@ -218,7 +229,8 @@ public class CodecComponent {
218 229 /**
219 230 * Adds the single position.
220 231 *
221   - * @param prefix the prefix
  232 + * @param prefix
  233 + * the prefix
222 234 */
223 235 public void addSinglePosition(String prefix) {
224 236 if (!prefix.trim().isEmpty() && !singlePositionList.contains(prefix)
... ... @@ -230,7 +242,8 @@ public class CodecComponent {
230 242 /**
231 243 * Adds the multiple position.
232 244 *
233   - * @param prefix the prefix
  245 + * @param prefix
  246 + * the prefix
234 247 */
235 248 public void addMultiplePosition(String prefix) {
236 249 if (!prefix.trim().isEmpty()) {
... ... @@ -248,7 +261,8 @@ public class CodecComponent {
248 261 /**
249 262 * Adds the set position.
250 263 *
251   - * @param prefix the prefix
  264 + * @param prefix
  265 + * the prefix
252 266 */
253 267 public void addSetPosition(String prefix) {
254 268 if (!prefix.trim().isEmpty()) {
... ... @@ -266,7 +280,8 @@ public class CodecComponent {
266 280 /**
267 281 * Adds the intersecting.
268 282 *
269   - * @param prefix the prefix
  283 + * @param prefix
  284 + * the prefix
270 285 */
271 286 public void addIntersecting(String prefix) {
272 287 if (!prefix.trim().isEmpty()) {
... ... @@ -335,19 +350,32 @@ public class CodecComponent {
335 350 /**
336 351 * Instantiates a new component document.
337 352 *
338   - * @param key the key
339   - * @param prefix the prefix
340   - * @param statsType the stats type
341   - * @param regexp the regexp
342   - * @param list the list
343   - * @param listNumber the list number
344   - * @param listRegexp the list regexp
345   - * @param listExpand the list expand
346   - * @param listExpandNumber the list expand number
347   - * @param ignoreRegexp the ignore regexp
348   - * @param ignoreList the ignore list
349   - * @param ignoreListRegexp the ignore list regexp
350   - * @throws IOException Signals that an I/O exception has occurred.
  353 + * @param key
  354 + * the key
  355 + * @param prefix
  356 + * the prefix
  357 + * @param statsType
  358 + * the stats type
  359 + * @param regexp
  360 + * the regexp
  361 + * @param list
  362 + * the list
  363 + * @param listNumber
  364 + * the list number
  365 + * @param listRegexp
  366 + * the list regexp
  367 + * @param listExpand
  368 + * the list expand
  369 + * @param listExpandNumber
  370 + * the list expand number
  371 + * @param ignoreRegexp
  372 + * the ignore regexp
  373 + * @param ignoreList
  374 + * the ignore list
  375 + * @param ignoreListRegexp
  376 + * the ignore list regexp
  377 + * @throws IOException
  378 + * Signals that an I/O exception has occurred.
351 379 */
352 380 public ComponentDocument(String key, String prefix, String statsType,
353 381 String regexp, String[] list, int listNumber, Boolean listRegexp,
... ... @@ -451,15 +479,24 @@ public class CodecComponent {
451 479 /**
452 480 * Instantiates a new component kwic.
453 481 *
454   - * @param query the query
455   - * @param key the key
456   - * @param prefixes the prefixes
457   - * @param number the number
458   - * @param start the start
459   - * @param left the left
460   - * @param right the right
461   - * @param output the output
462   - * @throws IOException Signals that an I/O exception has occurred.
  482 + * @param query
  483 + * the query
  484 + * @param key
  485 + * the key
  486 + * @param prefixes
  487 + * the prefixes
  488 + * @param number
  489 + * the number
  490 + * @param start
  491 + * the start
  492 + * @param left
  493 + * the left
  494 + * @param right
  495 + * the right
  496 + * @param output
  497 + * the output
  498 + * @throws IOException
  499 + * Signals that an I/O exception has occurred.
463 500 */
464 501 public ComponentKwic(MtasSpanQuery query, String key, String prefixes,
465 502 Integer number, int start, int left, int right, String output)
... ... @@ -585,22 +622,38 @@ public class CodecComponent {
585 622 /**
586 623 * Instantiates a new component list.
587 624 *
588   - * @param spanQuery the span query
589   - * @param field the field
590   - * @param queryValue the query value
591   - * @param queryType the query type
592   - * @param queryPrefix the query prefix
593   - * @param queryVariables the query variables
594   - * @param queryIgnore the query ignore
595   - * @param queryMaximumIgnoreLength the query maximum ignore length
596   - * @param key the key
597   - * @param prefix the prefix
598   - * @param start the start
599   - * @param number the number
600   - * @param left the left
601   - * @param right the right
602   - * @param output the output
603   - * @throws IOException Signals that an I/O exception has occurred.
  625 + * @param spanQuery
  626 + * the span query
  627 + * @param field
  628 + * the field
  629 + * @param queryValue
  630 + * the query value
  631 + * @param queryType
  632 + * the query type
  633 + * @param queryPrefix
  634 + * the query prefix
  635 + * @param queryVariables
  636 + * the query variables
  637 + * @param queryIgnore
  638 + * the query ignore
  639 + * @param queryMaximumIgnoreLength
  640 + * the query maximum ignore length
  641 + * @param key
  642 + * the key
  643 + * @param prefix
  644 + * the prefix
  645 + * @param start
  646 + * the start
  647 + * @param number
  648 + * the number
  649 + * @param left
  650 + * the left
  651 + * @param right
  652 + * the right
  653 + * @param output
  654 + * the output
  655 + * @throws IOException
  656 + * Signals that an I/O exception has occurred.
604 657 */
605 658 public ComponentList(MtasSpanQuery spanQuery, String field,
606 659 String queryValue, String queryType, String queryPrefix,
... ... @@ -715,23 +768,40 @@ public class CodecComponent {
715 768 /**
716 769 * Instantiates a new component group.
717 770 *
718   - * @param spanQuery the span query
719   - * @param key the key
720   - * @param number the number
721   - * @param groupingHitInsidePrefixes the grouping hit inside prefixes
722   - * @param groupingHitInsideLeftPosition the grouping hit inside left position
723   - * @param groupingHitInsideLeftPrefixes the grouping hit inside left prefixes
724   - * @param groupingHitInsideRightPosition the grouping hit inside right position
725   - * @param groupingHitInsideRightPrefixes the grouping hit inside right prefixes
726   - * @param groupingHitLeftPosition the grouping hit left position
727   - * @param groupingHitLeftPrefixes the grouping hit left prefixes
728   - * @param groupingHitRightPosition the grouping hit right position
729   - * @param groupingHitRightPrefixes the grouping hit right prefixes
730   - * @param groupingLeftPosition the grouping left position
731   - * @param groupingLeftPrefixes the grouping left prefixes
732   - * @param groupingRightPosition the grouping right position
733   - * @param groupingRightPrefixes the grouping right prefixes
734   - * @throws IOException Signals that an I/O exception has occurred.
  771 + * @param spanQuery
  772 + * the span query
  773 + * @param key
  774 + * the key
  775 + * @param number
  776 + * the number
  777 + * @param groupingHitInsidePrefixes
  778 + * the grouping hit inside prefixes
  779 + * @param groupingHitInsideLeftPosition
  780 + * the grouping hit inside left position
  781 + * @param groupingHitInsideLeftPrefixes
  782 + * the grouping hit inside left prefixes
  783 + * @param groupingHitInsideRightPosition
  784 + * the grouping hit inside right position
  785 + * @param groupingHitInsideRightPrefixes
  786 + * the grouping hit inside right prefixes
  787 + * @param groupingHitLeftPosition
  788 + * the grouping hit left position
  789 + * @param groupingHitLeftPrefixes
  790 + * the grouping hit left prefixes
  791 + * @param groupingHitRightPosition
  792 + * the grouping hit right position
  793 + * @param groupingHitRightPrefixes
  794 + * the grouping hit right prefixes
  795 + * @param groupingLeftPosition
  796 + * the grouping left position
  797 + * @param groupingLeftPrefixes
  798 + * the grouping left prefixes
  799 + * @param groupingRightPosition
  800 + * the grouping right position
  801 + * @param groupingRightPrefixes
  802 + * the grouping right prefixes
  803 + * @throws IOException
  804 + * Signals that an I/O exception has occurred.
735 805 */
736 806 public ComponentGroup(MtasSpanQuery spanQuery, String key, int number,
737 807 String groupingHitInsidePrefixes,
... ... @@ -791,11 +861,15 @@ public class CodecComponent {
791 861 /**
792 862 * Creates the positioned prefixes.
793 863 *
794   - * @param prefixList the prefix list
795   - * @param position the position
796   - * @param prefixes the prefixes
  864 + * @param prefixList
  865 + * the prefix list
  866 + * @param position
  867 + * the position
  868 + * @param prefixes
  869 + * the prefixes
797 870 * @return the hash set[]
798   - * @throws IOException Signals that an I/O exception has occurred.
  871 + * @throws IOException
  872 + * Signals that an I/O exception has occurred.
799 873 */
800 874 private static HashSet<String>[] createPositionedPrefixes(
801 875 HashSet<String> prefixList, String[] position, String[] prefixes)
... ... @@ -958,24 +1032,42 @@ public class CodecComponent {
958 1032 /**
959 1033 * Instantiates a new component facet.
960 1034 *
961   - * @param spanQueries the span queries
962   - * @param field the field
963   - * @param key the key
964   - * @param baseFields the base fields
965   - * @param baseFieldTypes the base field types
966   - * @param baseTypes the base types
967   - * @param baseRangeSizes the base range sizes
968   - * @param baseRangeBases the base range bases
969   - * @param baseSortTypes the base sort types
970   - * @param baseSortDirections the base sort directions
971   - * @param baseNumbers the base numbers
972   - * @param baseMinimumDoubles the base minimum doubles
973   - * @param baseMaximumDoubles the base maximum doubles
974   - * @param baseFunctionKeys the base function keys
975   - * @param baseFunctionExpressions the base function expressions
976   - * @param baseFunctionTypes the base function types
977   - * @throws IOException Signals that an I/O exception has occurred.
978   - * @throws ParseException the parse exception
  1035 + * @param spanQueries
  1036 + * the span queries
  1037 + * @param field
  1038 + * the field
  1039 + * @param key
  1040 + * the key
  1041 + * @param baseFields
  1042 + * the base fields
  1043 + * @param baseFieldTypes
  1044 + * the base field types
  1045 + * @param baseTypes
  1046 + * the base types
  1047 + * @param baseRangeSizes
  1048 + * the base range sizes
  1049 + * @param baseRangeBases
  1050 + * the base range bases
  1051 + * @param baseSortTypes
  1052 + * the base sort types
  1053 + * @param baseSortDirections
  1054 + * the base sort directions
  1055 + * @param baseNumbers
  1056 + * the base numbers
  1057 + * @param baseMinimumDoubles
  1058 + * the base minimum doubles
  1059 + * @param baseMaximumDoubles
  1060 + * the base maximum doubles
  1061 + * @param baseFunctionKeys
  1062 + * the base function keys
  1063 + * @param baseFunctionExpressions
  1064 + * the base function expressions
  1065 + * @param baseFunctionTypes
  1066 + * the base function types
  1067 + * @throws IOException
  1068 + * Signals that an I/O exception has occurred.
  1069 + * @throws ParseException
  1070 + * the parse exception
979 1071 */
980 1072 @SuppressWarnings("unchecked")
981 1073 public ComponentFacet(MtasSpanQuery[] spanQueries, String field, String key,
... ... @@ -1235,26 +1327,46 @@ public class CodecComponent {
1235 1327 /**
1236 1328 * Instantiates a new component term vector.
1237 1329 *
1238   - * @param key the key
1239   - * @param prefix the prefix
1240   - * @param regexp the regexp
1241   - * @param full the full
1242   - * @param type the type
1243   - * @param sortType the sort type
1244   - * @param sortDirection the sort direction
1245   - * @param startValue the start value
1246   - * @param number the number
1247   - * @param functionKey the function key
1248   - * @param functionExpression the function expression
1249   - * @param functionType the function type
1250   - * @param boundary the boundary
1251   - * @param list the list
1252   - * @param listRegexp the list regexp
1253   - * @param ignoreRegexp the ignore regexp
1254   - * @param ignoreList the ignore list
1255   - * @param ignoreListRegexp the ignore list regexp
1256   - * @throws IOException Signals that an I/O exception has occurred.
1257   - * @throws ParseException the parse exception
  1330 + * @param key
  1331 + * the key
  1332 + * @param prefix
  1333 + * the prefix
  1334 + * @param regexp
  1335 + * the regexp
  1336 + * @param full
  1337 + * the full
  1338 + * @param type
  1339 + * the type
  1340 + * @param sortType
  1341 + * the sort type
  1342 + * @param sortDirection
  1343 + * the sort direction
  1344 + * @param startValue
  1345 + * the start value
  1346 + * @param number
  1347 + * the number
  1348 + * @param functionKey
  1349 + * the function key
  1350 + * @param functionExpression
  1351 + * the function expression
  1352 + * @param functionType
  1353 + * the function type
  1354 + * @param boundary
  1355 + * the boundary
  1356 + * @param list
  1357 + * the list
  1358 + * @param listRegexp
  1359 + * the list regexp
  1360 + * @param ignoreRegexp
  1361 + * the ignore regexp
  1362 + * @param ignoreList
  1363 + * the ignore list
  1364 + * @param ignoreListRegexp
  1365 + * the ignore list regexp
  1366 + * @throws IOException
  1367 + * Signals that an I/O exception has occurred.
  1368 + * @throws ParseException
  1369 + * the parse exception
1258 1370 */
1259 1371 @SuppressWarnings({ "unchecked", "rawtypes" })
1260 1372 public ComponentTermVector(String key, String prefix, String regexp,
... ... @@ -1461,16 +1573,26 @@ public class CodecComponent {
1461 1573 /**
1462 1574 * Instantiates a new component span.
1463 1575 *
1464   - * @param queries the queries
1465   - * @param key the key
1466   - * @param minimumDouble the minimum double
1467   - * @param maximumDouble the maximum double
1468   - * @param type the type
1469   - * @param functionKey the function key
1470   - * @param functionExpression the function expression
1471   - * @param functionType the function type
1472   - * @throws IOException Signals that an I/O exception has occurred.
1473   - * @throws ParseException the parse exception
  1576 + * @param queries
  1577 + * the queries
  1578 + * @param key
  1579 + * the key
  1580 + * @param minimumDouble
  1581 + * the minimum double
  1582 + * @param maximumDouble
  1583 + * the maximum double
  1584 + * @param type
  1585 + * the type
  1586 + * @param functionKey
  1587 + * the function key
  1588 + * @param functionExpression
  1589 + * the function expression
  1590 + * @param functionType
  1591 + * the function type
  1592 + * @throws IOException
  1593 + * Signals that an I/O exception has occurred.
  1594 + * @throws ParseException
  1595 + * the parse exception
1474 1596 */
1475 1597 public ComponentSpan(MtasSpanQuery[] queries, String key,
1476 1598 Double minimumDouble, Double maximumDouble, String type,
... ... @@ -1603,12 +1725,18 @@ public class CodecComponent {
1603 1725 /**
1604 1726 * Instantiates a new component position.
1605 1727 *
1606   - * @param key the key
1607   - * @param minimumDouble the minimum double
1608   - * @param maximumDouble the maximum double
1609   - * @param statsType the stats type
1610   - * @throws IOException Signals that an I/O exception has occurred.
1611   - * @throws ParseException the parse exception
  1728 + * @param key
  1729 + * the key
  1730 + * @param minimumDouble
  1731 + * the minimum double
  1732 + * @param maximumDouble
  1733 + * the maximum double
  1734 + * @param statsType
  1735 + * the stats type
  1736 + * @throws IOException
  1737 + * Signals that an I/O exception has occurred.
  1738 + * @throws ParseException
  1739 + * the parse exception
1612 1740 */
1613 1741 public ComponentPosition(String key, Double minimumDouble,
1614 1742 Double maximumDouble, String statsType)
... ... @@ -1662,12 +1790,18 @@ public class CodecComponent {
1662 1790 /**
1663 1791 * Instantiates a new component token.
1664 1792 *
1665   - * @param key the key
1666   - * @param minimumDouble the minimum double
1667   - * @param maximumDouble the maximum double
1668   - * @param statsType the stats type
1669   - * @throws IOException Signals that an I/O exception has occurred.
1670   - * @throws ParseException the parse exception
  1793 + * @param key
  1794 + * the key
  1795 + * @param minimumDouble
  1796 + * the minimum double
  1797 + * @param maximumDouble
  1798 + * the maximum double
  1799 + * @param statsType
  1800 + * the stats type
  1801 + * @throws IOException
  1802 + * Signals that an I/O exception has occurred.
  1803 + * @throws ParseException
  1804 + * the parse exception
1671 1805 */
1672 1806 public ComponentToken(String key, Double minimumDouble,
1673 1807 Double maximumDouble, String statsType)
... ... @@ -1693,65 +1827,267 @@ public class CodecComponent {
1693 1827 }
1694 1828  
1695 1829 /**
1696   - * The Class ComponentJoin.
  1830 + * The Class ComponentCollection.
1697 1831 */
1698   - public static class ComponentJoin implements BasicComponent {
  1832 + public static class ComponentCollection implements BasicComponent {
  1833 +
  1834 + /** The Constant ACTION_CREATE. */
  1835 + public static final String ACTION_CREATE = "create";
  1836 +
  1837 + /** The Constant ACTION_CHECK. */
  1838 + public static final String ACTION_CHECK = "check";
  1839 +
  1840 + /** The Constant ACTION_LIST. */
  1841 + public static final String ACTION_LIST = "list";
  1842 +
  1843 + /** The Constant ACTION_POST. */
  1844 + public static final String ACTION_POST = "post";
  1845 +
  1846 + /** The Constant ACTION_IMPORT. */
  1847 + public static final String ACTION_IMPORT = "import";
  1848 +
  1849 + /** The Constant ACTION_DELETE. */
  1850 + public static final String ACTION_DELETE = "delete";
  1851 +
  1852 + /** The Constant ACTION_EMPTY. */
  1853 + public static final String ACTION_EMPTY = "empty";
  1854 +
  1855 + /** The Constant ACTION_GET. */
  1856 + public static final String ACTION_GET = "get";
  1857 +
  1858 + /** The key. */
  1859 + public String key;
  1860 +
  1861 + /** The version. */
  1862 + public String version;
  1863 +
  1864 + /** The id. */
  1865 + public String id;
  1866 +
  1867 + /** The action. */
  1868 + private String action;
1699 1869  
1700 1870 /** The fields. */
1701 1871 private Set<String> fields;
1702 1872  
1703 1873 /** The values. */
1704   - private Set<String> values;
1705   -
1706   - /** The key. */
1707   - private String key;
  1874 + private HashSet<String> values;
1708 1875  
1709 1876 /**
1710   - * Instantiates a new component join.
  1877 + * Instantiates a new component collection.
1711 1878 *
1712   - * @param fields the fields
1713   - * @param key the key
  1879 + * @param key
  1880 + * the key
  1881 + * @param action
  1882 + * the action
1714 1883 */
1715   - public ComponentJoin(Set<String> fields, String key) {
1716   - this.fields = fields;
  1884 + public ComponentCollection(String key, String action) {
1717 1885 this.key = key;
1718   - this.values = new HashSet<>();
  1886 + this.action = action;
  1887 + this.version = null;
  1888 + values = new HashSet<>();
1719 1889 }
1720 1890  
1721 1891 /**
1722   - * Adds the.
  1892 + * Sets the list variables.
1723 1893 *
1724   - * @param value the value
  1894 + * @throws IOException
  1895 + * Signals that an I/O exception has occurred.
1725 1896 */
1726   - public void add(String value) {
1727   - values.add(value);
  1897 + public void setListVariables() throws IOException {
  1898 + if (action.equals(ACTION_LIST)) {
  1899 + // do nothing
  1900 + } else {
  1901 + throw new IOException("not allowed with action " + action);
  1902 + }
1728 1903 }
1729 1904  
1730 1905 /**
1731   - * Adds the.
  1906 + * Sets the create variables.
1732 1907 *
1733   - * @param values the values
  1908 + * @param id
  1909 + * the id
  1910 + * @param fields
  1911 + * the fields
  1912 + * @throws IOException
  1913 + * Signals that an I/O exception has occurred.
1734 1914 */
1735   - public void add(Set<String> values) {
1736   - this.values.addAll(values);
  1915 + public void setCreateVariables(String id, Set<String> fields)
  1916 + throws IOException {
  1917 + if (action.equals(ACTION_CREATE)) {
  1918 + this.id = id;
  1919 + this.fields = fields;
  1920 + } else {
  1921 + throw new IOException("not allowed with action " + action);
  1922 + }
1737 1923 }
1738 1924  
1739 1925 /**
1740   - * Values.
  1926 + * Sets the check variables.
1741 1927 *
1742   - * @return the sets the
  1928 + * @param id
  1929 + * the new check variables
  1930 + * @throws IOException
  1931 + * Signals that an I/O exception has occurred.
1743 1932 */
1744   - public Set<String> values() {
1745   - return values;
  1933 + public void setCheckVariables(String id) throws IOException {
  1934 + if (action.equals(ACTION_CHECK)) {
  1935 + this.id = id;
  1936 + } else {
  1937 + throw new IOException("not allowed with action " + action);
  1938 + }
  1939 + }
  1940 +
  1941 + /**
  1942 + * Sets the gets the variables.
  1943 + *
  1944 + * @param id
  1945 + * the new gets the variables
  1946 + * @throws IOException
  1947 + * Signals that an I/O exception has occurred.
  1948 + */
  1949 + public void setGetVariables(String id) throws IOException {
  1950 + if (action.equals(ACTION_GET)) {
  1951 + this.id = id;
  1952 + } else {
  1953 + throw new IOException("not allowed with action " + action);
  1954 + }
  1955 + }
  1956 +
  1957 + /**
  1958 + * Sets the post variables.
  1959 + *
  1960 + * @param id
  1961 + * the id
  1962 + * @param values
  1963 + * the values
  1964 + * @throws IOException
  1965 + * Signals that an I/O exception has occurred.
  1966 + */
  1967 + public void setPostVariables(String id, HashSet<String> values)
  1968 + throws IOException {
  1969 + if (action.equals(ACTION_POST)) {
  1970 + this.id = id;
  1971 + this.values = values;
  1972 + } else {
  1973 + throw new IOException("not allowed with action " + action);
  1974 + }
  1975 + }
  1976 +
  1977 + public void setImportVariables(String id, String url, String collection)
  1978 + throws IOException {
  1979 + if (action.equals(ACTION_IMPORT)) {
  1980 + this.id = id;
  1981 + StringBuilder importUrlBuffer = new StringBuilder(url);
  1982 + importUrlBuffer.append("select");
  1983 + importUrlBuffer.append("?q=*:*&rows=0&wt=json");
  1984 + importUrlBuffer.append("&mtas=true&mtas.collection=true");
  1985 + importUrlBuffer.append("&mtas.collection.0.key=0");
  1986 + importUrlBuffer.append("&mtas.collection.0.action=get");
  1987 + importUrlBuffer.append(
  1988 + "&mtas.collection.0.id=" + URLEncoder.encode(collection, "UTF-8"));
  1989 + Map<String, Object> params = getImport(importUrlBuffer.toString());
  1990 + try {
  1991 + if (params.containsKey("mtas")
  1992 + && params.get("mtas") instanceof Map) {
  1993 + Map<String, Object> mtasParams = (Map<String, Object>) params
  1994 + .get("mtas");
  1995 + if (mtasParams.containsKey("collection")
  1996 + && mtasParams.get("collection") instanceof List) {
  1997 + List<Object> mtasCollectionList = (List<Object>) mtasParams
  1998 + .get("collection");
  1999 + if (mtasCollectionList.size() == 1
  2000 + && mtasCollectionList.get(0) instanceof Map) {
  2001 + Map<String, Object> collectionData = (Map<String, Object>) mtasCollectionList
  2002 + .get(0);
  2003 + if (collectionData.containsKey("values")
  2004 + && collectionData.get("values") instanceof List) {
  2005 + List<String> valuesList = (List<String>) collectionData
  2006 + .get("values");
  2007 + for (String valueItem : valuesList) {
  2008 + values.add(valueItem);
  2009 + }
  2010 + } else {
  2011 + throw new IOException("no values in response");
  2012 + }
  2013 + } else {
  2014 + throw new IOException(
  2015 + "no valid mtas collection item in response");
  2016 + }
  2017 + } else {
  2018 + throw new IOException("no valid mtas collection in response");
  2019 + }
  2020 + } else {
  2021 + throw new IOException("no valid mtas in response");
  2022 + }
  2023 + } catch (ClassCastException e) {
  2024 + throw new IOException("unexpected response", e);
  2025 + }
  2026 + } else {
  2027 + throw new IOException("not allowed with action " + action);
  2028 + }
  2029 + }
  2030 +
  2031 + private Map<String, Object> getImport(String collectionGetUrl)
  2032 + throws IOException {
  2033 + // get data
  2034 + URL url = new URL(collectionGetUrl);
  2035 + HttpURLConnection connection = (HttpURLConnection) url.openConnection();
  2036 + connection.setDoOutput(false);
  2037 + connection.setDoInput(true);
  2038 + connection.setInstanceFollowRedirects(false);
  2039 + connection.setRequestMethod("GET");
  2040 + connection.setRequestProperty("Content-Type",
  2041 + "application/json; charset=UTF-8");
  2042 + connection.setRequestProperty("charset", "utf-8");
  2043 + connection.setUseCaches(false);
  2044 + // process response
  2045 + InputStream is = null;
  2046 + try {
  2047 + is = connection.getInputStream();
  2048 + } catch (IOException ioe) {
  2049 + throw new IOException("Couldn't get data from url");
  2050 + }
  2051 + InputStreamReader in = new InputStreamReader((InputStream) is, "UTF8");
  2052 + Map<String, Object> params = new HashMap<>();
  2053 + getParamsFromJSON(params,
  2054 + IOUtils.toString(in));
  2055 + connection.disconnect();
  2056 + return params;
1746 2057 }
1747 2058  
1748 2059 /**
1749   - * Key.
  2060 + * Sets the delete variables.
  2061 + *
  2062 + * @param id
  2063 + * the new delete variables
  2064 + * @throws IOException
  2065 + * Signals that an I/O exception has occurred.
  2066 + */
  2067 + public void setDeleteVariables(String id) throws IOException {
  2068 + if (action.equals(ACTION_DELETE)) {
  2069 + this.id = id;
  2070 + } else {
  2071 + throw new IOException("not allowed with action " + action);
  2072 + }
  2073 + }
  2074 +
  2075 + /**
  2076 + * Action.
1750 2077 *
1751 2078 * @return the string
1752 2079 */
1753   - public String key() {
1754   - return key;
  2080 + public String action() {
  2081 + return action;
  2082 + }
  2083 +
  2084 + /**
  2085 + * Values.
  2086 + *
  2087 + * @return the hash set
  2088 + */
  2089 + public HashSet<String> values() {
  2090 + return values;
1755 2091 }
1756 2092  
1757 2093 /**
... ... @@ -1763,6 +2099,61 @@ public class CodecComponent {
1763 2099 return fields;
1764 2100 }
1765 2101  
  2102 + /**
  2103 + * Adds the value.
  2104 + *
  2105 + * @param value
  2106 + * the value
  2107 + * @throws IOException
  2108 + * Signals that an I/O exception has occurred.
  2109 + */
  2110 + public void addValue(String value) throws IOException {
  2111 + if (action.equals(ACTION_CREATE)) {
  2112 + if (version == null) {
  2113 + values.add(value);
  2114 + } else {
  2115 + throw new IOException("version already set");
  2116 + }
  2117 + } else {
  2118 + throw new IOException("not allowed for action '" + action + "'");
  2119 + }
  2120 + }
  2121 +
  2122 + private static void getParamsFromJSON(Map<String, Object> params,
  2123 + String json) {
  2124 + JSONParser parser = new JSONParser(json);
  2125 + try {
  2126 + Object o = ObjectBuilder.getVal(parser);
  2127 + if (!(o instanceof Map))
  2128 + return;
  2129 + Map<String, Object> map = (Map<String, Object>) o;
  2130 + // To make consistent with json.param handling, we should make query
  2131 + // params come after json params (i.e. query params should
  2132 + // appear to overwrite json params.
  2133 +
  2134 + // Solr params are based on String though, so we need to convert
  2135 + for (Map.Entry<String, Object> entry : map.entrySet()) {
  2136 + String key = entry.getKey();
  2137 + Object val = entry.getValue();
  2138 + if (params.get(key) != null) {
  2139 + continue;
  2140 + }
  2141 +
  2142 + if (val == null) {
  2143 + params.remove(key);
  2144 + } else {
  2145 + params.put(key, val);
  2146 + }
  2147 + }
  2148 +
  2149 + } catch (Exception e) {
  2150 + // ignore parse exceptions at this stage, they may be caused by incomplete
  2151 + // macro expansions
  2152 + return;
  2153 + }
  2154 +
  2155 + }
  2156 +
1766 2157 }
1767 2158  
1768 2159 /**
... ... @@ -1803,18 +2194,30 @@ public class CodecComponent {
1803 2194 /**
1804 2195 * Instantiates a new sub component function.
1805 2196 *
1806   - * @param collectorType the collector type
1807   - * @param key the key
1808   - * @param type the type
1809   - * @param parserFunction the parser function
1810   - * @param sortType the sort type
1811   - * @param sortDirection the sort direction
1812   - * @param start the start
1813   - * @param number the number
1814   - * @param segmentRegistration the segment registration
1815   - * @param boundary the boundary
1816   - * @throws ParseException the parse exception
1817   - * @throws IOException Signals that an I/O exception has occurred.
  2197 + * @param collectorType
  2198 + * the collector type
  2199 + * @param key
  2200 + * the key
  2201 + * @param type
  2202 + * the type
  2203 + * @param parserFunction
  2204 + * the parser function
  2205 + * @param sortType
  2206 + * the sort type
  2207 + * @param sortDirection
  2208 + * the sort direction
  2209 + * @param start
  2210 + * the start
  2211 + * @param number
  2212 + * the number
  2213 + * @param segmentRegistration
  2214 + * the segment registration
  2215 + * @param boundary
  2216 + * the boundary
  2217 + * @throws ParseException
  2218 + * the parse exception
  2219 + * @throws IOException
  2220 + * Signals that an I/O exception has occurred.
1818 2221 */
1819 2222 public SubComponentFunction(String collectorType, String key, String type,
1820 2223 MtasFunctionParserFunction parserFunction, String sortType,
... ... @@ -1847,12 +2250,18 @@ public class CodecComponent {
1847 2250 /**
1848 2251 * Instantiates a new sub component function.
1849 2252 *
1850   - * @param collectorType the collector type
1851   - * @param key the key
1852   - * @param expression the expression
1853   - * @param type the type
1854   - * @throws ParseException the parse exception
1855   - * @throws IOException Signals that an I/O exception has occurred.
  2253 + * @param collectorType
  2254 + * the collector type
  2255 + * @param key
  2256 + * the key
  2257 + * @param expression
  2258 + * the expression
  2259 + * @param type
  2260 + * the type
  2261 + * @throws ParseException
  2262 + * the parse exception
  2263 + * @throws IOException
  2264 + * Signals that an I/O exception has occurred.
1856 2265 */
1857 2266 public SubComponentFunction(String collectorType, String key,
1858 2267 String expression, String type) throws ParseException, IOException {
... ... @@ -1895,8 +2304,10 @@ public class CodecComponent {
1895 2304 /**
1896 2305 * Instantiates a new kwic token.
1897 2306 *
1898   - * @param match the match
1899   - * @param tokens the tokens
  2307 + * @param match
  2308 + * the match
  2309 + * @param tokens
  2310 + * the tokens
1900 2311 */
1901 2312 public KwicToken(Match match, List<MtasTokenString> tokens) {
1902 2313 startPosition = match.startPosition;
... ... @@ -1923,8 +2334,10 @@ public class CodecComponent {
1923 2334 /**
1924 2335 * Instantiates a new kwic hit.
1925 2336 *
1926   - * @param match the match
1927   - * @param hits the hits
  2337 + * @param match
  2338 + * the match
  2339 + * @param hits
  2340 + * the hits
1928 2341 */
1929 2342 public KwicHit(Match match, Map<Integer, List<String>> hits) {
1930 2343 startPosition = match.startPosition;
... ... @@ -1996,7 +2409,8 @@ public class CodecComponent {
1996 2409 /**
1997 2410 * Sort.
1998 2411 *
1999   - * @param data the data
  2412 + * @param data
  2413 + * the data
2000 2414 * @return the list
2001 2415 */
2002 2416 private List<MtasTreeHit<String>> sort(List<MtasTreeHit<String>> data) {
... ... @@ -2015,30 +2429,35 @@ public class CodecComponent {
2015 2429 /**
2016 2430 * Instantiates a new group hit.
2017 2431 *
2018   - * @param list the list
2019   - * @param start the start
2020   - * @param end the end
2021   - * @param hitStart the hit start
2022   - * @param hitEnd the hit end
2023   - * @param group the group
2024   - * @param knownPrefixes the known prefixes
2025   - * @throws UnsupportedEncodingException the unsupported encoding exception
  2432 + * @param list
  2433 + * the list
  2434 + * @param start
  2435 + * the start
  2436 + * @param end
  2437 + * the end
  2438 + * @param hitStart
  2439 + * the hit start
  2440 + * @param hitEnd
  2441 + * the hit end
  2442 + * @param group
  2443 + * the group
  2444 + * @param knownPrefixes
  2445 + * the known prefixes
  2446 + * @throws UnsupportedEncodingException
  2447 + * the unsupported encoding exception
2026 2448 */
2027 2449 @SuppressWarnings("unchecked")
2028 2450 public GroupHit(List<MtasTreeHit<String>> list, int start, int end,
2029 2451 int hitStart, int hitEnd, ComponentGroup group,
2030 2452 Set<String> knownPrefixes) throws UnsupportedEncodingException {
2031   - // System.out.println("init: "+start+"-"+end+"\t"+hitStart+"-"+hitEnd);
2032 2453 // compute dimensions
2033 2454 int leftRangeStart = start;
2034   - int leftRangeEnd = Math.min(end - 1, hitStart - 1);
  2455 + int leftRangeEnd = Math.min(end, hitStart - 1);
2035 2456 int leftRangeLength = Math.max(0, 1 + leftRangeEnd - leftRangeStart);
2036 2457 int hitLength = 1 + hitEnd - hitStart;
2037 2458 int rightRangeStart = Math.max(start, hitEnd + 1);
2038 2459 int rightRangeEnd = end;
2039 2460 int rightRangeLength = Math.max(0, 1 + rightRangeEnd - rightRangeStart);
2040   - // System.out.println(leftRangeStart+"\t"+leftRangeEnd+"\t"+leftRangeLength+"
2041   - // - "+rightRangeStart+"\t"+rightRangeEnd+"\t"+rightRangeLength);
2042 2461 // create initial arrays
2043 2462 if (leftRangeLength > 0) {
2044 2463 keyLeft = "";
... ... @@ -2112,15 +2531,8 @@ public class CodecComponent {
2112 2531 }
2113 2532 }
2114 2533 if (group.hitInsideRight != null) {
2115   - // System.out.println(missingHit.length + " items in missingHit");
2116   - // System.out.println(
2117   - // group.hitInsideRight.length + " items in group.hitInsideRight");
2118   - for (int p = 0; p < group.hitInsideRight.length; p++) {
2119   - // System.out.println(" - " + group.hitInsideRight[p]);
2120   - }
2121 2534 for (int p = Math.max(hitStart,
2122 2535 hitEnd - group.hitInsideRight.length + 1); p <= hitEnd; p++) {
2123   - // System.out.println("Test voor p is " + (p - hitStart));
2124 2536 if (group.hitInsideRight[hitEnd - p] != null) {
2125 2537 missingHit[p - hitStart].addAll(group.hitInsideRight[hitEnd - p]);
2126 2538 }
... ... @@ -2142,7 +2554,7 @@ public class CodecComponent {
2142 2554 }
2143 2555 }
2144 2556 if (group.hitRight != null) {
2145   - for (int p = 0; p <= Math.min(leftRangeLength,
  2557 + for (int p = 0; p < Math.min(leftRangeLength,
2146 2558 group.hitRight.length - dataHit.length); p++) {
2147 2559 if (group.hitRight[p + dataHit.length] != null) {
2148 2560 missingLeft[p].addAll(group.hitRight[p + dataHit.length]);
... ... @@ -2152,13 +2564,17 @@ public class CodecComponent {
2152 2564 if (group.right != null) {
2153 2565 for (int p = 0; p < Math.min(rightRangeLength,
2154 2566 group.right.length); p++) {
2155   - missingRight[p].addAll(group.right[p]);
  2567 + if (group.right[p] != null) {
  2568 + missingRight[p].addAll(group.right[p]);
  2569 + }
2156 2570 }
2157 2571 }
2158   - if (group.hitRight != null) {
2159   - for (int p = 0; p <= Math.min(rightRangeLength,
  2572 + if (group.hitLeft != null) {
  2573 + for (int p = 0; p < Math.min(rightRangeLength,
2160 2574 group.hitLeft.length - dataHit.length); p++) {
2161   - missingRight[p].addAll(group.hitLeft[p + dataHit.length]);
  2575 + if(group.hitLeft[p + dataHit.length]!=null) {
  2576 + missingRight[p].addAll(group.hitLeft[p + dataHit.length]);
  2577 + }
2162 2578 }
2163 2579 }
2164 2580  
... ... @@ -2170,12 +2586,9 @@ public class CodecComponent {
2170 2586 && group.hitInside.contains(hit.idData)) {
2171 2587 for (int p = Math.max(hitStart, hit.startPosition); p <= Math
2172 2588 .min(hitEnd, hit.endPosition); p++) {
2173   - // keyHit += hit.refData;
2174 2589 dataHit[p - hitStart].add(hit.refData);
2175 2590 missingHit[p - hitStart]
2176   - .remove(MtasToken.getPrefixFromValue(hit.refData));
2177   - // System.out.print(p + "." + hit.idData + ":" + hit.refData +
2178   - // "\t");
  2591 + .remove(MtasToken.getPrefixFromValue(hit.refData));
2179 2592 }
2180 2593 } else if ((group.hitInsideLeft != null || group.hitLeft != null
2181 2594 || group.hitInsideRight != null || group.hitRight != null)
... ... @@ -2191,9 +2604,7 @@ public class CodecComponent {
2191 2604 // keyHit += hit.refData;
2192 2605 dataHit[p - hitStart].add(hit.refData);
2193 2606 missingHit[p - hitStart]
2194   - .remove(MtasToken.getPrefixFromValue(hit.refData));
2195   - // System.out.print(p+"."+hit.idData + ":" + hit.additionalRef +
2196   - // "\t");
  2607 + .remove(MtasToken.getPrefixFromValue(hit.refData));
2197 2608 } else if (group.hitLeft != null
2198 2609 && pHitLeft <= (group.hitLeft.length - 1)
2199 2610 && group.hitLeft[pHitLeft] != null
... ... @@ -2201,29 +2612,21 @@ public class CodecComponent {
2201 2612 // keyHit += hit.refData;
2202 2613 dataHit[p - hitStart].add(hit.refData);
2203 2614 missingHit[p - hitStart]
2204   - .remove(MtasToken.getPrefixFromValue(hit.refData));
2205   - // System.out.print(p+"."+hit.idData + ":" + hit.additionalRef +
2206   - // "\t");
  2615 + .remove(MtasToken.getPrefixFromValue(hit.refData));
2207 2616 } else if (group.hitInsideRight != null
2208 2617 && pHitRight <= (group.hitInsideRight.length - 1)
2209 2618 && group.hitInsideRight[pHitRight] != null
2210 2619 && group.hitInsideRight[pHitRight].contains(hit.idData)) {
2211   - // keyHit += hit.refData;
2212 2620 dataHit[p - hitStart].add(hit.refData);
2213 2621 missingHit[p - hitStart]
2214   - .remove(MtasToken.getPrefixFromValue(hit.refData));
2215   - // System.out.print(p+"."+hit.idData + ":" + hit.additionalRef +
2216   - // "\t");
  2622 + .remove(MtasToken.getPrefixFromValue(hit.refData));
2217 2623 } else if (group.hitRight != null
2218 2624 && pHitRight <= (group.hitRight.length - 1)
2219 2625 && group.hitRight[pHitRight] != null
2220 2626 && group.hitRight[pHitRight].contains(hit.idData)) {
2221   - // keyHit += hit.refData;
2222 2627 dataHit[p - hitStart].add(hit.refData);
2223 2628 missingHit[p - hitStart]
2224   - .remove(MtasToken.getPrefixFromValue(hit.refData));
2225   - // System.out.print(p+"."+hit.idData + ":" + hit.additionalRef +
2226   - // "\t");
  2629 + .remove(MtasToken.getPrefixFromValue(hit.refData));
2227 2630 }
2228 2631 }
2229 2632 }
... ... @@ -2239,18 +2642,16 @@ public class CodecComponent {
2239 2642 if (group.left != null && pLeft <= (group.left.length - 1)
2240 2643 && group.left[pLeft] != null
2241 2644 && group.left[pLeft].contains(hit.idData)) {
2242   - dataLeft[p - leftRangeStart].add(hit.refData);
2243   - missingLeft[p - leftRangeStart]
  2645 + dataLeft[hitStart - 1 - p].add(hit.refData);
  2646 + missingLeft[hitStart - 1 - p]
2244 2647 .remove(MtasToken.getPrefixFromValue(hit.refData));
2245   - // System.out.print("L"+p+"."+prefix + ":" + value + "\t");
2246 2648 } else if (group.hitRight != null
2247 2649 && pHitRight <= (group.hitRight.length - 1)
2248 2650 && group.hitRight[pHitRight] != null
2249 2651 && group.hitRight[pHitRight].contains(hit.idData)) {
2250   - dataLeft[p - leftRangeStart].add(hit.refData);
2251   - missingLeft[p - leftRangeStart]
  2652 + dataLeft[hitStart - 1 - p].add(hit.refData);
  2653 + missingLeft[hitStart - 1 - p]
2252 2654 .remove(MtasToken.getPrefixFromValue(hit.refData));
2253   - // System.out.print("L"+p+"."+prefix + ":" + value + "\t");
2254 2655 }
2255 2656 }
2256 2657 }
... ... @@ -2270,7 +2671,6 @@ public class CodecComponent {
2270 2671 dataRight[p - rightRangeStart].add(hit.refData);
2271 2672 missingRight[p - rightRangeStart]
2272 2673 .remove(MtasToken.getPrefixFromValue(hit.refData));
2273   - // System.out.print("R"+p+"."+prefix + ":" + value + "\t");
2274 2674 } else if (group.hitLeft != null
2275 2675 && pHitLeft <= (group.hitLeft.length - 1)
2276 2676 && group.hitLeft[pHitLeft] != null
... ... @@ -2278,7 +2678,6 @@ public class CodecComponent {
2278 2678 dataRight[p - rightRangeStart].add(hit.refData);
2279 2679 missingRight[p - rightRangeStart]
2280 2680 .remove(MtasToken.getPrefixFromValue(hit.refData));
2281   - // System.out.print("R"+p+"."+prefix + ":" + value + "\t");
2282 2681 }
2283 2682 }
2284 2683 }
... ... @@ -2354,8 +2753,10 @@ public class CodecComponent {
2354 2753 /**
2355 2754 * Data equals.
2356 2755 *
2357   - * @param d1 the d 1
2358   - * @param d2 the d 2
  2756 + * @param d1
  2757 + * the d 1
  2758 + * @param d2
  2759 + * the d 2
2359 2760 * @return true, if successful
2360 2761 */
2361 2762 private boolean dataEquals(List<String>[] d1, List<String>[] d2) {
... ... @@ -2415,10 +2816,13 @@ public class CodecComponent {
2415 2816 /**
2416 2817 * Data to string.
2417 2818 *
2418   - * @param data the data
2419   - * @param missing the missing
  2819 + * @param data
  2820 + * the data
  2821 + * @param missing
  2822 + * the missing
2420 2823 * @return the string
2421   - * @throws UnsupportedEncodingException the unsupported encoding exception
  2824 + * @throws UnsupportedEncodingException
  2825 + * the unsupported encoding exception
2422 2826 */
2423 2827 private String dataToString(List<String>[] data, Set<String>[] missing)
2424 2828 throws UnsupportedEncodingException {
... ... @@ -2475,8 +2879,10 @@ public class CodecComponent {
2475 2879 /**
2476 2880 * Key to sub sub object.
2477 2881 *
2478   - * @param key the key
2479   - * @param newKey the new key
  2882 + * @param key
  2883 + * the key
  2884 + * @param newKey
  2885 + * the new key
2480 2886 * @return the map[]
2481 2887 */
2482 2888 private static Map<String, String>[] keyToSubSubObject(String key,
... ... @@ -2543,8 +2949,10 @@ public class CodecComponent {
2543 2949 /**
2544 2950 * Key to sub object.
2545 2951 *
2546   - * @param key the key
2547   - * @param newKey the new key
  2952 + * @param key
  2953 + * the key
  2954 + * @param newKey
  2955 + * the new key
2548 2956 * @return the map
2549 2957 */
2550 2958 private static Map<Integer, Map<String, String>[]> keyToSubObject(
... ... @@ -2568,8 +2976,10 @@ public class CodecComponent {
2568 2976 /**
2569 2977 * Key to object.
2570 2978 *
2571   - * @param key the key
2572   - * @param newKey the new key
  2979 + * @param key
  2980 + * the key
  2981 + * @param newKey
  2982 + * the new key
2573 2983 * @return the map
2574 2984 */
2575 2985 public static Map<String, Map<Integer, Map<String, String>[]>> keyToObject(
... ... @@ -2634,10 +3044,14 @@ public class CodecComponent {
2634 3044 /**
2635 3045 * Instantiates a new list token.
2636 3046 *
2637   - * @param docId the doc id
2638   - * @param docPosition the doc position
2639   - * @param match the match
2640   - * @param tokens the tokens
  3047 + * @param docId
  3048 + * the doc id
  3049 + * @param docPosition
  3050 + * the doc position
  3051 + * @param match
  3052 + * the match
  3053 + * @param tokens
  3054 + * the tokens
2641 3055 */
2642 3056 public ListToken(Integer docId, Integer docPosition, Match match,
2643 3057 List<MtasTokenString> tokens) {
... ... @@ -2672,10 +3086,14 @@ public class CodecComponent {
2672 3086 /**
2673 3087 * Instantiates a new list hit.
2674 3088 *
2675   - * @param docId the doc id
2676   - * @param docPosition the doc position
2677   - * @param match the match
2678   - * @param hits the hits
  3089 + * @param docId
  3090 + * the doc id
  3091 + * @param docPosition
  3092 + * the doc position
  3093 + * @param match
  3094 + * the match
  3095 + * @param hits
  3096 + * the hits
2679 3097 */
2680 3098 public ListHit(Integer docId, Integer docPosition, Match match,
2681 3099 Map<Integer, List<String>> hits) {
... ... @@ -2701,8 +3119,10 @@ public class CodecComponent {
2701 3119 /**
2702 3120 * Instantiates a new match.
2703 3121 *
2704   - * @param startPosition the start position
2705   - * @param endPosition the end position
  3122 + * @param startPosition
  3123 + * the start position
  3124 + * @param endPosition
  3125 + * the end position
2706 3126 */
2707 3127 public Match(int startPosition, int endPosition) {
2708 3128 this.startPosition = startPosition;
... ...
src/mtas/codec/util/CodecUtil.java
... ... @@ -16,7 +16,7 @@ import mtas.codec.MtasCodecPostingsFormat;
16 16 import mtas.parser.function.util.MtasFunctionParserFunction;
17 17 import mtas.search.spans.util.MtasSpanQuery;
18 18 import mtas.codec.util.CodecComponent.ComponentField;
19   -import mtas.codec.util.CodecComponent.ComponentJoin;
  19 +import mtas.codec.util.CodecComponent.ComponentCollection;
20 20  
21 21 import org.apache.lucene.index.FieldInfo;
22 22 import org.apache.lucene.index.IndexReader;
... ... @@ -247,18 +247,18 @@ public class CodecUtil {
247 247 }
248 248  
249 249 /**
250   - * Collect join.
  250 + * Collect collection.
251 251 *
252 252 * @param reader the reader
253 253 * @param fullDocSet the full doc set
254   - * @param joinInfo the join info
  254 + * @param collectionInfo the collection info
255 255 * @throws IOException Signals that an I/O exception has occurred.
256 256 */
257   - public static void collectJoin(IndexReader reader,
258   - ArrayList<Integer> fullDocSet, ComponentJoin joinInfo)
  257 + public static void collectCollection(IndexReader reader,
  258 + List<Integer> fullDocSet, ComponentCollection collectionInfo)
259 259 throws IOException {
260   - if (joinInfo != null) {
261   - CodecCollector.collectJoin(reader, fullDocSet, joinInfo);
  260 + if (collectionInfo != null) {
  261 + CodecCollector.collectCollection(reader, fullDocSet, collectionInfo);
262 262 }
263 263 }
264 264  
... ...
src/mtas/codec/util/collector/MtasDataCollector.java
... ... @@ -1336,7 +1336,7 @@ public abstract class MtasDataCollector&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 e
1336 1336 text.append("\tclosed: " + closed + "\n");
1337 1337 text.append("\tkeylist: " + Arrays.asList(keyList) + "\n");
1338 1338 text.append("\tkeylist: " + Arrays.asList(keyList).contains("1") + "\n");
1339   - text.append("\tsegmentKeys: " + segmentKeys.contains("1") + "\n");
  1339 + text.append("\tsegmentKeys: " + (segmentKeys!=null?segmentKeys.contains("1"):"null") + "\n");
1340 1340 text.append("\tnewKeys: " + Arrays.asList(newKeyList).contains("1") + "\n");
1341 1341 return text.toString().trim();
1342 1342 }
... ...
src/mtas/solr/handler/component/MtasSolrSearchComponent.java
... ... @@ -11,6 +11,7 @@ import mtas.codec.util.CodecComponent.ComponentDocument;
11 11 import mtas.codec.util.CodecComponent.ComponentFacet;
12 12 import mtas.codec.util.CodecComponent.ComponentFields;
13 13 import mtas.codec.util.CodecComponent.ComponentGroup;
  14 +import mtas.codec.util.CodecComponent.ComponentCollection;
14 15 import mtas.codec.util.CodecComponent.ComponentKwic;
15 16 import mtas.codec.util.CodecComponent.ComponentList;
16 17 import mtas.codec.util.CodecComponent.ComponentPosition;
... ... @@ -19,11 +20,11 @@ import mtas.codec.util.CodecComponent.ComponentTermVector;
19 20 import mtas.codec.util.CodecComponent.ComponentToken;
20 21 import mtas.codec.util.CodecUtil;
21 22 import mtas.solr.handler.component.util.MtasSolrResultMerge;
22   -import mtas.solr.search.MtasSolrJoinCache;
  23 +import mtas.solr.search.MtasSolrCollectionCache;
23 24 import mtas.solr.handler.component.util.MtasSolrComponentDocument;
24 25 import mtas.solr.handler.component.util.MtasSolrComponentFacet;
25 26 import mtas.solr.handler.component.util.MtasSolrComponentGroup;
26   -import mtas.solr.handler.component.util.MtasSolrComponentJoin;
  27 +import mtas.solr.handler.component.util.MtasSolrComponentCollection;
27 28 import mtas.solr.handler.component.util.MtasSolrComponentKwic;
28 29 import mtas.solr.handler.component.util.MtasSolrComponentList;
29 30 import mtas.solr.handler.component.util.MtasSolrComponentPrefix;
... ... @@ -51,17 +52,17 @@ public class MtasSolrSearchComponent extends SearchComponent {
51 52 /** The search component. */
52 53 MtasSolrSearchComponent searchComponent;
53 54  
54   - /** The Constant CONFIG_JOIN_CACHE_DIRECTORY. */
55   - public static final String CONFIG_JOIN_CACHE_DIRECTORY = "joinCacheDirectory";
  55 + /** The Constant CONFIG_COLLECTION_CACHE_DIRECTORY. */
  56 + public static final String CONFIG_COLLECTION_CACHE_DIRECTORY = "collectionCacheDirectory";
56 57  
57   - /** The Constant CONFIG_JOIN_LIFETIME. */
58   - public static final String CONFIG_JOIN_LIFETIME = "joinLifetime";
  58 + /** The Constant CONFIG_COLLECTION_LIFETIME. */
  59 + public static final String CONFIG_COLLECTION_LIFETIME = "collectionLifetime";
59 60  
60   - /** The Constant CONFIG_JOIN_MAXIMUM_NUMBER. */
61   - public static final String CONFIG_JOIN_MAXIMUM_NUMBER = "joinMaximumNumber";
  61 + /** The Constant CONFIG_COLLECTION_MAXIMUM_NUMBER. */
  62 + public static final String CONFIG_COLLECTION_MAXIMUM_NUMBER = "collectionMaximumNumber";
62 63  
63   - /** The Constant CONFIG_JOIN_MAXIMUM_OVERFLOW. */
64   - public static final String CONFIG_JOIN_MAXIMUM_OVERFLOW = "joinMaximumOverflow";
  64 + /** The Constant CONFIG_COLLECTION_MAXIMUM_OVERFLOW. */
  65 + public static final String CONFIG_COLLECTION_MAXIMUM_OVERFLOW = "collectionMaximumOverflow";
65 66  
66 67 /** The Constant PARAM_MTAS. */
67 68 public static final String PARAM_MTAS = "mtas";
... ... @@ -97,8 +98,13 @@ public class MtasSolrSearchComponent extends SearchComponent {
97 98 public static final int STAGE_GROUP = ResponseBuilder.STAGE_EXECUTE_QUERY
98 99 + 60;
99 100  
100   - /** The Constant STAGE_JOIN. */
101   - public static final int STAGE_JOIN = ResponseBuilder.STAGE_EXECUTE_QUERY + 70;
  101 + /** The Constant STAGE_COLLECTION_INIT. */
  102 + public static final int STAGE_COLLECTION_INIT = ResponseBuilder.STAGE_EXECUTE_QUERY
  103 + + 70;
  104 +
  105 + /** The Constant STAGE_COLLECTION_FINISH. */
  106 + public static final int STAGE_COLLECTION_FINISH = ResponseBuilder.STAGE_EXECUTE_QUERY
  107 + + 71;
102 108  
103 109 /** The Constant STAGE_DOCUMENT. */
104 110 public static final int STAGE_DOCUMENT = ResponseBuilder.STAGE_GET_FIELDS
... ... @@ -131,11 +137,11 @@ public class MtasSolrSearchComponent extends SearchComponent {
131 137 /** The search document. */
132 138 private MtasSolrComponentDocument searchDocument;
133 139  
134   - /** The search join. */
135   - private MtasSolrComponentJoin searchJoin;
  140 + /** The search collection. */
  141 + private MtasSolrComponentCollection searchCollection;
136 142  
137   - /** The join cache. */
138   - private MtasSolrJoinCache joinCache = null;
  143 + /** The collection cache. */
  144 + private MtasSolrCollectionCache collectionCache = null;
139 145  
140 146 /*
141 147 * (non-Javadoc)
... ... @@ -148,7 +154,7 @@ public class MtasSolrSearchComponent extends SearchComponent {
148 154 public void init(NamedList args) {
149 155 super.init(args);
150 156 // init components
151   - searchDocument = new MtasSolrComponentDocument();
  157 + searchDocument = new MtasSolrComponentDocument(this);
152 158 searchKwic = new MtasSolrComponentKwic(this);
153 159 searchList = new MtasSolrComponentList(this);
154 160 searchGroup = new MtasSolrComponentGroup(this);
... ... @@ -156,30 +162,33 @@ public class MtasSolrSearchComponent extends SearchComponent {
156 162 searchPrefix = new MtasSolrComponentPrefix(this);
157 163 searchStats = new MtasSolrComponentStats(this);
158 164 searchFacet = new MtasSolrComponentFacet(this);
159   - searchJoin = new MtasSolrComponentJoin(this);
160   - // init join
161   - String joinCacheDirectory = null;
162   - Long joinLifetime = null;
163   - Integer joinMaximumNumber = null;
164   - Integer joinMaximumOverflow = null;
165   - if (args.get(CONFIG_JOIN_CACHE_DIRECTORY) != null
166   - && args.get(CONFIG_JOIN_CACHE_DIRECTORY) instanceof String) {
167   - joinCacheDirectory = (String) args.get(CONFIG_JOIN_CACHE_DIRECTORY);
  165 + searchCollection = new MtasSolrComponentCollection(this);
  166 + // init collection
  167 + String collectionCacheDirectory = null;
  168 + Long collectionLifetime = null;
  169 + Integer collectionMaximumNumber = null;
  170 + Integer collectionMaximumOverflow = null;
  171 + if (args.get(CONFIG_COLLECTION_CACHE_DIRECTORY) != null
  172 + && args.get(CONFIG_COLLECTION_CACHE_DIRECTORY) instanceof String) {
  173 + collectionCacheDirectory = (String) args
  174 + .get(CONFIG_COLLECTION_CACHE_DIRECTORY);
168 175 }
169   - if (args.get(CONFIG_JOIN_LIFETIME) != null
170   - && args.get(CONFIG_JOIN_LIFETIME) instanceof Long) {
171   - joinLifetime = (Long) args.get(CONFIG_JOIN_LIFETIME);
  176 + if (args.get(CONFIG_COLLECTION_LIFETIME) != null
  177 + && args.get(CONFIG_COLLECTION_LIFETIME) instanceof Long) {
  178 + collectionLifetime = (Long) args.get(CONFIG_COLLECTION_LIFETIME);
172 179 }
173   - if (args.get(CONFIG_JOIN_MAXIMUM_NUMBER) != null
174   - && args.get(CONFIG_JOIN_MAXIMUM_NUMBER) instanceof Integer) {
175   - joinMaximumNumber = (Integer) args.get(CONFIG_JOIN_MAXIMUM_NUMBER);
  180 + if (args.get(CONFIG_COLLECTION_MAXIMUM_NUMBER) != null
  181 + && args.get(CONFIG_COLLECTION_MAXIMUM_NUMBER) instanceof Integer) {
  182 + collectionMaximumNumber = (Integer) args
  183 + .get(CONFIG_COLLECTION_MAXIMUM_NUMBER);
176 184 }
177   - if (args.get(CONFIG_JOIN_MAXIMUM_OVERFLOW) != null
178   - && args.get(CONFIG_JOIN_MAXIMUM_OVERFLOW) instanceof Integer) {
179   - joinMaximumNumber = (Integer) args.get(CONFIG_JOIN_MAXIMUM_OVERFLOW);
  185 + if (args.get(CONFIG_COLLECTION_MAXIMUM_OVERFLOW) != null
  186 + && args.get(CONFIG_COLLECTION_MAXIMUM_OVERFLOW) instanceof Integer) {
  187 + collectionMaximumNumber = (Integer) args
  188 + .get(CONFIG_COLLECTION_MAXIMUM_OVERFLOW);
180 189 }
181   - joinCache = new MtasSolrJoinCache(joinCacheDirectory, joinLifetime,
182   - joinMaximumNumber, joinMaximumOverflow);
  190 + collectionCache = new MtasSolrCollectionCache(collectionCacheDirectory,
  191 + collectionLifetime, collectionMaximumNumber, collectionMaximumOverflow);
183 192 }
184 193  
185 194 /*
... ... @@ -202,6 +211,15 @@ public class MtasSolrSearchComponent extends SearchComponent {
202 211 return "Mtas";
203 212 }
204 213  
  214 + /**
  215 + * Gets the collection cache.
  216 + *
  217 + * @return the collection cache
  218 + */
  219 + public MtasSolrCollectionCache getCollectionCache() {
  220 + return collectionCache;
  221 + }
  222 +
205 223 /*
206 224 * (non-Javadoc)
207 225 *
... ... @@ -258,10 +276,10 @@ public class MtasSolrSearchComponent extends SearchComponent {
258 276 false)) {
259 277 searchFacet.prepare(rb, mtasFields);
260 278 }
261   - // get settings join
262   - if (rb.req.getParams().getBool(MtasSolrComponentJoin.PARAM_MTAS_JOIN,
263   - false)) {
264   - searchJoin.prepare(rb, mtasFields);
  279 + // get settings collection
  280 + if (rb.req.getParams()
  281 + .getBool(MtasSolrComponentCollection.PARAM_MTAS_COLLECTION, false)) {
  282 + searchCollection.prepare(rb, mtasFields);
265 283 }
266 284 rb.req.getContext().put(ComponentFields.class, mtasFields);
267 285 }
... ... @@ -286,7 +304,7 @@ public class MtasSolrSearchComponent extends SearchComponent {
286 304 DocList docList = rb.getResults().docList;
287 305 if (mtasFields.doStats || mtasFields.doDocument || mtasFields.doKwic
288 306 || mtasFields.doList || mtasFields.doGroup || mtasFields.doFacet
289   - || mtasFields.doJoin || mtasFields.doTermVector
  307 + || mtasFields.doCollection || mtasFields.doTermVector
290 308 || mtasFields.doPrefix) {
291 309 SolrIndexSearcher searcher = rb.req.getSearcher();
292 310 ArrayList<Integer> docSetList = null;
... ... @@ -317,8 +335,10 @@ public class MtasSolrSearchComponent extends SearchComponent {
317 335 throw new IOException(e);
318 336 }
319 337 }
320   - CodecUtil.collectJoin(searcher.getRawReader(), docSetList,
321   - mtasFields.join);
  338 + for (ComponentCollection collection : mtasFields.collection) {
  339 + CodecUtil.collectCollection(searcher.getRawReader(), docSetList,
  340 + collection);
  341 + }
322 342 NamedList<Object> mtasResponse = new SimpleOrderedMap<>();
323 343 if (mtasFields.doDocument) {
324 344 ArrayList<NamedList<?>> mtasDocumentResponses = new ArrayList<>();
... ... @@ -355,13 +375,19 @@ public class MtasSolrSearchComponent extends SearchComponent {
355 375 // add to response
356 376 mtasResponse.add("facet", mtasFacetResponses);
357 377 }
358   - if (mtasFields.doJoin) {
359   - // add to response
360   - if (rb.req.getParams().getBool("isShard", false)) {
361   - mtasResponse.add("join", searchJoin.create(mtasFields.join, true));
362   - } else {
363   - mtasResponse.add("join", searchJoin.create(mtasFields.join, false));
  378 + if (mtasFields.doCollection) {
  379 + ArrayList<NamedList<?>> mtasCollectionResponses = new ArrayList<>();
  380 + for (ComponentCollection collection : mtasFields.collection) {
  381 + if (rb.req.getParams().getBool("isShard", false)) {
  382 + mtasCollectionResponses
  383 + .add(searchCollection.create(collection, true));
  384 + } else {
  385 + mtasCollectionResponses
  386 + .add(searchCollection.create(collection, false));
  387 + }
364 388 }
  389 + // add to response
  390 + mtasResponse.add("collection", mtasCollectionResponses);
365 391 }
366 392 if (mtasFields.doList) {
367 393 ArrayList<NamedList<?>> mtasListResponses = new ArrayList<>();
... ... @@ -492,7 +518,8 @@ public class MtasSolrSearchComponent extends SearchComponent {
492 518 @Override
493 519 public void modifyRequest(ResponseBuilder rb, SearchComponent who,
494 520 ShardRequest sreq) {
495   - // System.out.println(Thread.currentThread().getId() + " - "
  521 + // System.out.println(System.nanoTime() + " - " +
  522 + // Thread.currentThread().getId() + " - "
496 523 // + rb.req.getParams().getBool("isShard", false) + " MODIFY REQUEST "
497 524 // + rb.stage + " " + rb.req.getParamString());
498 525 if (sreq.params.getBool(PARAM_MTAS, false)) {
... ... @@ -510,8 +537,9 @@ public class MtasSolrSearchComponent extends SearchComponent {
510 537 if (sreq.params.getBool(MtasSolrComponentFacet.PARAM_MTAS_FACET, false)) {
511 538 searchFacet.modifyRequest(rb, who, sreq);
512 539 }
513   - if (sreq.params.getBool(MtasSolrComponentJoin.PARAM_MTAS_JOIN, false)) {
514   - searchJoin.modifyRequest(rb, who, sreq);
  540 + if (sreq.params.getBool(MtasSolrComponentCollection.PARAM_MTAS_COLLECTION,
  541 + false)) {
  542 + searchCollection.modifyRequest(rb, who, sreq);
515 543 }
516 544 if (sreq.params.getBool(MtasSolrComponentGroup.PARAM_MTAS_GROUP, false)) {
517 545 searchGroup.modifyRequest(rb, who, sreq);
... ... @@ -574,9 +602,9 @@ public class MtasSolrSearchComponent extends SearchComponent {
574 602 false)) {
575 603 searchFacet.finishStage(rb);
576 604 }
577   - if (rb.req.getParams().getBool(MtasSolrComponentJoin.PARAM_MTAS_JOIN,
578   - false)) {
579   - searchJoin.finishStage(rb);
  605 + if (rb.req.getParams()
  606 + .getBool(MtasSolrComponentCollection.PARAM_MTAS_COLLECTION, false)) {
  607 + searchCollection.finishStage(rb);
580 608 }
581 609 if (rb.req.getParams().getBool(MtasSolrComponentGroup.PARAM_MTAS_GROUP,
582 610 false)) {
... ... @@ -630,9 +658,10 @@ public class MtasSolrSearchComponent extends SearchComponent {
630 658 } else if (rb.stage == STAGE_FACET) {
631 659 ComponentFields mtasFields = getMtasFields(rb);
632 660 searchFacet.distributedProcess(rb, mtasFields);
633   - } else if (rb.stage == STAGE_JOIN) {
  661 + } else if (rb.stage == STAGE_COLLECTION_INIT
  662 + || rb.stage == STAGE_COLLECTION_FINISH) {
634 663 ComponentFields mtasFields = getMtasFields(rb);
635   - searchJoin.distributedProcess(rb, mtasFields);
  664 + searchCollection.distributedProcess(rb, mtasFields);
636 665 } else if (rb.stage == STAGE_GROUP) {
637 666 ComponentFields mtasFields = getMtasFields(rb);
638 667 searchGroup.distributedProcess(rb, mtasFields);
... ... @@ -670,9 +699,14 @@ public class MtasSolrSearchComponent extends SearchComponent {
670 699 } else if (rb.stage < STAGE_GROUP && rb.req.getParams()
671 700 .getBool(MtasSolrComponentGroup.PARAM_MTAS_GROUP, false)) {
672 701 return STAGE_GROUP;
673   - } else if (rb.stage < STAGE_JOIN && rb.req.getParams()
674   - .getBool(MtasSolrComponentJoin.PARAM_MTAS_JOIN, false)) {
675   - return STAGE_JOIN;
  702 + } else if (rb.stage < STAGE_COLLECTION_INIT
  703 + && rb.req.getParams().getBool(
  704 + MtasSolrComponentCollection.PARAM_MTAS_COLLECTION, false)) {
  705 + return STAGE_COLLECTION_INIT;
  706 + } else if (rb.stage < STAGE_COLLECTION_FINISH
  707 + && rb.req.getParams().getBool(
  708 + MtasSolrComponentCollection.PARAM_MTAS_COLLECTION, false)) {
  709 + return STAGE_COLLECTION_FINISH;
676 710 }
677 711 } else if (rb.stage >= ResponseBuilder.STAGE_GET_FIELDS
678 712 && rb.stage < ResponseBuilder.STAGE_DONE) {
... ...
src/mtas/solr/handler/component/util/MtasSolrCollectionResult.java 0 → 100644
  1 +package mtas.solr.handler.component.util;
  2 +
  3 +import java.io.IOException;
  4 +import java.io.Serializable;
  5 +import java.util.ArrayList;
  6 +import java.util.HashMap;
  7 +import java.util.HashSet;
  8 +import java.util.Iterator;
  9 +import java.util.List;
  10 +import java.util.Map.Entry;
  11 +import org.apache.solr.common.util.SimpleOrderedMap;
  12 +
  13 +import mtas.codec.util.CodecComponent.ComponentCollection;
  14 +import mtas.solr.handler.component.MtasSolrSearchComponent;
  15 +
  16 +/**
  17 + * The Class MtasSolrCollectionResult.
  18 + */
  19 +public class MtasSolrCollectionResult implements Serializable {
  20 +
  21 + /** The Constant serialVersionUID. */
  22 + private static final long serialVersionUID = 1L;
  23 +
  24 + /** The values. */
  25 + private HashSet<String> values;
  26 +
  27 + /** The id. */
  28 + private String id;
  29 +
  30 + /** The action. */
  31 + private String action;
  32 +
  33 + /** The now. */
  34 + private Long now;
  35 +
  36 + /** The list. */
  37 + private List<SimpleOrderedMap<Object>> list;
  38 +
  39 + /** The status. */
  40 + public SimpleOrderedMap<Object> status;
  41 +
  42 + /** The component collection. */
  43 + private transient ComponentCollection componentCollection = null;
  44 +
  45 + /**
  46 + * Instantiates a new mtas solr collection result.
  47 + *
  48 + * @param componentCollection the component collection
  49 + * @throws IOException Signals that an I/O exception has occurred.
  50 + */
  51 + public MtasSolrCollectionResult(ComponentCollection componentCollection)
  52 + throws IOException {
  53 + this.componentCollection = componentCollection;
  54 + if (componentCollection != null) {
  55 + action = componentCollection.action();
  56 + id = null;
  57 + values = null;
  58 + now = null;
  59 + list = null;
  60 + switch (action) {
  61 + case ComponentCollection.ACTION_CREATE:
  62 + values = componentCollection.values();
  63 + id = componentCollection.id;
  64 + break;
  65 + case ComponentCollection.ACTION_CHECK:
  66 + case ComponentCollection.ACTION_GET:
  67 + case ComponentCollection.ACTION_DELETE:
  68 + id = componentCollection.id;
  69 + break;
  70 + case ComponentCollection.ACTION_POST:
  71 + case ComponentCollection.ACTION_IMPORT:
  72 + id = componentCollection.id;
  73 + values = componentCollection.values();
  74 + break;
  75 + case ComponentCollection.ACTION_LIST:
  76 + case ComponentCollection.ACTION_EMPTY:
  77 + // do nothing
  78 + break;
  79 + default:
  80 + throw new IOException("action " + action + " not allowed");
  81 + }
  82 + } else {
  83 + throw new IOException("no componentCollection available");
  84 + }
  85 + }
  86 +
  87 + /**
  88 + * Sets the list.
  89 + *
  90 + * @param now the now
  91 + * @param list the list
  92 + * @throws IOException Signals that an I/O exception has occurred.
  93 + */
  94 + public void setList(long now, List<SimpleOrderedMap<Object>> list)
  95 + throws IOException {
  96 + if (action.equals(ComponentCollection.ACTION_LIST)) {
  97 + this.now = now;
  98 + this.list = list;
  99 + } else {
  100 + throw new IOException("not allowed with action '" + action + "'");
  101 + }
  102 + }
  103 +
  104 + /**
  105 + * Sets the check.
  106 + *
  107 + * @param now the now
  108 + * @param status the status
  109 + * @throws IOException Signals that an I/O exception has occurred.
  110 + */
  111 + public void setCheck(long now, SimpleOrderedMap<Object> status)
  112 + throws IOException {
  113 + if (action.equals(ComponentCollection.ACTION_CHECK)) {
  114 + this.now = now;
  115 + this.status = status;
  116 + } else {
  117 + throw new IOException("not allowed with action '" + action + "'");
  118 + }
  119 + }
  120 +
  121 + /**
  122 + * Sets the get.
  123 + *
  124 + * @param now the now
  125 + * @param status the status
  126 + * @param stringValues the string values
  127 + * @throws IOException Signals that an I/O exception has occurred.
  128 + */
  129 + public void setGet(long now, SimpleOrderedMap<Object> status,
  130 + HashSet<String> stringValues) throws IOException {
  131 + if (action.equals(ComponentCollection.ACTION_GET)) {
  132 + this.now = now;
  133 + this.status = status;
  134 + this.values = stringValues;
  135 + } else {
  136 + throw new IOException("not allowed with action '" + action + "'");
  137 + }
  138 + }
  139 +
  140 + /**
  141 + * Sets the post.
  142 + *
  143 + * @param now the now
  144 + * @param status the status
  145 + * @throws IOException Signals that an I/O exception has occurred.
  146 + */
  147 + public void setPost(long now, SimpleOrderedMap<Object> status)
  148 + throws IOException {
  149 + if (action.equals(ComponentCollection.ACTION_POST)) {
  150 + this.now = now;
  151 + this.status = status;
  152 + } else {
  153 + throw new IOException("not allowed with action '" + action + "'");
  154 + }
  155 + }
  156 +
  157 + public void setImport(long now, SimpleOrderedMap<Object> status)
  158 + throws IOException {
  159 + if (action.equals(ComponentCollection.ACTION_IMPORT)) {
  160 + this.now = now;
  161 + this.status = status;
  162 + } else {
  163 + throw new IOException("not allowed with action '" + action + "'");
  164 + }
  165 + }
  166 +
  167 + /**
  168 + * Sets the create.
  169 + *
  170 + * @param now the now
  171 + * @param status the status
  172 + * @throws IOException Signals that an I/O exception has occurred.
  173 + */
  174 + public void setCreate(long now, SimpleOrderedMap<Object> status)
  175 + throws IOException {
  176 + if (action.equals(ComponentCollection.ACTION_CREATE)) {
  177 + this.now = now;
  178 + this.status = status;
  179 + } else {
  180 + throw new IOException("not allowed with action '" + action + "'");
  181 + }
  182 + }
  183 +
  184 + /**
  185 + * Id.
  186 + *
  187 + * @return the string
  188 + */
  189 + public String id() {
  190 + return id;
  191 + }
  192 +
  193 + /**
  194 + * Action.
  195 + *
  196 + * @return the string
  197 + */
  198 + public String action() {
  199 + return action;
  200 + }
  201 +
  202 + /**
  203 + * Rewrite.
  204 + *
  205 + * @param searchComponent the search component
  206 + * @return the simple ordered map
  207 + * @throws IOException Signals that an I/O exception has occurred.
  208 + */
  209 + public SimpleOrderedMap<Object> rewrite(
  210 + MtasSolrSearchComponent searchComponent) throws IOException {
  211 + SimpleOrderedMap<Object> response = new SimpleOrderedMap<>();
  212 + Iterator<Entry<String, Object>> it;
  213 + switch (action) {
  214 + case ComponentCollection.ACTION_LIST:
  215 + response.add("now", now);
  216 + response.add("list", list);
  217 + break;
  218 + case ComponentCollection.ACTION_CREATE:
  219 + case ComponentCollection.ACTION_POST:
  220 + case ComponentCollection.ACTION_IMPORT:
  221 + if (componentCollection != null && status != null) {
  222 + it = status.iterator();
  223 + while (it.hasNext()) {
  224 + Entry<String, Object> entry = it.next();
  225 + response.add(entry.getKey(), entry.getValue());
  226 + }
  227 + }
  228 + break;
  229 + case ComponentCollection.ACTION_CHECK:
  230 + if (status != null) {
  231 + it = status.iterator();
  232 + while (it.hasNext()) {
  233 + Entry<String, Object> entry = it.next();
  234 + response.add(entry.getKey(), entry.getValue());
  235 + }
  236 + }
  237 + break;
  238 + case ComponentCollection.ACTION_GET:
  239 + if (status != null) {
  240 + it = status.iterator();
  241 + while (it.hasNext()) {
  242 + Entry<String, Object> entry = it.next();
  243 + response.add(entry.getKey(), entry.getValue());
  244 + }
  245 + }
  246 + if (values != null) {
  247 + response.add("values", values);
  248 + }
  249 + break;
  250 + default:
  251 + break;
  252 + }
  253 + return response;
  254 + }
  255 +
  256 + /**
  257 + * Merge.
  258 + *
  259 + * @param newItem the new item
  260 + * @throws IOException Signals that an I/O exception has occurred.
  261 + */
  262 + public void merge(MtasSolrCollectionResult newItem) throws IOException {
  263 + if (action != null && newItem.action != null) {
  264 + if (action.equals(ComponentCollection.ACTION_CREATE)
  265 + && newItem.action.equals(ComponentCollection.ACTION_CREATE)) {
  266 + values.addAll(newItem.values);
  267 + if (id != null && (newItem.id == null || !newItem.id.equals(id))) {
  268 + id = null;
  269 + }
  270 + } else if (action.equals(ComponentCollection.ACTION_LIST)) {
  271 + if (list != null) {
  272 + HashMap<String, SimpleOrderedMap<Object>> index = new HashMap<>();
  273 + for (SimpleOrderedMap<Object> item : list) {
  274 + if (item.get("id") != null && item.get("id") instanceof String) {
  275 + index.put((String) item.get("id"), item);
  276 + if (item.get("shards") == null
  277 + || !(item.get("shards") instanceof List)) {
  278 + item.add("shards", new ArrayList<>());
  279 + }
  280 + }
  281 + }
  282 + for (SimpleOrderedMap<Object> item : newItem.list) {
  283 + if (item.get("id") != null && item.get("id") instanceof String) {
  284 + String id = (String) item.get("id");
  285 + if (index.containsKey(id)) {
  286 + SimpleOrderedMap<Object> indexItem = index.get(id);
  287 + List<SimpleOrderedMap<Object>> shards;
  288 + if (indexItem.get("shards") != null
  289 + && indexItem.get("shards") instanceof List) {
  290 + shards = (List<SimpleOrderedMap<Object>>) indexItem
  291 + .get("shards");
  292 + } else {
  293 + shards = new ArrayList<>();
  294 + indexItem.add("shards", shards);
  295 + }
  296 + shards.add(item);
  297 + }
  298 + }
  299 + }
  300 + }
  301 + } else if (action.equals(ComponentCollection.ACTION_CHECK)
  302 + || action.equals(ComponentCollection.ACTION_POST)
  303 + || action.equals(ComponentCollection.ACTION_IMPORT)
  304 + || action.equals(ComponentCollection.ACTION_CREATE)
  305 + || action.equals(ComponentCollection.ACTION_GET)) {
  306 + if (status != null && status.get("id") != null
  307 + && status.get("id") instanceof String) {
  308 + String id = (String) status.get("id");
  309 + if (id.equals(newItem.id)) {
  310 + List<SimpleOrderedMap<Object>> shards;
  311 + if (status.get("shards") != null
  312 + && status.get("shards") instanceof List) {
  313 + shards = (List<SimpleOrderedMap<Object>>) status.get("shards");
  314 + } else {
  315 + shards = new ArrayList<>();
  316 + status.add("shards", shards);
  317 + }
  318 + if (newItem.status != null) {
  319 + if (action.equals(ComponentCollection.ACTION_GET)) {
  320 + newItem.status.add("values", newItem.values);
  321 + }
  322 + shards.add(newItem.status);
  323 + }
  324 + }
  325 + }
  326 + } else {
  327 + throw new IOException("not allowed for action '" + action + "'");
  328 + }
  329 + }
  330 + }
  331 +
  332 + /*
  333 + * (non-Javadoc)
  334 + *
  335 + * @see java.lang.Object#toString()
  336 + */
  337 + @Override
  338 + public String toString() {
  339 + StringBuilder text = new StringBuilder("");
  340 + text.append(MtasSolrCollectionResult.class.getSimpleName() + "[");
  341 + text.append(action + ", ");
  342 + text.append(id + ", ");
  343 + if (componentCollection != null) {
  344 + text.append(componentCollection.version + ", ");
  345 + } else if (status != null) {
  346 + text.append(status.get("version") + ", ");
  347 + } else {
  348 + text.append("null, ");
  349 + }
  350 + text.append((values != null) ? values.size() : "null");
  351 + text.append("]");
  352 + return text.toString();
  353 + }
  354 +
  355 +}
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentCollection.java 0 → 100644
  1 +package mtas.solr.handler.component.util;
  2 +
  3 +import java.io.IOException;
  4 +import java.io.Serializable;
  5 +import java.util.ArrayList;
  6 +import java.util.Arrays;
  7 +import java.util.HashMap;
  8 +import java.util.HashSet;
  9 +import java.util.Map;
  10 +import java.util.Set;
  11 +import java.util.Map.Entry;
  12 +
  13 +import org.apache.commons.logging.Log;
  14 +import org.apache.commons.logging.LogFactory;
  15 +import org.apache.lucene.util.BytesRef;
  16 +import org.apache.solr.common.params.ModifiableSolrParams;
  17 +import org.apache.solr.common.util.NamedList;
  18 +import org.apache.solr.common.util.SimpleOrderedMap;
  19 +import org.apache.solr.handler.component.ResponseBuilder;
  20 +import org.apache.solr.handler.component.SearchComponent;
  21 +import org.apache.solr.handler.component.ShardRequest;
  22 +import org.apache.solr.handler.component.ShardResponse;
  23 +import org.noggit.JSONParser;
  24 +import org.noggit.JSONUtil;
  25 +
  26 +import mtas.codec.util.CodecComponent.ComponentFields;
  27 +import mtas.codec.util.CodecComponent.ComponentCollection;
  28 +import mtas.solr.handler.component.MtasSolrSearchComponent;
  29 +
  30 +/**
  31 + * The Class MtasSolrComponentCollection.
  32 + */
  33 +public class MtasSolrComponentCollection
  34 + implements MtasSolrComponent<ComponentCollection> {
  35 +
  36 + /** The Constant log. */
  37 + private static final Log log = LogFactory
  38 + .getLog(MtasSolrComponentCollection.class);
  39 +
  40 + /** The Constant PARAM_MTAS_COLLECTION. */
  41 + public static final String PARAM_MTAS_COLLECTION = MtasSolrSearchComponent.PARAM_MTAS
  42 + + ".collection";
  43 +
  44 + /** The Constant NAME_MTAS_COLLECTION_ACTION. */
  45 + public static final String NAME_MTAS_COLLECTION_ACTION = "action";
  46 +
  47 + /** The Constant NAME_MTAS_COLLECTION_ID. */
  48 + public static final String NAME_MTAS_COLLECTION_ID = "id";
  49 +
  50 + /** The Constant NAME_MTAS_COLLECTION_FIELD. */
  51 + public static final String NAME_MTAS_COLLECTION_FIELD = "field";
  52 +
  53 + /** The Constant NAME_MTAS_COLLECTION_POST. */
  54 + public static final String NAME_MTAS_COLLECTION_POST = "post";
  55 +
  56 + /** The Constant NAME_MTAS_COLLECTION_URL. */
  57 + public static final String NAME_MTAS_COLLECTION_URL = "url";
  58 +
  59 + /** The Constant NAME_MTAS_COLLECTION_COLLECTION. */
  60 + public static final String NAME_MTAS_COLLECTION_COLLECTION = "collection";
  61 +
  62 + /** The Constant NAME_MTAS_COLLECTION_KEY. */
  63 + public static final String NAME_MTAS_COLLECTION_KEY = "key";
  64 +
  65 + /** The search component. */
  66 + private MtasSolrSearchComponent searchComponent;
  67 +
  68 + /**
  69 + * Instantiates a new mtas solr component collection.
  70 + *
  71 + * @param searchComponent
  72 + * the search component
  73 + */
  74 + public MtasSolrComponentCollection(MtasSolrSearchComponent searchComponent) {
  75 + this.searchComponent = searchComponent;
  76 + }
  77 +
  78 + /*
  79 + * (non-Javadoc)
  80 + *
  81 + * @see
  82 + * mtas.solr.handler.component.util.MtasSolrComponent#prepare(org.apache.solr.
  83 + * handler.component.ResponseBuilder,
  84 + * mtas.codec.util.CodecComponent.ComponentFields)
  85 + */
  86 + public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
  87 + throws IOException {
  88 + // System.out.println(
  89 + // "collection: " + System.nanoTime() + " - " +
  90 + // Thread.currentThread().getId()
  91 + // + " - " + rb.req.getParams().getBool("isShard", false) + " PREPARE "
  92 + // + rb.stage + " " + rb.req.getParamString());
  93 + Set<String> ids = MtasSolrResultUtil
  94 + .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_COLLECTION);
  95 + if (!ids.isEmpty()) {
  96 + int tmpCounter = 0;
  97 + String[] keys = new String[ids.size()];
  98 + String[] actions = new String[ids.size()];
  99 + String[] fields = new String[ids.size()];
  100 + String[] collectionIds = new String[ids.size()];
  101 + String[] posts = new String[ids.size()];
  102 + String[] urls = new String[ids.size()];
  103 + String[] collections = new String[ids.size()];
  104 + for (String id : ids) {
  105 + actions[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_COLLECTION + "."
  106 + + id + "." + NAME_MTAS_COLLECTION_ACTION, null);
  107 + keys[tmpCounter] = rb.req.getParams().get(
  108 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_KEY,
  109 + String.valueOf(tmpCounter)).trim();
  110 + fields[tmpCounter] = rb.req.getParams().get(
  111 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_FIELD,
  112 + null);
  113 + collectionIds[tmpCounter] = rb.req.getParams().get(
  114 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_ID,
  115 + null);
  116 + posts[tmpCounter] = rb.req.getParams().get(
  117 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_POST,
  118 + null);
  119 + urls[tmpCounter] = rb.req.getParams().get(
  120 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_URL,
  121 + null);
  122 + collections[tmpCounter] = rb.req.getParams().get(
  123 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_COLLECTION,
  124 + null);
  125 + tmpCounter++;
  126 + }
  127 + mtasFields.doCollection = true;
  128 + MtasSolrResultUtil.compareAndCheck(keys, actions,
  129 + NAME_MTAS_COLLECTION_KEY, NAME_MTAS_COLLECTION_ACTION, true);
  130 + MtasSolrResultUtil.compareAndCheck(keys, fields, NAME_MTAS_COLLECTION_KEY,
  131 + NAME_MTAS_COLLECTION_FIELD, false);
  132 + MtasSolrResultUtil.compareAndCheck(keys, collectionIds,
  133 + NAME_MTAS_COLLECTION_KEY, NAME_MTAS_COLLECTION_ID, false);
  134 + MtasSolrResultUtil.compareAndCheck(keys, posts, NAME_MTAS_COLLECTION_KEY,
  135 + NAME_MTAS_COLLECTION_POST, false);
  136 + MtasSolrResultUtil.compareAndCheck(keys, urls, NAME_MTAS_COLLECTION_KEY,
  137 + NAME_MTAS_COLLECTION_URL, false);
  138 + MtasSolrResultUtil.compareAndCheck(keys, collections, NAME_MTAS_COLLECTION_KEY,
  139 + NAME_MTAS_COLLECTION_COLLECTION, false);
  140 + for (int i = 0; i < actions.length; i++) {
  141 + if (actions[i] != null) {
  142 + ComponentCollection componentCollection;
  143 + switch (actions[i]) {
  144 + case ComponentCollection.ACTION_LIST:
  145 + componentCollection = new ComponentCollection(keys[i],
  146 + ComponentCollection.ACTION_LIST);
  147 + componentCollection.setListVariables();
  148 + mtasFields.collection.add(componentCollection);
  149 + break;
  150 + case ComponentCollection.ACTION_CHECK:
  151 + if (collectionIds[i] != null) {
  152 + componentCollection = new ComponentCollection(keys[i],
  153 + ComponentCollection.ACTION_CHECK);
  154 + componentCollection.setCheckVariables(collectionIds[i]);
  155 + mtasFields.collection.add(componentCollection);
  156 + } else {
  157 + throw new IOException(
  158 + "no id defined for collection (" + actions[i] + ")");
  159 + }
  160 + break;
  161 + case ComponentCollection.ACTION_GET:
  162 + if (collectionIds[i] != null) {
  163 + componentCollection = new ComponentCollection(keys[i],
  164 + ComponentCollection.ACTION_GET);
  165 + componentCollection.setGetVariables(collectionIds[i]);
  166 + mtasFields.collection.add(componentCollection);
  167 + } else {
  168 + throw new IOException(
  169 + "no id defined for collection (" + actions[i] + ")");
  170 + }
  171 + break;
  172 + case ComponentCollection.ACTION_CREATE:
  173 + if (fields[i] != null) {
  174 + Set<String> fieldList = new HashSet<>(
  175 + Arrays.asList(fields[i].split(",")));
  176 + componentCollection = new ComponentCollection(keys[i],
  177 + ComponentCollection.ACTION_CREATE);
  178 + componentCollection.setCreateVariables(collectionIds[i],
  179 + fieldList);
  180 + mtasFields.doCollection = true;
  181 + mtasFields.collection.add(componentCollection);
  182 + rb.setNeedDocSet(true);
  183 + } else {
  184 + throw new IOException(
  185 + "no field defined for collection (" + actions[i] + ")");
  186 + }
  187 + break;
  188 + case ComponentCollection.ACTION_POST:
  189 + if (posts[i] != null) {
  190 + componentCollection = new ComponentCollection(keys[i],
  191 + ComponentCollection.ACTION_POST);
  192 + componentCollection.setPostVariables(collectionIds[i],
  193 + stringToStringValues(posts[i]));
  194 + mtasFields.collection.add(componentCollection);
  195 + } else {
  196 + throw new IOException(
  197 + "no post defined for collection (" + actions[i] + ")");
  198 + }
  199 + break;
  200 + case ComponentCollection.ACTION_IMPORT:
  201 + if (urls[i] != null && collections[i]!=null) {
  202 + componentCollection = new ComponentCollection(keys[i],
  203 + ComponentCollection.ACTION_IMPORT);
  204 + componentCollection.setImportVariables(collectionIds[i],
  205 + urls[i], collections[i]);
  206 + mtasFields.collection.add(componentCollection);
  207 + } else {
  208 + throw new IOException(
  209 + "no post defined for collection (" + actions[i] + ")");
  210 + }
  211 + break;
  212 + case ComponentCollection.ACTION_DELETE:
  213 + if (collectionIds[i] != null) {
  214 + componentCollection = new ComponentCollection(keys[i],
  215 + ComponentCollection.ACTION_DELETE);
  216 + componentCollection.setDeleteVariables(collectionIds[i]);
  217 + mtasFields.collection.add(componentCollection);
  218 + } else {
  219 + throw new IOException(
  220 + "no id defined for collection (" + actions[i] + ")");
  221 + }
  222 + break;
  223 + case ComponentCollection.ACTION_EMPTY:
  224 + componentCollection = new ComponentCollection(keys[i],
  225 + ComponentCollection.ACTION_EMPTY);
  226 + mtasFields.collection.add(componentCollection);
  227 + break;
  228 + default:
  229 + throw new IOException(
  230 + "unrecognized action '" + actions[i] + "' for collection");
  231 + }
  232 + } else {
  233 + throw new IOException("no action defined for collection");
  234 + }
  235 + }
  236 + }
  237 + }
  238 +
  239 + /*
  240 + * (non-Javadoc)
  241 + *
  242 + * @see
  243 + * mtas.solr.handler.component.util.MtasSolrComponent#modifyRequest(org.apache
  244 + * .solr.handler.component.ResponseBuilder,
  245 + * org.apache.solr.handler.component.SearchComponent,
  246 + * org.apache.solr.handler.component.ShardRequest)
  247 + */
  248 + public void modifyRequest(ResponseBuilder rb, SearchComponent who,
  249 + ShardRequest sreq) {
  250 + // System.out.println(
  251 + // "collection: " + System.nanoTime() + " - " +
  252 + // Thread.currentThread().getId()
  253 + // + " - " + rb.req.getParams().getBool("isShard", false)
  254 + // + " MODIFYREQUEST " + rb.stage + " " + rb.req.getParamString());
  255 + if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
  256 + && sreq.params.getBool(PARAM_MTAS_COLLECTION, false)) {
  257 + if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
  258 + // do nothing
  259 + } else {
  260 + // remove for other requests
  261 + Set<String> keys = MtasSolrResultUtil
  262 + .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_COLLECTION);
  263 + sreq.params.remove(PARAM_MTAS_COLLECTION);
  264 + for (String key : keys) {
  265 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  266 + + NAME_MTAS_COLLECTION_ACTION);
  267 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  268 + + NAME_MTAS_COLLECTION_ID);
  269 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  270 + + NAME_MTAS_COLLECTION_FIELD);
  271 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  272 + + NAME_MTAS_COLLECTION_POST);
  273 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  274 + + NAME_MTAS_COLLECTION_KEY);
  275 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  276 + + NAME_MTAS_COLLECTION_URL);
  277 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  278 + + NAME_MTAS_COLLECTION_COLLECTION);
  279 + }
  280 + }
  281 + }
  282 + }
  283 +
  284 + /*
  285 + * (non-Javadoc)
  286 + *
  287 + * @see
  288 + * mtas.solr.handler.component.util.MtasSolrComponent#create(mtas.codec.util.
  289 + * CodecComponent.BasicComponent, java.lang.Boolean)
  290 + */
  291 + public SimpleOrderedMap<Object> create(
  292 + ComponentCollection componentCollection, Boolean encode)
  293 + throws IOException {
  294 + MtasSolrCollectionResult data = createMtasSolrCollectionResult(
  295 + componentCollection, encode ? false : true);
  296 + // Create response
  297 + SimpleOrderedMap<Object> mtasCollectionResponse = new SimpleOrderedMap<>();
  298 + mtasCollectionResponse.add("key", componentCollection.key);
  299 + if (encode) {
  300 + mtasCollectionResponse.add("_encoded_data",
  301 + MtasSolrResultUtil.encode(data));
  302 + } else {
  303 + mtasCollectionResponse.add("data", data);
  304 + MtasSolrResultUtil.rewrite(mtasCollectionResponse, searchComponent);
  305 + }
  306 + return mtasCollectionResponse;
  307 + }
  308 +
  309 + /**
  310 + * Creates the mtas solr collection result.
  311 + *
  312 + * @param componentCollection the component collection
  313 + * @param storeIfRelevant the store if relevant
  314 + * @return the mtas solr collection result
  315 + * @throws IOException Signals that an I/O exception has occurred.
  316 + */
  317 + /*
  318 + * (non-Javadoc)
  319 + *
  320 + * @see
  321 + * mtas.solr.handler.component.util.MtasSolrComponent#create(mtas.codec.util.
  322 + * CodecComponent.BasicComponent, java.lang.Boolean)
  323 + */
  324 + private MtasSolrCollectionResult createMtasSolrCollectionResult(
  325 + ComponentCollection componentCollection, boolean storeIfRelevant)
  326 + throws IOException {
  327 + // System.out.println("collection: " + System.nanoTime() + " - "
  328 + // + Thread.currentThread().getId() + " - " + " CREATE ");
  329 + if (componentCollection != null) {
  330 + // Create response
  331 + MtasSolrCollectionResult data = new MtasSolrCollectionResult(
  332 + componentCollection);
  333 + if (componentCollection.action()
  334 + .equals(ComponentCollection.ACTION_CREATE)) {
  335 + if (storeIfRelevant && componentCollection.version == null) {
  336 + componentCollection.version = searchComponent.getCollectionCache()
  337 + .create(componentCollection.id,
  338 + componentCollection.values().size(),
  339 + componentCollection.values());
  340 + }
  341 + data.setCreate(searchComponent.getCollectionCache().now(),
  342 + searchComponent.getCollectionCache().check(componentCollection.id));
  343 + } else if (componentCollection.action()
  344 + .equals(ComponentCollection.ACTION_LIST)) {
  345 + // retrieve and add list to result
  346 + data.setList(searchComponent.getCollectionCache().now(),
  347 + searchComponent.getCollectionCache().list());
  348 + } else if (componentCollection.action()
  349 + .equals(ComponentCollection.ACTION_CHECK)) {
  350 + // retrieve and add status to result
  351 + data.setCheck(searchComponent.getCollectionCache().now(),
  352 + searchComponent.getCollectionCache().check(componentCollection.id));
  353 + } else if (componentCollection.action()
  354 + .equals(ComponentCollection.ACTION_GET)) {
  355 + // retrieve and add status to result
  356 + HashSet<String> values = searchComponent.getCollectionCache()
  357 + .getDataById(componentCollection.id);
  358 + if (values != null) {
  359 + data.setGet(searchComponent.getCollectionCache().now(),
  360 + searchComponent.getCollectionCache()
  361 + .check(componentCollection.id),
  362 + values);
  363 + }
  364 + } else if (componentCollection.action()
  365 + .equals(ComponentCollection.ACTION_EMPTY)) {
  366 + // empty
  367 + searchComponent.getCollectionCache().empty();
  368 + } else if (componentCollection.action()
  369 + .equals(ComponentCollection.ACTION_POST)) {
  370 + // store if not already stored
  371 + if (componentCollection.version == null) {
  372 + componentCollection.version = searchComponent.getCollectionCache()
  373 + .create(componentCollection.id,
  374 + componentCollection.values().size(),
  375 + componentCollection.values());
  376 + }
  377 + // add status to result
  378 + data.setPost(searchComponent.getCollectionCache().now(),
  379 + searchComponent.getCollectionCache().check(componentCollection.id));
  380 + } else if (componentCollection.action()
  381 + .equals(ComponentCollection.ACTION_IMPORT)) {
  382 + // import if not already stored
  383 + if (componentCollection.version == null) {
  384 + componentCollection.version = searchComponent.getCollectionCache()
  385 + .create(componentCollection.id,
  386 + componentCollection.values().size(),
  387 + componentCollection.values());
  388 + }
  389 + // add status to result
  390 + data.setImport(searchComponent.getCollectionCache().now(),
  391 + searchComponent.getCollectionCache().check(componentCollection.id));
  392 + } else if (componentCollection.action()
  393 + .equals(ComponentCollection.ACTION_DELETE)) {
  394 + searchComponent.getCollectionCache().deleteById(componentCollection.id);
  395 + }
  396 + return data;
  397 + } else {
  398 + throw new IOException("no componentCollection available");
  399 + }
  400 + }
  401 +
  402 + /*
  403 + * (non-Javadoc)
  404 + *
  405 + * @see
  406 + * mtas.solr.handler.component.util.MtasSolrComponent#finishStage(org.apache.
  407 + * solr.handler.component.ResponseBuilder)
  408 + */
  409 + public void finishStage(ResponseBuilder rb) {
  410 + // System.out.println(
  411 + // "collection: " + System.nanoTime() + " - " + Thread.currentThread().getId()
  412 + // + " - " + rb.req.getParams().getBool("isShard", false)
  413 + // + " FINISHSTAGE " + rb.stage + " " + rb.req.getParamString());
  414 + if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
  415 + if (rb.stage >= ResponseBuilder.STAGE_EXECUTE_QUERY
  416 + && rb.stage < ResponseBuilder.STAGE_GET_FIELDS) {
  417 + ComponentFields mtasFields = getMtasFields(rb);
  418 + if (mtasFields.doCollection) {
  419 + if (rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY) {
  420 + // mtas response
  421 + NamedList<Object> mtasResponse = null;
  422 + try {
  423 + mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
  424 + } catch (ClassCastException e) {
  425 + log.debug(e);
  426 + mtasResponse = null;
  427 + }
  428 + if (mtasResponse == null) {
  429 + mtasResponse = new SimpleOrderedMap<>();
  430 + rb.rsp.add("mtas", mtasResponse);
  431 + }
  432 + ArrayList<Object> mtasCollectionResponses;
  433 + if (mtasResponse.get("collection") != null
  434 + && mtasResponse.get("collection") instanceof ArrayList) {
  435 + mtasCollectionResponses = (ArrayList<Object>) mtasResponse.get("collection");
  436 + } else {
  437 + mtasCollectionResponses = new ArrayList<>();
  438 + mtasResponse.add("collection", mtasCollectionResponses);
  439 + }
  440 + MtasSolrCollectionResult collectionResult;
  441 + for (ComponentCollection componentCollection : mtasFields.collection) {
  442 + try {
  443 + collectionResult = createMtasSolrCollectionResult(componentCollection,
  444 + false);
  445 + // Create response
  446 + SimpleOrderedMap<Object> mtasCollectionResponse = new SimpleOrderedMap<>();
  447 + mtasCollectionResponse.add("key", componentCollection.key);
  448 + mtasCollectionResponse.add("data", collectionResult);
  449 + mtasCollectionResponses.add(mtasCollectionResponse);
  450 + } catch (IOException e) {
  451 + log.debug(e);
  452 + }
  453 + }
  454 + }
  455 + // decode shard responses
  456 + for (ShardRequest sreq : rb.finished) {
  457 + if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
  458 + && sreq.params.getBool(PARAM_MTAS_COLLECTION, false)) {
  459 + for (ShardResponse shardResponse : sreq.responses) {
  460 + NamedList<Object> solrShardResponse = shardResponse
  461 + .getSolrResponse().getResponse();
  462 + try {
  463 + ArrayList<SimpleOrderedMap<Object>> data = (ArrayList<SimpleOrderedMap<Object>>) solrShardResponse
  464 + .findRecursive("mtas", "collection");
  465 + if (data != null) {
  466 + MtasSolrResultUtil.decode(data);
  467 + if (rb.stage > ResponseBuilder.STAGE_EXECUTE_QUERY) {
  468 + ArrayList<SimpleOrderedMap<Object>> filteredData = new ArrayList<>();
  469 + for (SimpleOrderedMap<Object> dataItem : data) {
  470 + if (dataItem.get("data") != null && dataItem
  471 + .get("data") instanceof MtasSolrCollectionResult) {
  472 + MtasSolrCollectionResult collectionResult = (MtasSolrCollectionResult) dataItem
  473 + .get("data");
  474 + if (rb.stage <= MtasSolrSearchComponent.STAGE_COLLECTION_INIT) {
  475 + if (!collectionResult.action()
  476 + .equals(ComponentCollection.ACTION_CREATE)
  477 + && !collectionResult.action()
  478 + .equals(ComponentCollection.ACTION_LIST)
  479 + && !collectionResult.action()
  480 + .equals(ComponentCollection.ACTION_CHECK)) {
  481 + filteredData.add(dataItem);
  482 + }
  483 + } else if (rb.stage <= MtasSolrSearchComponent.STAGE_COLLECTION_FINISH) {
  484 + if (!collectionResult.action()
  485 + .equals(ComponentCollection.ACTION_POST)
  486 + && !collectionResult.action()
  487 + .equals(ComponentCollection.ACTION_IMPORT) && !collectionResult.action()
  488 + .equals(ComponentCollection.ACTION_CHECK)) {
  489 + filteredData.add(dataItem);
  490 + }
  491 + }
  492 + } else {
  493 + filteredData.add(dataItem);
  494 + }
  495 + }
  496 + data.clear();
  497 + data.addAll(filteredData);
  498 + }
  499 + }
  500 + } catch (ClassCastException e) {
  501 + log.debug(e);
  502 + // shouldn't happen
  503 + }
  504 + }
  505 +
  506 + }
  507 + }
  508 + }
  509 + }
  510 + }
  511 + }
  512 +
  513 + /*
  514 + * (non-Javadoc)
  515 + *
  516 + * @see
  517 + * mtas.solr.handler.component.util.MtasSolrComponent#distributedProcess(org.
  518 + * apache.solr.handler.component.ResponseBuilder,
  519 + * mtas.codec.util.CodecComponent.ComponentFields)
  520 + */
  521 + @SuppressWarnings("unchecked")
  522 + public void distributedProcess(ResponseBuilder rb, ComponentFields mtasFields)
  523 + throws IOException {
  524 + // System.out.println("collection: " + System.nanoTime() + " - "
  525 + // + Thread.currentThread().getId() + " - "
  526 + // + rb.req.getParams().getBool("isShard", false) + " DISTRIBUTEDPROCESS "
  527 + // + rb.stage + " " + rb.req.getParamString());
  528 + NamedList<Object> mtasResponse = null;
  529 + try {
  530 + mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
  531 + } catch (ClassCastException e) {
  532 + log.debug(e);
  533 + mtasResponse = null;
  534 + }
  535 + if (mtasResponse != null) {
  536 + if (rb.stage == MtasSolrSearchComponent.STAGE_COLLECTION_INIT) {
  537 + // build index
  538 + Map<String, MtasSolrCollectionResult> index = new HashMap<>();
  539 + ArrayList<Object> mtasResponseCollection;
  540 + try {
  541 + mtasResponseCollection = (ArrayList<Object>) mtasResponse.get("collection");
  542 + for (Object item : mtasResponseCollection) {
  543 + if (item instanceof SimpleOrderedMap) {
  544 + SimpleOrderedMap<Object> itemMap = (SimpleOrderedMap<Object>) item;
  545 + if (itemMap.get("data") != null
  546 + && itemMap.get("data") instanceof MtasSolrCollectionResult) {
  547 + MtasSolrCollectionResult collectionItem = (MtasSolrCollectionResult) itemMap
  548 + .get("data");
  549 + index.put(collectionItem.id(), collectionItem);
  550 + }
  551 + }
  552 + }
  553 + } catch (ClassCastException e) {
  554 + log.debug(e);
  555 + mtasResponse.remove("collection");
  556 + }
  557 + // check and remove previous responses
  558 + Map<String, Set<String>> createPostAfterMissingCheckResult = new HashMap<>();
  559 + for (ShardRequest sreq : rb.finished) {
  560 + if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
  561 + && sreq.params.getBool(PARAM_MTAS_COLLECTION, false)) {
  562 + for (ShardResponse shardResponse : sreq.responses) {
  563 + NamedList<Object> solrShardResponse = shardResponse
  564 + .getSolrResponse().getResponse();
  565 + try {
  566 + ArrayList<SimpleOrderedMap<Object>> data = (ArrayList<SimpleOrderedMap<Object>>) solrShardResponse
  567 + .findRecursive("mtas", "collection");
  568 + if (data != null) {
  569 + for (SimpleOrderedMap<Object> dataItem : data) {
  570 + if (dataItem.get("data") != null && dataItem
  571 + .get("data") instanceof MtasSolrCollectionResult) {
  572 + MtasSolrCollectionResult dataItemResult = (MtasSolrCollectionResult) dataItem
  573 + .get("data");
  574 + if (index.containsKey(dataItemResult.id())
  575 + && index.get(dataItemResult.id()).action()
  576 + .equals(ComponentCollection.ACTION_CHECK)) {
  577 + if (dataItemResult.status == null) {
  578 + if (!createPostAfterMissingCheckResult
  579 + .containsKey(shardResponse.getShard())) {
  580 + createPostAfterMissingCheckResult
  581 + .put(shardResponse.getShard(), new HashSet<>());
  582 + }
  583 + createPostAfterMissingCheckResult
  584 + .get(shardResponse.getShard())
  585 + .add(dataItemResult.id());
  586 + }
  587 + }
  588 + }
  589 + }
  590 + data.clear();
  591 + }
  592 + } catch (ClassCastException e) {
  593 + log.debug(e);
  594 + // shouldn't happen
  595 + }
  596 + }
  597 + }
  598 + }
  599 + // construct new requests
  600 + HashMap<String, ModifiableSolrParams> requestParamList = new HashMap<>();
  601 + int id = 0;
  602 + for (ComponentCollection componentCollection : mtasFields.collection) {
  603 + if (componentCollection.action().equals(ComponentCollection.ACTION_CHECK)) {
  604 + for (String shardAddress : rb.shards) {
  605 + if (createPostAfterMissingCheckResult.containsKey(shardAddress)) {
  606 + if (createPostAfterMissingCheckResult.get(shardAddress)
  607 + .contains(componentCollection.id)) {
  608 + HashSet<String> values = searchComponent.getCollectionCache()
  609 + .getDataById(componentCollection.id);
  610 + if (values != null) {
  611 + ModifiableSolrParams paramsNewRequest;
  612 + if (!requestParamList.containsKey(shardAddress)) {
  613 + paramsNewRequest = new ModifiableSolrParams();
  614 + requestParamList.put(shardAddress, paramsNewRequest);
  615 + } else {
  616 + paramsNewRequest = requestParamList.get(shardAddress);
  617 + }
  618 + paramsNewRequest.add(PARAM_MTAS_COLLECTION + "." + id + "."
  619 + + NAME_MTAS_COLLECTION_KEY, componentCollection.key);
  620 + paramsNewRequest.add(PARAM_MTAS_COLLECTION + "." + id + "."
  621 + + NAME_MTAS_COLLECTION_ID, componentCollection.id);
  622 + paramsNewRequest.add(
  623 + PARAM_MTAS_COLLECTION + "." + id + "."
  624 + + NAME_MTAS_COLLECTION_ACTION,
  625 + ComponentCollection.ACTION_POST);
  626 + paramsNewRequest.add(
  627 + PARAM_MTAS_COLLECTION + "." + id + "."
  628 + + NAME_MTAS_COLLECTION_POST,
  629 + stringValuesToString(values));
  630 + id++;
  631 + }
  632 + }
  633 + }
  634 + }
  635 + } else if (componentCollection.action()
  636 + .equals(ComponentCollection.ACTION_CREATE)) {
  637 + if (componentCollection.version == null) {
  638 + componentCollection.version = searchComponent.getCollectionCache()
  639 + .create(componentCollection.id, componentCollection.values().size(),
  640 + componentCollection.values());
  641 + }
  642 + if (index.containsKey(componentCollection.id)) {
  643 + index.get(componentCollection.id).setCreate(
  644 + searchComponent.getCollectionCache().now(),
  645 + searchComponent.getCollectionCache().check(componentCollection.id));
  646 + }
  647 + for (String shardAddress : rb.shards) {
  648 + ModifiableSolrParams paramsNewRequest;
  649 + if (!requestParamList.containsKey(shardAddress)) {
  650 + paramsNewRequest = new ModifiableSolrParams();
  651 + requestParamList.put(shardAddress, paramsNewRequest);
  652 + } else {
  653 + paramsNewRequest = requestParamList.get(shardAddress);
  654 + }
  655 + paramsNewRequest.add(PARAM_MTAS_COLLECTION + "." + id + "."
  656 + + NAME_MTAS_COLLECTION_KEY, componentCollection.key);
  657 + paramsNewRequest.add(PARAM_MTAS_COLLECTION + "." + id + "."
  658 + + NAME_MTAS_COLLECTION_ID, componentCollection.id);
  659 + paramsNewRequest.add(
  660 + PARAM_MTAS_COLLECTION + "." + id + "."
  661 + + NAME_MTAS_COLLECTION_ACTION,
  662 + ComponentCollection.ACTION_POST);
  663 + paramsNewRequest.add(
  664 + PARAM_MTAS_COLLECTION + "." + id + "."
  665 + + NAME_MTAS_COLLECTION_POST,
  666 + stringValuesToString(componentCollection.values()));
  667 + }
  668 + }
  669 + id++;
  670 + }
  671 + // add new requests
  672 + for (Entry<String, ModifiableSolrParams> entry : requestParamList
  673 + .entrySet()) {
  674 + ShardRequest newSreq = new ShardRequest();
  675 + newSreq.shards = new String[] { entry.getKey() };
  676 + newSreq.purpose = ShardRequest.PURPOSE_PRIVATE;
  677 + newSreq.params = entry.getValue();
  678 + newSreq.params.add("q", "*");
  679 + newSreq.params.add("rows", "0");
  680 + newSreq.params.add(MtasSolrSearchComponent.PARAM_MTAS,
  681 + rb.req.getOriginalParams()
  682 + .getParams(MtasSolrSearchComponent.PARAM_MTAS));
  683 + newSreq.params.add(PARAM_MTAS_COLLECTION,
  684 + rb.req.getOriginalParams().getParams(PARAM_MTAS_COLLECTION));
  685 + rb.addRequest(searchComponent, newSreq);
  686 + }
  687 + } else if (rb.stage == MtasSolrSearchComponent.STAGE_COLLECTION_FINISH) {
  688 + // just rewrite
  689 + ArrayList<Object> mtasResponseCollection;
  690 + try {
  691 + mtasResponseCollection = (ArrayList<Object>) mtasResponse.get("collection");
  692 + if (mtasResponseCollection != null) {
  693 + MtasSolrResultUtil.rewrite(mtasResponseCollection, searchComponent);
  694 + }
  695 + } catch (ClassCastException e) {
  696 + log.debug(e);
  697 + mtasResponse.remove("collection");
  698 + }
  699 + }
  700 + }
  701 + }
  702 +
  703 + /**
  704 + * Gets the mtas fields.
  705 + *
  706 + * @param rb
  707 + * the rb
  708 + * @return the mtas fields
  709 + */
  710 + private ComponentFields getMtasFields(ResponseBuilder rb) {
  711 + return (ComponentFields) rb.req.getContext().get(ComponentFields.class);
  712 + }
  713 +
  714 + /**
  715 + * String values to string.
  716 + *
  717 + * @param stringValues
  718 + * the string values
  719 + * @return the string
  720 + */
  721 + private static String stringValuesToString(HashSet<String> stringValues) {
  722 + return JSONUtil.toJSON(stringValues);
  723 + }
  724 +
  725 + /**
  726 + * String to string values.
  727 + *
  728 + * @param stringValue
  729 + * the string value
  730 + * @return the hash set
  731 + * @throws IOException
  732 + * Signals that an I/O exception has occurred.
  733 + */
  734 + private static HashSet<String> stringToStringValues(String stringValue)
  735 + throws IOException {
  736 + // should be improved to support escaped characters
  737 + HashSet<String> stringValues = new HashSet<>();
  738 + JSONParser jsonParser = new JSONParser(stringValue);
  739 + int event = jsonParser.nextEvent();
  740 + if (event == JSONParser.ARRAY_START) {
  741 + while ((event = jsonParser.nextEvent()) != JSONParser.ARRAY_END) {
  742 + if (jsonParser.getLevel() == 1) {
  743 + switch (event) {
  744 + case JSONParser.STRING:
  745 + stringValues.add(jsonParser.getString());
  746 + break;
  747 + case JSONParser.BIGNUMBER:
  748 + case JSONParser.NUMBER:
  749 + case JSONParser.LONG:
  750 + stringValues.add(jsonParser.getNumberChars().toString());
  751 + break;
  752 + case JSONParser.BOOLEAN:
  753 + stringValues.add(Boolean.toString(jsonParser.getBoolean()));
  754 + break;
  755 + default:
  756 + // do nothing
  757 + }
  758 + }
  759 + }
  760 + } else {
  761 + throw new IOException("unsupported json structure");
  762 + }
  763 + return stringValues;
  764 + }
  765 +
  766 +}
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentDocument.java
... ... @@ -71,12 +71,14 @@ public class MtasSolrComponentDocument
71 71  
72 72 /** The Constant NAME_MTAS_DOCUMENT_NUMBER. */
73 73 public static final String NAME_MTAS_DOCUMENT_NUMBER = "number";
  74 +
  75 + private MtasSolrSearchComponent searchComponent;
74 76  
75 77 /**
76 78 * Instantiates a new mtas solr component document.
77 79 */
78   - public MtasSolrComponentDocument() {
79   - // do nothing for now
  80 + public MtasSolrComponentDocument(MtasSolrSearchComponent searchComponent) {
  81 + this.searchComponent = searchComponent;
80 82 }
81 83  
82 84 /*
... ... @@ -257,7 +259,7 @@ public class MtasSolrComponentDocument
257 259 mtasDocumentItemResponses.add(mtasDocumentItemResponse);
258 260 }
259 261 mtasDocumentResponse.add("list", mtasDocumentItemResponses);
260   - MtasSolrResultUtil.rewrite(mtasDocumentResponse);
  262 + MtasSolrResultUtil.rewrite(mtasDocumentResponse, searchComponent);
261 263 return mtasDocumentResponse;
262 264 }
263 265  
... ... @@ -337,7 +339,7 @@ public class MtasSolrComponentDocument
337 339 try {
338 340 mtasResponseDocument = (ArrayList<Object>) mtasResponse.get("document");
339 341 if (mtasResponseDocument != null) {
340   - MtasSolrResultUtil.rewrite(mtasResponseDocument);
  342 + MtasSolrResultUtil.rewrite(mtasResponseDocument, searchComponent);
341 343 }
342 344 } catch (ClassCastException e) {
343 345 log.debug(e);
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentFacet.java
... ... @@ -573,7 +573,7 @@ public class MtasSolrComponentFacet
573 573 mtasFacetResponse.add("_encoded_list", MtasSolrResultUtil.encode(data));
574 574 } else {
575 575 mtasFacetResponse.add("list", data);
576   - MtasSolrResultUtil.rewrite(mtasFacetResponse);
  576 + MtasSolrResultUtil.rewrite(mtasFacetResponse, searchComponent);
577 577 }
578 578 return mtasFacetResponse;
579 579 }
... ... @@ -636,7 +636,7 @@ public class MtasSolrComponentFacet
636 636 try {
637 637 mtasResponseFacet = (ArrayList<Object>) mtasResponse.get("facet");
638 638 if (mtasResponseFacet != null) {
639   - MtasSolrResultUtil.rewrite(mtasResponseFacet);
  639 + MtasSolrResultUtil.rewrite(mtasResponseFacet, searchComponent);
640 640 }
641 641 } catch (ClassCastException e) {
642 642 log.debug(e);
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentGroup.java
... ... @@ -333,9 +333,10 @@ public class MtasSolrComponentGroup
333 333 * @param name the name
334 334 * @param positions the positions
335 335 * @param prefixes the prefixes
  336 + * @throws IOException
336 337 */
337 338 private void prepare(SolrParams solrParams, SortedSet<String> gids,
338   - String name, String[] positions, String[] prefixes) {
  339 + String name, String[] positions, String[] prefixes) throws IOException {
339 340 if (!gids.isEmpty()) {
340 341 int tmpSubCounter = 0;
341 342 for (String gid : gids) {
... ... @@ -343,6 +344,11 @@ public class MtasSolrComponentGroup
343 344 name + "." + gid + "." + NAME_MTAS_GROUP_GROUPING_POSITION, null);
344 345 prefixes[tmpSubCounter] = solrParams.get(
345 346 name + "." + gid + "." + NAME_MTAS_GROUP_GROUPING_PREFIXES, null);
  347 + if(positions[tmpSubCounter]==null) {
  348 + throw new IOException("no position for "+gid);
  349 + } else if(prefixes[tmpSubCounter]==null) {
  350 + throw new IOException("no prefix for "+gid);
  351 + }
346 352 tmpSubCounter++;
347 353 }
348 354 }
... ... @@ -465,7 +471,7 @@ public class MtasSolrComponentGroup
465 471 mtasGroupResponse.add("_encoded_list", MtasSolrResultUtil.encode(data));
466 472 } else {
467 473 mtasGroupResponse.add("list", data);
468   - MtasSolrResultUtil.rewrite(mtasGroupResponse);
  474 + MtasSolrResultUtil.rewrite(mtasGroupResponse, searchComponent);
469 475 }
470 476 return mtasGroupResponse;
471 477 }
... ... @@ -528,7 +534,7 @@ public class MtasSolrComponentGroup
528 534 try {
529 535 mtasResponseGroup = (ArrayList<Object>) mtasResponse.get("group");
530 536 if (mtasResponseGroup != null) {
531   - MtasSolrResultUtil.rewrite(mtasResponseGroup);
  537 + MtasSolrResultUtil.rewrite(mtasResponseGroup, searchComponent);
532 538 }
533 539 } catch (ClassCastException e) {
534 540 log.debug(e);
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentJoin.java deleted
1   -package mtas.solr.handler.component.util;
2   -
3   -import java.io.IOException;
4   -import java.util.Arrays;
5   -import java.util.HashSet;
6   -import java.util.Set;
7   -
8   -import org.apache.commons.logging.Log;
9   -import org.apache.commons.logging.LogFactory;
10   -import org.apache.solr.common.util.NamedList;
11   -import org.apache.solr.common.util.SimpleOrderedMap;
12   -import org.apache.solr.handler.component.ResponseBuilder;
13   -import org.apache.solr.handler.component.SearchComponent;
14   -import org.apache.solr.handler.component.ShardRequest;
15   -import org.apache.solr.handler.component.ShardResponse;
16   -
17   -import mtas.codec.util.CodecComponent.ComponentFields;
18   -import mtas.codec.util.CodecComponent.ComponentJoin;
19   -import mtas.solr.handler.component.MtasSolrSearchComponent;
20   -
21   -/**
22   - * The Class MtasSolrComponentJoin.
23   - */
24   -@SuppressWarnings("deprecation")
25   -public class MtasSolrComponentJoin implements MtasSolrComponent<ComponentJoin> {
26   -
27   - /** The Constant log. */
28   - private static final Log log = LogFactory.getLog(MtasSolrComponentJoin.class);
29   -
30   - /** The Constant PARAM_MTAS_JOIN. */
31   - public static final String PARAM_MTAS_JOIN = MtasSolrSearchComponent.PARAM_MTAS
32   - + ".join";
33   -
34   - /** The Constant NAME_MTAS_JOIN_FIELD. */
35   - public static final String NAME_MTAS_JOIN_FIELD = "field";
36   -
37   - /**
38   - * Instantiates a new mtas solr component join.
39   - *
40   - * @param searchComponent the search component
41   - */
42   - public MtasSolrComponentJoin(MtasSolrSearchComponent searchComponent) {
43   - }
44   -
45   - /*
46   - * (non-Javadoc)
47   - *
48   - * @see
49   - * mtas.solr.handler.component.util.MtasSolrComponent#prepare(org.apache.solr.
50   - * handler.component.ResponseBuilder,
51   - * mtas.codec.util.CodecComponent.ComponentFields)
52   - */
53   - public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
54   - throws IOException {
55   - if (rb.req.getParams().get(PARAM_MTAS_JOIN + "." + NAME_MTAS_JOIN_FIELD,
56   - null) != null) {
57   - Set<String> fields = new HashSet<>(Arrays.asList(rb.req.getParams()
58   - .get(PARAM_MTAS_JOIN + "." + NAME_MTAS_JOIN_FIELD).split(",")));
59   - String key = createKeyFromRequest(rb);
60   - mtasFields.doJoin = true;
61   - mtasFields.join = new ComponentJoin(fields, key);
62   - rb.setNeedDocSet(true);
63   - }
64   -
65   - }
66   -
67   - /*
68   - * (non-Javadoc)
69   - *
70   - * @see
71   - * mtas.solr.handler.component.util.MtasSolrComponent#modifyRequest(org.apache
72   - * .solr.handler.component.ResponseBuilder,
73   - * org.apache.solr.handler.component.SearchComponent,
74   - * org.apache.solr.handler.component.ShardRequest)
75   - */
76   - public void modifyRequest(ResponseBuilder rb, SearchComponent who,
77   - ShardRequest sreq) {
78   - if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
79   - && sreq.params.getBool(PARAM_MTAS_JOIN, false)) {
80   - if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
81   - // do nothing
82   - } else {
83   - // remove for other requests
84   - Set<String> keys = MtasSolrResultUtil
85   - .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_JOIN);
86   - sreq.params.remove(PARAM_MTAS_JOIN);
87   - for (String key : keys) {
88   - sreq.params.remove(PARAM_MTAS_JOIN + "." + key);
89   - }
90   - }
91   - }
92   - }
93   -
94   - /*
95   - * (non-Javadoc)
96   - *
97   - * @see
98   - * mtas.solr.handler.component.util.MtasSolrComponent#create(mtas.codec.util.
99   - * CodecComponent.BasicComponent, java.lang.Boolean)
100   - */
101   - public SimpleOrderedMap<Object> create(ComponentJoin join, Boolean encode)
102   - throws IOException {
103   - MtasSolrJoinResult data = new MtasSolrJoinResult(join);
104   - SimpleOrderedMap<Object> mtasJoinResponse = new SimpleOrderedMap<>();
105   - if (encode) {
106   - mtasJoinResponse.add("_encoded_data", MtasSolrResultUtil.encode(data));
107   - } else {
108   - mtasJoinResponse.add("data", data.rewrite());
109   - }
110   - return mtasJoinResponse;
111   - }
112   -
113   - /*
114   - * (non-Javadoc)
115   - *
116   - * @see
117   - * mtas.solr.handler.component.util.MtasSolrComponent#finishStage(org.apache.
118   - * solr.handler.component.ResponseBuilder)
119   - */
120   - @SuppressWarnings("unchecked")
121   - public void finishStage(ResponseBuilder rb) {
122   - if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
123   - && rb.stage == MtasSolrSearchComponent.STAGE_JOIN) {
124   - for (ShardRequest sreq : rb.finished) {
125   - if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
126   - && sreq.params.getBool(PARAM_MTAS_JOIN, false)) {
127   - for (ShardResponse shardResponse : sreq.responses) {
128   - NamedList<Object> response = shardResponse.getSolrResponse()
129   - .getResponse();
130   - try {
131   - Object data = response.findRecursive("mtas", "join");
132   - if (data != null && data instanceof String) {
133   - NamedList<Object> mtasResponse = (NamedList<Object>) response
134   - .get("mtas");
135   - mtasResponse.remove("join");
136   - mtasResponse.add("join",
137   - MtasSolrResultUtil.decode((String) data));
138   - }
139   - } catch (ClassCastException e) {
140   - log.debug(e);
141   - // shouldn't happen
142   - }
143   - }
144   - }
145   - }
146   - }
147   - }
148   -
149   - /*
150   - * (non-Javadoc)
151   - *
152   - * @see
153   - * mtas.solr.handler.component.util.MtasSolrComponent#distributedProcess(org.
154   - * apache.solr.handler.component.ResponseBuilder,
155   - * mtas.codec.util.CodecComponent.ComponentFields)
156   - */
157   - @SuppressWarnings("unchecked")
158   - public void distributedProcess(ResponseBuilder rb, ComponentFields mtasFields)
159   - throws IOException {
160   - // rewrite
161   - NamedList<Object> mtasResponse = null;
162   - try {
163   - mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
164   - } catch (ClassCastException e) {
165   - log.debug(e);
166   - mtasResponse = null;
167   - }
168   - if (mtasResponse != null) {
169   - MtasSolrJoinResult mtasSolrJoinResult;
170   - try {
171   - mtasSolrJoinResult = (MtasSolrJoinResult) mtasResponse.get("join");
172   - if (mtasSolrJoinResult != null) {
173   - mtasResponse.removeAll("join");
174   - mtasResponse.add("join", mtasSolrJoinResult.rewrite());
175   - }
176   - } catch (ClassCastException e) {
177   - log.debug(e);
178   - mtasResponse.remove("join");
179   - }
180   - }
181   - }
182   -
183   - /**
184   - * Creates the key from request.
185   - *
186   - * @param rb the rb
187   - * @return the string
188   - */
189   - private String createKeyFromRequest(ResponseBuilder rb) {
190   - return rb.req.getParams().toQueryString();
191   - }
192   -
193   -}
src/mtas/solr/handler/component/util/MtasSolrComponentPrefix.java
... ... @@ -136,7 +136,7 @@ public class MtasSolrComponentPrefix
136 136 * CodecComponent.BasicComponent, java.lang.Boolean)
137 137 */
138 138 public SimpleOrderedMap<Object> create(ComponentPrefix prefix,
139   - Boolean encode) {
  139 + Boolean encode) throws IOException {
140 140 SimpleOrderedMap<Object> mtasPrefixResponse = new SimpleOrderedMap<>();
141 141 mtasPrefixResponse.add("key", prefix.key);
142 142 if (encode) {
... ... @@ -219,7 +219,7 @@ public class MtasSolrComponentPrefix
219 219 for (Object mtasResponsePrefixItemRaw : mtasResponsePrefix) {
220 220 mtasResponsePrefixItem = (NamedList<Object>) mtasResponsePrefixItemRaw;
221 221 repairPrefixItems(mtasResponsePrefixItem);
222   - MtasSolrResultUtil.rewrite(mtasResponsePrefixItem);
  222 + MtasSolrResultUtil.rewrite(mtasResponsePrefixItem, searchComponent);
223 223 }
224 224 }
225 225 } catch (ClassCastException e) {
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentStats.java
... ... @@ -878,7 +878,7 @@ public class MtasSolrComponentStats
878 878 MtasSolrResultUtil.encode(data));
879 879 } else {
880 880 mtasPositionResponse.add(position.dataCollector.getCollectorType(), data);
881   - MtasSolrResultUtil.rewrite(mtasPositionResponse);
  881 + MtasSolrResultUtil.rewrite(mtasPositionResponse, searchComponent);
882 882 }
883 883 return mtasPositionResponse;
884 884 }
... ... @@ -903,7 +903,7 @@ public class MtasSolrComponentStats
903 903 mtasTokenResponse.add("_encoded_data", MtasSolrResultUtil.encode(data));
904 904 } else {
905 905 mtasTokenResponse.add(token.dataCollector.getCollectorType(), data);
906   - MtasSolrResultUtil.rewrite(mtasTokenResponse);
  906 + MtasSolrResultUtil.rewrite(mtasTokenResponse, searchComponent);
907 907 }
908 908 return mtasTokenResponse;
909 909 }
... ... @@ -944,7 +944,7 @@ public class MtasSolrComponentStats
944 944 mtasSpanResponse.add("_encoded_data", MtasSolrResultUtil.encode(data));
945 945 } else {
946 946 mtasSpanResponse.add(span.dataCollector.getCollectorType(), data);
947   - MtasSolrResultUtil.rewrite(mtasSpanResponse);
  947 + MtasSolrResultUtil.rewrite(mtasSpanResponse, searchComponent);
948 948 }
949 949 return mtasSpanResponse;
950 950 }
... ... @@ -1007,7 +1007,7 @@ public class MtasSolrComponentStats
1007 1007 try {
1008 1008 mtasResponseStats = (NamedList<Object>) mtasResponse.get("stats");
1009 1009 if (mtasResponseStats != null) {
1010   - MtasSolrResultUtil.rewrite(mtasResponseStats);
  1010 + MtasSolrResultUtil.rewrite(mtasResponseStats, searchComponent);
1011 1011 }
1012 1012 } catch (ClassCastException e) {
1013 1013 log.debug(e);
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentTermvector.java
... ... @@ -476,7 +476,7 @@ public class MtasSolrComponentTermvector
476 476 MtasSolrResultUtil.encode(data));
477 477 } else {
478 478 mtasTermVectorResponse.add("list", data);
479   - MtasSolrResultUtil.rewrite(mtasTermVectorResponse);
  479 + MtasSolrResultUtil.rewrite(mtasTermVectorResponse, searchComponent);
480 480 }
481 481 return mtasTermVectorResponse;
482 482 }
... ... @@ -627,7 +627,7 @@ public class MtasSolrComponentTermvector
627 627 if ((mtasResponseTermvectorRaw = mtasResponse.get("termvector")) != null
628 628 && mtasResponseTermvectorRaw instanceof ArrayList) {
629 629 MtasSolrResultUtil
630   - .rewrite((ArrayList<Object>) mtasResponseTermvectorRaw);
  630 + .rewrite((ArrayList<Object>) mtasResponseTermvectorRaw, searchComponent);
631 631 }
632 632 }
633 633 }
... ...
src/mtas/solr/handler/component/util/MtasSolrJoinResult.java deleted
1   -package mtas.solr.handler.component.util;
2   -
3   -import java.io.Serializable;
4   -import java.util.Set;
5   -
6   -import org.apache.solr.common.util.NamedList;
7   -
8   -import mtas.codec.util.CodecComponent.ComponentJoin;
9   -
10   -/**
11   - * The Class MtasSolrJoinResult.
12   - */
13   -public class MtasSolrJoinResult implements Serializable {
14   -
15   - /** The Constant serialVersionUID. */
16   - private static final long serialVersionUID = 1L;
17   -
18   - /** The values. */
19   - private Set<String> values;
20   -
21   - /** The key. */
22   - private String key;
23   -
24   - /**
25   - * Instantiates a new mtas solr join result.
26   - *
27   - * @param join the join
28   - */
29   - public MtasSolrJoinResult(ComponentJoin join) {
30   - values = join.values();
31   - key = join.key();
32   - }
33   -
34   - /**
35   - * Rewrite.
36   - *
37   - * @return the named list
38   - */
39   - public NamedList<Object> rewrite() {
40   - NamedList<Object> response = new NamedList<>();
41   - response.add("values", values);
42   - response.add("key", key);
43   - return response;
44   - }
45   -
46   - /**
47   - * Merge.
48   - *
49   - * @param newItem the new item
50   - */
51   - public void merge(MtasSolrJoinResult newItem) {
52   - values.addAll(newItem.values);
53   - }
54   -
55   -}
src/mtas/solr/handler/component/util/MtasSolrResultMerge.java
... ... @@ -15,7 +15,6 @@ import org.apache.solr.handler.component.ResponseBuilder;
15 15 import org.apache.solr.handler.component.ShardRequest;
16 16 import org.apache.solr.handler.component.ShardResponse;
17 17  
18   -import mtas.codec.util.CodecComponent.ComponentFields;
19 18 import mtas.solr.handler.component.MtasSolrSearchComponent;
20 19  
21 20 /**
... ... @@ -64,21 +63,22 @@ public class MtasSolrResultMerge {
64 63 .getBool(MtasSolrComponentFacet.PARAM_MTAS_FACET, false)) {
65 64 mergeArrayList(sreq, mtasResponse, "facet", null, false);
66 65 }
67   - // merge join
68   - if (rb.req.getParams().getBool(MtasSolrComponentJoin.PARAM_MTAS_JOIN,
  66 + // merge collection
  67 + if (rb.req.getParams().getBool(MtasSolrComponentCollection.PARAM_MTAS_COLLECTION,
69 68 false)) {
70   - ComponentFields componentFields = (ComponentFields) rb.req
71   - .getContext().get(ComponentFields.class);
72   - mtasResponse.add("join",
73   - new MtasSolrJoinResult(componentFields.join));
74   - mergeJoinResult(sreq, mtasResponse, "join", null);
75   -
  69 + mergeArrayList(sreq, mtasResponse, "collection", null, false);
76 70 }
77 71 // merge prefix
78 72 if (rb.req.getParams()
79 73 .getBool(MtasSolrComponentPrefix.PARAM_MTAS_PREFIX, false)) {
80 74 mergeArrayList(sreq, mtasResponse, "prefix", null, false);
81 75 }
  76 + } else if (rb.stage == MtasSolrSearchComponent.STAGE_COLLECTION_INIT) {
  77 + // merge collection
  78 + if (rb.req.getParams().getBool(
  79 + MtasSolrComponentCollection.PARAM_MTAS_COLLECTION, false)) {
  80 + mergeArrayList(sreq, mtasResponse, "collection", null, false);
  81 + }
82 82 } else if (rb.stage == MtasSolrSearchComponent.STAGE_TERMVECTOR_MISSING_KEY) {
83 83 // merge termvector
84 84 if (rb.req.getParams().getBool(
... ... @@ -109,57 +109,7 @@ public class MtasSolrResultMerge {
109 109 }
110 110 }
111 111 }
112   -
113   - /**
114   - * Merge join result.
115   - *
116   - * @param sreq the sreq
117   - * @param mtasResponse the mtas response
118   - * @param key the key
119   - * @param preferredPurpose the preferred purpose
120   - */
121   - @SuppressWarnings("unchecked")
122   - private void mergeJoinResult(ShardRequest sreq,
123   - NamedList<Object> mtasResponse, String key, Integer preferredPurpose) {
124   - Object o = mtasResponse.get(key);
125   - MtasSolrJoinResult mtasJoinResponse;
126   - if (o instanceof MtasSolrJoinResult) {
127   - mtasJoinResponse = (MtasSolrJoinResult) o;
128   - } else {
129   - mtasJoinResponse = null;
130   - }
131   - // collect responses for each shard
132   - HashMap<String, NamedList<Object>> mtasListShardResponses = new HashMap<>();
133   - for (ShardResponse response : sreq.responses) {
134   - // only continue if new shard or preferred purpose
135   - if (mtasListShardResponses.containsKey(response.getShard())
136   - && ((preferredPurpose == null)
137   - || (sreq.purpose != preferredPurpose))) {
138   - break;
139   - }
140   - // update
141   - try {
142   - NamedList<Object> result = response.getSolrResponse().getResponse();
143   - String data = (String) result.findRecursive("mtas", key);
144   - if (data != null) {
145   - MtasSolrJoinResult decodedData = (MtasSolrJoinResult) MtasSolrResultUtil
146   - .decode(data);
147   - if (mtasJoinResponse == null) {
148   - mtasJoinResponse = decodedData;
149   - } else {
150   - mtasJoinResponse.merge(decodedData);
151   - }
152   - }
153   - } catch (ClassCastException e) {
154   - log.debug(e);
155   - }
156   - }
157   - if (mtasJoinResponse != null) {
158   - mtasResponse.removeAll(key);
159   - mtasResponse.add(key, mtasJoinResponse);
160   - }
161   - }
162   -
  112 +
163 113 /**
164 114 * Merge named list.
165 115 *
... ... @@ -381,9 +331,9 @@ public class MtasSolrResultMerge {
381 331 } else if (original instanceof MtasSolrMtasResult) {
382 332 MtasSolrMtasResult originalComponentResult = (MtasSolrMtasResult) original;
383 333 originalComponentResult.merge((MtasSolrMtasResult) shardValue);
384   - } else if (original instanceof MtasSolrJoinResult) {
385   - MtasSolrJoinResult originalComponentResult = (MtasSolrJoinResult) original;
386   - originalComponentResult.merge((MtasSolrJoinResult) shardValue);
  334 + } else if (original instanceof MtasSolrCollectionResult) {
  335 + MtasSolrCollectionResult originalComponentResult = (MtasSolrCollectionResult) original;
  336 + originalComponentResult.merge((MtasSolrCollectionResult) shardValue);
387 337 } else if (original instanceof String) {
388 338 // ignore?
389 339 } else if (original instanceof Integer) {
... ...