dariah / mtas

Commit 7179e4b769f7bea15982a6bc97cdd9f6e2a4adf0

Authored by Matthijs Brouwer
1 parent 347732b5
master ... 7.1.0 group_cql publications 20171021 20171016 20171012 20171007 20170929 20170925 20170913 20170911 20170908 v7.1.0.4 v7.1.0.3 v7.1.0.2 v7.1.0.1 v7.1.0.0 v7.0.1.2 v7.0.1.1 v7.0.1.0

documentation, bugfixes

Inline Side-by-side
Showing 65 changed files with 4601 additions and 1323 deletions

Too many changes to show.
Reload with full diff Plain diff Email patch

To preserve performance only 30 of 65 files are displayed.

conf/parser/mtas/chat_test.xml
  View file @7179e4b
... ... @@ -103,15 +103,6 @@
103 103 <item type="text" />
104 104 </post>
105 105 </token>
106   - <token type="string" offset="false">
107   - <pre>
108   - <item type="name" />
109   - <item type="string" value="_lc" />
110   - </pre>
111   - <post>
112   - <item type="text" filter="ascii,lowercase" />
113   - </post>
114   - </token>
115 106 </mapping>
116 107 <mapping type="word" name="t">
117 108 <token type="string" offset="false">
... ...
conf/parser/mtas/folia_pm.xml
  View file @7179e4b
... ... @@ -25,7 +25,7 @@
25 25 <!-- START REFERENCES -->
26 26 <references>
27 27 <reference name="wref" ref="id" />
28   - </references>
  28 + </references>
29 29 <!-- END REFERENCES -->
30 30  
31 31 <!-- START MAPPINGS -->
... ... @@ -284,6 +284,7 @@
284 284 <token type="string" offset="false" realoffset="false" parent="false">
285 285 <pre>
286 286 <item type="name" />
  287 + <item type="ancestorGroupAttribute" name="class" prefix="." />
287 288 <item type="attribute" name="subset" prefix="." />
288 289 </pre>
289 290 <post>
... ...
conf/solr/solrconfig.xml
  View file @7179e4b
... ... @@ -1171,7 +1171,8 @@
1171 1171  
1172 1172 http://wiki.apache.org/solr/TermVectorComponent
1173 1173 -->
1174   - <searchComponent name="tvComponent" class="solr.TermVectorComponent"/>
  1174 + <!-- <searchComponent name="tvComponent" class="solr.TermVectorComponent"/> -->
  1175 + <searchComponent name="tvComponent" class="org.apache.solr.handler.component.TermVectorComponent"/>
1175 1176  
1176 1177 <!-- A request handler for demonstrating the term vector component
1177 1178  
... ... @@ -1347,10 +1348,10 @@
1347 1348 </searchComponent>
1348 1349  
1349 1350 <searchComponent name="mtas" class="mtas.solr.handler.component.MtasSolrSearchComponent">
1350   - <!-- <str name="joinCacheDirectory">${solr.core.instanceDir}/cache/join</str>
1351   - <long name="joinLifetime">86400</long>
1352   - <int name="joinMaximumNumber">1000</int>
1353   - <int name="joinMaximumOverflow">10</int> -->
  1351 + <str name="collectionCacheDirectory">${solr.core.instanceDir}/cache/collection</str>
  1352 + <long name="collectionLifetime">86400</long>
  1353 + <int name="collectionMaximumNumber">1000</int>
  1354 + <int name="collectionMaximumOverflow">10</int>
1354 1355 </searchComponent>
1355 1356  
1356 1357 <!-- Update Processors
... ...
docker/Dockerfile
  View file @7179e4b
1 1 # Automatically generated Dockerfile
2   -# - Build 2017-07-13 14:19
  2 +# - Build 2017-09-06 06:47
3 3 # - Lucene/Solr version 6.6.0
4 4 # - Mtas release 20170713
5 5 #
... ... @@ -74,7 +74,7 @@ RUN service apache2 stop &amp;&amp; \
74 74 chmod -R 755 /var/www/html && \
75 75 printf "echo\n" >> /start.sh && \
76 76 printf "echo \"================ Mtas -- Multi Tier Annotation Search =================\"\n" >> /start.sh && \
77   - printf "echo \" Timestamp 2017-07-13 14:19\"\n" >> /start.sh && \
  77 + printf "echo \" Timestamp 2017-09-06 06:47\"\n" >> /start.sh && \
78 78 printf "echo \" Lucene/Solr version 6.6.0\"\n" >> /start.sh && \
79 79 printf "echo \" Mtas release 20170713\"\n" >> /start.sh && \
80 80 printf "echo \" See https://meertensinstituut.github.io/mtas/ for more information\"\n" >> /start.sh && \
... ...
docker/solrconfig.xml
  View file @7179e4b
... ... @@ -1246,10 +1246,10 @@
1246 1246  
1247 1247 <!-- MTAS: searchComponent -->
1248 1248 <searchComponent name="mtas" class="mtas.solr.handler.component.MtasSolrSearchComponent">
1249   - <!-- <str name="joinCacheDirectory">${solr.core.instanceDir}/cache/join</str>
1250   - <long name="joinLifetime">86400</long>
1251   - <int name="joinMaximumNumber">1000</int>
1252   - <int name="joinMaximumOverflow">10</int> -->
  1249 + <str name="collectionCacheDirectory">${solr.core.instanceDir}/cache/collection</str>
  1250 + <long name="collectionLifetime">86400</long>
  1251 + <int name="collectionMaximumNumber">1000</int>
  1252 + <int name="collectionMaximumOverflow">10</int>
1253 1253 </searchComponent>
1254 1254  
1255 1255  
... ...
junit/data/conf/solrconfig.xml
  View file @7179e4b
... ... @@ -1347,10 +1347,10 @@
1347 1347 </searchComponent>
1348 1348  
1349 1349 <searchComponent name="mtas" class="mtas.solr.handler.component.MtasSolrSearchComponent">
1350   - <!-- <str name="joinCacheDirectory">${solr.core.instanceDir}/cache/join</str>
1351   - <long name="joinLifetime">86400</long>
1352   - <int name="joinMaximumNumber">1000</int>
1353   - <int name="joinMaximumOverflow">10</int> -->
  1350 + <str name="collectionCacheDirectory">${solr.core.instanceDir}/cache/collection</str>
  1351 + <long name="collectionLifetime">86400</long>
  1352 + <int name="collectionMaximumNumber">1000</int>
  1353 + <int name="collectionMaximumOverflow">10</int>
1354 1354 </searchComponent>
1355 1355  
1356 1356 <!-- Update Processors
... ...
junit/mtas/solr/MtasSolrBase.java
  View file @7179e4b
... ... @@ -14,6 +14,7 @@ import java.util.Map.Entry;
14 14  
15 15 import org.apache.commons.logging.Log;
16 16 import org.apache.commons.logging.LogFactory;
  17 +import org.apache.solr.common.SolrDocumentList;
17 18 import org.apache.solr.common.SolrInputDocument;
18 19 import org.apache.solr.common.util.NamedList;
19 20  
... ... @@ -22,6 +23,24 @@ import org.apache.solr.common.util.NamedList;
22 23 */
23 24 public class MtasSolrBase {
24 25  
  26 + /** The field id. */
  27 + public static String FIELD_ID = "id";
  28 +
  29 + /** The field title. */
  30 + public static String FIELD_TITLE = "title";
  31 +
  32 + /** The field text. */
  33 + public static String FIELD_TEXT = "text";
  34 +
  35 + /** The field mtas. */
  36 + public static String FIELD_MTAS = "mtas";
  37 +
  38 + /** The field mtas advanced. */
  39 + public static String FIELD_MTAS_ADVANCED = "mtasAdvanced";
  40 +
  41 + /** The field source. */
  42 + public static String FIELD_SOURCE = "source";
  43 +
25 44 /**
26 45 * Instantiates a new mtas solr base.
27 46 */
... ... @@ -33,6 +52,28 @@ public class MtasSolrBase {
33 52 private static Log log = LogFactory.getLog(MtasSolrBase.class);
34 53  
35 54 /**
  55 + * Gets the num found.
  56 + *
  57 + * @param response the response
  58 + * @return the num found
  59 + */
  60 + public static long getNumFound(NamedList<Object> response) {
  61 + if (response == null) {
  62 + log.error("no (valid); response");
  63 + } else {
  64 + Object mtasResponseRaw = response.get("response");
  65 + if (mtasResponseRaw != null
  66 + && mtasResponseRaw instanceof SolrDocumentList) {
  67 + SolrDocumentList mtasResponse = (SolrDocumentList) mtasResponseRaw;
  68 + return mtasResponse.getNumFound();
  69 + } else {
  70 + log.error("unexpected " + mtasResponseRaw);
  71 + }
  72 + }
  73 + return 0;
  74 + }
  75 +
  76 + /**
36 77 * Gets the from stats.
37 78 *
38 79 * @param response the response
... ... @@ -52,11 +93,11 @@ public class MtasSolrBase {
52 93 Object mtasStatsFieldsRaw = mtasStats.get("stats_fields");
53 94 if (mtasStatsFieldsRaw != null
54 95 && mtasStatsFieldsRaw instanceof NamedList) {
55   - NamedList<Object> mtasStatsFields = (NamedList) mtasStatsFieldsRaw;
  96 + NamedList<Object> mtasStatsFields = (NamedList<Object>) mtasStatsFieldsRaw;
56 97 Object mtasStatsFieldsFieldRaw = mtasStatsFields.get(field);
57 98 if (mtasStatsFieldsFieldRaw != null
58 99 && mtasStatsFieldsFieldRaw instanceof NamedList) {
59   - NamedList<Object> mtasStatsFieldsField = (NamedList) mtasStatsFieldsFieldRaw;
  100 + NamedList<Object> mtasStatsFieldsField = (NamedList<Object>) mtasStatsFieldsFieldRaw;
60 101 Object mtasStatsFieldsFieldNameRaw = mtasStatsFieldsField.get(name);
61 102 if (mtasStatsFieldsFieldNameRaw != null
62 103 && mtasStatsFieldsFieldNameRaw instanceof Number) {
... ... @@ -148,18 +189,19 @@ public class MtasSolrBase {
148 189 * @param key the key
149 190 * @return the from mtas termvector
150 191 */
151   - public static List<NamedList> getFromMtasTermvector(
  192 + public static List<NamedList<Object>> getFromMtasTermvector(
152 193 NamedList<Object> response, String key) {
153 194 if (response == null) {
154 195 log.error("no (valid); response");
155 196 } else {
156 197 Object mtasResponseRaw = response.get("mtas");
157 198 if (mtasResponseRaw != null && mtasResponseRaw instanceof NamedList) {
158   - NamedList<Object> mtasResponse = (NamedList) response.get("mtas");
  199 + NamedList<Object> mtasResponse = (NamedList<Object>) response
  200 + .get("mtas");
159 201 Object mtasTermvectorResponseRaw = mtasResponse.get("termvector");
160 202 if (mtasTermvectorResponseRaw != null
161 203 && mtasTermvectorResponseRaw instanceof List) {
162   - List<NamedList> mtasTermvectorResponse = (List) mtasTermvectorResponseRaw;
  204 + List<NamedList<Object>> mtasTermvectorResponse = (List) mtasTermvectorResponseRaw;
163 205 if (mtasTermvectorResponse.isEmpty()) {
164 206 log.error("no (valid) mtas termvector response");
165 207 } else {
... ... @@ -173,9 +215,9 @@ public class MtasSolrBase {
173 215 }
174 216 }
175 217 assertFalse("no item with key " + key, item == null);
176   - if (item.get("list") != null
  218 + if (item != null && item.get("list") != null
177 219 && (item.get("list") instanceof List)) {
178   - return (List<NamedList>) item.get("list");
  220 + return (List<NamedList<Object>>) item.get("list");
179 221 }
180 222 }
181 223 } else {
... ... @@ -221,14 +263,16 @@ public class MtasSolrBase {
221 263 }
222 264 assertFalse("no item with key " + key, item == null);
223 265 Map<String, List<String>> result = new HashMap<>();
224   - Iterator<Entry<String, Object>> it = item.iterator();
225   - Entry<String, Object> entry;
226   - while (it.hasNext()) {
227   - entry = it.next();
228   - if (!entry.getKey().equals("key")) {
229   - assertTrue("invalid entry prefix",
230   - entry.getValue() instanceof List);
231   - result.put(entry.getKey(), (List) entry.getValue());
  266 + if (item != null) {
  267 + Iterator<Entry<String, Object>> it = item.iterator();
  268 + Entry<String, Object> entry;
  269 + while (it.hasNext()) {
  270 + entry = it.next();
  271 + if (!entry.getKey().equals("key")) {
  272 + assertTrue("invalid entry prefix",
  273 + entry.getValue() instanceof List);
  274 + result.put(entry.getKey(), (List) entry.getValue());
  275 + }
232 276 }
233 277 }
234 278 return result;
... ... @@ -240,6 +284,78 @@ public class MtasSolrBase {
240 284 }
241 285  
242 286 /**
  287 + * Gets the from mtas collection.
  288 + *
  289 + * @param response the response
  290 + * @param key the key
  291 + * @return the from mtas collection
  292 + */
  293 + public static NamedList<Object> getFromMtasCollection(
  294 + NamedList<Object> response, String key) {
  295 + if (response == null) {
  296 + log.error("no (valid); response");
  297 + } else {
  298 + Object mtasResponseRaw = response.get("mtas");
  299 + if (mtasResponseRaw != null && mtasResponseRaw instanceof NamedList) {
  300 + NamedList<Object> mtasResponse = (NamedList<Object>) response
  301 + .get("mtas");
  302 + Object mtasCollectionResponseRaw = mtasResponse.get("collection");
  303 + if (mtasCollectionResponseRaw != null
  304 + && mtasCollectionResponseRaw instanceof List) {
  305 + List<NamedList<Object>> mtasCollectionResponse = (List<NamedList<Object>>) mtasCollectionResponseRaw;
  306 + if (mtasCollectionResponse.isEmpty()) {
  307 + log.error("no (valid) mtas join response");
  308 + } else {
  309 + for (NamedList<Object> mtasCollectionResponseItem : mtasCollectionResponse) {
  310 + if (mtasCollectionResponseItem.get("key") != null
  311 + && (mtasCollectionResponseItem.get("key") instanceof String)
  312 + && mtasCollectionResponseItem.get("key").equals(key)) {
  313 + return mtasCollectionResponseItem;
  314 + }
  315 + }
  316 + }
  317 + } else {
  318 + log.error("unexpected " + mtasCollectionResponseRaw);
  319 + }
  320 + } else {
  321 + log.error("unexpected " + mtasResponseRaw);
  322 + }
  323 + }
  324 + return null;
  325 + }
  326 +
  327 + /**
  328 + * Gets the from mtas collection list.
  329 + *
  330 + * @param response the response
  331 + * @param key the key
  332 + * @param id the id
  333 + * @return the from mtas collection list
  334 + */
  335 + public static NamedList<Object> getFromMtasCollectionList(
  336 + NamedList<Object> response, String key, String id) {
  337 + NamedList<Object> collectionResponse = getFromMtasCollection(response, key);
  338 + if (collectionResponse != null) {
  339 + Object collectionResponseListRaw = collectionResponse.get("list");
  340 + if (collectionResponseListRaw != null && collectionResponseListRaw instanceof List) {
  341 + List<NamedList<Object>> collectionResponseList = (List<NamedList<Object>>) collectionResponseListRaw;
  342 + for (NamedList<Object> item : collectionResponseList) {
  343 + if (item.get("id") != null && item.get("id") instanceof String) {
  344 + if (id.equals((String) item.get("id"))) {
  345 + return item;
  346 + }
  347 + }
  348 + }
  349 + } else {
  350 + log.error("unexpected " + collectionResponseListRaw + " (searching list)");
  351 + }
  352 + } else {
  353 + log.error("no collectionResponse (searching key " + key + ")");
  354 + }
  355 + return null;
  356 + }
  357 +
  358 + /**
243 359 * Delete directory.
244 360 *
245 361 * @param directory the directory
... ... @@ -273,38 +389,38 @@ public class MtasSolrBase {
273 389 Path dataPath = Paths.get("junit").resolve("data");
274 390 // data
275 391 SolrInputDocument newDoc1 = new SolrInputDocument();
276   - newDoc1.addField("id", "1");
277   - newDoc1.addField("title", "Een onaangenaam mens in de Haarlemmerhout");
278   - newDoc1.addField("text", "Een onaangenaam mens in de Haarlemmerhout");
279   - newDoc1.addField("mtas", dataPath.resolve("resources")
  392 + newDoc1.addField(FIELD_ID, "1");
  393 + newDoc1.addField(FIELD_TITLE, "Een onaangenaam mens in de Haarlemmerhout");
  394 + newDoc1.addField(FIELD_TEXT, "Een onaangenaam mens in de Haarlemmerhout");
  395 + newDoc1.addField(FIELD_MTAS, dataPath.resolve("resources")
280 396 .resolve("beets1.xml.gz").toFile().getAbsolutePath());
281 397 if (includeAdvanced) {
282   - newDoc1.addField("source", "source1");
283   - newDoc1.addField("mtasAdvanced", dataPath.resolve("resources")
  398 + newDoc1.addField(FIELD_SOURCE, "source1");
  399 + newDoc1.addField(FIELD_MTAS_ADVANCED, dataPath.resolve("resources")
284 400 .resolve("beets1").toFile().getAbsolutePath());
285 401 }
286 402 solrDocuments.put(1, newDoc1);
287 403 SolrInputDocument newDoc2 = new SolrInputDocument();
288   - newDoc2.addField("id", "2");
289   - newDoc2.addField("title", "Een oude kennis");
290   - newDoc2.addField("text", "Een oude kennis");
291   - newDoc2.addField("mtas", dataPath.resolve("resources")
  404 + newDoc2.addField(FIELD_ID, "2");
  405 + newDoc2.addField(FIELD_TITLE, "Een oude kennis");
  406 + newDoc2.addField(FIELD_TEXT, "Een oude kennis");
  407 + newDoc2.addField(FIELD_MTAS, dataPath.resolve("resources")
292 408 .resolve("beets2.xml.gz").toFile().getAbsolutePath());
293 409 if (includeAdvanced) {
294   - newDoc2.addField("source", "source2");
295   - newDoc2.addField("mtasAdvanced", dataPath.resolve("resources")
  410 + newDoc2.addField(FIELD_SOURCE, "source2");
  411 + newDoc2.addField(FIELD_MTAS_ADVANCED, dataPath.resolve("resources")
296 412 .resolve("beets2.xml").toFile().getAbsolutePath());
297 413 }
298 414 SolrInputDocument newDoc3 = new SolrInputDocument();
299 415 solrDocuments.put(2, newDoc2);
300   - newDoc3.addField("id", "3");
301   - newDoc3.addField("title", "Varen en Rijden");
302   - newDoc3.addField("text", "Varen en Rijden");
303   - newDoc3.addField("mtas", dataPath.resolve("resources")
  416 + newDoc3.addField(FIELD_ID, "3");
  417 + newDoc3.addField(FIELD_TITLE, "Varen en Rijden");
  418 + newDoc3.addField(FIELD_TEXT, "Varen en Rijden");
  419 + newDoc3.addField(FIELD_MTAS, dataPath.resolve("resources")
304 420 .resolve("beets3.xml.gz").toFile().getAbsolutePath());
305 421 if (includeAdvanced) {
306   - newDoc3.addField("source", "source3");
307   - newDoc3.addField("mtasAdvanced", dataPath.resolve("resources")
  422 + newDoc3.addField(FIELD_SOURCE, "source3");
  423 + newDoc3.addField(FIELD_MTAS_ADVANCED, dataPath.resolve("resources")
308 424 .resolve("beets3.xml.gz").toFile().getAbsolutePath());
309 425 }
310 426 solrDocuments.put(3, newDoc3);
... ...
junit/mtas/solr/MtasSolrTestDistributedSearchConsistency.java
  View file @7179e4b
... ... @@ -225,7 +225,7 @@ public class MtasSolrTestDistributedSearchConsistency {
225 225 list.get(COLLECTION_DISTRIBUTED).getResponse(), "tv",
226 226 new String[] { "n", "sum" });
227 227 for (Entry<String, QueryResponse> entry : list.entrySet()) {
228   - List<NamedList> tv = MtasSolrBase
  228 + List<NamedList<Object>> tv = MtasSolrBase
229 229 .getFromMtasTermvector(entry.getValue().getResponse(), "tv");
230 230 for (NamedList<Object> item : tv) {
231 231 String key = item.get("key").toString();
... ... @@ -311,6 +311,310 @@ public class MtasSolrTestDistributedSearchConsistency {
311 311 }
312 312  
313 313 /**
  314 + * Mtas request handler collection 1.
  315 + *
  316 + * @throws IOException Signals that an I/O exception has occurred.
  317 + */
  318 + @org.junit.Test
  319 + public void mtasRequestHandlerCollection1() throws IOException {
  320 + String[] collections = new String[] { COLLECTION_ALL_OPTIMIZED,
  321 + COLLECTION_ALL_MULTIPLE_SEGMENTS, COLLECTION_DISTRIBUTED };
  322 + String[] collectionsParts = new String[] { COLLECTION_PART1_OPTIMIZED,
  323 + COLLECTION_PART2_MULTIPLE_SEGMENTS };
  324 + Map<String, String> listCreateVersion = new HashMap<>();
  325 + Map<String, Number> listCreateSize = new HashMap<>();
  326 + Map<String, String> listPostVersion = new HashMap<>();
  327 + Map<String, Number> listPostSize = new HashMap<>();
  328 + // create
  329 + ModifiableSolrParams paramsCreate = new ModifiableSolrParams();
  330 + paramsCreate.set("q", "*:*");
  331 + paramsCreate.set("rows", "0");
  332 + paramsCreate.set("mtas", "true");
  333 + paramsCreate.set("mtas.collection", "true");
  334 + paramsCreate.set("mtas.collection.0.key", "create");
  335 + paramsCreate.set("mtas.collection.0.action", "create");
  336 + paramsCreate.set("mtas.collection.0.id", "idCreate");
  337 + paramsCreate.set("mtas.collection.0.field", MtasSolrBase.FIELD_ID);
  338 + Map<String, QueryResponse> listCreate = createResults(paramsCreate,
  339 + Arrays.asList(collections));
  340 + for (Entry<String, QueryResponse> entry : listCreate.entrySet()) {
  341 + long size = MtasSolrBase.getNumFound(entry.getValue().getResponse());
  342 + NamedList<Object> create = MtasSolrBase
  343 + .getFromMtasCollection(entry.getValue().getResponse(), "create");
  344 + createCollectionAssertions(create, entry.getKey(), "idCreate", null, size,
  345 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  346 + listCreateVersion.put(entry.getKey(), (String) create.get("version"));
  347 + listCreateSize.put(entry.getKey(), (Number) create.get("size"));
  348 + }
  349 + // post
  350 + ModifiableSolrParams paramsPost = new ModifiableSolrParams();
  351 + paramsPost.set("q", "*:*");
  352 + paramsPost.set("rows", "0");
  353 + paramsPost.set("mtas", "true");
  354 + paramsPost.set("mtas.collection", "true");
  355 + paramsPost.set("mtas.collection.0.key", "post");
  356 + paramsPost.set("mtas.collection.0.action", "post");
  357 + paramsPost.set("mtas.collection.0.id", "idPost");
  358 + paramsPost.set("mtas.collection.0.post", "[1,2,3,4]");
  359 + Map<String, QueryResponse> listPost = createResults(paramsPost,
  360 + Arrays.asList(collections));
  361 + for (Entry<String, QueryResponse> entry : listPost.entrySet()) {
  362 + long size = 4;
  363 + NamedList<Object> post = MtasSolrBase
  364 + .getFromMtasCollection(entry.getValue().getResponse(), "post");
  365 + createCollectionAssertions(post, entry.getKey(), "idPost", null, size,
  366 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  367 + listPostVersion.put(entry.getKey(), (String) post.get("version"));
  368 + listPostSize.put(entry.getKey(), (Number) post.get("size"));
  369 + }
  370 + // list
  371 + ModifiableSolrParams paramsList = new ModifiableSolrParams();
  372 + paramsList.set("q", "*:*");
  373 + paramsList.set("rows", "0");
  374 + paramsList.set("mtas", "true");
  375 + paramsList.set("mtas.collection", "true");
  376 + paramsList.set("mtas.collection.0.key", "list");
  377 + paramsList.set("mtas.collection.0.action", "list");
  378 + Map<String, QueryResponse> listList = createResults(paramsList,
  379 + Arrays.asList(collections));
  380 + for (Entry<String, QueryResponse> entry : listList.entrySet()) {
  381 + // check create
  382 + NamedList<Object> listCreateItem1 = MtasSolrBase
  383 + .getFromMtasCollectionList(entry.getValue().getResponse(), "list",
  384 + "idCreate");
  385 + createCollectionAssertions(listCreateItem1, entry.getKey(), "idCreate",
  386 + listCreateVersion.get(entry.getKey()),
  387 + listCreateSize.get(entry.getKey()),
  388 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  389 + // check post
  390 + NamedList<Object> listPostItem1 = MtasSolrBase.getFromMtasCollectionList(
  391 + entry.getValue().getResponse(), "list", "idPost");
  392 + createCollectionAssertions(listPostItem1, entry.getKey(), "idPost",
  393 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  394 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  395 + }
  396 + // check
  397 + ModifiableSolrParams paramsCheck = new ModifiableSolrParams();
  398 + paramsCheck.set("q", "*:*");
  399 + paramsCheck.set("rows", "0");
  400 + paramsCheck.set("mtas", "true");
  401 + paramsCheck.set("mtas.collection", "true");
  402 + paramsCheck.set("mtas.collection.0.key", "check1");
  403 + paramsCheck.set("mtas.collection.0.action", "check");
  404 + paramsCheck.set("mtas.collection.0.id", "idCreate");
  405 + paramsCheck.set("mtas.collection.1.key", "check2");
  406 + paramsCheck.set("mtas.collection.1.action", "check");
  407 + paramsCheck.set("mtas.collection.1.id", "idPost");
  408 + // check on all
  409 + Map<String, QueryResponse> listCheck = createResults(paramsCheck,
  410 + Arrays.asList(collections));
  411 + for (Entry<String, QueryResponse> entry : listCheck.entrySet()) {
  412 + NamedList<Object> listItemCheck1 = MtasSolrBase
  413 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  414 + createCollectionAssertions(listItemCheck1, entry.getKey(), "idCreate",
  415 + listCreateVersion.get(entry.getKey()),
  416 + listCreateSize.get(entry.getKey()),
  417 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  418 + NamedList<Object> listItemCheck2 = MtasSolrBase
  419 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  420 + createCollectionAssertions(listItemCheck2, entry.getKey(), "idPost",
  421 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  422 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  423 + }
  424 + // check on parts
  425 + createResults(paramsCheck, Arrays.asList(collectionsParts));
  426 + for (Entry<String, QueryResponse> entry : listCheck.entrySet()) {
  427 + NamedList<Object> listItemCheck1 = MtasSolrBase
  428 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  429 + createCollectionAssertions(listItemCheck1, entry.getKey(), "idCreate",
  430 + listCreateVersion.get(entry.getKey()),
  431 + listCreateSize.get(entry.getKey()),
  432 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  433 + NamedList<Object> listItemCheck2 = MtasSolrBase
  434 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  435 + createCollectionAssertions(listItemCheck2, entry.getKey(), "idPost",
  436 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  437 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  438 + }
  439 + // delete
  440 + ModifiableSolrParams paramsDelete = new ModifiableSolrParams();
  441 + paramsDelete.set("q", "*:*");
  442 + paramsDelete.set("rows", "0");
  443 + paramsDelete.set("mtas", "true");
  444 + paramsDelete.set("mtas.collection", "true");
  445 + paramsDelete.set("mtas.collection.0.key", "delete1");
  446 + paramsDelete.set("mtas.collection.0.action", "delete");
  447 + paramsDelete.set("mtas.collection.0.id", "idCreate");
  448 + paramsDelete.set("mtas.collection.1.key", "delete2");
  449 + paramsDelete.set("mtas.collection.1.action", "delete");
  450 + paramsDelete.set("mtas.collection.1.id", "idPost");
  451 + // delete on parts
  452 + createResults(paramsDelete, Arrays.asList(collectionsParts));
  453 + // recheck on parts
  454 + Map<String, QueryResponse> listCheckParts = createResults(paramsCheck,
  455 + Arrays.asList(collectionsParts));
  456 + for (Entry<String, QueryResponse> entry : listCheckParts.entrySet()) {
  457 + NamedList<Object> listItemCheck1 = MtasSolrBase
  458 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  459 + assertTrue(
  460 + entry.getKey() + " - create - should be removed: " + listItemCheck1,
  461 + listItemCheck1 != null && listItemCheck1.get("id") == null);
  462 + NamedList<Object> listItemCheck2 = MtasSolrBase
  463 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  464 + assertTrue(
  465 + entry.getKey() + " - post - should be removed: " + listItemCheck2,
  466 + listItemCheck2 != null && listItemCheck2.get("id") == null);
  467 + }
  468 + // list with empty parts
  469 + listList = createResults(paramsList, Arrays.asList(collections));
  470 + for (Entry<String, QueryResponse> entry : listList.entrySet()) {
  471 + // check create
  472 + NamedList<Object> listCreateItem1 = MtasSolrBase
  473 + .getFromMtasCollectionList(entry.getValue().getResponse(), "list",
  474 + "idCreate");
  475 + createCollectionAssertions(listCreateItem1, entry.getKey(), "idCreate",
  476 + listCreateVersion.get(entry.getKey()),
  477 + listCreateSize.get(entry.getKey()), 0);
  478 + // check post
  479 + NamedList<Object> listPostItem1 = MtasSolrBase.getFromMtasCollectionList(
  480 + entry.getValue().getResponse(), "list", "idPost");
  481 + createCollectionAssertions(listPostItem1, entry.getKey(), "idPost",
  482 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  483 + 0);
  484 + }
  485 + // recheck on all, assuming empty parts, autofix
  486 + listCheck = createResults(paramsCheck, Arrays.asList(collections));
  487 + for (Entry<String, QueryResponse> entry : listCheck.entrySet()) {
  488 + NamedList<Object> listItemCheck1 = MtasSolrBase
  489 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  490 + createCollectionAssertions(listItemCheck1, entry.getKey(), "idCreate",
  491 + listCreateVersion.get(entry.getKey()),
  492 + listCreateSize.get(entry.getKey()),
  493 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  494 + NamedList<Object> listItemCheck2 = MtasSolrBase
  495 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  496 + createCollectionAssertions(listItemCheck2, entry.getKey(), "idPost",
  497 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  498 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  499 + }
  500 + // recheck on parts
  501 + listCheckParts = createResults(paramsCheck,
  502 + Arrays.asList(collectionsParts));
  503 + for (Entry<String, QueryResponse> entry : listCheck.entrySet()) {
  504 + NamedList<Object> listItemCheck1 = MtasSolrBase
  505 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  506 + createCollectionAssertions(listItemCheck1, entry.getKey(), "idCreate",
  507 + listCreateVersion.get(entry.getKey()),
  508 + listCreateSize.get(entry.getKey()),
  509 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  510 + NamedList<Object> listItemCheck2 = MtasSolrBase
  511 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  512 + createCollectionAssertions(listItemCheck2, entry.getKey(), "idPost",
  513 + listPostVersion.get(entry.getKey()), listPostSize.get(entry.getKey()),
  514 + entry.getKey().equals(COLLECTION_DISTRIBUTED) ? 2 : 0);
  515 + }
  516 + // full delete
  517 + createResults(paramsDelete, Arrays.asList(collections));
  518 + // final check
  519 + listCheck = createResults(paramsCheck, Arrays.asList(collections));
  520 + for (Entry<String, QueryResponse> entry : listCheck.entrySet()) {
  521 + NamedList<Object> listItemCheck1 = MtasSolrBase
  522 + .getFromMtasCollection(entry.getValue().getResponse(), "check1");
  523 + assertTrue(
  524 + entry.getKey() + " - create - should be removed: " + listItemCheck1,
  525 + listItemCheck1 != null && listItemCheck1.get("id") == null);
  526 + NamedList<Object> listItemCheck2 = MtasSolrBase
  527 + .getFromMtasCollection(entry.getValue().getResponse(), "check2");
  528 + assertTrue(
  529 + entry.getKey() + " - post - should be removed: " + listItemCheck2,
  530 + listItemCheck2 != null && listItemCheck2.get("id") == null);
  531 + }
  532 + }
  533 +
  534 + /**
  535 + * Mtas request handler collection 2.
  536 + *
  537 + * @throws IOException Signals that an I/O exception has occurred.
  538 + */
  539 + @org.junit.Test
  540 + public void mtasRequestHandlerCollection2() throws IOException {
  541 + String[] collections = new String[] { COLLECTION_ALL_OPTIMIZED,
  542 + COLLECTION_ALL_MULTIPLE_SEGMENTS, COLLECTION_DISTRIBUTED };
  543 + // post
  544 + ModifiableSolrParams paramsPost = new ModifiableSolrParams();
  545 + paramsPost.set("q", "*:*");
  546 + paramsPost.set("mtas", "true");
  547 + paramsPost.set("mtas.collection", "true");
  548 + paramsPost.set("mtas.collection.0.key", "postKey1");
  549 + paramsPost.set("mtas.collection.0.action", "post");
  550 + paramsPost.set("mtas.collection.0.id", "postSet1");
  551 + paramsPost.set("mtas.collection.0.post", "[1,3,4]");
  552 + paramsPost.set("mtas.collection.1.key", "postKey2");
  553 + paramsPost.set("mtas.collection.1.action", "post");
  554 + paramsPost.set("mtas.collection.1.id", "postSet2");
  555 + paramsPost.set("mtas.collection.1.post", "[2]");
  556 + paramsPost.set("mtas.collection.2.key", "createKey1");
  557 + paramsPost.set("mtas.collection.2.action", "create");
  558 + paramsPost.set("mtas.collection.2.id", "createSet1");
  559 + paramsPost.set("mtas.collection.2.field", MtasSolrBase.FIELD_ID);
  560 + createResults(paramsPost, Arrays.asList(collections));
  561 + // query set1
  562 + ModifiableSolrParams paramsSelect1 = new ModifiableSolrParams();
  563 + paramsSelect1.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  564 + + "\" collection=\"postSet1\"}");
  565 + paramsSelect1.set("rows", "0");
  566 + Map<String, QueryResponse> listPost1 = createResults(paramsSelect1,
  567 + Arrays.asList(collections));
  568 + for (Entry<String, QueryResponse> entry : listPost1.entrySet()) {
  569 + long n = MtasSolrBase.getNumFound(entry.getValue().getResponse());
  570 + assertTrue(
  571 + entry.getKey() + " - incorrect number of matching documents : " + n,
  572 + n == 2);
  573 + }
  574 + // query set2
  575 + ModifiableSolrParams paramsSelect2 = new ModifiableSolrParams();
  576 + paramsSelect2.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  577 + + "\" collection=\"postSet2\"}");
  578 + paramsSelect2.set("rows", "0");
  579 + Map<String, QueryResponse> listPost2 = createResults(paramsSelect2,
  580 + Arrays.asList(collections));
  581 + for (Entry<String, QueryResponse> entry : listPost2.entrySet()) {
  582 + long n = MtasSolrBase.getNumFound(entry.getValue().getResponse());
  583 + assertTrue(
  584 + entry.getKey() + " - incorrect number of matching documents : " + n,
  585 + n == 1);
  586 + }
  587 + // query set3
  588 + ModifiableSolrParams paramsSelect3 = new ModifiableSolrParams();
  589 + paramsSelect3.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  590 + + "\" collection=\"createSet1\"}");
  591 + paramsSelect3.set("rows", "0");
  592 + Map<String, QueryResponse> listPost3 = createResults(paramsSelect3,
  593 + Arrays.asList(collections));
  594 + for (Entry<String, QueryResponse> entry : listPost3.entrySet()) {
  595 + long n = MtasSolrBase.getNumFound(entry.getValue().getResponse());
  596 + assertTrue(
  597 + entry.getKey() + " - incorrect number of matching documents : " + n,
  598 + n == 3);
  599 + }
  600 + // query set1 or set2
  601 + ModifiableSolrParams paramsSelect4 = new ModifiableSolrParams();
  602 + paramsSelect4.set("q",
  603 + "({!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  604 + + "\" collection=\"postSet1\"}) OR ({!mtas_join field=\""
  605 + + MtasSolrBase.FIELD_ID + "\" collection=\"postSet2\"})");
  606 + paramsSelect4.set("rows", "0");
  607 + Map<String, QueryResponse> listPost4 = createResults(paramsSelect4,
  608 + Arrays.asList(collections));
  609 + for (Entry<String, QueryResponse> entry : listPost4.entrySet()) {
  610 + long n = MtasSolrBase.getNumFound(entry.getValue().getResponse());
  611 + assertTrue(
  612 + entry.getKey() + " - incorrect number of matching documents : " + n,
  613 + n == 3);
  614 + }
  615 + }
  616 +
  617 + /**
314 618 * Mtas request handler prefix.
315 619 *
316 620 * @throws IOException Signals that an I/O exception has occurred.
... ... @@ -336,12 +640,16 @@ public class MtasSolrTestDistributedSearchConsistency {
336 640 /**
337 641 * Creates the results.
338 642 *
339   - * @param params the params
  643 + * @param initialParams the initial params
340 644 * @param collections the collections
341 645 * @return the hash map
342 646 */
343 647 private static HashMap<String, QueryResponse> createResults(
344   - final ModifiableSolrParams params, List<String> collections) {
  648 + final ModifiableSolrParams initialParams, List<String> collections) {
  649 + // use initial params
  650 + ModifiableSolrParams params = new ModifiableSolrParams();
  651 + params.add(initialParams);
  652 + // continue
345 653 HashMap<String, QueryResponse> list = new HashMap<>();
346 654 CloudSolrClient client = cloudCluster.getSolrClient();
347 655 try {
... ... @@ -450,8 +758,10 @@ public class MtasSolrTestDistributedSearchConsistency {
450 758 */
451 759 private static void createTermvectorAssertions(NamedList<Object> response1,
452 760 NamedList<Object> response2, String key, String[] names) {
453   - List<NamedList> list1 = MtasSolrBase.getFromMtasTermvector(response1, key);
454   - List<NamedList> list2 = MtasSolrBase.getFromMtasTermvector(response2, key);
  761 + List<NamedList<Object>> list1 = MtasSolrBase
  762 + .getFromMtasTermvector(response1, key);
  763 + List<NamedList<Object>> list2 = MtasSolrBase
  764 + .getFromMtasTermvector(response2, key);
455 765 assertFalse("list should be defined", list1 == null || list2 == null);
456 766 if (list1 != null && list2 != null) {
457 767 assertEquals("lists should have equal size", list1.size(), list2.size());
... ... @@ -479,6 +789,57 @@ public class MtasSolrTestDistributedSearchConsistency {
479 789 }
480 790  
481 791 /**
  792 + * Creates the collection assertions.
  793 + *
  794 + * @param create the create
  795 + * @param collection the collection
  796 + * @param id the id
  797 + * @param version the version
  798 + * @param size the size
  799 + * @param shards the shards
  800 + */
  801 + private static void createCollectionAssertions(NamedList<Object> create,
  802 + String collection, String id, String version, Number size, int shards) {
  803 + assertFalse(collection + ": create - not found", create == null);
  804 + assertTrue(collection + ": create - no valid version",
  805 + create.get("id") != null && create.get("id") instanceof String);
  806 + assertTrue(collection + ": create - id incorrect, '" + id
  807 + + "' not equal to '" + create.get("id") + "'",
  808 + ((String) create.get("id")).equals(id));
  809 + assertTrue(
  810 + collection + ": create - no valid version, '" + version
  811 + + "' not equal to '" + create.get("version") + "'",
  812 + create.get("version") != null
  813 + && create.get("version") instanceof String);
  814 + if (version != null) {
  815 + assertTrue(collection + ": create - version incorrect",
  816 + ((String) create.get("version")).equals(version));
  817 + }
  818 + assertTrue(collection + ": create - no valid size",
  819 + create.get("size") != null && create.get("size") instanceof Number);
  820 + Number createSize = (Number) create.get("size");
  821 + assertEquals(collection + ": number of values", size.longValue(),
  822 + createSize.longValue());
  823 + if (shards > 0) {
  824 + assertTrue("no (valid) shards",
  825 + create.get("shards") != null && create.get("shards") instanceof List
  826 + && ((List) create.get("shards")).size() == shards);
  827 + for (Object shardItem : (List<Object>) create.get("shards")) {
  828 + assertTrue(collection + ": invalid shardItem",
  829 + shardItem instanceof NamedList);
  830 + Object sizeRaw = ((NamedList<Object>) shardItem).get("size");
  831 + assertTrue(collection + ": incorrect size",
  832 + sizeRaw != null && sizeRaw instanceof Number
  833 + && ((Number) sizeRaw).longValue() == createSize.longValue());
  834 + }
  835 + } else {
  836 + assertFalse(collection + ": shards found : " + create.get("shards"),
  837 + create.get("shards") != null && create.get("shards") instanceof List
  838 + && !((List) create.get("shards")).isEmpty());
  839 + }
  840 + }
  841 +
  842 + /**
482 843 * Creates the cloud.
483 844 */
484 845 private static void createCloud() {
... ...
junit/mtas/solr/MtasSolrTestSearchConsistency.java
  View file @7179e4b
... ... @@ -221,13 +221,13 @@ public class MtasSolrTestSearchConsistency {
221 221 params.set("mtas", "true");
222 222 params.set("mtas.stats", "true");
223 223 params.set("mtas.stats.spans", "true");
224   - params.set("mtas.stats.spans.0.field", "mtas");
  224 + params.set("mtas.stats.spans.0.field", MtasSolrBase.FIELD_MTAS);
225 225 params.set("mtas.stats.spans.0.key", "statsKey");
226 226 params.set("mtas.stats.spans.0.query.0.type", "cql");
227 227 params.set("mtas.stats.spans.0.query.0.value", "[]");
228 228 params.set("mtas.stats.spans.0.type", "n,sum,mean");
229 229 params.set("mtas.stats.positions", "true");
230   - params.set("mtas.stats.positions.0.field", "mtas");
  230 + params.set("mtas.stats.positions.0.field", MtasSolrBase.FIELD_MTAS);
231 231 params.set("mtas.stats.positions.0.key", "statsKey");
232 232 params.set("mtas.stats.positions.0.type", "n,sum,mean");
233 233 params.set("rows", "0");
... ... @@ -264,7 +264,7 @@ public class MtasSolrTestSearchConsistency {
264 264 params.set("mtas", "true");
265 265 params.set("mtas.stats", "true");
266 266 params.set("mtas.stats.tokens", "true");
267   - params.set("mtas.stats.tokens.0.field", "mtas");
  267 + params.set("mtas.stats.tokens.0.field", MtasSolrBase.FIELD_MTAS);
268 268 params.set("mtas.stats.tokens.0.key", "statsKey");
269 269 params.set("mtas.stats.tokens.0.type", String.join(",", types));
270 270 params.set("mtas.stats.tokens.0.minimum", 1);
... ... @@ -293,7 +293,7 @@ public class MtasSolrTestSearchConsistency {
293 293 params.set("rows", 0);
294 294 params.set("mtas", "true");
295 295 params.set("mtas.termvector", "true");
296   - params.set("mtas.termvector.0.field", "mtas");
  296 + params.set("mtas.termvector.0.field", MtasSolrBase.FIELD_MTAS);
297 297 params.set("mtas.termvector.0.prefix", "t_lc");
298 298 params.set("mtas.termvector.0.key", "tv");
299 299 params.set("mtas.termvector.0.sort.type", "sum");
... ... @@ -342,7 +342,7 @@ public class MtasSolrTestSearchConsistency {
342 342 params.set("rows", 0);
343 343 params.set("mtas", "true");
344 344 params.set("mtas.termvector", "true");
345   - params.set("mtas.termvector.0.field", "mtas");
  345 + params.set("mtas.termvector.0.field", MtasSolrBase.FIELD_MTAS);
346 346 params.set("mtas.termvector.0.prefix", "t_lc");
347 347 params.set("mtas.termvector.0.key", "tv");
348 348 params.set("mtas.termvector.0.type", String.join(",", types));
... ... @@ -355,7 +355,8 @@ public class MtasSolrTestSearchConsistency {
355 355 } catch (SolrServerException e) {
356 356 throw new IOException(e);
357 357 }
358   - List<NamedList> tv = MtasSolrBase.getFromMtasTermvector(response, "tv");
  358 + List<NamedList<Object>> tv = MtasSolrBase.getFromMtasTermvector(response,
  359 + "tv");
359 360 for (String key : list) {
360 361 params.clear();
361 362 params.set("q", "*:*");
... ... @@ -363,13 +364,13 @@ public class MtasSolrTestSearchConsistency {
363 364 params.set("mtas", "true");
364 365 params.set("mtas.stats", "true");
365 366 params.set("mtas.stats.spans", "true");
366   - params.set("mtas.stats.spans.0.field", "mtas");
  367 + params.set("mtas.stats.spans.0.field", MtasSolrBase.FIELD_MTAS);
367 368 params.set("mtas.stats.spans.0.key", "statsKey0");
368 369 params.set("mtas.stats.spans.0.minimum", 1);
369 370 params.set("mtas.stats.spans.0.query.0.type", "cql");
370 371 params.set("mtas.stats.spans.0.query.0.value", "[t_lc=\"" + key + "\"]");
371 372 params.set("mtas.stats.spans.0.type", String.join(",", types));
372   - params.set("mtas.stats.spans.1.field", "mtas");
  373 + params.set("mtas.stats.spans.1.field", MtasSolrBase.FIELD_MTAS);
373 374 params.set("mtas.stats.spans.1.key", "statsKey1");
374 375 params.set("mtas.stats.spans.1.minimum", 0);
375 376 params.set("mtas.stats.spans.1.query.0.type", "cql");
... ... @@ -419,7 +420,7 @@ public class MtasSolrTestSearchConsistency {
419 420 params.set("rows", 0);
420 421 params.set("mtas", "true");
421 422 params.set("mtas.termvector", "true");
422   - params.set("mtas.termvector.0.field", "mtas");
  423 + params.set("mtas.termvector.0.field", MtasSolrBase.FIELD_MTAS);
423 424 params.set("mtas.termvector.0.prefix", "t_lc");
424 425 params.set("mtas.termvector.0.key", "tv");
425 426 params.set("mtas.termvector.0.regexp", "een[a-z]*");
... ... @@ -435,7 +436,8 @@ public class MtasSolrTestSearchConsistency {
435 436 } catch (SolrServerException e) {
436 437 throw new IOException(e);
437 438 }
438   - List<NamedList> tv = MtasSolrBase.getFromMtasTermvector(response, "tv");
  439 + List<NamedList<Object>> tv = MtasSolrBase.getFromMtasTermvector(response,
  440 + "tv");
439 441 Set<String> keys = new HashSet<>();
440 442 for (NamedList<Object> item : tv) {
441 443 if (item != null && item.get("key") != null
... ... @@ -455,6 +457,350 @@ public class MtasSolrTestSearchConsistency {
455 457 }
456 458  
457 459 /**
  460 + * Mtas request handler collection 1.
  461 + *
  462 + * @throws IOException Signals that an I/O exception has occurred.
  463 + */
  464 + @org.junit.Test
  465 + public void mtasRequestHandlerCollection1() throws IOException {
  466 + // create
  467 + ModifiableSolrParams paramsCreate = new ModifiableSolrParams();
  468 + paramsCreate.set("q", "*:*");
  469 + paramsCreate.set("mtas", "true");
  470 + paramsCreate.set("mtas.collection", "true");
  471 + paramsCreate.set("mtas.collection.0.key", "create");
  472 + paramsCreate.set("mtas.collection.0.action", "create");
  473 + paramsCreate.set("mtas.collection.0.id", "idCreate");
  474 + paramsCreate.set("mtas.collection.0.field", "id");
  475 + SolrRequest<?> requestCreate = new QueryRequest(paramsCreate, METHOD.POST);
  476 + NamedList<Object> responseCreate;
  477 + try {
  478 + responseCreate = server.request(requestCreate, "collection1");
  479 + } catch (SolrServerException e) {
  480 + throw new IOException(e);
  481 + }
  482 + long n = MtasSolrBase.getNumFound(responseCreate);
  483 + NamedList<Object> create = MtasSolrBase
  484 + .getFromMtasCollection(responseCreate, "create");
  485 + assertFalse("create - id not found", create == null);
  486 + assertTrue("create - no valid version", create.get("version") != null
  487 + && create.get("version") instanceof String);
  488 + assertTrue("create - no valid size",
  489 + create.get("size") != null && create.get("size") instanceof Number);
  490 + String createVersion = (String) create.get("version");
  491 + Number createSize = (Number) create.get("size");
  492 + assertEquals("number of values", n, createSize.longValue());
  493 + // post
  494 + ModifiableSolrParams paramsPost = new ModifiableSolrParams();
  495 + paramsPost.set("q", "*:*");
  496 + paramsPost.set("mtas", "true");
  497 + paramsPost.set("mtas.collection", "true");
  498 + paramsPost.set("mtas.collection.0.key", "post");
  499 + paramsPost.set("mtas.collection.0.action", "post");
  500 + paramsPost.set("mtas.collection.0.id", "idPost");
  501 + paramsPost.set("mtas.collection.0.post", "[1,2,3,4]");
  502 + SolrRequest<?> requestPost = new QueryRequest(paramsPost, METHOD.POST);
  503 + NamedList<Object> responsePost;
  504 + try {
  505 + responsePost = server.request(requestPost, "collection1");
  506 + } catch (SolrServerException e) {
  507 + throw new IOException(e);
  508 + }
  509 + NamedList<Object> post = MtasSolrBase.getFromMtasCollection(responsePost,
  510 + "post");
  511 + assertFalse("post - id not found", post == null);
  512 + assertTrue("post - no valid version",
  513 + post.get("version") != null && post.get("version") instanceof String);
  514 + assertTrue("post - no valid size",
  515 + post.get("size") != null && post.get("size") instanceof Number);
  516 + String postVersion = (String) post.get("version");
  517 + Number postSize = (Number) post.get("size");
  518 + assertTrue("post - incorrect size", postSize.longValue() == 4);
  519 + // list
  520 + ModifiableSolrParams paramsList = new ModifiableSolrParams();
  521 + paramsList.set("q", "*:*");
  522 + paramsList.set("mtas", "true");
  523 + paramsList.set("mtas.collection", "true");
  524 + paramsList.set("mtas.collection.0.key", "list");
  525 + paramsList.set("mtas.collection.0.action", "list");
  526 + SolrRequest<?> requestList1 = new QueryRequest(paramsList, METHOD.POST);
  527 + NamedList<Object> responseList1;
  528 + try {
  529 + responseList1 = server.request(requestList1, "collection1");
  530 + } catch (SolrServerException e) {
  531 + throw new IOException(e);
  532 + }
  533 + // check create
  534 + NamedList<Object> listCreateItem1 = MtasSolrBase
  535 + .getFromMtasCollectionList(responseList1, "list", "idCreate");
  536 + assertFalse("list - create - id not found", listCreateItem1 == null);
  537 + assertTrue("list - create - incorrect version",
  538 + listCreateItem1.get("version") != null
  539 + && listCreateItem1.get("version") instanceof String
  540 + && listCreateItem1.get("version").equals(createVersion));
  541 + assertTrue("list - create - incorrect size",
  542 + listCreateItem1.get("size") != null
  543 + && listCreateItem1.get("size") instanceof Number
  544 + && ((Number) listCreateItem1.get("size")).longValue() == createSize
  545 + .longValue());
  546 + // check post
  547 + NamedList<Object> listPostItem1 = MtasSolrBase
  548 + .getFromMtasCollectionList(responseList1, "list", "idPost");
  549 + assertFalse("list - post - id not found", listPostItem1 == null);
  550 + assertTrue("list - post - incorrect version",
  551 + listPostItem1.get("version") != null
  552 + && listPostItem1.get("version") instanceof String
  553 + && listPostItem1.get("version").equals(postVersion));
  554 + assertTrue("list - post - incorrect size",
  555 + listPostItem1.get("size") != null
  556 + && listPostItem1.get("size") instanceof Number
  557 + && ((Number) listPostItem1.get("size")).longValue() == postSize
  558 + .longValue());
  559 + // check
  560 + ModifiableSolrParams paramsCheck = new ModifiableSolrParams();
  561 + paramsCheck.set("q", "*:*");
  562 + paramsCheck.set("mtas", "true");
  563 + paramsCheck.set("mtas.collection", "true");
  564 + paramsCheck.set("mtas.collection.0.key", "check1");
  565 + paramsCheck.set("mtas.collection.0.action", "check");
  566 + paramsCheck.set("mtas.collection.0.id", "idCreate");
  567 + paramsCheck.set("mtas.collection.1.key", "check2");
  568 + paramsCheck.set("mtas.collection.1.action", "check");
  569 + paramsCheck.set("mtas.collection.1.id", "idPost");
  570 + SolrRequest<?> requestCheck = new QueryRequest(paramsCheck, METHOD.POST);
  571 + NamedList<Object> responseCheck;
  572 + try {
  573 + responseCheck = server.request(requestCheck, "collection1");
  574 + } catch (SolrServerException e) {
  575 + throw new IOException(e);
  576 + }
  577 + // check create
  578 + NamedList<Object> check1 = MtasSolrBase.getFromMtasCollection(responseCheck,
  579 + "check1");
  580 + assertFalse("check - create - no response", check1 == null);
  581 + assertTrue("check - create - no valid version",
  582 + check1.get("version") != null
  583 + && check1.get("version") instanceof String);
  584 + assertTrue("check - create - no valid size",
  585 + check1.get("size") != null && check1.get("size") instanceof Number);
  586 + String check1Version = (String) check1.get("version");
  587 + Number check1Size = (Number) check1.get("size");
  588 + assertEquals("check - create - version", check1Version, createVersion);
  589 + assertEquals("check - create - number of values", check1Size.longValue(),
  590 + createSize.longValue());
  591 + // check post
  592 + NamedList<Object> check2 = MtasSolrBase.getFromMtasCollection(responseCheck,
  593 + "check2");
  594 + assertFalse("check - post - no response", check2 == null);
  595 + assertTrue("check - post - no valid version", check2.get("version") != null
  596 + && check2.get("version") instanceof String);
  597 + assertTrue("check - post - no valid size",
  598 + check2.get("size") != null && check2.get("size") instanceof Number);
  599 + String check2Version = (String) check2.get("version");
  600 + Number check2Size = (Number) check2.get("size");
  601 + assertEquals("check - post - version", check2Version, postVersion);
  602 + assertEquals("check - post - number of values", check2Size.longValue(), 4);
  603 + // delete
  604 + ModifiableSolrParams paramsDelete = new ModifiableSolrParams();
  605 + paramsDelete.set("q", "*:*");
  606 + paramsDelete.set("mtas", "true");
  607 + paramsDelete.set("mtas.collection", "true");
  608 + paramsDelete.set("mtas.collection.0.key", "delete1");
  609 + paramsDelete.set("mtas.collection.0.action", "delete");
  610 + paramsDelete.set("mtas.collection.0.id", "idCreate");
  611 + paramsDelete.set("mtas.collection.1.key", "delete2");
  612 + paramsDelete.set("mtas.collection.1.action", "delete");
  613 + paramsDelete.set("mtas.collection.1.id", "idPost");
  614 + SolrRequest<?> requestDelete = new QueryRequest(paramsDelete, METHOD.POST);
  615 + NamedList<Object> responseDelete;
  616 + try {
  617 + responseDelete = server.request(requestDelete, "collection1");
  618 + } catch (SolrServerException e) {
  619 + throw new IOException(e);
  620 + }
  621 + // check create
  622 + NamedList<Object> delete1 = MtasSolrBase
  623 + .getFromMtasCollection(responseDelete, "delete1");
  624 + assertFalse("delete - create - no response", delete1 == null);
  625 + // check post
  626 + NamedList<Object> delete2 = MtasSolrBase
  627 + .getFromMtasCollection(responseDelete, "delete2");
  628 + assertFalse("delete - post - no response", delete2 == null);
  629 + // list (again)
  630 + SolrRequest<?> requestList2 = new QueryRequest(paramsList, METHOD.POST);
  631 + NamedList<Object> responseList2;
  632 + try {
  633 + responseList2 = server.request(requestList2, "collection1");
  634 + } catch (SolrServerException e) {
  635 + throw new IOException(e);
  636 + }
  637 + // check create
  638 + NamedList<Object> listCreateItem2 = MtasSolrBase
  639 + .getFromMtasCollectionList(responseList2, "list", "idCreate");
  640 + assertTrue("list - create - id found", listCreateItem2 == null);
  641 + // check post
  642 + NamedList<Object> listPostItem2 = MtasSolrBase
  643 + .getFromMtasCollectionList(responseList2, "list", "idPost");
  644 + assertTrue("list - post - id found", listPostItem2 == null);
  645 + }
  646 +
  647 + /**
  648 + * Mtas request handler collection 2.
  649 + *
  650 + * @throws IOException Signals that an I/O exception has occurred.
  651 + */
  652 + @org.junit.Test
  653 + public void mtasRequestHandlerCollection2() throws IOException {
  654 + // post
  655 + ModifiableSolrParams paramsPost = new ModifiableSolrParams();
  656 + paramsPost.set("q", "*:*");
  657 + paramsPost.set("mtas", "true");
  658 + paramsPost.set("mtas.collection", "true");
  659 + paramsPost.set("mtas.collection.0.key", "postKey1");
  660 + paramsPost.set("mtas.collection.0.action", "post");
  661 + paramsPost.set("mtas.collection.0.id", "postSet1");
  662 + paramsPost.set("mtas.collection.0.post", "[1,3,4]");
  663 + paramsPost.set("mtas.collection.1.key", "postKey2");
  664 + paramsPost.set("mtas.collection.1.action", "post");
  665 + paramsPost.set("mtas.collection.1.id", "postSet2");
  666 + paramsPost.set("mtas.collection.1.post", "[2]");
  667 + paramsPost.set("mtas.collection.2.key", "createKey1");
  668 + paramsPost.set("mtas.collection.2.action", "create");
  669 + paramsPost.set("mtas.collection.2.id", "createSet1");
  670 + paramsPost.set("mtas.collection.2.field", MtasSolrBase.FIELD_ID);
  671 + SolrRequest<?> requestPost = new QueryRequest(paramsPost, METHOD.POST);
  672 + NamedList<Object> responsePost;
  673 + try {
  674 + responsePost = server.request(requestPost, "collection1");
  675 + } catch (SolrServerException e) {
  676 + throw new IOException(e);
  677 + }
  678 + MtasSolrBase.getFromMtasCollection(responsePost, "post");
  679 + // query set1
  680 + ModifiableSolrParams paramsSelect1 = new ModifiableSolrParams();
  681 + paramsSelect1.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  682 + + "\" collection=\"postSet1\"}");
  683 + paramsSelect1.set("rows", "0");
  684 + SolrRequest<?> request1 = new QueryRequest(paramsSelect1, METHOD.POST);
  685 + NamedList<Object> response1;
  686 + try {
  687 + response1 = server.request(request1, "collection1");
  688 + } catch (SolrServerException e) {
  689 + throw new IOException(e);
  690 + }
  691 + long n1 = MtasSolrBase.getNumFound(response1);
  692 + assertTrue("incorrect number of matching documents : " + n1, n1 == 2);
  693 + // query set2
  694 + ModifiableSolrParams paramsSelect2 = new ModifiableSolrParams();
  695 + paramsSelect2.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  696 + + "\" collection=\"postSet2\"}");
  697 + paramsSelect2.set("rows", "0");
  698 + SolrRequest<?> request2 = new QueryRequest(paramsSelect2, METHOD.POST);
  699 + NamedList<Object> response2;
  700 + try {
  701 + response2 = server.request(request2, "collection1");
  702 + } catch (SolrServerException e) {
  703 + throw new IOException(e);
  704 + }
  705 + long n2 = MtasSolrBase.getNumFound(response2);
  706 + assertTrue("incorrect number of matching documents : " + n2, n2 == 1);
  707 + // query set3
  708 + ModifiableSolrParams paramsSelect3 = new ModifiableSolrParams();
  709 + paramsSelect3.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  710 + + "\" collection=\"createSet1\"}");
  711 + paramsSelect3.set("rows", "0");
  712 + SolrRequest<?> request3 = new QueryRequest(paramsSelect3, METHOD.POST);
  713 + NamedList<Object> response3;
  714 + try {
  715 + response3 = server.request(request3, "collection1");
  716 + } catch (SolrServerException e) {
  717 + throw new IOException(e);
  718 + }
  719 + long n3 = MtasSolrBase.getNumFound(response3);
  720 + assertTrue("incorrect number of matching documents : " + n3, n3 == 3);
  721 + // query set1 or set2
  722 + ModifiableSolrParams paramsSelect4 = new ModifiableSolrParams();
  723 + paramsSelect4.set("q",
  724 + "({!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  725 + + "\" collection=\"postSet1\"}) OR ({!mtas_join field=\""
  726 + + MtasSolrBase.FIELD_ID + "\" collection=\"postSet2\"})");
  727 + paramsSelect4.set("rows", "0");
  728 + SolrRequest<?> request4 = new QueryRequest(paramsSelect4, METHOD.POST);
  729 + NamedList<Object> response4;
  730 + try {
  731 + response4 = server.request(request4, "collection1");
  732 + } catch (SolrServerException e) {
  733 + throw new IOException(e);
  734 + }
  735 + long n4 = MtasSolrBase.getNumFound(response4);
  736 + assertTrue("incorrect number of matching documents : " + n4, n4 == 3);
  737 + }
  738 +
  739 + @org.junit.Test
  740 + public void mtasRequestHandlerCollection3() throws IOException {
  741 + // post
  742 + ModifiableSolrParams paramsPost = new ModifiableSolrParams();
  743 + paramsPost.set("q", "*:*");
  744 + paramsPost.set("mtas", "true");
  745 + paramsPost.set("mtas.collection", "true");
  746 + paramsPost.set("mtas.collection.0.key", "setCreatedByPost");
  747 + paramsPost.set("mtas.collection.0.action", "post");
  748 + paramsPost.set("mtas.collection.0.id", "setCreatedByPost");
  749 + paramsPost.set("mtas.collection.0.post", "[1,3,4]");
  750 + SolrRequest<?> requestPost = new QueryRequest(paramsPost, METHOD.POST);
  751 + try {
  752 + server.request(requestPost, "collection1");
  753 + } catch (SolrServerException e) {
  754 + throw new IOException(e);
  755 + }
  756 + // import
  757 + ModifiableSolrParams paramsImport = new ModifiableSolrParams();
  758 + paramsImport.set("q", "*:*");
  759 + paramsImport.set("mtas", "true");
  760 + paramsImport.set("mtas.collection", "true");
  761 + paramsImport.set("mtas.collection.0.key", "setCreatedByImport");
  762 + paramsImport.set("mtas.collection.0.action", "post");
  763 + paramsImport.set("mtas.collection.0.id", "setCreatedByImport");
  764 + paramsImport.set("mtas.collection.0.post", "[1,3,4]");
  765 + SolrRequest<?> requestImport = new QueryRequest(paramsImport, METHOD.POST);
  766 + try {
  767 + server.request(requestImport, "collection1");
  768 + } catch (SolrServerException e) {
  769 + throw new IOException(e);
  770 + }
  771 + // query post
  772 + ModifiableSolrParams paramsSelect1 = new ModifiableSolrParams();
  773 + paramsSelect1.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  774 + + "\" collection=\"setCreatedByPost\"}");
  775 + paramsSelect1.set("rows", "0");
  776 + SolrRequest<?> request1 = new QueryRequest(paramsSelect1, METHOD.POST);
  777 + NamedList<Object> response1;
  778 + try {
  779 + response1 = server.request(request1, "collection1");
  780 + } catch (SolrServerException e) {
  781 + throw new IOException(e);
  782 + }
  783 + long n1 = MtasSolrBase.getNumFound(response1);
  784 + assertTrue ("no matching documents for posted set: " + n1, n1>0);
  785 + // query import
  786 + ModifiableSolrParams paramsSelect2 = new ModifiableSolrParams();
  787 + paramsSelect2.set("q", "{!mtas_join field=\"" + MtasSolrBase.FIELD_ID
  788 + + "\" collection=\"setCreatedByImport\"}");
  789 + paramsSelect2.set("rows", "0");
  790 + SolrRequest<?> request2 = new QueryRequest(paramsSelect2, METHOD.POST);
  791 + NamedList<Object> response2;
  792 + try {
  793 + response2 = server.request(request2, "collection1");
  794 + } catch (SolrServerException e) {
  795 + throw new IOException(e);
  796 + }
  797 + long n2 = MtasSolrBase.getNumFound(response2);
  798 + assertTrue ("no matching documents for imported set: " + n2, n2>0);
  799 + //compare
  800 + assertTrue("posted set and imported set give different results : "+n1+" and "+n2, n1==n2);
  801 + }
  802 +
  803 + /**
458 804 * Mtas solr schema pre analyzed parser and field.
459 805 *
460 806 * @throws IOException Signals that an I/O exception has occurred.
... ... @@ -468,17 +814,17 @@ public class MtasSolrTestSearchConsistency {
468 814 params.set("mtas", "true");
469 815 params.set("mtas.stats", "true");
470 816 params.set("mtas.stats.spans", "true");
471   - params.set("mtas.stats.spans.0.field", "mtas");
  817 + params.set("mtas.stats.spans.0.field", MtasSolrBase.FIELD_MTAS);
472 818 params.set("mtas.stats.spans.0.key", "statsKey");
473 819 params.set("mtas.stats.spans.0.query.0.type", "cql");
474 820 params.set("mtas.stats.spans.0.query.0.value", "[]");
475 821 params.set("mtas.stats.spans.0.type", "n,sum,sumsq");
476 822 params.set("mtas.stats.positions", "true");
477   - params.set("mtas.stats.positions.0.field", "mtas");
  823 + params.set("mtas.stats.positions.0.field", MtasSolrBase.FIELD_MTAS);
478 824 params.set("mtas.stats.positions.0.key", "statsKey");
479 825 params.set("mtas.stats.positions.0.type", "n,sum,sumsq");
480 826 params.set("mtas.stats.tokens", "true");
481   - params.set("mtas.stats.tokens.0.field", "mtas");
  827 + params.set("mtas.stats.tokens.0.field", MtasSolrBase.FIELD_MTAS);
482 828 params.set("mtas.stats.tokens.0.key", "statsKey");
483 829 params.set("mtas.stats.tokens.0.type", "n,sum,sumsq");
484 830 params.set("rows", "0");
... ... @@ -493,9 +839,10 @@ public class MtasSolrTestSearchConsistency {
493 839 params.remove("mtas.stats.spans.0.field");
494 840 params.remove("mtas.stats.positions.0.field");
495 841 params.remove("mtas.stats.tokens.0.field");
496   - params.set("mtas.stats.spans.0.field", "mtasAdvanced");
497   - params.set("mtas.stats.positions.0.field", "mtasAdvanced");
498   - params.set("mtas.stats.tokens.0.field", "mtasAdvanced");
  842 + params.set("mtas.stats.spans.0.field", MtasSolrBase.FIELD_MTAS_ADVANCED);
  843 + params.set("mtas.stats.positions.0.field",
  844 + MtasSolrBase.FIELD_MTAS_ADVANCED);
  845 + params.set("mtas.stats.tokens.0.field", MtasSolrBase.FIELD_MTAS_ADVANCED);
499 846 try {
500 847 response2 = server.request(request, "collection1");
501 848 } catch (SolrServerException e) {
... ... @@ -523,8 +870,10 @@ public class MtasSolrTestSearchConsistency {
523 870 */
524 871 private static void createTermvectorAssertions(NamedList<Object> response1,
525 872 NamedList<Object> response2, String key, String[] names) {
526   - List<NamedList> list1 = MtasSolrBase.getFromMtasTermvector(response1, key);
527   - List<NamedList> list2 = MtasSolrBase.getFromMtasTermvector(response2, key);
  873 + List<NamedList<Object>> list1 = MtasSolrBase
  874 + .getFromMtasTermvector(response1, key);
  875 + List<NamedList<Object>> list2 = MtasSolrBase
  876 + .getFromMtasTermvector(response2, key);
528 877 assertFalse("list should be defined", list1 == null || list2 == null);
529 878 if (list1 != null && list2 != null) {
530 879 assertFalse("first list should not be longer",
... ...
... ... @@ -144,7 +144,8 @@
144 144 <artifactId>maven-site-plugin</artifactId>
145 145 <version>3.6</version>
146 146 <configuration>
147   - <outputDirectory>${project.basedir}/gh-pages/</outputDirectory>
  147 + <siteDirectory>${project.basedir}/src/site/</siteDirectory>
  148 + <outputDirectory>${project.basedir}/gh-pages/</outputDirectory>
148 149 </configuration>
149 150 </plugin>
150 151 <plugin>
... ... @@ -306,7 +307,7 @@
306 307 <!-- <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>versions-maven-plugin</artifactId>
307 308 <version>2.2</version> <reportSets> <reportSet> <reports> <report>dependency-updates-report</report>
308 309 <report>plugin-updates-report</report> <report>property-updates-report</report>
309   - </reports> </reportSet> </reportSets> </plugin> -->
  310 + </reports> </reportSet> </reportSets> </plugin> -->
310 311 </plugins>
311 312 </reporting>
312 313 <repositories>
... ...
src/mtas/analysis/parser/MtasXMLParser.java
  View file @7179e4b
... ... @@ -126,7 +126,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
126 126 super(config);
127 127 try {
128 128 initParser();
129   - // System.out.print(printConfig());
  129 + //System.out.print(printConfig());
130 130 } catch (MtasConfigException e) {
131 131 log.error(e);
132 132 }
... ...
src/mtas/analysis/util/MtasCharFilterFactory.java
  View file @7179e4b
... ... @@ -66,8 +66,10 @@ public class MtasCharFilterFactory extends CharFilterFactory
66 66 /**
67 67 * Instantiates a new mtas char filter factory.
68 68 *
69   - * @param args the args
70   - * @throws IOException Signals that an I/O exception has occurred.
  69 + * @param args
  70 + * the args
  71 + * @throws IOException
  72 + * Signals that an I/O exception has occurred.
71 73 */
72 74 public MtasCharFilterFactory(Map<String, String> args) throws IOException {
73 75 this(args, null);
... ... @@ -76,12 +78,15 @@ public class MtasCharFilterFactory extends CharFilterFactory
76 78 /**
77 79 * Instantiates a new mtas char filter factory.
78 80 *
79   - * @param args the args
80   - * @param resourceLoader the resource loader
81   - * @throws IOException Signals that an I/O exception has occurred.
  81 + * @param args
  82 + * the args
  83 + * @param resourceLoader
  84 + * the resource loader
  85 + * @throws IOException
  86 + * Signals that an I/O exception has occurred.
82 87 */
83 88 public MtasCharFilterFactory(Map<String, String> args,
84   - SolrResourceLoader resourceLoader) throws IOException {
  89 + ResourceLoader resourceLoader) throws IOException {
85 90 super(args);
86 91 typeArgument = get(args, ARGUMENT_TYPE);
87 92 prefixArgument = get(args, ARGUMENT_PREFIX);
... ... @@ -110,8 +115,10 @@ public class MtasCharFilterFactory extends CharFilterFactory
110 115 /**
111 116 * Inits the.
112 117 *
113   - * @param resourceLoader the resource loader
114   - * @throws IOException Signals that an I/O exception has occurred.
  118 + * @param resourceLoader
  119 + * the resource loader
  120 + * @throws IOException
  121 + * Signals that an I/O exception has occurred.
115 122 */
116 123 private void init(ResourceLoader resourceLoader) throws IOException {
117 124 if (config == null && configs == null) {
... ... @@ -164,10 +171,13 @@ public class MtasCharFilterFactory extends CharFilterFactory
164 171 /**
165 172 * Creates the.
166 173 *
167   - * @param input the input
168   - * @param configuration the configuration
  174 + * @param input
  175 + * the input
  176 + * @param configuration
  177 + * the configuration
169 178 * @return the reader
170   - * @throws IOException Signals that an I/O exception has occurred.
  179 + * @throws IOException
  180 + * Signals that an I/O exception has occurred.
171 181 */
172 182 public Reader create(Reader input, String configuration) throws IOException {
173 183 if (configs != null && configs.size() > 0) {
... ... @@ -210,10 +220,13 @@ public class MtasCharFilterFactory extends CharFilterFactory
210 220 /**
211 221 * Creates the.
212 222 *
213   - * @param input the input
214   - * @param config the config
  223 + * @param input
  224 + * the input
  225 + * @param config
  226 + * the config
215 227 * @return the reader
216   - * @throws IOException Signals that an I/O exception has occurred.
  228 + * @throws IOException
  229 + * Signals that an I/O exception has occurred.
217 230 */
218 231 public Reader create(Reader input, MtasConfiguration config)
219 232 throws IOException {
... ...
src/mtas/analysis/util/MtasTokenizerFactory.java
  View file @7179e4b
... ... @@ -9,8 +9,6 @@ import org.apache.lucene.analysis.util.ResourceLoader;
9 9 import org.apache.lucene.analysis.util.ResourceLoaderAware;
10 10 import org.apache.lucene.analysis.util.TokenizerFactory;
11 11 import org.apache.lucene.util.AttributeFactory;
12   -import org.apache.solr.core.SolrResourceLoader;
13   -
14 12 import java.io.IOException;
15 13 import java.util.HashMap;
16 14 import java.util.Map;
... ... @@ -66,7 +64,7 @@ public class MtasTokenizerFactory extends TokenizerFactory
66 64 * @throws IOException Signals that an I/O exception has occurred.
67 65 */
68 66 public MtasTokenizerFactory(Map<String, String> args,
69   - SolrResourceLoader resourceLoader) throws IOException {
  67 + ResourceLoader resourceLoader) throws IOException {
70 68 super(args);
71 69 configFileArgument = get(args, ARGUMENT_CONFIGFILE);
72 70 configArgument = get(args, ARGUMENT_CONFIG);
... ...
src/mtas/codec/MtasFieldsProducer.java
  View file @7179e4b
1 1 package mtas.codec;
2 2  
  3 +import java.io.EOFException;
  4 +import java.io.FileNotFoundException;
3 5 import java.io.IOException;
  6 +import java.nio.file.NoSuchFileException;
4 7 import java.util.ArrayList;
5 8 import java.util.Collection;
6 9 import java.util.Collections;
... ... @@ -52,8 +55,8 @@ public class MtasFieldsProducer extends FieldsProducer {
52 55 public MtasFieldsProducer(SegmentReadState state, String name)
53 56 throws IOException {
54 57 String postingsFormatName = null;
55   - indexInputList = new HashMap<String, IndexInput>();
56   - indexInputOffsetList = new HashMap<String, Long>();
  58 + indexInputList = new HashMap<>();
  59 + indexInputOffsetList = new HashMap<>();
57 60 version = MtasCodecPostingsFormat.VERSION_CURRENT;
58 61  
59 62 postingsFormatName = addIndexInputToList("object", openMtasFile(state, name,
... ... @@ -109,20 +112,25 @@ public class MtasFieldsProducer extends FieldsProducer {
109 112 * @throws IOException Signals that an I/O exception has occurred.
110 113 */
111 114 private String addIndexInputToList(String name, IndexInput in,
112   - String postingsFormatName) throws IOException {
  115 + String postingsFormatName) throws IOException {
113 116 if (indexInputList.get(name) != null) {
114 117 indexInputList.get(name).close();
115 118 }
116   - String localPostingsFormatName = postingsFormatName;
117   - if (localPostingsFormatName == null) {
118   - localPostingsFormatName = in.readString();
119   - } else if (!in.readString().equals(localPostingsFormatName)) {
120   - throw new IOException("delegate codec " + name + " doesn't equal "
121   - + localPostingsFormatName);
  119 + if(in!=null) {
  120 + String localPostingsFormatName = postingsFormatName;
  121 + if (localPostingsFormatName == null) {
  122 + localPostingsFormatName = in.readString();
  123 + } else if (!in.readString().equals(localPostingsFormatName)) {
  124 + throw new IOException("delegate codec " + name + " doesn't equal "
  125 + + localPostingsFormatName);
  126 + }
  127 + indexInputList.put(name, in);
  128 + indexInputOffsetList.put(name, in.getFilePointer());
  129 + return localPostingsFormatName;
  130 + } else {
  131 + log.debug("no "+name+" registered");
  132 + return null;
122 133 }
123   - indexInputList.put(name, in);
124   - indexInputOffsetList.put(name, in.getFilePointer());
125   - return localPostingsFormatName;
126 134 }
127 135  
128 136 /*
... ... @@ -232,7 +240,13 @@ public class MtasFieldsProducer extends FieldsProducer {
232 240 String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
233 241 state.segmentSuffix, extension);
234 242 IndexInput object;
235   - object = state.directory.openInput(fileName, state.context);
  243 + try {
  244 + object = state.directory.openInput(fileName, state.context);
  245 + } catch (FileNotFoundException | NoSuchFileException e) {
  246 + log.debug(e);
  247 + //throw new NoSuchFileException(e.getMessage());
  248 + return null;
  249 + }
236 250 int minVersion = (minimum == null) ? MtasCodecPostingsFormat.VERSION_START
237 251 : minimum.intValue();
238 252 int maxVersion = (maximum == null) ? MtasCodecPostingsFormat.VERSION_CURRENT
... ... @@ -245,7 +259,12 @@ public class MtasFieldsProducer extends FieldsProducer {
245 259 log.debug(e);
246 260 throw new IndexFormatTooOldException(e.getMessage(), e.getVersion(),
247 261 e.getMinVersion(), e.getMaxVersion());
248   - }
  262 + } catch (EOFException e) {
  263 + object.close();
  264 + log.debug(e);
  265 + //throw new EOFException(e.getMessage());
  266 + return null;
  267 + }
249 268 return object;
250 269 }
251 270  
... ...
src/mtas/codec/util/CodecCollector.java
  View file @7179e4b
... ... @@ -27,7 +27,7 @@ import mtas.codec.util.CodecComponent.ComponentDocument;
27 27 import mtas.codec.util.CodecComponent.ComponentFacet;
28 28 import mtas.codec.util.CodecComponent.ComponentField;
29 29 import mtas.codec.util.CodecComponent.ComponentGroup;
30   -import mtas.codec.util.CodecComponent.ComponentJoin;
  30 +import mtas.codec.util.CodecComponent.ComponentCollection;
31 31 import mtas.codec.util.CodecComponent.ComponentKwic;
32 32 import mtas.codec.util.CodecComponent.ComponentList;
33 33 import mtas.codec.util.CodecComponent.ComponentPosition;
... ... @@ -203,40 +203,52 @@ public class CodecCollector {
203 203 }
204 204  
205 205 /**
206   - * Collect join.
  206 + * Collect collection.
207 207 *
208 208 * @param reader the reader
209 209 * @param docSet the doc set
210   - * @param joinInfo the join info
  210 + * @param collectionInfo the collection info
211 211 * @throws IOException Signals that an I/O exception has occurred.
212 212 */
213   - public static void collectJoin(IndexReader reader, List<Integer> docSet,
214   - ComponentJoin joinInfo) throws IOException {
215   - BytesRef term = null;
216   - PostingsEnum postingsEnum = null;
217   - Integer docId;
218   - Integer termDocId = -1;
219   - Terms terms;
220   - LeafReaderContext lrc;
221   - LeafReader r;
222   - ListIterator<LeafReaderContext> iterator = reader.leaves().listIterator();
223   - while (iterator.hasNext()) {
224   - lrc = iterator.next();
225   - r = lrc.reader();
226   - for (String field : joinInfo.fields()) {
227   - if ((terms = r.fields().terms(field)) != null) {
228   - TermsEnum termsEnum = terms.iterator();
229   - termDocId = -1;
230   - while ((term = termsEnum.next()) != null) {
231   - Iterator<Integer> docIterator = docSet.iterator();
232   - postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE);
233   - while (docIterator.hasNext()) {
234   - docId = docIterator.next() - lrc.docBase;
235   - if ((docId >= termDocId) && ((docId.equals(termDocId))
236   - || ((termDocId = postingsEnum.advance(docId))
237   - .equals(docId)))) {
238   - joinInfo.add(term.utf8ToString());
239   - break;
  213 + public static void collectCollection(IndexReader reader, List<Integer> docSet,
  214 + ComponentCollection collectionInfo) throws IOException {
  215 + if (collectionInfo.action().equals(ComponentCollection.ACTION_CHECK)) {
  216 + // can't do anything in lucene for check
  217 + } else if (collectionInfo.action()
  218 + .equals(ComponentCollection.ACTION_LIST)) {
  219 + // can't do anything in lucene for list
  220 + } else if (collectionInfo.action()
  221 + .equals(ComponentCollection.ACTION_CREATE)) {
  222 + BytesRef term = null;
  223 + PostingsEnum postingsEnum = null;
  224 + Integer docId;
  225 + Integer termDocId = -1;
  226 + Terms terms;
  227 + LeafReaderContext lrc;
  228 + LeafReader r;
  229 + ListIterator<LeafReaderContext> iterator = reader.leaves().listIterator();
  230 + while (iterator.hasNext()) {
  231 + lrc = iterator.next();
  232 + r = lrc.reader();
  233 + for (String field : collectionInfo.fields()) {
  234 + if ((terms = r.fields().terms(field)) != null) {
  235 + TermsEnum termsEnum = terms.iterator();
  236 + while ((term = termsEnum.next()) != null) {
  237 + Iterator<Integer> docIterator = docSet.iterator();
  238 + postingsEnum = termsEnum.postings(postingsEnum,
  239 + PostingsEnum.NONE);
  240 + termDocId = -1;
  241 + while (docIterator.hasNext()) {
  242 + docId = docIterator.next() - lrc.docBase;
  243 + if ((docId >= termDocId) && ((docId.equals(termDocId))
  244 + || ((termDocId = postingsEnum.advance(docId))
  245 + .equals(docId)))) {
  246 + collectionInfo.addValue(term.utf8ToString());
  247 + break;
  248 + }
  249 + if (termDocId.equals(PostingsEnum.NO_MORE_DOCS)) {
  250 + break;
  251 + }
240 252 }
241 253 }
242 254 }
... ... @@ -270,7 +282,7 @@ public class CodecCollector {
270 282 boolean needSpans = false;
271 283 boolean needPositions = false;
272 284 boolean needTokens = false;
273   -
  285 +
274 286 // results
275 287 Map<Integer, Integer> positionsData = null;
276 288 Map<Integer, Integer> tokensData = null;
... ... @@ -1517,7 +1529,6 @@ public class CodecCollector {
1517 1529 }
1518 1530 }
1519 1531 }
1520   -
1521 1532 } else {
1522 1533 int maximumNumberOfDocuments = 0;
1523 1534 int boundaryMinimumNumberOfDocuments = 1;
... ... @@ -1671,9 +1682,9 @@ public class CodecCollector {
1671 1682 m.endPosition - 1);
1672 1683 }
1673 1684 if (group.hitRight != null) {
1674   - start = Math.min(m.endPosition - group.hitRight.length + 1,
  1685 + start = Math.min(m.endPosition - group.hitRight.length,
1675 1686 m.startPosition);
1676   - end = end == null ? m.endPosition : Math.max(end, m.endPosition);
  1687 + end = end == null ? (m.endPosition - 1) : Math.max(end, (m.endPosition - 1));
1677 1688 }
1678 1689 if (group.left != null) {
1679 1690 start = start == null ? m.startPosition - group.left.length
... ... @@ -1683,8 +1694,8 @@ public class CodecCollector {
1683 1694 }
1684 1695 if (group.right != null) {
1685 1696 start = start == null ? m.endPosition : Math.min(m.endPosition, start);
1686   - end = end == null ? m.endPosition + group.right.length
1687   - : Math.max(m.endPosition + group.right.length, end);
  1697 + end = end == null ? m.endPosition + group.right.length - 1
  1698 + : Math.max(m.endPosition + group.right.length - 1, end);
1688 1699 }
1689 1700 return new IntervalTreeNodeData<>(start, end, m.startPosition,
1690 1701 m.endPosition - 1);
... ...
src/mtas/codec/util/CodecComponent.java
  View file @7179e4b
... ... @@ -2,8 +2,13 @@ package mtas.codec.util;
2 2  
3 3 import java.io.BufferedReader;
4 4 import java.io.IOException;
  5 +import java.io.InputStream;
  6 +import java.io.InputStreamReader;
5 7 import java.io.StringReader;
6 8 import java.io.UnsupportedEncodingException;
  9 +import java.net.HttpURLConnection;
  10 +import java.net.URL;
  11 +import java.net.URLEncoder;
7 12 import java.nio.charset.StandardCharsets;
8 13 import java.util.ArrayList;
9 14 import java.util.Arrays;
... ... @@ -31,8 +36,12 @@ import mtas.parser.function.util.MtasFunctionParserFunction;
31 36 import mtas.parser.function.util.MtasFunctionParserFunctionDefault;
32 37 import mtas.search.spans.util.MtasSpanQuery;
33 38  
  39 +import org.apache.commons.io.IOUtils;
34 40 import org.apache.commons.lang.ArrayUtils;
35 41 import org.apache.lucene.util.BytesRef;
  42 +import org.noggit.JSONParser;
  43 +import org.noggit.ObjectBuilder;
  44 +
36 45  
37 46 /**
38 47 * The Class CodecComponent.
... ... @@ -53,8 +62,8 @@ public class CodecComponent {
53 62 /** The list. */
54 63 public Map<String, ComponentField> list;
55 64  
56   - /** The join. */
57   - public ComponentJoin join;
  65 + /** The collection. */
  66 + public List<ComponentCollection> collection;
58 67  
59 68 /** The do document. */
60 69 public boolean doDocument;
... ... @@ -89,15 +98,15 @@ public class CodecComponent {
89 98 /** The do facet. */
90 99 public boolean doFacet;
91 100  
92   - /** The do join. */
93   - public boolean doJoin;
  101 + /** The do collection. */
  102 + public boolean doCollection;
94 103  
95 104 /**
96 105 * Instantiates a new component fields.
97 106 */
98 107 public ComponentFields() {
99 108 list = new HashMap<>();
100   - join = null;
  109 + collection = new ArrayList<>();
101 110 doDocument = false;
102 111 doKwic = false;
103 112 doList = false;
... ... @@ -109,7 +118,7 @@ public class CodecComponent {
109 118 doStatsTokens = false;
110 119 doPrefix = false;
111 120 doFacet = false;
112   - doJoin = false;
  121 + doCollection = false;
113 122 }
114 123 }
115 124  
... ... @@ -163,7 +172,8 @@ public class CodecComponent {
163 172 /**
164 173 * Instantiates a new component field.
165 174 *
166   - * @param uniqueKeyField the unique key field
  175 + * @param uniqueKeyField
  176 + * the unique key field
167 177 */
168 178 public ComponentField(String uniqueKeyField) {
169 179 this.uniqueKeyField = uniqueKeyField;
... ... @@ -205,7 +215,8 @@ public class CodecComponent {
205 215 /**
206 216 * Instantiates a new component prefix.
207 217 *
208   - * @param key the key
  218 + * @param key
  219 + * the key
209 220 */
210 221 public ComponentPrefix(String key) {
211 222 this.key = key;
... ... @@ -218,7 +229,8 @@ public class CodecComponent {
218 229 /**
219 230 * Adds the single position.
220 231 *
221   - * @param prefix the prefix
  232 + * @param prefix
  233 + * the prefix
222 234 */
223 235 public void addSinglePosition(String prefix) {
224 236 if (!prefix.trim().isEmpty() && !singlePositionList.contains(prefix)
... ... @@ -230,7 +242,8 @@ public class CodecComponent {
230 242 /**
231 243 * Adds the multiple position.
232 244 *
233   - * @param prefix the prefix
  245 + * @param prefix
  246 + * the prefix
234 247 */
235 248 public void addMultiplePosition(String prefix) {
236 249 if (!prefix.trim().isEmpty()) {
... ... @@ -248,7 +261,8 @@ public class CodecComponent {
248 261 /**
249 262 * Adds the set position.
250 263 *
251   - * @param prefix the prefix
  264 + * @param prefix
  265 + * the prefix
252 266 */
253 267 public void addSetPosition(String prefix) {
254 268 if (!prefix.trim().isEmpty()) {
... ... @@ -266,7 +280,8 @@ public class CodecComponent {
266 280 /**
267 281 * Adds the intersecting.
268 282 *
269   - * @param prefix the prefix
  283 + * @param prefix
  284 + * the prefix
270 285 */
271 286 public void addIntersecting(String prefix) {
272 287 if (!prefix.trim().isEmpty()) {
... ... @@ -335,19 +350,32 @@ public class CodecComponent {
335 350 /**
336 351 * Instantiates a new component document.
337 352 *
338   - * @param key the key
339   - * @param prefix the prefix
340   - * @param statsType the stats type
341   - * @param regexp the regexp
342   - * @param list the list
343   - * @param listNumber the list number
344   - * @param listRegexp the list regexp
345   - * @param listExpand the list expand
346   - * @param listExpandNumber the list expand number
347   - * @param ignoreRegexp the ignore regexp
348   - * @param ignoreList the ignore list
349   - * @param ignoreListRegexp the ignore list regexp
350   - * @throws IOException Signals that an I/O exception has occurred.
  353 + * @param key
  354 + * the key
  355 + * @param prefix
  356 + * the prefix
  357 + * @param statsType
  358 + * the stats type
  359 + * @param regexp
  360 + * the regexp
  361 + * @param list
  362 + * the list
  363 + * @param listNumber
  364 + * the list number
  365 + * @param listRegexp
  366 + * the list regexp
  367 + * @param listExpand
  368 + * the list expand
  369 + * @param listExpandNumber
  370 + * the list expand number
  371 + * @param ignoreRegexp
  372 + * the ignore regexp
  373 + * @param ignoreList
  374 + * the ignore list
  375 + * @param ignoreListRegexp
  376 + * the ignore list regexp
  377 + * @throws IOException
  378 + * Signals that an I/O exception has occurred.
351 379 */
352 380 public ComponentDocument(String key, String prefix, String statsType,
353 381 String regexp, String[] list, int listNumber, Boolean listRegexp,
... ... @@ -451,15 +479,24 @@ public class CodecComponent {
451 479 /**
452 480 * Instantiates a new component kwic.
453 481 *
454   - * @param query the query
455   - * @param key the key
456   - * @param prefixes the prefixes
457   - * @param number the number
458   - * @param start the start
459   - * @param left the left
460   - * @param right the right
461   - * @param output the output
462   - * @throws IOException Signals that an I/O exception has occurred.
  482 + * @param query
  483 + * the query
  484 + * @param key
  485 + * the key
  486 + * @param prefixes
  487 + * the prefixes
  488 + * @param number
  489 + * the number
  490 + * @param start
  491 + * the start
  492 + * @param left
  493 + * the left
  494 + * @param right
  495 + * the right
  496 + * @param output
  497 + * the output
  498 + * @throws IOException
  499 + * Signals that an I/O exception has occurred.
463 500 */
464 501 public ComponentKwic(MtasSpanQuery query, String key, String prefixes,
465 502 Integer number, int start, int left, int right, String output)
... ... @@ -585,22 +622,38 @@ public class CodecComponent {
585 622 /**
586 623 * Instantiates a new component list.
587 624 *
588   - * @param spanQuery the span query
589   - * @param field the field
590   - * @param queryValue the query value
591   - * @param queryType the query type
592   - * @param queryPrefix the query prefix
593   - * @param queryVariables the query variables
594   - * @param queryIgnore the query ignore
595   - * @param queryMaximumIgnoreLength the query maximum ignore length
596   - * @param key the key
597   - * @param prefix the prefix
598   - * @param start the start
599   - * @param number the number
600   - * @param left the left
601   - * @param right the right
602   - * @param output the output
603   - * @throws IOException Signals that an I/O exception has occurred.
  625 + * @param spanQuery
  626 + * the span query
  627 + * @param field
  628 + * the field
  629 + * @param queryValue
  630 + * the query value
  631 + * @param queryType
  632 + * the query type
  633 + * @param queryPrefix
  634 + * the query prefix
  635 + * @param queryVariables
  636 + * the query variables
  637 + * @param queryIgnore
  638 + * the query ignore
  639 + * @param queryMaximumIgnoreLength
  640 + * the query maximum ignore length
  641 + * @param key
  642 + * the key
  643 + * @param prefix
  644 + * the prefix
  645 + * @param start
  646 + * the start
  647 + * @param number
  648 + * the number
  649 + * @param left
  650 + * the left
  651 + * @param right
  652 + * the right
  653 + * @param output
  654 + * the output
  655 + * @throws IOException
  656 + * Signals that an I/O exception has occurred.
604 657 */
605 658 public ComponentList(MtasSpanQuery spanQuery, String field,
606 659 String queryValue, String queryType, String queryPrefix,
... ... @@ -715,23 +768,40 @@ public class CodecComponent {
715 768 /**
716 769 * Instantiates a new component group.
717 770 *
718   - * @param spanQuery the span query
719   - * @param key the key
720   - * @param number the number
721   - * @param groupingHitInsidePrefixes the grouping hit inside prefixes
722   - * @param groupingHitInsideLeftPosition the grouping hit inside left position
723   - * @param groupingHitInsideLeftPrefixes the grouping hit inside left prefixes
724   - * @param groupingHitInsideRightPosition the grouping hit inside right position
725   - * @param groupingHitInsideRightPrefixes the grouping hit inside right prefixes
726   - * @param groupingHitLeftPosition the grouping hit left position
727   - * @param groupingHitLeftPrefixes the grouping hit left prefixes
728   - * @param groupingHitRightPosition the grouping hit right position
729   - * @param groupingHitRightPrefixes the grouping hit right prefixes
730   - * @param groupingLeftPosition the grouping left position
731   - * @param groupingLeftPrefixes the grouping left prefixes
732   - * @param groupingRightPosition the grouping right position
733   - * @param groupingRightPrefixes the grouping right prefixes
734   - * @throws IOException Signals that an I/O exception has occurred.
  771 + * @param spanQuery
  772 + * the span query
  773 + * @param key
  774 + * the key
  775 + * @param number
  776 + * the number
  777 + * @param groupingHitInsidePrefixes
  778 + * the grouping hit inside prefixes
  779 + * @param groupingHitInsideLeftPosition
  780 + * the grouping hit inside left position
  781 + * @param groupingHitInsideLeftPrefixes
  782 + * the grouping hit inside left prefixes
  783 + * @param groupingHitInsideRightPosition
  784 + * the grouping hit inside right position
  785 + * @param groupingHitInsideRightPrefixes
  786 + * the grouping hit inside right prefixes
  787 + * @param groupingHitLeftPosition
  788 + * the grouping hit left position
  789 + * @param groupingHitLeftPrefixes
  790 + * the grouping hit left prefixes
  791 + * @param groupingHitRightPosition
  792 + * the grouping hit right position
  793 + * @param groupingHitRightPrefixes
  794 + * the grouping hit right prefixes
  795 + * @param groupingLeftPosition
  796 + * the grouping left position
  797 + * @param groupingLeftPrefixes
  798 + * the grouping left prefixes
  799 + * @param groupingRightPosition
  800 + * the grouping right position
  801 + * @param groupingRightPrefixes
  802 + * the grouping right prefixes
  803 + * @throws IOException
  804 + * Signals that an I/O exception has occurred.
735 805 */
736 806 public ComponentGroup(MtasSpanQuery spanQuery, String key, int number,
737 807 String groupingHitInsidePrefixes,
... ... @@ -791,11 +861,15 @@ public class CodecComponent {
791 861 /**
792 862 * Creates the positioned prefixes.
793 863 *
794   - * @param prefixList the prefix list
795   - * @param position the position
796   - * @param prefixes the prefixes
  864 + * @param prefixList
  865 + * the prefix list
  866 + * @param position
  867 + * the position
  868 + * @param prefixes
  869 + * the prefixes
797 870 * @return the hash set[]
798   - * @throws IOException Signals that an I/O exception has occurred.
  871 + * @throws IOException
  872 + * Signals that an I/O exception has occurred.
799 873 */
800 874 private static HashSet<String>[] createPositionedPrefixes(
801 875 HashSet<String> prefixList, String[] position, String[] prefixes)
... ... @@ -958,24 +1032,42 @@ public class CodecComponent {
958 1032 /**
959 1033 * Instantiates a new component facet.
960 1034 *
961   - * @param spanQueries the span queries
962   - * @param field the field
963   - * @param key the key
964   - * @param baseFields the base fields
965   - * @param baseFieldTypes the base field types
966   - * @param baseTypes the base types
967   - * @param baseRangeSizes the base range sizes
968   - * @param baseRangeBases the base range bases
969   - * @param baseSortTypes the base sort types
970   - * @param baseSortDirections the base sort directions
971   - * @param baseNumbers the base numbers
972   - * @param baseMinimumDoubles the base minimum doubles
973   - * @param baseMaximumDoubles the base maximum doubles
974   - * @param baseFunctionKeys the base function keys
975   - * @param baseFunctionExpressions the base function expressions
976   - * @param baseFunctionTypes the base function types
977   - * @throws IOException Signals that an I/O exception has occurred.
978   - * @throws ParseException the parse exception
  1035 + * @param spanQueries
  1036 + * the span queries
  1037 + * @param field
  1038 + * the field
  1039 + * @param key
  1040 + * the key
  1041 + * @param baseFields
  1042 + * the base fields
  1043 + * @param baseFieldTypes
  1044 + * the base field types
  1045 + * @param baseTypes
  1046 + * the base types
  1047 + * @param baseRangeSizes
  1048 + * the base range sizes
  1049 + * @param baseRangeBases
  1050 + * the base range bases
  1051 + * @param baseSortTypes
  1052 + * the base sort types
  1053 + * @param baseSortDirections
  1054 + * the base sort directions
  1055 + * @param baseNumbers
  1056 + * the base numbers
  1057 + * @param baseMinimumDoubles
  1058 + * the base minimum doubles
  1059 + * @param baseMaximumDoubles
  1060 + * the base maximum doubles
  1061 + * @param baseFunctionKeys
  1062 + * the base function keys
  1063 + * @param baseFunctionExpressions
  1064 + * the base function expressions
  1065 + * @param baseFunctionTypes
  1066 + * the base function types
  1067 + * @throws IOException
  1068 + * Signals that an I/O exception has occurred.
  1069 + * @throws ParseException
  1070 + * the parse exception
979 1071 */
980 1072 @SuppressWarnings("unchecked")
981 1073 public ComponentFacet(MtasSpanQuery[] spanQueries, String field, String key,
... ... @@ -1235,26 +1327,46 @@ public class CodecComponent {
1235 1327 /**
1236 1328 * Instantiates a new component term vector.
1237 1329 *
1238   - * @param key the key
1239   - * @param prefix the prefix
1240   - * @param regexp the regexp
1241   - * @param full the full
1242   - * @param type the type
1243   - * @param sortType the sort type
1244   - * @param sortDirection the sort direction
1245   - * @param startValue the start value
1246   - * @param number the number
1247   - * @param functionKey the function key
1248   - * @param functionExpression the function expression
1249   - * @param functionType the function type
1250   - * @param boundary the boundary
1251   - * @param list the list
1252   - * @param listRegexp the list regexp
1253   - * @param ignoreRegexp the ignore regexp
1254   - * @param ignoreList the ignore list
1255   - * @param ignoreListRegexp the ignore list regexp
1256   - * @throws IOException Signals that an I/O exception has occurred.
1257   - * @throws ParseException the parse exception
  1330 + * @param key
  1331 + * the key
  1332 + * @param prefix
  1333 + * the prefix
  1334 + * @param regexp
  1335 + * the regexp
  1336 + * @param full
  1337 + * the full
  1338 + * @param type
  1339 + * the type
  1340 + * @param sortType
  1341 + * the sort type
  1342 + * @param sortDirection
  1343 + * the sort direction
  1344 + * @param startValue
  1345 + * the start value
  1346 + * @param number
  1347 + * the number
  1348 + * @param functionKey
  1349 + * the function key
  1350 + * @param functionExpression
  1351 + * the function expression
  1352 + * @param functionType
  1353 + * the function type
  1354 + * @param boundary
  1355 + * the boundary
  1356 + * @param list
  1357 + * the list
  1358 + * @param listRegexp
  1359 + * the list regexp
  1360 + * @param ignoreRegexp
  1361 + * the ignore regexp
  1362 + * @param ignoreList
  1363 + * the ignore list
  1364 + * @param ignoreListRegexp
  1365 + * the ignore list regexp
  1366 + * @throws IOException
  1367 + * Signals that an I/O exception has occurred.
  1368 + * @throws ParseException
  1369 + * the parse exception
1258 1370 */
1259 1371 @SuppressWarnings({ "unchecked", "rawtypes" })
1260 1372 public ComponentTermVector(String key, String prefix, String regexp,
... ... @@ -1461,16 +1573,26 @@ public class CodecComponent {
1461 1573 /**
1462 1574 * Instantiates a new component span.
1463 1575 *
1464   - * @param queries the queries
1465   - * @param key the key
1466   - * @param minimumDouble the minimum double
1467   - * @param maximumDouble the maximum double
1468   - * @param type the type
1469   - * @param functionKey the function key
1470   - * @param functionExpression the function expression
1471   - * @param functionType the function type
1472   - * @throws IOException Signals that an I/O exception has occurred.
1473   - * @throws ParseException the parse exception
  1576 + * @param queries
  1577 + * the queries
  1578 + * @param key
  1579 + * the key
  1580 + * @param minimumDouble
  1581 + * the minimum double
  1582 + * @param maximumDouble
  1583 + * the maximum double
  1584 + * @param type
  1585 + * the type
  1586 + * @param functionKey
  1587 + * the function key
  1588 + * @param functionExpression
  1589 + * the function expression
  1590 + * @param functionType
  1591 + * the function type
  1592 + * @throws IOException
  1593 + * Signals that an I/O exception has occurred.
  1594 + * @throws ParseException
  1595 + * the parse exception
1474 1596 */
1475 1597 public ComponentSpan(MtasSpanQuery[] queries, String key,
1476 1598 Double minimumDouble, Double maximumDouble, String type,
... ... @@ -1603,12 +1725,18 @@ public class CodecComponent {
1603 1725 /**
1604 1726 * Instantiates a new component position.
1605 1727 *
1606   - * @param key the key
1607   - * @param minimumDouble the minimum double
1608   - * @param maximumDouble the maximum double
1609   - * @param statsType the stats type
1610   - * @throws IOException Signals that an I/O exception has occurred.
1611   - * @throws ParseException the parse exception
  1728 + * @param key
  1729 + * the key
  1730 + * @param minimumDouble
  1731 + * the minimum double
  1732 + * @param maximumDouble
  1733 + * the maximum double
  1734 + * @param statsType
  1735 + * the stats type
  1736 + * @throws IOException
  1737 + * Signals that an I/O exception has occurred.
  1738 + * @throws ParseException
  1739 + * the parse exception
1612 1740 */
1613 1741 public ComponentPosition(String key, Double minimumDouble,
1614 1742 Double maximumDouble, String statsType)
... ... @@ -1662,12 +1790,18 @@ public class CodecComponent {
1662 1790 /**
1663 1791 * Instantiates a new component token.
1664 1792 *
1665   - * @param key the key
1666   - * @param minimumDouble the minimum double
1667   - * @param maximumDouble the maximum double
1668   - * @param statsType the stats type
1669   - * @throws IOException Signals that an I/O exception has occurred.
1670   - * @throws ParseException the parse exception
  1793 + * @param key
  1794 + * the key
  1795 + * @param minimumDouble
  1796 + * the minimum double
  1797 + * @param maximumDouble
  1798 + * the maximum double
  1799 + * @param statsType
  1800 + * the stats type
  1801 + * @throws IOException
  1802 + * Signals that an I/O exception has occurred.
  1803 + * @throws ParseException
  1804 + * the parse exception
1671 1805 */
1672 1806 public ComponentToken(String key, Double minimumDouble,
1673 1807 Double maximumDouble, String statsType)
... ... @@ -1693,65 +1827,267 @@ public class CodecComponent {
1693 1827 }
1694 1828  
1695 1829 /**
1696   - * The Class ComponentJoin.
  1830 + * The Class ComponentCollection.
1697 1831 */
1698   - public static class ComponentJoin implements BasicComponent {
  1832 + public static class ComponentCollection implements BasicComponent {
  1833 +
  1834 + /** The Constant ACTION_CREATE. */
  1835 + public static final String ACTION_CREATE = "create";
  1836 +
  1837 + /** The Constant ACTION_CHECK. */
  1838 + public static final String ACTION_CHECK = "check";
  1839 +
  1840 + /** The Constant ACTION_LIST. */
  1841 + public static final String ACTION_LIST = "list";
  1842 +
  1843 + /** The Constant ACTION_POST. */
  1844 + public static final String ACTION_POST = "post";
  1845 +
  1846 + /** The Constant ACTION_IMPORT. */
  1847 + public static final String ACTION_IMPORT = "import";
  1848 +
  1849 + /** The Constant ACTION_DELETE. */
  1850 + public static final String ACTION_DELETE = "delete";
  1851 +
  1852 + /** The Constant ACTION_EMPTY. */
  1853 + public static final String ACTION_EMPTY = "empty";
  1854 +
  1855 + /** The Constant ACTION_GET. */
  1856 + public static final String ACTION_GET = "get";
  1857 +
  1858 + /** The key. */
  1859 + public String key;
  1860 +
  1861 + /** The version. */
  1862 + public String version;
  1863 +
  1864 + /** The id. */
  1865 + public String id;
  1866 +
  1867 + /** The action. */
  1868 + private String action;
1699 1869  
1700 1870 /** The fields. */
1701 1871 private Set<String> fields;
1702 1872  
1703 1873 /** The values. */
1704   - private Set<String> values;
1705   -
1706   - /** The key. */
1707   - private String key;
  1874 + private HashSet<String> values;
1708 1875  
1709 1876 /**
1710   - * Instantiates a new component join.
  1877 + * Instantiates a new component collection.
1711 1878 *
1712   - * @param fields the fields
1713   - * @param key the key
  1879 + * @param key
  1880 + * the key
  1881 + * @param action
  1882 + * the action
1714 1883 */
1715   - public ComponentJoin(Set<String> fields, String key) {
1716   - this.fields = fields;
  1884 + public ComponentCollection(String key, String action) {
1717 1885 this.key = key;
1718   - this.values = new HashSet<>();
  1886 + this.action = action;
  1887 + this.version = null;
  1888 + values = new HashSet<>();
1719 1889 }
1720 1890  
1721 1891 /**
1722   - * Adds the.
  1892 + * Sets the list variables.
1723 1893 *
1724   - * @param value the value
  1894 + * @throws IOException
  1895 + * Signals that an I/O exception has occurred.
1725 1896 */
1726   - public void add(String value) {
1727   - values.add(value);
  1897 + public void setListVariables() throws IOException {
  1898 + if (action.equals(ACTION_LIST)) {
  1899 + // do nothing
  1900 + } else {
  1901 + throw new IOException("not allowed with action " + action);
  1902 + }
1728 1903 }
1729 1904  
1730 1905 /**
1731   - * Adds the.
  1906 + * Sets the create variables.
1732 1907 *
1733   - * @param values the values
  1908 + * @param id
  1909 + * the id
  1910 + * @param fields
  1911 + * the fields
  1912 + * @throws IOException
  1913 + * Signals that an I/O exception has occurred.
1734 1914 */
1735   - public void add(Set<String> values) {
1736   - this.values.addAll(values);
  1915 + public void setCreateVariables(String id, Set<String> fields)
  1916 + throws IOException {
  1917 + if (action.equals(ACTION_CREATE)) {
  1918 + this.id = id;
  1919 + this.fields = fields;
  1920 + } else {
  1921 + throw new IOException("not allowed with action " + action);
  1922 + }
1737 1923 }
1738 1924  
1739 1925 /**
1740   - * Values.
  1926 + * Sets the check variables.
1741 1927 *
1742   - * @return the sets the
  1928 + * @param id
  1929 + * the new check variables
  1930 + * @throws IOException
  1931 + * Signals that an I/O exception has occurred.
1743 1932 */
1744   - public Set<String> values() {
1745   - return values;
  1933 + public void setCheckVariables(String id) throws IOException {
  1934 + if (action.equals(ACTION_CHECK)) {
  1935 + this.id = id;
  1936 + } else {
  1937 + throw new IOException("not allowed with action " + action);
  1938 + }
  1939 + }
  1940 +
  1941 + /**
  1942 + * Sets the gets the variables.
  1943 + *
  1944 + * @param id
  1945 + * the new gets the variables
  1946 + * @throws IOException
  1947 + * Signals that an I/O exception has occurred.
  1948 + */
  1949 + public void setGetVariables(String id) throws IOException {
  1950 + if (action.equals(ACTION_GET)) {
  1951 + this.id = id;
  1952 + } else {
  1953 + throw new IOException("not allowed with action " + action);
  1954 + }
  1955 + }
  1956 +
  1957 + /**
  1958 + * Sets the post variables.
  1959 + *
  1960 + * @param id
  1961 + * the id
  1962 + * @param values
  1963 + * the values
  1964 + * @throws IOException
  1965 + * Signals that an I/O exception has occurred.
  1966 + */
  1967 + public void setPostVariables(String id, HashSet<String> values)
  1968 + throws IOException {
  1969 + if (action.equals(ACTION_POST)) {
  1970 + this.id = id;
  1971 + this.values = values;
  1972 + } else {
  1973 + throw new IOException("not allowed with action " + action);
  1974 + }
  1975 + }
  1976 +
  1977 + public void setImportVariables(String id, String url, String collection)
  1978 + throws IOException {
  1979 + if (action.equals(ACTION_IMPORT)) {
  1980 + this.id = id;
  1981 + StringBuilder importUrlBuffer = new StringBuilder(url);
  1982 + importUrlBuffer.append("select");
  1983 + importUrlBuffer.append("?q=*:*&rows=0&wt=json");
  1984 + importUrlBuffer.append("&mtas=true&mtas.collection=true");
  1985 + importUrlBuffer.append("&mtas.collection.0.key=0");
  1986 + importUrlBuffer.append("&mtas.collection.0.action=get");
  1987 + importUrlBuffer.append(
  1988 + "&mtas.collection.0.id=" + URLEncoder.encode(collection, "UTF-8"));
  1989 + Map<String, Object> params = getImport(importUrlBuffer.toString());
  1990 + try {
  1991 + if (params.containsKey("mtas")
  1992 + && params.get("mtas") instanceof Map) {
  1993 + Map<String, Object> mtasParams = (Map<String, Object>) params
  1994 + .get("mtas");
  1995 + if (mtasParams.containsKey("collection")
  1996 + && mtasParams.get("collection") instanceof List) {
  1997 + List<Object> mtasCollectionList = (List<Object>) mtasParams
  1998 + .get("collection");
  1999 + if (mtasCollectionList.size() == 1
  2000 + && mtasCollectionList.get(0) instanceof Map) {
  2001 + Map<String, Object> collectionData = (Map<String, Object>) mtasCollectionList
  2002 + .get(0);
  2003 + if (collectionData.containsKey("values")
  2004 + && collectionData.get("values") instanceof List) {
  2005 + List<String> valuesList = (List<String>) collectionData
  2006 + .get("values");
  2007 + for (String valueItem : valuesList) {
  2008 + values.add(valueItem);
  2009 + }
  2010 + } else {
  2011 + throw new IOException("no values in response");
  2012 + }
  2013 + } else {
  2014 + throw new IOException(
  2015 + "no valid mtas collection item in response");
  2016 + }
  2017 + } else {
  2018 + throw new IOException("no valid mtas collection in response");
  2019 + }
  2020 + } else {
  2021 + throw new IOException("no valid mtas in response");
  2022 + }
  2023 + } catch (ClassCastException e) {
  2024 + throw new IOException("unexpected response", e);
  2025 + }
  2026 + } else {
  2027 + throw new IOException("not allowed with action " + action);
  2028 + }
  2029 + }
  2030 +
  2031 + private Map<String, Object> getImport(String collectionGetUrl)
  2032 + throws IOException {
  2033 + // get data
  2034 + URL url = new URL(collectionGetUrl);
  2035 + HttpURLConnection connection = (HttpURLConnection) url.openConnection();
  2036 + connection.setDoOutput(false);
  2037 + connection.setDoInput(true);
  2038 + connection.setInstanceFollowRedirects(false);
  2039 + connection.setRequestMethod("GET");
  2040 + connection.setRequestProperty("Content-Type",
  2041 + "application/json; charset=UTF-8");
  2042 + connection.setRequestProperty("charset", "utf-8");
  2043 + connection.setUseCaches(false);
  2044 + // process response
  2045 + InputStream is = null;
  2046 + try {
  2047 + is = connection.getInputStream();
  2048 + } catch (IOException ioe) {
  2049 + throw new IOException("Couldn't get data from url");
  2050 + }
  2051 + InputStreamReader in = new InputStreamReader((InputStream) is, "UTF8");
  2052 + Map<String, Object> params = new HashMap<>();
  2053 + getParamsFromJSON(params,
  2054 + IOUtils.toString(in));
  2055 + connection.disconnect();
  2056 + return params;
1746 2057 }
1747 2058  
1748 2059 /**
1749   - * Key.
  2060 + * Sets the delete variables.
  2061 + *
  2062 + * @param id
  2063 + * the new delete variables
  2064 + * @throws IOException
  2065 + * Signals that an I/O exception has occurred.
  2066 + */
  2067 + public void setDeleteVariables(String id) throws IOException {
  2068 + if (action.equals(ACTION_DELETE)) {
  2069 + this.id = id;
  2070 + } else {
  2071 + throw new IOException("not allowed with action " + action);
  2072 + }
  2073 + }
  2074 +
  2075 + /**
  2076 + * Action.
1750 2077 *
1751 2078 * @return the string
1752 2079 */
1753   - public String key() {
1754   - return key;
  2080 + public String action() {
  2081 + return action;
  2082 + }
  2083 +
  2084 + /**
  2085 + * Values.
  2086 + *
  2087 + * @return the hash set
  2088 + */
  2089 + public HashSet<String> values() {
  2090 + return values;
1755 2091 }
1756 2092  
1757 2093 /**
... ... @@ -1763,6 +2099,61 @@ public class CodecComponent {
1763 2099 return fields;
1764 2100 }
1765 2101  
  2102 + /**
  2103 + * Adds the value.
  2104 + *
  2105 + * @param value
  2106 + * the value
  2107 + * @throws IOException
  2108 + * Signals that an I/O exception has occurred.
  2109 + */
  2110 + public void addValue(String value) throws IOException {
  2111 + if (action.equals(ACTION_CREATE)) {
  2112 + if (version == null) {
  2113 + values.add(value);
  2114 + } else {
  2115 + throw new IOException("version already set");
  2116 + }
  2117 + } else {
  2118 + throw new IOException("not allowed for action '" + action + "'");
  2119 + }
  2120 + }
  2121 +
  2122 + private static void getParamsFromJSON(Map<String, Object> params,
  2123 + String json) {
  2124 + JSONParser parser = new JSONParser(json);
  2125 + try {
  2126 + Object o = ObjectBuilder.getVal(parser);
  2127 + if (!(o instanceof Map))
  2128 + return;
  2129 + Map<String, Object> map = (Map<String, Object>) o;
  2130 + // To make consistent with json.param handling, we should make query
  2131 + // params come after json params (i.e. query params should
  2132 + // appear to overwrite json params.
  2133 +
  2134 + // Solr params are based on String though, so we need to convert
  2135 + for (Map.Entry<String, Object> entry : map.entrySet()) {
  2136 + String key = entry.getKey();
  2137 + Object val = entry.getValue();
  2138 + if (params.get(key) != null) {
  2139 + continue;
  2140 + }
  2141 +
  2142 + if (val == null) {
  2143 + params.remove(key);
  2144 + } else {
  2145 + params.put(key, val);
  2146 + }
  2147 + }
  2148 +
  2149 + } catch (Exception e) {
  2150 + // ignore parse exceptions at this stage, they may be caused by incomplete
  2151 + // macro expansions
  2152 + return;
  2153 + }
  2154 +
  2155 + }
  2156 +
1766 2157 }
1767 2158  
1768 2159 /**
... ... @@ -1803,18 +2194,30 @@ public class CodecComponent {
1803 2194 /**
1804 2195 * Instantiates a new sub component function.
1805 2196 *
1806   - * @param collectorType the collector type
1807   - * @param key the key
1808   - * @param type the type
1809   - * @param parserFunction the parser function
1810   - * @param sortType the sort type
1811   - * @param sortDirection the sort direction
1812   - * @param start the start
1813   - * @param number the number
1814   - * @param segmentRegistration the segment registration
1815   - * @param boundary the boundary
1816   - * @throws ParseException the parse exception
1817   - * @throws IOException Signals that an I/O exception has occurred.
  2197 + * @param collectorType
  2198 + * the collector type
  2199 + * @param key
  2200 + * the key
  2201 + * @param type
  2202 + * the type
  2203 + * @param parserFunction
  2204 + * the parser function
  2205 + * @param sortType
  2206 + * the sort type
  2207 + * @param sortDirection
  2208 + * the sort direction
  2209 + * @param start
  2210 + * the start
  2211 + * @param number
  2212 + * the number
  2213 + * @param segmentRegistration
  2214 + * the segment registration
  2215 + * @param boundary
  2216 + * the boundary
  2217 + * @throws ParseException
  2218 + * the parse exception
  2219 + * @throws IOException
  2220 + * Signals that an I/O exception has occurred.
1818 2221 */
1819 2222 public SubComponentFunction(String collectorType, String key, String type,
1820 2223 MtasFunctionParserFunction parserFunction, String sortType,
... ... @@ -1847,12 +2250,18 @@ public class CodecComponent {
1847 2250 /**
1848 2251 * Instantiates a new sub component function.
1849 2252 *
1850   - * @param collectorType the collector type
1851   - * @param key the key
1852   - * @param expression the expression
1853   - * @param type the type
1854   - * @throws ParseException the parse exception
1855   - * @throws IOException Signals that an I/O exception has occurred.
  2253 + * @param collectorType
  2254 + * the collector type
  2255 + * @param key
  2256 + * the key
  2257 + * @param expression
  2258 + * the expression
  2259 + * @param type
  2260 + * the type
  2261 + * @throws ParseException
  2262 + * the parse exception
  2263 + * @throws IOException
  2264 + * Signals that an I/O exception has occurred.
1856 2265 */
1857 2266 public SubComponentFunction(String collectorType, String key,
1858 2267 String expression, String type) throws ParseException, IOException {
... ... @@ -1895,8 +2304,10 @@ public class CodecComponent {
1895 2304 /**
1896 2305 * Instantiates a new kwic token.
1897 2306 *
1898   - * @param match the match
1899   - * @param tokens the tokens
  2307 + * @param match
  2308 + * the match
  2309 + * @param tokens
  2310 + * the tokens
1900 2311 */
1901 2312 public KwicToken(Match match, List<MtasTokenString> tokens) {
1902 2313 startPosition = match.startPosition;
... ... @@ -1923,8 +2334,10 @@ public class CodecComponent {
1923 2334 /**
1924 2335 * Instantiates a new kwic hit.
1925 2336 *
1926   - * @param match the match
1927   - * @param hits the hits
  2337 + * @param match
  2338 + * the match
  2339 + * @param hits
  2340 + * the hits
1928 2341 */
1929 2342 public KwicHit(Match match, Map<Integer, List<String>> hits) {
1930 2343 startPosition = match.startPosition;
... ... @@ -1996,7 +2409,8 @@ public class CodecComponent {
1996 2409 /**
1997 2410 * Sort.
1998 2411 *
1999   - * @param data the data
  2412 + * @param data
  2413 + * the data
2000 2414 * @return the list
2001 2415 */
2002 2416 private List<MtasTreeHit<String>> sort(List<MtasTreeHit<String>> data) {
... ... @@ -2015,30 +2429,35 @@ public class CodecComponent {
2015 2429 /**
2016 2430 * Instantiates a new group hit.
2017 2431 *
2018   - * @param list the list
2019   - * @param start the start
2020   - * @param end the end
2021   - * @param hitStart the hit start
2022   - * @param hitEnd the hit end
2023   - * @param group the group
2024   - * @param knownPrefixes the known prefixes
2025   - * @throws UnsupportedEncodingException the unsupported encoding exception
  2432 + * @param list
  2433 + * the list
  2434 + * @param start
  2435 + * the start
  2436 + * @param end
  2437 + * the end
  2438 + * @param hitStart
  2439 + * the hit start
  2440 + * @param hitEnd
  2441 + * the hit end
  2442 + * @param group
  2443 + * the group
  2444 + * @param knownPrefixes
  2445 + * the known prefixes
  2446 + * @throws UnsupportedEncodingException
  2447 + * the unsupported encoding exception
2026 2448 */
2027 2449 @SuppressWarnings("unchecked")
2028 2450 public GroupHit(List<MtasTreeHit<String>> list, int start, int end,
2029 2451 int hitStart, int hitEnd, ComponentGroup group,
2030 2452 Set<String> knownPrefixes) throws UnsupportedEncodingException {
2031   - // System.out.println("init: "+start+"-"+end+"\t"+hitStart+"-"+hitEnd);
2032 2453 // compute dimensions
2033 2454 int leftRangeStart = start;
2034   - int leftRangeEnd = Math.min(end - 1, hitStart - 1);
  2455 + int leftRangeEnd = Math.min(end, hitStart - 1);
2035 2456 int leftRangeLength = Math.max(0, 1 + leftRangeEnd - leftRangeStart);
2036 2457 int hitLength = 1 + hitEnd - hitStart;
2037 2458 int rightRangeStart = Math.max(start, hitEnd + 1);
2038 2459 int rightRangeEnd = end;
2039 2460 int rightRangeLength = Math.max(0, 1 + rightRangeEnd - rightRangeStart);
2040   - // System.out.println(leftRangeStart+"\t"+leftRangeEnd+"\t"+leftRangeLength+"
2041   - // - "+rightRangeStart+"\t"+rightRangeEnd+"\t"+rightRangeLength);
2042 2461 // create initial arrays
2043 2462 if (leftRangeLength > 0) {
2044 2463 keyLeft = "";
... ... @@ -2112,15 +2531,8 @@ public class CodecComponent {
2112 2531 }
2113 2532 }
2114 2533 if (group.hitInsideRight != null) {
2115   - // System.out.println(missingHit.length + " items in missingHit");
2116   - // System.out.println(
2117   - // group.hitInsideRight.length + " items in group.hitInsideRight");
2118   - for (int p = 0; p < group.hitInsideRight.length; p++) {
2119   - // System.out.println(" - " + group.hitInsideRight[p]);
2120   - }
2121 2534 for (int p = Math.max(hitStart,
2122 2535 hitEnd - group.hitInsideRight.length + 1); p <= hitEnd; p++) {
2123   - // System.out.println("Test voor p is " + (p - hitStart));
2124 2536 if (group.hitInsideRight[hitEnd - p] != null) {
2125 2537 missingHit[p - hitStart].addAll(group.hitInsideRight[hitEnd - p]);
2126 2538 }
... ... @@ -2142,7 +2554,7 @@ public class CodecComponent {
2142 2554 }
2143 2555 }
2144 2556 if (group.hitRight != null) {
2145   - for (int p = 0; p <= Math.min(leftRangeLength,
  2557 + for (int p = 0; p < Math.min(leftRangeLength,
2146 2558 group.hitRight.length - dataHit.length); p++) {
2147 2559 if (group.hitRight[p + dataHit.length] != null) {
2148 2560 missingLeft[p].addAll(group.hitRight[p + dataHit.length]);
... ... @@ -2152,13 +2564,17 @@ public class CodecComponent {
2152 2564 if (group.right != null) {
2153 2565 for (int p = 0; p < Math.min(rightRangeLength,
2154 2566 group.right.length); p++) {
2155   - missingRight[p].addAll(group.right[p]);
  2567 + if (group.right[p] != null) {
  2568 + missingRight[p].addAll(group.right[p]);
  2569 + }
2156 2570 }
2157 2571 }
2158   - if (group.hitRight != null) {
2159   - for (int p = 0; p <= Math.min(rightRangeLength,
  2572 + if (group.hitLeft != null) {
  2573 + for (int p = 0; p < Math.min(rightRangeLength,
2160 2574 group.hitLeft.length - dataHit.length); p++) {
2161   - missingRight[p].addAll(group.hitLeft[p + dataHit.length]);
  2575 + if(group.hitLeft[p + dataHit.length]!=null) {
  2576 + missingRight[p].addAll(group.hitLeft[p + dataHit.length]);
  2577 + }
2162 2578 }
2163 2579 }
2164 2580  
... ... @@ -2170,12 +2586,9 @@ public class CodecComponent {
2170 2586 && group.hitInside.contains(hit.idData)) {
2171 2587 for (int p = Math.max(hitStart, hit.startPosition); p <= Math
2172 2588 .min(hitEnd, hit.endPosition); p++) {
2173   - // keyHit += hit.refData;
2174 2589 dataHit[p - hitStart].add(hit.refData);
2175 2590 missingHit[p - hitStart]
2176   - .remove(MtasToken.getPrefixFromValue(hit.refData));
2177   - // System.out.print(p + "." + hit.idData + ":" + hit.refData +
2178   - // "\t");
  2591 + .remove(MtasToken.getPrefixFromValue(hit.refData));
2179 2592 }
2180 2593 } else if ((group.hitInsideLeft != null || group.hitLeft != null
2181 2594 || group.hitInsideRight != null || group.hitRight != null)
... ... @@ -2191,9 +2604,7 @@ public class CodecComponent {
2191 2604 // keyHit += hit.refData;
2192 2605 dataHit[p - hitStart].add(hit.refData);
2193 2606 missingHit[p - hitStart]
2194   - .remove(MtasToken.getPrefixFromValue(hit.refData));
2195   - // System.out.print(p+"."+hit.idData + ":" + hit.additionalRef +
2196   - // "\t");
  2607 + .remove(MtasToken.getPrefixFromValue(hit.refData));
2197 2608 } else if (group.hitLeft != null
2198 2609 && pHitLeft <= (group.hitLeft.length - 1)
2199 2610 && group.hitLeft[pHitLeft] != null
... ... @@ -2201,29 +2612,21 @@ public class CodecComponent {
2201 2612 // keyHit += hit.refData;
2202 2613 dataHit[p - hitStart].add(hit.refData);
2203 2614 missingHit[p - hitStart]
2204   - .remove(MtasToken.getPrefixFromValue(hit.refData));
2205   - // System.out.print(p+"."+hit.idData + ":" + hit.additionalRef +
2206   - // "\t");
  2615 + .remove(MtasToken.getPrefixFromValue(hit.refData));
2207 2616 } else if (group.hitInsideRight != null
2208 2617 && pHitRight <= (group.hitInsideRight.length - 1)
2209 2618 && group.hitInsideRight[pHitRight] != null
2210 2619 && group.hitInsideRight[pHitRight].contains(hit.idData)) {
2211   - // keyHit += hit.refData;
2212 2620 dataHit[p - hitStart].add(hit.refData);
2213 2621 missingHit[p - hitStart]
2214   - .remove(MtasToken.getPrefixFromValue(hit.refData));
2215   - // System.out.print(p+"."+hit.idData + ":" + hit.additionalRef +
2216   - // "\t");
  2622 + .remove(MtasToken.getPrefixFromValue(hit.refData));
2217 2623 } else if (group.hitRight != null
2218 2624 && pHitRight <= (group.hitRight.length - 1)
2219 2625 && group.hitRight[pHitRight] != null
2220 2626 && group.hitRight[pHitRight].contains(hit.idData)) {
2221   - // keyHit += hit.refData;
2222 2627 dataHit[p - hitStart].add(hit.refData);
2223 2628 missingHit[p - hitStart]
2224   - .remove(MtasToken.getPrefixFromValue(hit.refData));
2225   - // System.out.print(p+"."+hit.idData + ":" + hit.additionalRef +
2226   - // "\t");
  2629 + .remove(MtasToken.getPrefixFromValue(hit.refData));
2227 2630 }
2228 2631 }
2229 2632 }
... ... @@ -2239,18 +2642,16 @@ public class CodecComponent {
2239 2642 if (group.left != null && pLeft <= (group.left.length - 1)
2240 2643 && group.left[pLeft] != null
2241 2644 && group.left[pLeft].contains(hit.idData)) {
2242   - dataLeft[p - leftRangeStart].add(hit.refData);
2243   - missingLeft[p - leftRangeStart]
  2645 + dataLeft[hitStart - 1 - p].add(hit.refData);
  2646 + missingLeft[hitStart - 1 - p]
2244 2647 .remove(MtasToken.getPrefixFromValue(hit.refData));
2245   - // System.out.print("L"+p+"."+prefix + ":" + value + "\t");
2246 2648 } else if (group.hitRight != null
2247 2649 && pHitRight <= (group.hitRight.length - 1)
2248 2650 && group.hitRight[pHitRight] != null
2249 2651 && group.hitRight[pHitRight].contains(hit.idData)) {
2250   - dataLeft[p - leftRangeStart].add(hit.refData);
2251   - missingLeft[p - leftRangeStart]
  2652 + dataLeft[hitStart - 1 - p].add(hit.refData);
  2653 + missingLeft[hitStart - 1 - p]
2252 2654 .remove(MtasToken.getPrefixFromValue(hit.refData));
2253   - // System.out.print("L"+p+"."+prefix + ":" + value + "\t");
2254 2655 }
2255 2656 }
2256 2657 }
... ... @@ -2270,7 +2671,6 @@ public class CodecComponent {
2270 2671 dataRight[p - rightRangeStart].add(hit.refData);
2271 2672 missingRight[p - rightRangeStart]
2272 2673 .remove(MtasToken.getPrefixFromValue(hit.refData));
2273   - // System.out.print("R"+p+"."+prefix + ":" + value + "\t");
2274 2674 } else if (group.hitLeft != null
2275 2675 && pHitLeft <= (group.hitLeft.length - 1)
2276 2676 && group.hitLeft[pHitLeft] != null
... ... @@ -2278,7 +2678,6 @@ public class CodecComponent {
2278 2678 dataRight[p - rightRangeStart].add(hit.refData);
2279 2679 missingRight[p - rightRangeStart]
2280 2680 .remove(MtasToken.getPrefixFromValue(hit.refData));
2281   - // System.out.print("R"+p+"."+prefix + ":" + value + "\t");
2282 2681 }
2283 2682 }
2284 2683 }
... ... @@ -2354,8 +2753,10 @@ public class CodecComponent {
2354 2753 /**
2355 2754 * Data equals.
2356 2755 *
2357   - * @param d1 the d 1
2358   - * @param d2 the d 2
  2756 + * @param d1
  2757 + * the d 1
  2758 + * @param d2
  2759 + * the d 2
2359 2760 * @return true, if successful
2360 2761 */
2361 2762 private boolean dataEquals(List<String>[] d1, List<String>[] d2) {
... ... @@ -2415,10 +2816,13 @@ public class CodecComponent {
2415 2816 /**
2416 2817 * Data to string.
2417 2818 *
2418   - * @param data the data
2419   - * @param missing the missing
  2819 + * @param data
  2820 + * the data
  2821 + * @param missing
  2822 + * the missing
2420 2823 * @return the string
2421   - * @throws UnsupportedEncodingException the unsupported encoding exception
  2824 + * @throws UnsupportedEncodingException
  2825 + * the unsupported encoding exception
2422 2826 */
2423 2827 private String dataToString(List<String>[] data, Set<String>[] missing)
2424 2828 throws UnsupportedEncodingException {
... ... @@ -2475,8 +2879,10 @@ public class CodecComponent {
2475 2879 /**
2476 2880 * Key to sub sub object.
2477 2881 *
2478   - * @param key the key
2479   - * @param newKey the new key
  2882 + * @param key
  2883 + * the key
  2884 + * @param newKey
  2885 + * the new key
2480 2886 * @return the map[]
2481 2887 */
2482 2888 private static Map<String, String>[] keyToSubSubObject(String key,
... ... @@ -2543,8 +2949,10 @@ public class CodecComponent {
2543 2949 /**
2544 2950 * Key to sub object.
2545 2951 *
2546   - * @param key the key
2547   - * @param newKey the new key
  2952 + * @param key
  2953 + * the key
  2954 + * @param newKey
  2955 + * the new key
2548 2956 * @return the map
2549 2957 */
2550 2958 private static Map<Integer, Map<String, String>[]> keyToSubObject(
... ... @@ -2568,8 +2976,10 @@ public class CodecComponent {
2568 2976 /**
2569 2977 * Key to object.
2570 2978 *
2571   - * @param key the key
2572   - * @param newKey the new key
  2979 + * @param key
  2980 + * the key
  2981 + * @param newKey
  2982 + * the new key
2573 2983 * @return the map
2574 2984 */
2575 2985 public static Map<String, Map<Integer, Map<String, String>[]>> keyToObject(
... ... @@ -2634,10 +3044,14 @@ public class CodecComponent {
2634 3044 /**
2635 3045 * Instantiates a new list token.
2636 3046 *
2637   - * @param docId the doc id
2638   - * @param docPosition the doc position
2639   - * @param match the match
2640   - * @param tokens the tokens
  3047 + * @param docId
  3048 + * the doc id
  3049 + * @param docPosition
  3050 + * the doc position
  3051 + * @param match
  3052 + * the match
  3053 + * @param tokens
  3054 + * the tokens
2641 3055 */
2642 3056 public ListToken(Integer docId, Integer docPosition, Match match,
2643 3057 List<MtasTokenString> tokens) {
... ... @@ -2672,10 +3086,14 @@ public class CodecComponent {
2672 3086 /**
2673 3087 * Instantiates a new list hit.
2674 3088 *
2675   - * @param docId the doc id
2676   - * @param docPosition the doc position
2677   - * @param match the match
2678   - * @param hits the hits
  3089 + * @param docId
  3090 + * the doc id
  3091 + * @param docPosition
  3092 + * the doc position
  3093 + * @param match
  3094 + * the match
  3095 + * @param hits
  3096 + * the hits
2679 3097 */
2680 3098 public ListHit(Integer docId, Integer docPosition, Match match,
2681 3099 Map<Integer, List<String>> hits) {
... ... @@ -2701,8 +3119,10 @@ public class CodecComponent {
2701 3119 /**
2702 3120 * Instantiates a new match.
2703 3121 *
2704   - * @param startPosition the start position
2705   - * @param endPosition the end position
  3122 + * @param startPosition
  3123 + * the start position
  3124 + * @param endPosition
  3125 + * the end position
2706 3126 */
2707 3127 public Match(int startPosition, int endPosition) {
2708 3128 this.startPosition = startPosition;
... ...
src/mtas/codec/util/CodecUtil.java
  View file @7179e4b
... ... @@ -16,7 +16,7 @@ import mtas.codec.MtasCodecPostingsFormat;
16 16 import mtas.parser.function.util.MtasFunctionParserFunction;
17 17 import mtas.search.spans.util.MtasSpanQuery;
18 18 import mtas.codec.util.CodecComponent.ComponentField;
19   -import mtas.codec.util.CodecComponent.ComponentJoin;
  19 +import mtas.codec.util.CodecComponent.ComponentCollection;
20 20  
21 21 import org.apache.lucene.index.FieldInfo;
22 22 import org.apache.lucene.index.IndexReader;
... ... @@ -247,18 +247,18 @@ public class CodecUtil {
247 247 }
248 248  
249 249 /**
250   - * Collect join.
  250 + * Collect collection.
251 251 *
252 252 * @param reader the reader
253 253 * @param fullDocSet the full doc set
254   - * @param joinInfo the join info
  254 + * @param collectionInfo the collection info
255 255 * @throws IOException Signals that an I/O exception has occurred.
256 256 */
257   - public static void collectJoin(IndexReader reader,
258   - ArrayList<Integer> fullDocSet, ComponentJoin joinInfo)
  257 + public static void collectCollection(IndexReader reader,
  258 + List<Integer> fullDocSet, ComponentCollection collectionInfo)
259 259 throws IOException {
260   - if (joinInfo != null) {
261   - CodecCollector.collectJoin(reader, fullDocSet, joinInfo);
  260 + if (collectionInfo != null) {
  261 + CodecCollector.collectCollection(reader, fullDocSet, collectionInfo);
262 262 }
263 263 }
264 264  
... ...
src/mtas/codec/util/collector/MtasDataCollector.java
  View file @7179e4b
... ... @@ -1336,7 +1336,7 @@ public abstract class MtasDataCollector&lt;T1 extends Number &amp; Comparable&lt;T1&gt;, T2 e
1336 1336 text.append("\tclosed: " + closed + "\n");
1337 1337 text.append("\tkeylist: " + Arrays.asList(keyList) + "\n");
1338 1338 text.append("\tkeylist: " + Arrays.asList(keyList).contains("1") + "\n");
1339   - text.append("\tsegmentKeys: " + segmentKeys.contains("1") + "\n");
  1339 + text.append("\tsegmentKeys: " + (segmentKeys!=null?segmentKeys.contains("1"):"null") + "\n");
1340 1340 text.append("\tnewKeys: " + Arrays.asList(newKeyList).contains("1") + "\n");
1341 1341 return text.toString().trim();
1342 1342 }
... ...
src/mtas/solr/handler/component/MtasSolrSearchComponent.java
  View file @7179e4b
... ... @@ -11,6 +11,7 @@ import mtas.codec.util.CodecComponent.ComponentDocument;
11 11 import mtas.codec.util.CodecComponent.ComponentFacet;
12 12 import mtas.codec.util.CodecComponent.ComponentFields;
13 13 import mtas.codec.util.CodecComponent.ComponentGroup;
  14 +import mtas.codec.util.CodecComponent.ComponentCollection;
14 15 import mtas.codec.util.CodecComponent.ComponentKwic;
15 16 import mtas.codec.util.CodecComponent.ComponentList;
16 17 import mtas.codec.util.CodecComponent.ComponentPosition;
... ... @@ -19,11 +20,11 @@ import mtas.codec.util.CodecComponent.ComponentTermVector;
19 20 import mtas.codec.util.CodecComponent.ComponentToken;
20 21 import mtas.codec.util.CodecUtil;
21 22 import mtas.solr.handler.component.util.MtasSolrResultMerge;
22   -import mtas.solr.search.MtasSolrJoinCache;
  23 +import mtas.solr.search.MtasSolrCollectionCache;
23 24 import mtas.solr.handler.component.util.MtasSolrComponentDocument;
24 25 import mtas.solr.handler.component.util.MtasSolrComponentFacet;
25 26 import mtas.solr.handler.component.util.MtasSolrComponentGroup;
26   -import mtas.solr.handler.component.util.MtasSolrComponentJoin;
  27 +import mtas.solr.handler.component.util.MtasSolrComponentCollection;
27 28 import mtas.solr.handler.component.util.MtasSolrComponentKwic;
28 29 import mtas.solr.handler.component.util.MtasSolrComponentList;
29 30 import mtas.solr.handler.component.util.MtasSolrComponentPrefix;
... ... @@ -51,17 +52,17 @@ public class MtasSolrSearchComponent extends SearchComponent {
51 52 /** The search component. */
52 53 MtasSolrSearchComponent searchComponent;
53 54  
54   - /** The Constant CONFIG_JOIN_CACHE_DIRECTORY. */
55   - public static final String CONFIG_JOIN_CACHE_DIRECTORY = "joinCacheDirectory";
  55 + /** The Constant CONFIG_COLLECTION_CACHE_DIRECTORY. */
  56 + public static final String CONFIG_COLLECTION_CACHE_DIRECTORY = "collectionCacheDirectory";
56 57  
57   - /** The Constant CONFIG_JOIN_LIFETIME. */
58   - public static final String CONFIG_JOIN_LIFETIME = "joinLifetime";
  58 + /** The Constant CONFIG_COLLECTION_LIFETIME. */
  59 + public static final String CONFIG_COLLECTION_LIFETIME = "collectionLifetime";
59 60  
60   - /** The Constant CONFIG_JOIN_MAXIMUM_NUMBER. */
61   - public static final String CONFIG_JOIN_MAXIMUM_NUMBER = "joinMaximumNumber";
  61 + /** The Constant CONFIG_COLLECTION_MAXIMUM_NUMBER. */
  62 + public static final String CONFIG_COLLECTION_MAXIMUM_NUMBER = "collectionMaximumNumber";
62 63  
63   - /** The Constant CONFIG_JOIN_MAXIMUM_OVERFLOW. */
64   - public static final String CONFIG_JOIN_MAXIMUM_OVERFLOW = "joinMaximumOverflow";
  64 + /** The Constant CONFIG_COLLECTION_MAXIMUM_OVERFLOW. */
  65 + public static final String CONFIG_COLLECTION_MAXIMUM_OVERFLOW = "collectionMaximumOverflow";
65 66  
66 67 /** The Constant PARAM_MTAS. */
67 68 public static final String PARAM_MTAS = "mtas";
... ... @@ -97,8 +98,13 @@ public class MtasSolrSearchComponent extends SearchComponent {
97 98 public static final int STAGE_GROUP = ResponseBuilder.STAGE_EXECUTE_QUERY
98 99 + 60;
99 100  
100   - /** The Constant STAGE_JOIN. */
101   - public static final int STAGE_JOIN = ResponseBuilder.STAGE_EXECUTE_QUERY + 70;
  101 + /** The Constant STAGE_COLLECTION_INIT. */
  102 + public static final int STAGE_COLLECTION_INIT = ResponseBuilder.STAGE_EXECUTE_QUERY
  103 + + 70;
  104 +
  105 + /** The Constant STAGE_COLLECTION_FINISH. */
  106 + public static final int STAGE_COLLECTION_FINISH = ResponseBuilder.STAGE_EXECUTE_QUERY
  107 + + 71;
102 108  
103 109 /** The Constant STAGE_DOCUMENT. */
104 110 public static final int STAGE_DOCUMENT = ResponseBuilder.STAGE_GET_FIELDS
... ... @@ -131,11 +137,11 @@ public class MtasSolrSearchComponent extends SearchComponent {
131 137 /** The search document. */
132 138 private MtasSolrComponentDocument searchDocument;
133 139  
134   - /** The search join. */
135   - private MtasSolrComponentJoin searchJoin;
  140 + /** The search collection. */
  141 + private MtasSolrComponentCollection searchCollection;
136 142  
137   - /** The join cache. */
138   - private MtasSolrJoinCache joinCache = null;
  143 + /** The collection cache. */
  144 + private MtasSolrCollectionCache collectionCache = null;
139 145  
140 146 /*
141 147 * (non-Javadoc)
... ... @@ -148,7 +154,7 @@ public class MtasSolrSearchComponent extends SearchComponent {
148 154 public void init(NamedList args) {
149 155 super.init(args);
150 156 // init components
151   - searchDocument = new MtasSolrComponentDocument();
  157 + searchDocument = new MtasSolrComponentDocument(this);
152 158 searchKwic = new MtasSolrComponentKwic(this);
153 159 searchList = new MtasSolrComponentList(this);
154 160 searchGroup = new MtasSolrComponentGroup(this);
... ... @@ -156,30 +162,33 @@ public class MtasSolrSearchComponent extends SearchComponent {
156 162 searchPrefix = new MtasSolrComponentPrefix(this);
157 163 searchStats = new MtasSolrComponentStats(this);
158 164 searchFacet = new MtasSolrComponentFacet(this);
159   - searchJoin = new MtasSolrComponentJoin(this);
160   - // init join
161   - String joinCacheDirectory = null;
162   - Long joinLifetime = null;
163   - Integer joinMaximumNumber = null;
164   - Integer joinMaximumOverflow = null;
165   - if (args.get(CONFIG_JOIN_CACHE_DIRECTORY) != null
166   - && args.get(CONFIG_JOIN_CACHE_DIRECTORY) instanceof String) {
167   - joinCacheDirectory = (String) args.get(CONFIG_JOIN_CACHE_DIRECTORY);
  165 + searchCollection = new MtasSolrComponentCollection(this);
  166 + // init collection
  167 + String collectionCacheDirectory = null;
  168 + Long collectionLifetime = null;
  169 + Integer collectionMaximumNumber = null;
  170 + Integer collectionMaximumOverflow = null;
  171 + if (args.get(CONFIG_COLLECTION_CACHE_DIRECTORY) != null
  172 + && args.get(CONFIG_COLLECTION_CACHE_DIRECTORY) instanceof String) {
  173 + collectionCacheDirectory = (String) args
  174 + .get(CONFIG_COLLECTION_CACHE_DIRECTORY);
168 175 }
169   - if (args.get(CONFIG_JOIN_LIFETIME) != null
170   - && args.get(CONFIG_JOIN_LIFETIME) instanceof Long) {
171   - joinLifetime = (Long) args.get(CONFIG_JOIN_LIFETIME);
  176 + if (args.get(CONFIG_COLLECTION_LIFETIME) != null
  177 + && args.get(CONFIG_COLLECTION_LIFETIME) instanceof Long) {
  178 + collectionLifetime = (Long) args.get(CONFIG_COLLECTION_LIFETIME);
172 179 }
173   - if (args.get(CONFIG_JOIN_MAXIMUM_NUMBER) != null
174   - && args.get(CONFIG_JOIN_MAXIMUM_NUMBER) instanceof Integer) {
175   - joinMaximumNumber = (Integer) args.get(CONFIG_JOIN_MAXIMUM_NUMBER);
  180 + if (args.get(CONFIG_COLLECTION_MAXIMUM_NUMBER) != null
  181 + && args.get(CONFIG_COLLECTION_MAXIMUM_NUMBER) instanceof Integer) {
  182 + collectionMaximumNumber = (Integer) args
  183 + .get(CONFIG_COLLECTION_MAXIMUM_NUMBER);
176 184 }
177   - if (args.get(CONFIG_JOIN_MAXIMUM_OVERFLOW) != null
178   - && args.get(CONFIG_JOIN_MAXIMUM_OVERFLOW) instanceof Integer) {
179   - joinMaximumNumber = (Integer) args.get(CONFIG_JOIN_MAXIMUM_OVERFLOW);
  185 + if (args.get(CONFIG_COLLECTION_MAXIMUM_OVERFLOW) != null
  186 + && args.get(CONFIG_COLLECTION_MAXIMUM_OVERFLOW) instanceof Integer) {
  187 + collectionMaximumNumber = (Integer) args
  188 + .get(CONFIG_COLLECTION_MAXIMUM_OVERFLOW);
180 189 }
181   - joinCache = new MtasSolrJoinCache(joinCacheDirectory, joinLifetime,
182   - joinMaximumNumber, joinMaximumOverflow);
  190 + collectionCache = new MtasSolrCollectionCache(collectionCacheDirectory,
  191 + collectionLifetime, collectionMaximumNumber, collectionMaximumOverflow);
183 192 }
184 193  
185 194 /*
... ... @@ -202,6 +211,15 @@ public class MtasSolrSearchComponent extends SearchComponent {
202 211 return "Mtas";
203 212 }
204 213  
  214 + /**
  215 + * Gets the collection cache.
  216 + *
  217 + * @return the collection cache
  218 + */
  219 + public MtasSolrCollectionCache getCollectionCache() {
  220 + return collectionCache;
  221 + }
  222 +
205 223 /*
206 224 * (non-Javadoc)
207 225 *
... ... @@ -258,10 +276,10 @@ public class MtasSolrSearchComponent extends SearchComponent {
258 276 false)) {
259 277 searchFacet.prepare(rb, mtasFields);
260 278 }
261   - // get settings join
262   - if (rb.req.getParams().getBool(MtasSolrComponentJoin.PARAM_MTAS_JOIN,
263   - false)) {
264   - searchJoin.prepare(rb, mtasFields);
  279 + // get settings collection
  280 + if (rb.req.getParams()
  281 + .getBool(MtasSolrComponentCollection.PARAM_MTAS_COLLECTION, false)) {
  282 + searchCollection.prepare(rb, mtasFields);
265 283 }
266 284 rb.req.getContext().put(ComponentFields.class, mtasFields);
267 285 }
... ... @@ -286,7 +304,7 @@ public class MtasSolrSearchComponent extends SearchComponent {
286 304 DocList docList = rb.getResults().docList;
287 305 if (mtasFields.doStats || mtasFields.doDocument || mtasFields.doKwic
288 306 || mtasFields.doList || mtasFields.doGroup || mtasFields.doFacet
289   - || mtasFields.doJoin || mtasFields.doTermVector
  307 + || mtasFields.doCollection || mtasFields.doTermVector
290 308 || mtasFields.doPrefix) {
291 309 SolrIndexSearcher searcher = rb.req.getSearcher();
292 310 ArrayList<Integer> docSetList = null;
... ... @@ -317,8 +335,10 @@ public class MtasSolrSearchComponent extends SearchComponent {
317 335 throw new IOException(e);
318 336 }
319 337 }
320   - CodecUtil.collectJoin(searcher.getRawReader(), docSetList,
321   - mtasFields.join);
  338 + for (ComponentCollection collection : mtasFields.collection) {
  339 + CodecUtil.collectCollection(searcher.getRawReader(), docSetList,
  340 + collection);
  341 + }
322 342 NamedList<Object> mtasResponse = new SimpleOrderedMap<>();
323 343 if (mtasFields.doDocument) {
324 344 ArrayList<NamedList<?>> mtasDocumentResponses = new ArrayList<>();
... ... @@ -355,13 +375,19 @@ public class MtasSolrSearchComponent extends SearchComponent {
355 375 // add to response
356 376 mtasResponse.add("facet", mtasFacetResponses);
357 377 }
358   - if (mtasFields.doJoin) {
359   - // add to response
360   - if (rb.req.getParams().getBool("isShard", false)) {
361   - mtasResponse.add("join", searchJoin.create(mtasFields.join, true));
362   - } else {
363   - mtasResponse.add("join", searchJoin.create(mtasFields.join, false));
  378 + if (mtasFields.doCollection) {
  379 + ArrayList<NamedList<?>> mtasCollectionResponses = new ArrayList<>();
  380 + for (ComponentCollection collection : mtasFields.collection) {
  381 + if (rb.req.getParams().getBool("isShard", false)) {
  382 + mtasCollectionResponses
  383 + .add(searchCollection.create(collection, true));
  384 + } else {
  385 + mtasCollectionResponses
  386 + .add(searchCollection.create(collection, false));
  387 + }
364 388 }
  389 + // add to response
  390 + mtasResponse.add("collection", mtasCollectionResponses);
365 391 }
366 392 if (mtasFields.doList) {
367 393 ArrayList<NamedList<?>> mtasListResponses = new ArrayList<>();
... ... @@ -492,7 +518,8 @@ public class MtasSolrSearchComponent extends SearchComponent {
492 518 @Override
493 519 public void modifyRequest(ResponseBuilder rb, SearchComponent who,
494 520 ShardRequest sreq) {
495   - // System.out.println(Thread.currentThread().getId() + " - "
  521 + // System.out.println(System.nanoTime() + " - " +
  522 + // Thread.currentThread().getId() + " - "
496 523 // + rb.req.getParams().getBool("isShard", false) + " MODIFY REQUEST "
497 524 // + rb.stage + " " + rb.req.getParamString());
498 525 if (sreq.params.getBool(PARAM_MTAS, false)) {
... ... @@ -510,8 +537,9 @@ public class MtasSolrSearchComponent extends SearchComponent {
510 537 if (sreq.params.getBool(MtasSolrComponentFacet.PARAM_MTAS_FACET, false)) {
511 538 searchFacet.modifyRequest(rb, who, sreq);
512 539 }
513   - if (sreq.params.getBool(MtasSolrComponentJoin.PARAM_MTAS_JOIN, false)) {
514   - searchJoin.modifyRequest(rb, who, sreq);
  540 + if (sreq.params.getBool(MtasSolrComponentCollection.PARAM_MTAS_COLLECTION,
  541 + false)) {
  542 + searchCollection.modifyRequest(rb, who, sreq);
515 543 }
516 544 if (sreq.params.getBool(MtasSolrComponentGroup.PARAM_MTAS_GROUP, false)) {
517 545 searchGroup.modifyRequest(rb, who, sreq);
... ... @@ -574,9 +602,9 @@ public class MtasSolrSearchComponent extends SearchComponent {
574 602 false)) {
575 603 searchFacet.finishStage(rb);
576 604 }
577   - if (rb.req.getParams().getBool(MtasSolrComponentJoin.PARAM_MTAS_JOIN,
578   - false)) {
579   - searchJoin.finishStage(rb);
  605 + if (rb.req.getParams()
  606 + .getBool(MtasSolrComponentCollection.PARAM_MTAS_COLLECTION, false)) {
  607 + searchCollection.finishStage(rb);
580 608 }
581 609 if (rb.req.getParams().getBool(MtasSolrComponentGroup.PARAM_MTAS_GROUP,
582 610 false)) {
... ... @@ -630,9 +658,10 @@ public class MtasSolrSearchComponent extends SearchComponent {
630 658 } else if (rb.stage == STAGE_FACET) {
631 659 ComponentFields mtasFields = getMtasFields(rb);
632 660 searchFacet.distributedProcess(rb, mtasFields);
633   - } else if (rb.stage == STAGE_JOIN) {
  661 + } else if (rb.stage == STAGE_COLLECTION_INIT
  662 + || rb.stage == STAGE_COLLECTION_FINISH) {
634 663 ComponentFields mtasFields = getMtasFields(rb);
635   - searchJoin.distributedProcess(rb, mtasFields);
  664 + searchCollection.distributedProcess(rb, mtasFields);
636 665 } else if (rb.stage == STAGE_GROUP) {
637 666 ComponentFields mtasFields = getMtasFields(rb);
638 667 searchGroup.distributedProcess(rb, mtasFields);
... ... @@ -670,9 +699,14 @@ public class MtasSolrSearchComponent extends SearchComponent {
670 699 } else if (rb.stage < STAGE_GROUP && rb.req.getParams()
671 700 .getBool(MtasSolrComponentGroup.PARAM_MTAS_GROUP, false)) {
672 701 return STAGE_GROUP;
673   - } else if (rb.stage < STAGE_JOIN && rb.req.getParams()
674   - .getBool(MtasSolrComponentJoin.PARAM_MTAS_JOIN, false)) {
675   - return STAGE_JOIN;
  702 + } else if (rb.stage < STAGE_COLLECTION_INIT
  703 + && rb.req.getParams().getBool(
  704 + MtasSolrComponentCollection.PARAM_MTAS_COLLECTION, false)) {
  705 + return STAGE_COLLECTION_INIT;
  706 + } else if (rb.stage < STAGE_COLLECTION_FINISH
  707 + && rb.req.getParams().getBool(
  708 + MtasSolrComponentCollection.PARAM_MTAS_COLLECTION, false)) {
  709 + return STAGE_COLLECTION_FINISH;
676 710 }
677 711 } else if (rb.stage >= ResponseBuilder.STAGE_GET_FIELDS
678 712 && rb.stage < ResponseBuilder.STAGE_DONE) {
... ...
src/mtas/solr/handler/component/util/MtasSolrCollectionResult.java 0 → 100644
  View file @7179e4b
  1 +package mtas.solr.handler.component.util;
  2 +
  3 +import java.io.IOException;
  4 +import java.io.Serializable;
  5 +import java.util.ArrayList;
  6 +import java.util.HashMap;
  7 +import java.util.HashSet;
  8 +import java.util.Iterator;
  9 +import java.util.List;
  10 +import java.util.Map.Entry;
  11 +import org.apache.solr.common.util.SimpleOrderedMap;
  12 +
  13 +import mtas.codec.util.CodecComponent.ComponentCollection;
  14 +import mtas.solr.handler.component.MtasSolrSearchComponent;
  15 +
  16 +/**
  17 + * The Class MtasSolrCollectionResult.
  18 + */
  19 +public class MtasSolrCollectionResult implements Serializable {
  20 +
  21 + /** The Constant serialVersionUID. */
  22 + private static final long serialVersionUID = 1L;
  23 +
  24 + /** The values. */
  25 + private HashSet<String> values;
  26 +
  27 + /** The id. */
  28 + private String id;
  29 +
  30 + /** The action. */
  31 + private String action;
  32 +
  33 + /** The now. */
  34 + private Long now;
  35 +
  36 + /** The list. */
  37 + private List<SimpleOrderedMap<Object>> list;
  38 +
  39 + /** The status. */
  40 + public SimpleOrderedMap<Object> status;
  41 +
  42 + /** The component collection. */
  43 + private transient ComponentCollection componentCollection = null;
  44 +
  45 + /**
  46 + * Instantiates a new mtas solr collection result.
  47 + *
  48 + * @param componentCollection the component collection
  49 + * @throws IOException Signals that an I/O exception has occurred.
  50 + */
  51 + public MtasSolrCollectionResult(ComponentCollection componentCollection)
  52 + throws IOException {
  53 + this.componentCollection = componentCollection;
  54 + if (componentCollection != null) {
  55 + action = componentCollection.action();
  56 + id = null;
  57 + values = null;
  58 + now = null;
  59 + list = null;
  60 + switch (action) {
  61 + case ComponentCollection.ACTION_CREATE:
  62 + values = componentCollection.values();
  63 + id = componentCollection.id;
  64 + break;
  65 + case ComponentCollection.ACTION_CHECK:
  66 + case ComponentCollection.ACTION_GET:
  67 + case ComponentCollection.ACTION_DELETE:
  68 + id = componentCollection.id;
  69 + break;
  70 + case ComponentCollection.ACTION_POST:
  71 + case ComponentCollection.ACTION_IMPORT:
  72 + id = componentCollection.id;
  73 + values = componentCollection.values();
  74 + break;
  75 + case ComponentCollection.ACTION_LIST:
  76 + case ComponentCollection.ACTION_EMPTY:
  77 + // do nothing
  78 + break;
  79 + default:
  80 + throw new IOException("action " + action + " not allowed");
  81 + }
  82 + } else {
  83 + throw new IOException("no componentCollection available");
  84 + }
  85 + }
  86 +
  87 + /**
  88 + * Sets the list.
  89 + *
  90 + * @param now the now
  91 + * @param list the list
  92 + * @throws IOException Signals that an I/O exception has occurred.
  93 + */
  94 + public void setList(long now, List<SimpleOrderedMap<Object>> list)
  95 + throws IOException {
  96 + if (action.equals(ComponentCollection.ACTION_LIST)) {
  97 + this.now = now;
  98 + this.list = list;
  99 + } else {
  100 + throw new IOException("not allowed with action '" + action + "'");
  101 + }
  102 + }
  103 +
  104 + /**
  105 + * Sets the check.
  106 + *
  107 + * @param now the now
  108 + * @param status the status
  109 + * @throws IOException Signals that an I/O exception has occurred.
  110 + */
  111 + public void setCheck(long now, SimpleOrderedMap<Object> status)
  112 + throws IOException {
  113 + if (action.equals(ComponentCollection.ACTION_CHECK)) {
  114 + this.now = now;
  115 + this.status = status;
  116 + } else {
  117 + throw new IOException("not allowed with action '" + action + "'");
  118 + }
  119 + }
  120 +
  121 + /**
  122 + * Sets the get.
  123 + *
  124 + * @param now the now
  125 + * @param status the status
  126 + * @param stringValues the string values
  127 + * @throws IOException Signals that an I/O exception has occurred.
  128 + */
  129 + public void setGet(long now, SimpleOrderedMap<Object> status,
  130 + HashSet<String> stringValues) throws IOException {
  131 + if (action.equals(ComponentCollection.ACTION_GET)) {
  132 + this.now = now;
  133 + this.status = status;
  134 + this.values = stringValues;
  135 + } else {
  136 + throw new IOException("not allowed with action '" + action + "'");
  137 + }
  138 + }
  139 +
  140 + /**
  141 + * Sets the post.
  142 + *
  143 + * @param now the now
  144 + * @param status the status
  145 + * @throws IOException Signals that an I/O exception has occurred.
  146 + */
  147 + public void setPost(long now, SimpleOrderedMap<Object> status)
  148 + throws IOException {
  149 + if (action.equals(ComponentCollection.ACTION_POST)) {
  150 + this.now = now;
  151 + this.status = status;
  152 + } else {
  153 + throw new IOException("not allowed with action '" + action + "'");
  154 + }
  155 + }
  156 +
  157 + public void setImport(long now, SimpleOrderedMap<Object> status)
  158 + throws IOException {
  159 + if (action.equals(ComponentCollection.ACTION_IMPORT)) {
  160 + this.now = now;
  161 + this.status = status;
  162 + } else {
  163 + throw new IOException("not allowed with action '" + action + "'");
  164 + }
  165 + }
  166 +
  167 + /**
  168 + * Sets the create.
  169 + *
  170 + * @param now the now
  171 + * @param status the status
  172 + * @throws IOException Signals that an I/O exception has occurred.
  173 + */
  174 + public void setCreate(long now, SimpleOrderedMap<Object> status)
  175 + throws IOException {
  176 + if (action.equals(ComponentCollection.ACTION_CREATE)) {
  177 + this.now = now;
  178 + this.status = status;
  179 + } else {
  180 + throw new IOException("not allowed with action '" + action + "'");
  181 + }
  182 + }
  183 +
  184 + /**
  185 + * Id.
  186 + *
  187 + * @return the string
  188 + */
  189 + public String id() {
  190 + return id;
  191 + }
  192 +
  193 + /**
  194 + * Action.
  195 + *
  196 + * @return the string
  197 + */
  198 + public String action() {
  199 + return action;
  200 + }
  201 +
  202 + /**
  203 + * Rewrite.
  204 + *
  205 + * @param searchComponent the search component
  206 + * @return the simple ordered map
  207 + * @throws IOException Signals that an I/O exception has occurred.
  208 + */
  209 + public SimpleOrderedMap<Object> rewrite(
  210 + MtasSolrSearchComponent searchComponent) throws IOException {
  211 + SimpleOrderedMap<Object> response = new SimpleOrderedMap<>();
  212 + Iterator<Entry<String, Object>> it;
  213 + switch (action) {
  214 + case ComponentCollection.ACTION_LIST:
  215 + response.add("now", now);
  216 + response.add("list", list);
  217 + break;
  218 + case ComponentCollection.ACTION_CREATE:
  219 + case ComponentCollection.ACTION_POST:
  220 + case ComponentCollection.ACTION_IMPORT:
  221 + if (componentCollection != null && status != null) {
  222 + it = status.iterator();
  223 + while (it.hasNext()) {
  224 + Entry<String, Object> entry = it.next();
  225 + response.add(entry.getKey(), entry.getValue());
  226 + }
  227 + }
  228 + break;
  229 + case ComponentCollection.ACTION_CHECK:
  230 + if (status != null) {
  231 + it = status.iterator();
  232 + while (it.hasNext()) {
  233 + Entry<String, Object> entry = it.next();
  234 + response.add(entry.getKey(), entry.getValue());
  235 + }
  236 + }
  237 + break;
  238 + case ComponentCollection.ACTION_GET:
  239 + if (status != null) {
  240 + it = status.iterator();
  241 + while (it.hasNext()) {
  242 + Entry<String, Object> entry = it.next();
  243 + response.add(entry.getKey(), entry.getValue());
  244 + }
  245 + }
  246 + if (values != null) {
  247 + response.add("values", values);
  248 + }
  249 + break;
  250 + default:
  251 + break;
  252 + }
  253 + return response;
  254 + }
  255 +
  256 + /**
  257 + * Merge.
  258 + *
  259 + * @param newItem the new item
  260 + * @throws IOException Signals that an I/O exception has occurred.
  261 + */
  262 + public void merge(MtasSolrCollectionResult newItem) throws IOException {
  263 + if (action != null && newItem.action != null) {
  264 + if (action.equals(ComponentCollection.ACTION_CREATE)
  265 + && newItem.action.equals(ComponentCollection.ACTION_CREATE)) {
  266 + values.addAll(newItem.values);
  267 + if (id != null && (newItem.id == null || !newItem.id.equals(id))) {
  268 + id = null;
  269 + }
  270 + } else if (action.equals(ComponentCollection.ACTION_LIST)) {
  271 + if (list != null) {
  272 + HashMap<String, SimpleOrderedMap<Object>> index = new HashMap<>();
  273 + for (SimpleOrderedMap<Object> item : list) {
  274 + if (item.get("id") != null && item.get("id") instanceof String) {
  275 + index.put((String) item.get("id"), item);
  276 + if (item.get("shards") == null
  277 + || !(item.get("shards") instanceof List)) {
  278 + item.add("shards", new ArrayList<>());
  279 + }
  280 + }
  281 + }
  282 + for (SimpleOrderedMap<Object> item : newItem.list) {
  283 + if (item.get("id") != null && item.get("id") instanceof String) {
  284 + String id = (String) item.get("id");
  285 + if (index.containsKey(id)) {
  286 + SimpleOrderedMap<Object> indexItem = index.get(id);
  287 + List<SimpleOrderedMap<Object>> shards;
  288 + if (indexItem.get("shards") != null
  289 + && indexItem.get("shards") instanceof List) {
  290 + shards = (List<SimpleOrderedMap<Object>>) indexItem
  291 + .get("shards");
  292 + } else {
  293 + shards = new ArrayList<>();
  294 + indexItem.add("shards", shards);
  295 + }
  296 + shards.add(item);
  297 + }
  298 + }
  299 + }
  300 + }
  301 + } else if (action.equals(ComponentCollection.ACTION_CHECK)
  302 + || action.equals(ComponentCollection.ACTION_POST)
  303 + || action.equals(ComponentCollection.ACTION_IMPORT)
  304 + || action.equals(ComponentCollection.ACTION_CREATE)
  305 + || action.equals(ComponentCollection.ACTION_GET)) {
  306 + if (status != null && status.get("id") != null
  307 + && status.get("id") instanceof String) {
  308 + String id = (String) status.get("id");
  309 + if (id.equals(newItem.id)) {
  310 + List<SimpleOrderedMap<Object>> shards;
  311 + if (status.get("shards") != null
  312 + && status.get("shards") instanceof List) {
  313 + shards = (List<SimpleOrderedMap<Object>>) status.get("shards");
  314 + } else {
  315 + shards = new ArrayList<>();
  316 + status.add("shards", shards);
  317 + }
  318 + if (newItem.status != null) {
  319 + if (action.equals(ComponentCollection.ACTION_GET)) {
  320 + newItem.status.add("values", newItem.values);
  321 + }
  322 + shards.add(newItem.status);
  323 + }
  324 + }
  325 + }
  326 + } else {
  327 + throw new IOException("not allowed for action '" + action + "'");
  328 + }
  329 + }
  330 + }
  331 +
  332 + /*
  333 + * (non-Javadoc)
  334 + *
  335 + * @see java.lang.Object#toString()
  336 + */
  337 + @Override
  338 + public String toString() {
  339 + StringBuilder text = new StringBuilder("");
  340 + text.append(MtasSolrCollectionResult.class.getSimpleName() + "[");
  341 + text.append(action + ", ");
  342 + text.append(id + ", ");
  343 + if (componentCollection != null) {
  344 + text.append(componentCollection.version + ", ");
  345 + } else if (status != null) {
  346 + text.append(status.get("version") + ", ");
  347 + } else {
  348 + text.append("null, ");
  349 + }
  350 + text.append((values != null) ? values.size() : "null");
  351 + text.append("]");
  352 + return text.toString();
  353 + }
  354 +
  355 +}
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentCollection.java 0 → 100644
  View file @7179e4b
  1 +package mtas.solr.handler.component.util;
  2 +
  3 +import java.io.IOException;
  4 +import java.io.Serializable;
  5 +import java.util.ArrayList;
  6 +import java.util.Arrays;
  7 +import java.util.HashMap;
  8 +import java.util.HashSet;
  9 +import java.util.Map;
  10 +import java.util.Set;
  11 +import java.util.Map.Entry;
  12 +
  13 +import org.apache.commons.logging.Log;
  14 +import org.apache.commons.logging.LogFactory;
  15 +import org.apache.lucene.util.BytesRef;
  16 +import org.apache.solr.common.params.ModifiableSolrParams;
  17 +import org.apache.solr.common.util.NamedList;
  18 +import org.apache.solr.common.util.SimpleOrderedMap;
  19 +import org.apache.solr.handler.component.ResponseBuilder;
  20 +import org.apache.solr.handler.component.SearchComponent;
  21 +import org.apache.solr.handler.component.ShardRequest;
  22 +import org.apache.solr.handler.component.ShardResponse;
  23 +import org.noggit.JSONParser;
  24 +import org.noggit.JSONUtil;
  25 +
  26 +import mtas.codec.util.CodecComponent.ComponentFields;
  27 +import mtas.codec.util.CodecComponent.ComponentCollection;
  28 +import mtas.solr.handler.component.MtasSolrSearchComponent;
  29 +
  30 +/**
  31 + * The Class MtasSolrComponentCollection.
  32 + */
  33 +public class MtasSolrComponentCollection
  34 + implements MtasSolrComponent<ComponentCollection> {
  35 +
  36 + /** The Constant log. */
  37 + private static final Log log = LogFactory
  38 + .getLog(MtasSolrComponentCollection.class);
  39 +
  40 + /** The Constant PARAM_MTAS_COLLECTION. */
  41 + public static final String PARAM_MTAS_COLLECTION = MtasSolrSearchComponent.PARAM_MTAS
  42 + + ".collection";
  43 +
  44 + /** The Constant NAME_MTAS_COLLECTION_ACTION. */
  45 + public static final String NAME_MTAS_COLLECTION_ACTION = "action";
  46 +
  47 + /** The Constant NAME_MTAS_COLLECTION_ID. */
  48 + public static final String NAME_MTAS_COLLECTION_ID = "id";
  49 +
  50 + /** The Constant NAME_MTAS_COLLECTION_FIELD. */
  51 + public static final String NAME_MTAS_COLLECTION_FIELD = "field";
  52 +
  53 + /** The Constant NAME_MTAS_COLLECTION_POST. */
  54 + public static final String NAME_MTAS_COLLECTION_POST = "post";
  55 +
  56 + /** The Constant NAME_MTAS_COLLECTION_URL. */
  57 + public static final String NAME_MTAS_COLLECTION_URL = "url";
  58 +
  59 + /** The Constant NAME_MTAS_COLLECTION_COLLECTION. */
  60 + public static final String NAME_MTAS_COLLECTION_COLLECTION = "collection";
  61 +
  62 + /** The Constant NAME_MTAS_COLLECTION_KEY. */
  63 + public static final String NAME_MTAS_COLLECTION_KEY = "key";
  64 +
  65 + /** The search component. */
  66 + private MtasSolrSearchComponent searchComponent;
  67 +
  68 + /**
  69 + * Instantiates a new mtas solr component collection.
  70 + *
  71 + * @param searchComponent
  72 + * the search component
  73 + */
  74 + public MtasSolrComponentCollection(MtasSolrSearchComponent searchComponent) {
  75 + this.searchComponent = searchComponent;
  76 + }
  77 +
  78 + /*
  79 + * (non-Javadoc)
  80 + *
  81 + * @see
  82 + * mtas.solr.handler.component.util.MtasSolrComponent#prepare(org.apache.solr.
  83 + * handler.component.ResponseBuilder,
  84 + * mtas.codec.util.CodecComponent.ComponentFields)
  85 + */
  86 + public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
  87 + throws IOException {
  88 + // System.out.println(
  89 + // "collection: " + System.nanoTime() + " - " +
  90 + // Thread.currentThread().getId()
  91 + // + " - " + rb.req.getParams().getBool("isShard", false) + " PREPARE "
  92 + // + rb.stage + " " + rb.req.getParamString());
  93 + Set<String> ids = MtasSolrResultUtil
  94 + .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_COLLECTION);
  95 + if (!ids.isEmpty()) {
  96 + int tmpCounter = 0;
  97 + String[] keys = new String[ids.size()];
  98 + String[] actions = new String[ids.size()];
  99 + String[] fields = new String[ids.size()];
  100 + String[] collectionIds = new String[ids.size()];
  101 + String[] posts = new String[ids.size()];
  102 + String[] urls = new String[ids.size()];
  103 + String[] collections = new String[ids.size()];
  104 + for (String id : ids) {
  105 + actions[tmpCounter] = rb.req.getParams().get(PARAM_MTAS_COLLECTION + "."
  106 + + id + "." + NAME_MTAS_COLLECTION_ACTION, null);
  107 + keys[tmpCounter] = rb.req.getParams().get(
  108 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_KEY,
  109 + String.valueOf(tmpCounter)).trim();
  110 + fields[tmpCounter] = rb.req.getParams().get(
  111 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_FIELD,
  112 + null);
  113 + collectionIds[tmpCounter] = rb.req.getParams().get(
  114 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_ID,
  115 + null);
  116 + posts[tmpCounter] = rb.req.getParams().get(
  117 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_POST,
  118 + null);
  119 + urls[tmpCounter] = rb.req.getParams().get(
  120 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_URL,
  121 + null);
  122 + collections[tmpCounter] = rb.req.getParams().get(
  123 + PARAM_MTAS_COLLECTION + "." + id + "." + NAME_MTAS_COLLECTION_COLLECTION,
  124 + null);
  125 + tmpCounter++;
  126 + }
  127 + mtasFields.doCollection = true;
  128 + MtasSolrResultUtil.compareAndCheck(keys, actions,
  129 + NAME_MTAS_COLLECTION_KEY, NAME_MTAS_COLLECTION_ACTION, true);
  130 + MtasSolrResultUtil.compareAndCheck(keys, fields, NAME_MTAS_COLLECTION_KEY,
  131 + NAME_MTAS_COLLECTION_FIELD, false);
  132 + MtasSolrResultUtil.compareAndCheck(keys, collectionIds,
  133 + NAME_MTAS_COLLECTION_KEY, NAME_MTAS_COLLECTION_ID, false);
  134 + MtasSolrResultUtil.compareAndCheck(keys, posts, NAME_MTAS_COLLECTION_KEY,
  135 + NAME_MTAS_COLLECTION_POST, false);
  136 + MtasSolrResultUtil.compareAndCheck(keys, urls, NAME_MTAS_COLLECTION_KEY,
  137 + NAME_MTAS_COLLECTION_URL, false);
  138 + MtasSolrResultUtil.compareAndCheck(keys, collections, NAME_MTAS_COLLECTION_KEY,
  139 + NAME_MTAS_COLLECTION_COLLECTION, false);
  140 + for (int i = 0; i < actions.length; i++) {
  141 + if (actions[i] != null) {
  142 + ComponentCollection componentCollection;
  143 + switch (actions[i]) {
  144 + case ComponentCollection.ACTION_LIST:
  145 + componentCollection = new ComponentCollection(keys[i],
  146 + ComponentCollection.ACTION_LIST);
  147 + componentCollection.setListVariables();
  148 + mtasFields.collection.add(componentCollection);
  149 + break;
  150 + case ComponentCollection.ACTION_CHECK:
  151 + if (collectionIds[i] != null) {
  152 + componentCollection = new ComponentCollection(keys[i],
  153 + ComponentCollection.ACTION_CHECK);
  154 + componentCollection.setCheckVariables(collectionIds[i]);
  155 + mtasFields.collection.add(componentCollection);
  156 + } else {
  157 + throw new IOException(
  158 + "no id defined for collection (" + actions[i] + ")");
  159 + }
  160 + break;
  161 + case ComponentCollection.ACTION_GET:
  162 + if (collectionIds[i] != null) {
  163 + componentCollection = new ComponentCollection(keys[i],
  164 + ComponentCollection.ACTION_GET);
  165 + componentCollection.setGetVariables(collectionIds[i]);
  166 + mtasFields.collection.add(componentCollection);
  167 + } else {
  168 + throw new IOException(
  169 + "no id defined for collection (" + actions[i] + ")");
  170 + }
  171 + break;
  172 + case ComponentCollection.ACTION_CREATE:
  173 + if (fields[i] != null) {
  174 + Set<String> fieldList = new HashSet<>(
  175 + Arrays.asList(fields[i].split(",")));
  176 + componentCollection = new ComponentCollection(keys[i],
  177 + ComponentCollection.ACTION_CREATE);
  178 + componentCollection.setCreateVariables(collectionIds[i],
  179 + fieldList);
  180 + mtasFields.doCollection = true;
  181 + mtasFields.collection.add(componentCollection);
  182 + rb.setNeedDocSet(true);
  183 + } else {
  184 + throw new IOException(
  185 + "no field defined for collection (" + actions[i] + ")");
  186 + }
  187 + break;
  188 + case ComponentCollection.ACTION_POST:
  189 + if (posts[i] != null) {
  190 + componentCollection = new ComponentCollection(keys[i],
  191 + ComponentCollection.ACTION_POST);
  192 + componentCollection.setPostVariables(collectionIds[i],
  193 + stringToStringValues(posts[i]));
  194 + mtasFields.collection.add(componentCollection);
  195 + } else {
  196 + throw new IOException(
  197 + "no post defined for collection (" + actions[i] + ")");
  198 + }
  199 + break;
  200 + case ComponentCollection.ACTION_IMPORT:
  201 + if (urls[i] != null && collections[i]!=null) {
  202 + componentCollection = new ComponentCollection(keys[i],
  203 + ComponentCollection.ACTION_IMPORT);
  204 + componentCollection.setImportVariables(collectionIds[i],
  205 + urls[i], collections[i]);
  206 + mtasFields.collection.add(componentCollection);
  207 + } else {
  208 + throw new IOException(
  209 + "no post defined for collection (" + actions[i] + ")");
  210 + }
  211 + break;
  212 + case ComponentCollection.ACTION_DELETE:
  213 + if (collectionIds[i] != null) {
  214 + componentCollection = new ComponentCollection(keys[i],
  215 + ComponentCollection.ACTION_DELETE);
  216 + componentCollection.setDeleteVariables(collectionIds[i]);
  217 + mtasFields.collection.add(componentCollection);
  218 + } else {
  219 + throw new IOException(
  220 + "no id defined for collection (" + actions[i] + ")");
  221 + }
  222 + break;
  223 + case ComponentCollection.ACTION_EMPTY:
  224 + componentCollection = new ComponentCollection(keys[i],
  225 + ComponentCollection.ACTION_EMPTY);
  226 + mtasFields.collection.add(componentCollection);
  227 + break;
  228 + default:
  229 + throw new IOException(
  230 + "unrecognized action '" + actions[i] + "' for collection");
  231 + }
  232 + } else {
  233 + throw new IOException("no action defined for collection");
  234 + }
  235 + }
  236 + }
  237 + }
  238 +
  239 + /*
  240 + * (non-Javadoc)
  241 + *
  242 + * @see
  243 + * mtas.solr.handler.component.util.MtasSolrComponent#modifyRequest(org.apache
  244 + * .solr.handler.component.ResponseBuilder,
  245 + * org.apache.solr.handler.component.SearchComponent,
  246 + * org.apache.solr.handler.component.ShardRequest)
  247 + */
  248 + public void modifyRequest(ResponseBuilder rb, SearchComponent who,
  249 + ShardRequest sreq) {
  250 + // System.out.println(
  251 + // "collection: " + System.nanoTime() + " - " +
  252 + // Thread.currentThread().getId()
  253 + // + " - " + rb.req.getParams().getBool("isShard", false)
  254 + // + " MODIFYREQUEST " + rb.stage + " " + rb.req.getParamString());
  255 + if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
  256 + && sreq.params.getBool(PARAM_MTAS_COLLECTION, false)) {
  257 + if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
  258 + // do nothing
  259 + } else {
  260 + // remove for other requests
  261 + Set<String> keys = MtasSolrResultUtil
  262 + .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_COLLECTION);
  263 + sreq.params.remove(PARAM_MTAS_COLLECTION);
  264 + for (String key : keys) {
  265 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  266 + + NAME_MTAS_COLLECTION_ACTION);
  267 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  268 + + NAME_MTAS_COLLECTION_ID);
  269 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  270 + + NAME_MTAS_COLLECTION_FIELD);
  271 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  272 + + NAME_MTAS_COLLECTION_POST);
  273 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  274 + + NAME_MTAS_COLLECTION_KEY);
  275 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  276 + + NAME_MTAS_COLLECTION_URL);
  277 + sreq.params.remove(PARAM_MTAS_COLLECTION + "." + key + "."
  278 + + NAME_MTAS_COLLECTION_COLLECTION);
  279 + }
  280 + }
  281 + }
  282 + }
  283 +
  284 + /*
  285 + * (non-Javadoc)
  286 + *
  287 + * @see
  288 + * mtas.solr.handler.component.util.MtasSolrComponent#create(mtas.codec.util.
  289 + * CodecComponent.BasicComponent, java.lang.Boolean)
  290 + */
  291 + public SimpleOrderedMap<Object> create(
  292 + ComponentCollection componentCollection, Boolean encode)
  293 + throws IOException {
  294 + MtasSolrCollectionResult data = createMtasSolrCollectionResult(
  295 + componentCollection, encode ? false : true);
  296 + // Create response
  297 + SimpleOrderedMap<Object> mtasCollectionResponse = new SimpleOrderedMap<>();
  298 + mtasCollectionResponse.add("key", componentCollection.key);
  299 + if (encode) {
  300 + mtasCollectionResponse.add("_encoded_data",
  301 + MtasSolrResultUtil.encode(data));
  302 + } else {
  303 + mtasCollectionResponse.add("data", data);
  304 + MtasSolrResultUtil.rewrite(mtasCollectionResponse, searchComponent);
  305 + }
  306 + return mtasCollectionResponse;
  307 + }
  308 +
  309 + /**
  310 + * Creates the mtas solr collection result.
  311 + *
  312 + * @param componentCollection the component collection
  313 + * @param storeIfRelevant the store if relevant
  314 + * @return the mtas solr collection result
  315 + * @throws IOException Signals that an I/O exception has occurred.
  316 + */
  317 + /*
  318 + * (non-Javadoc)
  319 + *
  320 + * @see
  321 + * mtas.solr.handler.component.util.MtasSolrComponent#create(mtas.codec.util.
  322 + * CodecComponent.BasicComponent, java.lang.Boolean)
  323 + */
  324 + private MtasSolrCollectionResult createMtasSolrCollectionResult(
  325 + ComponentCollection componentCollection, boolean storeIfRelevant)
  326 + throws IOException {
  327 + // System.out.println("collection: " + System.nanoTime() + " - "
  328 + // + Thread.currentThread().getId() + " - " + " CREATE ");
  329 + if (componentCollection != null) {
  330 + // Create response
  331 + MtasSolrCollectionResult data = new MtasSolrCollectionResult(
  332 + componentCollection);
  333 + if (componentCollection.action()
  334 + .equals(ComponentCollection.ACTION_CREATE)) {
  335 + if (storeIfRelevant && componentCollection.version == null) {
  336 + componentCollection.version = searchComponent.getCollectionCache()
  337 + .create(componentCollection.id,
  338 + componentCollection.values().size(),
  339 + componentCollection.values());
  340 + }
  341 + data.setCreate(searchComponent.getCollectionCache().now(),
  342 + searchComponent.getCollectionCache().check(componentCollection.id));
  343 + } else if (componentCollection.action()
  344 + .equals(ComponentCollection.ACTION_LIST)) {
  345 + // retrieve and add list to result
  346 + data.setList(searchComponent.getCollectionCache().now(),
  347 + searchComponent.getCollectionCache().list());
  348 + } else if (componentCollection.action()
  349 + .equals(ComponentCollection.ACTION_CHECK)) {
  350 + // retrieve and add status to result
  351 + data.setCheck(searchComponent.getCollectionCache().now(),
  352 + searchComponent.getCollectionCache().check(componentCollection.id));
  353 + } else if (componentCollection.action()
  354 + .equals(ComponentCollection.ACTION_GET)) {
  355 + // retrieve and add status to result
  356 + HashSet<String> values = searchComponent.getCollectionCache()
  357 + .getDataById(componentCollection.id);
  358 + if (values != null) {
  359 + data.setGet(searchComponent.getCollectionCache().now(),
  360 + searchComponent.getCollectionCache()
  361 + .check(componentCollection.id),
  362 + values);
  363 + }
  364 + } else if (componentCollection.action()
  365 + .equals(ComponentCollection.ACTION_EMPTY)) {
  366 + // empty
  367 + searchComponent.getCollectionCache().empty();
  368 + } else if (componentCollection.action()
  369 + .equals(ComponentCollection.ACTION_POST)) {
  370 + // store if not already stored
  371 + if (componentCollection.version == null) {
  372 + componentCollection.version = searchComponent.getCollectionCache()
  373 + .create(componentCollection.id,
  374 + componentCollection.values().size(),
  375 + componentCollection.values());
  376 + }
  377 + // add status to result
  378 + data.setPost(searchComponent.getCollectionCache().now(),
  379 + searchComponent.getCollectionCache().check(componentCollection.id));
  380 + } else if (componentCollection.action()
  381 + .equals(ComponentCollection.ACTION_IMPORT)) {
  382 + // import if not already stored
  383 + if (componentCollection.version == null) {
  384 + componentCollection.version = searchComponent.getCollectionCache()
  385 + .create(componentCollection.id,
  386 + componentCollection.values().size(),
  387 + componentCollection.values());
  388 + }
  389 + // add status to result
  390 + data.setImport(searchComponent.getCollectionCache().now(),
  391 + searchComponent.getCollectionCache().check(componentCollection.id));
  392 + } else if (componentCollection.action()
  393 + .equals(ComponentCollection.ACTION_DELETE)) {
  394 + searchComponent.getCollectionCache().deleteById(componentCollection.id);
  395 + }
  396 + return data;
  397 + } else {
  398 + throw new IOException("no componentCollection available");
  399 + }
  400 + }
  401 +
  402 + /*
  403 + * (non-Javadoc)
  404 + *
  405 + * @see
  406 + * mtas.solr.handler.component.util.MtasSolrComponent#finishStage(org.apache.
  407 + * solr.handler.component.ResponseBuilder)
  408 + */
  409 + public void finishStage(ResponseBuilder rb) {
  410 + // System.out.println(
  411 + // "collection: " + System.nanoTime() + " - " + Thread.currentThread().getId()
  412 + // + " - " + rb.req.getParams().getBool("isShard", false)
  413 + // + " FINISHSTAGE " + rb.stage + " " + rb.req.getParamString());
  414 + if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)) {
  415 + if (rb.stage >= ResponseBuilder.STAGE_EXECUTE_QUERY
  416 + && rb.stage < ResponseBuilder.STAGE_GET_FIELDS) {
  417 + ComponentFields mtasFields = getMtasFields(rb);
  418 + if (mtasFields.doCollection) {
  419 + if (rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY) {
  420 + // mtas response
  421 + NamedList<Object> mtasResponse = null;
  422 + try {
  423 + mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
  424 + } catch (ClassCastException e) {
  425 + log.debug(e);
  426 + mtasResponse = null;
  427 + }
  428 + if (mtasResponse == null) {
  429 + mtasResponse = new SimpleOrderedMap<>();
  430 + rb.rsp.add("mtas", mtasResponse);
  431 + }
  432 + ArrayList<Object> mtasCollectionResponses;
  433 + if (mtasResponse.get("collection") != null
  434 + && mtasResponse.get("collection") instanceof ArrayList) {
  435 + mtasCollectionResponses = (ArrayList<Object>) mtasResponse.get("collection");
  436 + } else {
  437 + mtasCollectionResponses = new ArrayList<>();
  438 + mtasResponse.add("collection", mtasCollectionResponses);
  439 + }
  440 + MtasSolrCollectionResult collectionResult;
  441 + for (ComponentCollection componentCollection : mtasFields.collection) {
  442 + try {
  443 + collectionResult = createMtasSolrCollectionResult(componentCollection,
  444 + false);
  445 + // Create response
  446 + SimpleOrderedMap<Object> mtasCollectionResponse = new SimpleOrderedMap<>();
  447 + mtasCollectionResponse.add("key", componentCollection.key);
  448 + mtasCollectionResponse.add("data", collectionResult);
  449 + mtasCollectionResponses.add(mtasCollectionResponse);
  450 + } catch (IOException e) {
  451 + log.debug(e);
  452 + }
  453 + }
  454 + }
  455 + // decode shard responses
  456 + for (ShardRequest sreq : rb.finished) {
  457 + if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
  458 + && sreq.params.getBool(PARAM_MTAS_COLLECTION, false)) {
  459 + for (ShardResponse shardResponse : sreq.responses) {
  460 + NamedList<Object> solrShardResponse = shardResponse
  461 + .getSolrResponse().getResponse();
  462 + try {
  463 + ArrayList<SimpleOrderedMap<Object>> data = (ArrayList<SimpleOrderedMap<Object>>) solrShardResponse
  464 + .findRecursive("mtas", "collection");
  465 + if (data != null) {
  466 + MtasSolrResultUtil.decode(data);
  467 + if (rb.stage > ResponseBuilder.STAGE_EXECUTE_QUERY) {
  468 + ArrayList<SimpleOrderedMap<Object>> filteredData = new ArrayList<>();
  469 + for (SimpleOrderedMap<Object> dataItem : data) {
  470 + if (dataItem.get("data") != null && dataItem
  471 + .get("data") instanceof MtasSolrCollectionResult) {
  472 + MtasSolrCollectionResult collectionResult = (MtasSolrCollectionResult) dataItem
  473 + .get("data");
  474 + if (rb.stage <= MtasSolrSearchComponent.STAGE_COLLECTION_INIT) {
  475 + if (!collectionResult.action()
  476 + .equals(ComponentCollection.ACTION_CREATE)
  477 + && !collectionResult.action()
  478 + .equals(ComponentCollection.ACTION_LIST)
  479 + && !collectionResult.action()
  480 + .equals(ComponentCollection.ACTION_CHECK)) {
  481 + filteredData.add(dataItem);
  482 + }
  483 + } else if (rb.stage <= MtasSolrSearchComponent.STAGE_COLLECTION_FINISH) {
  484 + if (!collectionResult.action()
  485 + .equals(ComponentCollection.ACTION_POST)
  486 + && !collectionResult.action()
  487 + .equals(ComponentCollection.ACTION_IMPORT) && !collectionResult.action()
  488 + .equals(ComponentCollection.ACTION_CHECK)) {
  489 + filteredData.add(dataItem);
  490 + }
  491 + }
  492 + } else {
  493 + filteredData.add(dataItem);
  494 + }
  495 + }
  496 + data.clear();
  497 + data.addAll(filteredData);
  498 + }
  499 + }
  500 + } catch (ClassCastException e) {
  501 + log.debug(e);
  502 + // shouldn't happen
  503 + }
  504 + }
  505 +
  506 + }
  507 + }
  508 + }
  509 + }
  510 + }
  511 + }
  512 +
  513 + /*
  514 + * (non-Javadoc)
  515 + *
  516 + * @see
  517 + * mtas.solr.handler.component.util.MtasSolrComponent#distributedProcess(org.
  518 + * apache.solr.handler.component.ResponseBuilder,
  519 + * mtas.codec.util.CodecComponent.ComponentFields)
  520 + */
  521 + @SuppressWarnings("unchecked")
  522 + public void distributedProcess(ResponseBuilder rb, ComponentFields mtasFields)
  523 + throws IOException {
  524 + // System.out.println("collection: " + System.nanoTime() + " - "
  525 + // + Thread.currentThread().getId() + " - "
  526 + // + rb.req.getParams().getBool("isShard", false) + " DISTRIBUTEDPROCESS "
  527 + // + rb.stage + " " + rb.req.getParamString());
  528 + NamedList<Object> mtasResponse = null;
  529 + try {
  530 + mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
  531 + } catch (ClassCastException e) {
  532 + log.debug(e);
  533 + mtasResponse = null;
  534 + }
  535 + if (mtasResponse != null) {
  536 + if (rb.stage == MtasSolrSearchComponent.STAGE_COLLECTION_INIT) {
  537 + // build index
  538 + Map<String, MtasSolrCollectionResult> index = new HashMap<>();
  539 + ArrayList<Object> mtasResponseCollection;
  540 + try {
  541 + mtasResponseCollection = (ArrayList<Object>) mtasResponse.get("collection");
  542 + for (Object item : mtasResponseCollection) {
  543 + if (item instanceof SimpleOrderedMap) {
  544 + SimpleOrderedMap<Object> itemMap = (SimpleOrderedMap<Object>) item;
  545 + if (itemMap.get("data") != null
  546 + && itemMap.get("data") instanceof MtasSolrCollectionResult) {
  547 + MtasSolrCollectionResult collectionItem = (MtasSolrCollectionResult) itemMap
  548 + .get("data");
  549 + index.put(collectionItem.id(), collectionItem);
  550 + }
  551 + }
  552 + }
  553 + } catch (ClassCastException e) {
  554 + log.debug(e);
  555 + mtasResponse.remove("collection");
  556 + }
  557 + // check and remove previous responses
  558 + Map<String, Set<String>> createPostAfterMissingCheckResult = new HashMap<>();
  559 + for (ShardRequest sreq : rb.finished) {
  560 + if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
  561 + && sreq.params.getBool(PARAM_MTAS_COLLECTION, false)) {
  562 + for (ShardResponse shardResponse : sreq.responses) {
  563 + NamedList<Object> solrShardResponse = shardResponse
  564 + .getSolrResponse().getResponse();
  565 + try {
  566 + ArrayList<SimpleOrderedMap<Object>> data = (ArrayList<SimpleOrderedMap<Object>>) solrShardResponse
  567 + .findRecursive("mtas", "collection");
  568 + if (data != null) {
  569 + for (SimpleOrderedMap<Object> dataItem : data) {
  570 + if (dataItem.get("data") != null && dataItem
  571 + .get("data") instanceof MtasSolrCollectionResult) {
  572 + MtasSolrCollectionResult dataItemResult = (MtasSolrCollectionResult) dataItem
  573 + .get("data");
  574 + if (index.containsKey(dataItemResult.id())
  575 + && index.get(dataItemResult.id()).action()
  576 + .equals(ComponentCollection.ACTION_CHECK)) {
  577 + if (dataItemResult.status == null) {
  578 + if (!createPostAfterMissingCheckResult
  579 + .containsKey(shardResponse.getShard())) {
  580 + createPostAfterMissingCheckResult
  581 + .put(shardResponse.getShard(), new HashSet<>());
  582 + }
  583 + createPostAfterMissingCheckResult
  584 + .get(shardResponse.getShard())
  585 + .add(dataItemResult.id());
  586 + }
  587 + }
  588 + }
  589 + }
  590 + data.clear();
  591 + }
  592 + } catch (ClassCastException e) {
  593 + log.debug(e);
  594 + // shouldn't happen
  595 + }
  596 + }
  597 + }
  598 + }
  599 + // construct new requests
  600 + HashMap<String, ModifiableSolrParams> requestParamList = new HashMap<>();
  601 + int id = 0;
  602 + for (ComponentCollection componentCollection : mtasFields.collection) {
  603 + if (componentCollection.action().equals(ComponentCollection.ACTION_CHECK)) {
  604 + for (String shardAddress : rb.shards) {
  605 + if (createPostAfterMissingCheckResult.containsKey(shardAddress)) {
  606 + if (createPostAfterMissingCheckResult.get(shardAddress)
  607 + .contains(componentCollection.id)) {
  608 + HashSet<String> values = searchComponent.getCollectionCache()
  609 + .getDataById(componentCollection.id);
  610 + if (values != null) {
  611 + ModifiableSolrParams paramsNewRequest;
  612 + if (!requestParamList.containsKey(shardAddress)) {
  613 + paramsNewRequest = new ModifiableSolrParams();
  614 + requestParamList.put(shardAddress, paramsNewRequest);
  615 + } else {
  616 + paramsNewRequest = requestParamList.get(shardAddress);
  617 + }
  618 + paramsNewRequest.add(PARAM_MTAS_COLLECTION + "." + id + "."
  619 + + NAME_MTAS_COLLECTION_KEY, componentCollection.key);
  620 + paramsNewRequest.add(PARAM_MTAS_COLLECTION + "." + id + "."
  621 + + NAME_MTAS_COLLECTION_ID, componentCollection.id);
  622 + paramsNewRequest.add(
  623 + PARAM_MTAS_COLLECTION + "." + id + "."
  624 + + NAME_MTAS_COLLECTION_ACTION,
  625 + ComponentCollection.ACTION_POST);
  626 + paramsNewRequest.add(
  627 + PARAM_MTAS_COLLECTION + "." + id + "."
  628 + + NAME_MTAS_COLLECTION_POST,
  629 + stringValuesToString(values));
  630 + id++;
  631 + }
  632 + }
  633 + }
  634 + }
  635 + } else if (componentCollection.action()
  636 + .equals(ComponentCollection.ACTION_CREATE)) {
  637 + if (componentCollection.version == null) {
  638 + componentCollection.version = searchComponent.getCollectionCache()
  639 + .create(componentCollection.id, componentCollection.values().size(),
  640 + componentCollection.values());
  641 + }
  642 + if (index.containsKey(componentCollection.id)) {
  643 + index.get(componentCollection.id).setCreate(
  644 + searchComponent.getCollectionCache().now(),
  645 + searchComponent.getCollectionCache().check(componentCollection.id));
  646 + }
  647 + for (String shardAddress : rb.shards) {
  648 + ModifiableSolrParams paramsNewRequest;
  649 + if (!requestParamList.containsKey(shardAddress)) {
  650 + paramsNewRequest = new ModifiableSolrParams();
  651 + requestParamList.put(shardAddress, paramsNewRequest);
  652 + } else {
  653 + paramsNewRequest = requestParamList.get(shardAddress);
  654 + }
  655 + paramsNewRequest.add(PARAM_MTAS_COLLECTION + "." + id + "."
  656 + + NAME_MTAS_COLLECTION_KEY, componentCollection.key);
  657 + paramsNewRequest.add(PARAM_MTAS_COLLECTION + "." + id + "."
  658 + + NAME_MTAS_COLLECTION_ID, componentCollection.id);
  659 + paramsNewRequest.add(
  660 + PARAM_MTAS_COLLECTION + "." + id + "."
  661 + + NAME_MTAS_COLLECTION_ACTION,
  662 + ComponentCollection.ACTION_POST);
  663 + paramsNewRequest.add(
  664 + PARAM_MTAS_COLLECTION + "." + id + "."
  665 + + NAME_MTAS_COLLECTION_POST,
  666 + stringValuesToString(componentCollection.values()));
  667 + }
  668 + }
  669 + id++;
  670 + }
  671 + // add new requests
  672 + for (Entry<String, ModifiableSolrParams> entry : requestParamList
  673 + .entrySet()) {
  674 + ShardRequest newSreq = new ShardRequest();
  675 + newSreq.shards = new String[] { entry.getKey() };
  676 + newSreq.purpose = ShardRequest.PURPOSE_PRIVATE;
  677 + newSreq.params = entry.getValue();
  678 + newSreq.params.add("q", "*");
  679 + newSreq.params.add("rows", "0");
  680 + newSreq.params.add(MtasSolrSearchComponent.PARAM_MTAS,
  681 + rb.req.getOriginalParams()
  682 + .getParams(MtasSolrSearchComponent.PARAM_MTAS));
  683 + newSreq.params.add(PARAM_MTAS_COLLECTION,
  684 + rb.req.getOriginalParams().getParams(PARAM_MTAS_COLLECTION));
  685 + rb.addRequest(searchComponent, newSreq);
  686 + }
  687 + } else if (rb.stage == MtasSolrSearchComponent.STAGE_COLLECTION_FINISH) {
  688 + // just rewrite
  689 + ArrayList<Object> mtasResponseCollection;
  690 + try {
  691 + mtasResponseCollection = (ArrayList<Object>) mtasResponse.get("collection");
  692 + if (mtasResponseCollection != null) {
  693 + MtasSolrResultUtil.rewrite(mtasResponseCollection, searchComponent);
  694 + }
  695 + } catch (ClassCastException e) {
  696 + log.debug(e);
  697 + mtasResponse.remove("collection");
  698 + }
  699 + }
  700 + }
  701 + }
  702 +
  703 + /**
  704 + * Gets the mtas fields.
  705 + *
  706 + * @param rb
  707 + * the rb
  708 + * @return the mtas fields
  709 + */
  710 + private ComponentFields getMtasFields(ResponseBuilder rb) {
  711 + return (ComponentFields) rb.req.getContext().get(ComponentFields.class);
  712 + }
  713 +
  714 + /**
  715 + * String values to string.
  716 + *
  717 + * @param stringValues
  718 + * the string values
  719 + * @return the string
  720 + */
  721 + private static String stringValuesToString(HashSet<String> stringValues) {
  722 + return JSONUtil.toJSON(stringValues);
  723 + }
  724 +
  725 + /**
  726 + * String to string values.
  727 + *
  728 + * @param stringValue
  729 + * the string value
  730 + * @return the hash set
  731 + * @throws IOException
  732 + * Signals that an I/O exception has occurred.
  733 + */
  734 + private static HashSet<String> stringToStringValues(String stringValue)
  735 + throws IOException {
  736 + // should be improved to support escaped characters
  737 + HashSet<String> stringValues = new HashSet<>();
  738 + JSONParser jsonParser = new JSONParser(stringValue);
  739 + int event = jsonParser.nextEvent();
  740 + if (event == JSONParser.ARRAY_START) {
  741 + while ((event = jsonParser.nextEvent()) != JSONParser.ARRAY_END) {
  742 + if (jsonParser.getLevel() == 1) {
  743 + switch (event) {
  744 + case JSONParser.STRING:
  745 + stringValues.add(jsonParser.getString());
  746 + break;
  747 + case JSONParser.BIGNUMBER:
  748 + case JSONParser.NUMBER:
  749 + case JSONParser.LONG:
  750 + stringValues.add(jsonParser.getNumberChars().toString());
  751 + break;
  752 + case JSONParser.BOOLEAN:
  753 + stringValues.add(Boolean.toString(jsonParser.getBoolean()));
  754 + break;
  755 + default:
  756 + // do nothing
  757 + }
  758 + }
  759 + }
  760 + } else {
  761 + throw new IOException("unsupported json structure");
  762 + }
  763 + return stringValues;
  764 + }
  765 +
  766 +}
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentDocument.java
  View file @7179e4b
... ... @@ -71,12 +71,14 @@ public class MtasSolrComponentDocument
71 71  
72 72 /** The Constant NAME_MTAS_DOCUMENT_NUMBER. */
73 73 public static final String NAME_MTAS_DOCUMENT_NUMBER = "number";
  74 +
  75 + private MtasSolrSearchComponent searchComponent;
74 76  
75 77 /**
76 78 * Instantiates a new mtas solr component document.
77 79 */
78   - public MtasSolrComponentDocument() {
79   - // do nothing for now
  80 + public MtasSolrComponentDocument(MtasSolrSearchComponent searchComponent) {
  81 + this.searchComponent = searchComponent;
80 82 }
81 83  
82 84 /*
... ... @@ -257,7 +259,7 @@ public class MtasSolrComponentDocument
257 259 mtasDocumentItemResponses.add(mtasDocumentItemResponse);
258 260 }
259 261 mtasDocumentResponse.add("list", mtasDocumentItemResponses);
260   - MtasSolrResultUtil.rewrite(mtasDocumentResponse);
  262 + MtasSolrResultUtil.rewrite(mtasDocumentResponse, searchComponent);
261 263 return mtasDocumentResponse;
262 264 }
263 265  
... ... @@ -337,7 +339,7 @@ public class MtasSolrComponentDocument
337 339 try {
338 340 mtasResponseDocument = (ArrayList<Object>) mtasResponse.get("document");
339 341 if (mtasResponseDocument != null) {
340   - MtasSolrResultUtil.rewrite(mtasResponseDocument);
  342 + MtasSolrResultUtil.rewrite(mtasResponseDocument, searchComponent);
341 343 }
342 344 } catch (ClassCastException e) {
343 345 log.debug(e);
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentFacet.java
  View file @7179e4b
... ... @@ -573,7 +573,7 @@ public class MtasSolrComponentFacet
573 573 mtasFacetResponse.add("_encoded_list", MtasSolrResultUtil.encode(data));
574 574 } else {
575 575 mtasFacetResponse.add("list", data);
576   - MtasSolrResultUtil.rewrite(mtasFacetResponse);
  576 + MtasSolrResultUtil.rewrite(mtasFacetResponse, searchComponent);
577 577 }
578 578 return mtasFacetResponse;
579 579 }
... ... @@ -636,7 +636,7 @@ public class MtasSolrComponentFacet
636 636 try {
637 637 mtasResponseFacet = (ArrayList<Object>) mtasResponse.get("facet");
638 638 if (mtasResponseFacet != null) {
639   - MtasSolrResultUtil.rewrite(mtasResponseFacet);
  639 + MtasSolrResultUtil.rewrite(mtasResponseFacet, searchComponent);
640 640 }
641 641 } catch (ClassCastException e) {
642 642 log.debug(e);
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentGroup.java
  View file @7179e4b
... ... @@ -333,9 +333,10 @@ public class MtasSolrComponentGroup
333 333 * @param name the name
334 334 * @param positions the positions
335 335 * @param prefixes the prefixes
  336 + * @throws IOException
336 337 */
337 338 private void prepare(SolrParams solrParams, SortedSet<String> gids,
338   - String name, String[] positions, String[] prefixes) {
  339 + String name, String[] positions, String[] prefixes) throws IOException {
339 340 if (!gids.isEmpty()) {
340 341 int tmpSubCounter = 0;
341 342 for (String gid : gids) {
... ... @@ -343,6 +344,11 @@ public class MtasSolrComponentGroup
343 344 name + "." + gid + "." + NAME_MTAS_GROUP_GROUPING_POSITION, null);
344 345 prefixes[tmpSubCounter] = solrParams.get(
345 346 name + "." + gid + "." + NAME_MTAS_GROUP_GROUPING_PREFIXES, null);
  347 + if(positions[tmpSubCounter]==null) {
  348 + throw new IOException("no position for "+gid);
  349 + } else if(prefixes[tmpSubCounter]==null) {
  350 + throw new IOException("no prefix for "+gid);
  351 + }
346 352 tmpSubCounter++;
347 353 }
348 354 }
... ... @@ -465,7 +471,7 @@ public class MtasSolrComponentGroup
465 471 mtasGroupResponse.add("_encoded_list", MtasSolrResultUtil.encode(data));
466 472 } else {
467 473 mtasGroupResponse.add("list", data);
468   - MtasSolrResultUtil.rewrite(mtasGroupResponse);
  474 + MtasSolrResultUtil.rewrite(mtasGroupResponse, searchComponent);
469 475 }
470 476 return mtasGroupResponse;
471 477 }
... ... @@ -528,7 +534,7 @@ public class MtasSolrComponentGroup
528 534 try {
529 535 mtasResponseGroup = (ArrayList<Object>) mtasResponse.get("group");
530 536 if (mtasResponseGroup != null) {
531   - MtasSolrResultUtil.rewrite(mtasResponseGroup);
  537 + MtasSolrResultUtil.rewrite(mtasResponseGroup, searchComponent);
532 538 }
533 539 } catch (ClassCastException e) {
534 540 log.debug(e);
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentJoin.java deleted
View file @347732b
1   -package mtas.solr.handler.component.util;
2   -
3   -import java.io.IOException;
4   -import java.util.Arrays;
5   -import java.util.HashSet;
6   -import java.util.Set;
7   -
8   -import org.apache.commons.logging.Log;
9   -import org.apache.commons.logging.LogFactory;
10   -import org.apache.solr.common.util.NamedList;
11   -import org.apache.solr.common.util.SimpleOrderedMap;
12   -import org.apache.solr.handler.component.ResponseBuilder;
13   -import org.apache.solr.handler.component.SearchComponent;
14   -import org.apache.solr.handler.component.ShardRequest;
15   -import org.apache.solr.handler.component.ShardResponse;
16   -
17   -import mtas.codec.util.CodecComponent.ComponentFields;
18   -import mtas.codec.util.CodecComponent.ComponentJoin;
19   -import mtas.solr.handler.component.MtasSolrSearchComponent;
20   -
21   -/**
22   - * The Class MtasSolrComponentJoin.
23   - */
24   -@SuppressWarnings("deprecation")
25   -public class MtasSolrComponentJoin implements MtasSolrComponent<ComponentJoin> {
26   -
27   - /** The Constant log. */
28   - private static final Log log = LogFactory.getLog(MtasSolrComponentJoin.class);
29   -
30   - /** The Constant PARAM_MTAS_JOIN. */
31   - public static final String PARAM_MTAS_JOIN = MtasSolrSearchComponent.PARAM_MTAS
32   - + ".join";
33   -
34   - /** The Constant NAME_MTAS_JOIN_FIELD. */
35   - public static final String NAME_MTAS_JOIN_FIELD = "field";
36   -
37   - /**
38   - * Instantiates a new mtas solr component join.
39   - *
40   - * @param searchComponent the search component
41   - */
42   - public MtasSolrComponentJoin(MtasSolrSearchComponent searchComponent) {
43   - }
44   -
45   - /*
46   - * (non-Javadoc)
47   - *
48   - * @see
49   - * mtas.solr.handler.component.util.MtasSolrComponent#prepare(org.apache.solr.
50   - * handler.component.ResponseBuilder,
51   - * mtas.codec.util.CodecComponent.ComponentFields)
52   - */
53   - public void prepare(ResponseBuilder rb, ComponentFields mtasFields)
54   - throws IOException {
55   - if (rb.req.getParams().get(PARAM_MTAS_JOIN + "." + NAME_MTAS_JOIN_FIELD,
56   - null) != null) {
57   - Set<String> fields = new HashSet<>(Arrays.asList(rb.req.getParams()
58   - .get(PARAM_MTAS_JOIN + "." + NAME_MTAS_JOIN_FIELD).split(",")));
59   - String key = createKeyFromRequest(rb);
60   - mtasFields.doJoin = true;
61   - mtasFields.join = new ComponentJoin(fields, key);
62   - rb.setNeedDocSet(true);
63   - }
64   -
65   - }
66   -
67   - /*
68   - * (non-Javadoc)
69   - *
70   - * @see
71   - * mtas.solr.handler.component.util.MtasSolrComponent#modifyRequest(org.apache
72   - * .solr.handler.component.ResponseBuilder,
73   - * org.apache.solr.handler.component.SearchComponent,
74   - * org.apache.solr.handler.component.ShardRequest)
75   - */
76   - public void modifyRequest(ResponseBuilder rb, SearchComponent who,
77   - ShardRequest sreq) {
78   - if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
79   - && sreq.params.getBool(PARAM_MTAS_JOIN, false)) {
80   - if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
81   - // do nothing
82   - } else {
83   - // remove for other requests
84   - Set<String> keys = MtasSolrResultUtil
85   - .getIdsFromParameters(rb.req.getParams(), PARAM_MTAS_JOIN);
86   - sreq.params.remove(PARAM_MTAS_JOIN);
87   - for (String key : keys) {
88   - sreq.params.remove(PARAM_MTAS_JOIN + "." + key);
89   - }
90   - }
91   - }
92   - }
93   -
94   - /*
95   - * (non-Javadoc)
96   - *
97   - * @see
98   - * mtas.solr.handler.component.util.MtasSolrComponent#create(mtas.codec.util.
99   - * CodecComponent.BasicComponent, java.lang.Boolean)
100   - */
101   - public SimpleOrderedMap<Object> create(ComponentJoin join, Boolean encode)
102   - throws IOException {
103   - MtasSolrJoinResult data = new MtasSolrJoinResult(join);
104   - SimpleOrderedMap<Object> mtasJoinResponse = new SimpleOrderedMap<>();
105   - if (encode) {
106   - mtasJoinResponse.add("_encoded_data", MtasSolrResultUtil.encode(data));
107   - } else {
108   - mtasJoinResponse.add("data", data.rewrite());
109   - }
110   - return mtasJoinResponse;
111   - }
112   -
113   - /*
114   - * (non-Javadoc)
115   - *
116   - * @see
117   - * mtas.solr.handler.component.util.MtasSolrComponent#finishStage(org.apache.
118   - * solr.handler.component.ResponseBuilder)
119   - */
120   - @SuppressWarnings("unchecked")
121   - public void finishStage(ResponseBuilder rb) {
122   - if (rb.req.getParams().getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
123   - && rb.stage == MtasSolrSearchComponent.STAGE_JOIN) {
124   - for (ShardRequest sreq : rb.finished) {
125   - if (sreq.params.getBool(MtasSolrSearchComponent.PARAM_MTAS, false)
126   - && sreq.params.getBool(PARAM_MTAS_JOIN, false)) {
127   - for (ShardResponse shardResponse : sreq.responses) {
128   - NamedList<Object> response = shardResponse.getSolrResponse()
129   - .getResponse();
130   - try {
131   - Object data = response.findRecursive("mtas", "join");
132   - if (data != null && data instanceof String) {
133   - NamedList<Object> mtasResponse = (NamedList<Object>) response
134   - .get("mtas");
135   - mtasResponse.remove("join");
136   - mtasResponse.add("join",
137   - MtasSolrResultUtil.decode((String) data));
138   - }
139   - } catch (ClassCastException e) {
140   - log.debug(e);
141   - // shouldn't happen
142   - }
143   - }
144   - }
145   - }
146   - }
147   - }
148   -
149   - /*
150   - * (non-Javadoc)
151   - *
152   - * @see
153   - * mtas.solr.handler.component.util.MtasSolrComponent#distributedProcess(org.
154   - * apache.solr.handler.component.ResponseBuilder,
155   - * mtas.codec.util.CodecComponent.ComponentFields)
156   - */
157   - @SuppressWarnings("unchecked")
158   - public void distributedProcess(ResponseBuilder rb, ComponentFields mtasFields)
159   - throws IOException {
160   - // rewrite
161   - NamedList<Object> mtasResponse = null;
162   - try {
163   - mtasResponse = (NamedList<Object>) rb.rsp.getValues().get("mtas");
164   - } catch (ClassCastException e) {
165   - log.debug(e);
166   - mtasResponse = null;
167   - }
168   - if (mtasResponse != null) {
169   - MtasSolrJoinResult mtasSolrJoinResult;
170   - try {
171   - mtasSolrJoinResult = (MtasSolrJoinResult) mtasResponse.get("join");
172   - if (mtasSolrJoinResult != null) {
173   - mtasResponse.removeAll("join");
174   - mtasResponse.add("join", mtasSolrJoinResult.rewrite());
175   - }
176   - } catch (ClassCastException e) {
177   - log.debug(e);
178   - mtasResponse.remove("join");
179   - }
180   - }
181   - }
182   -
183   - /**
184   - * Creates the key from request.
185   - *
186   - * @param rb the rb
187   - * @return the string
188   - */
189   - private String createKeyFromRequest(ResponseBuilder rb) {
190   - return rb.req.getParams().toQueryString();
191   - }
192   -
193   -}
src/mtas/solr/handler/component/util/MtasSolrComponentPrefix.java
  View file @7179e4b
... ... @@ -136,7 +136,7 @@ public class MtasSolrComponentPrefix
136 136 * CodecComponent.BasicComponent, java.lang.Boolean)
137 137 */
138 138 public SimpleOrderedMap<Object> create(ComponentPrefix prefix,
139   - Boolean encode) {
  139 + Boolean encode) throws IOException {
140 140 SimpleOrderedMap<Object> mtasPrefixResponse = new SimpleOrderedMap<>();
141 141 mtasPrefixResponse.add("key", prefix.key);
142 142 if (encode) {
... ... @@ -219,7 +219,7 @@ public class MtasSolrComponentPrefix
219 219 for (Object mtasResponsePrefixItemRaw : mtasResponsePrefix) {
220 220 mtasResponsePrefixItem = (NamedList<Object>) mtasResponsePrefixItemRaw;
221 221 repairPrefixItems(mtasResponsePrefixItem);
222   - MtasSolrResultUtil.rewrite(mtasResponsePrefixItem);
  222 + MtasSolrResultUtil.rewrite(mtasResponsePrefixItem, searchComponent);
223 223 }
224 224 }
225 225 } catch (ClassCastException e) {
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentStats.java
  View file @7179e4b
... ... @@ -878,7 +878,7 @@ public class MtasSolrComponentStats
878 878 MtasSolrResultUtil.encode(data));
879 879 } else {
880 880 mtasPositionResponse.add(position.dataCollector.getCollectorType(), data);
881   - MtasSolrResultUtil.rewrite(mtasPositionResponse);
  881 + MtasSolrResultUtil.rewrite(mtasPositionResponse, searchComponent);
882 882 }
883 883 return mtasPositionResponse;
884 884 }
... ... @@ -903,7 +903,7 @@ public class MtasSolrComponentStats
903 903 mtasTokenResponse.add("_encoded_data", MtasSolrResultUtil.encode(data));
904 904 } else {
905 905 mtasTokenResponse.add(token.dataCollector.getCollectorType(), data);
906   - MtasSolrResultUtil.rewrite(mtasTokenResponse);
  906 + MtasSolrResultUtil.rewrite(mtasTokenResponse, searchComponent);
907 907 }
908 908 return mtasTokenResponse;
909 909 }
... ... @@ -944,7 +944,7 @@ public class MtasSolrComponentStats
944 944 mtasSpanResponse.add("_encoded_data", MtasSolrResultUtil.encode(data));
945 945 } else {
946 946 mtasSpanResponse.add(span.dataCollector.getCollectorType(), data);
947   - MtasSolrResultUtil.rewrite(mtasSpanResponse);
  947 + MtasSolrResultUtil.rewrite(mtasSpanResponse, searchComponent);
948 948 }
949 949 return mtasSpanResponse;
950 950 }
... ... @@ -1007,7 +1007,7 @@ public class MtasSolrComponentStats
1007 1007 try {
1008 1008 mtasResponseStats = (NamedList<Object>) mtasResponse.get("stats");
1009 1009 if (mtasResponseStats != null) {
1010   - MtasSolrResultUtil.rewrite(mtasResponseStats);
  1010 + MtasSolrResultUtil.rewrite(mtasResponseStats, searchComponent);
1011 1011 }
1012 1012 } catch (ClassCastException e) {
1013 1013 log.debug(e);
... ...
src/mtas/solr/handler/component/util/MtasSolrComponentTermvector.java
  View file @7179e4b
... ... @@ -476,7 +476,7 @@ public class MtasSolrComponentTermvector
476 476 MtasSolrResultUtil.encode(data));
477 477 } else {
478 478 mtasTermVectorResponse.add("list", data);
479   - MtasSolrResultUtil.rewrite(mtasTermVectorResponse);
  479 + MtasSolrResultUtil.rewrite(mtasTermVectorResponse, searchComponent);
480 480 }
481 481 return mtasTermVectorResponse;
482 482 }
... ... @@ -627,7 +627,7 @@ public class MtasSolrComponentTermvector
627 627 if ((mtasResponseTermvectorRaw = mtasResponse.get("termvector")) != null
628 628 && mtasResponseTermvectorRaw instanceof ArrayList) {
629 629 MtasSolrResultUtil
630   - .rewrite((ArrayList<Object>) mtasResponseTermvectorRaw);
  630 + .rewrite((ArrayList<Object>) mtasResponseTermvectorRaw, searchComponent);
631 631 }
632 632 }
633 633 }
... ...
src/mtas/solr/handler/component/util/MtasSolrJoinResult.java deleted
View file @347732b
1   -package mtas.solr.handler.component.util;
2   -
3   -import java.io.Serializable;
4   -import java.util.Set;
5   -
6   -import org.apache.solr.common.util.NamedList;
7   -
8   -import mtas.codec.util.CodecComponent.ComponentJoin;
9   -
10   -/**
11   - * The Class MtasSolrJoinResult.
12   - */
13   -public class MtasSolrJoinResult implements Serializable {
14   -
15   - /** The Constant serialVersionUID. */
16   - private static final long serialVersionUID = 1L;
17   -
18   - /** The values. */
19   - private Set<String> values;
20   -
21   - /** The key. */
22   - private String key;
23   -
24   - /**
25   - * Instantiates a new mtas solr join result.
26   - *
27   - * @param join the join
28   - */
29   - public MtasSolrJoinResult(ComponentJoin join) {
30   - values = join.values();
31   - key = join.key();
32   - }
33   -
34   - /**
35   - * Rewrite.
36   - *
37   - * @return the named list
38   - */
39   - public NamedList<Object> rewrite() {
40   - NamedList<Object> response = new NamedList<>();
41   - response.add("values", values);
42   - response.add("key", key);
43   - return response;
44   - }
45   -
46   - /**
47   - * Merge.
48   - *
49   - * @param newItem the new item
50   - */
51   - public void merge(MtasSolrJoinResult newItem) {
52   - values.addAll(newItem.values);
53   - }
54   -
55   -}
src/mtas/solr/handler/component/util/MtasSolrResultMerge.java
  View file @7179e4b
... ... @@ -15,7 +15,6 @@ import org.apache.solr.handler.component.ResponseBuilder;
15 15 import org.apache.solr.handler.component.ShardRequest;
16 16 import org.apache.solr.handler.component.ShardResponse;
17 17  
18   -import mtas.codec.util.CodecComponent.ComponentFields;
19 18 import mtas.solr.handler.component.MtasSolrSearchComponent;
20 19  
21 20 /**
... ... @@ -64,21 +63,22 @@ public class MtasSolrResultMerge {
64 63 .getBool(MtasSolrComponentFacet.PARAM_MTAS_FACET, false)) {
65 64 mergeArrayList(sreq, mtasResponse, "facet", null, false);
66 65 }
67   - // merge join
68   - if (rb.req.getParams().getBool(MtasSolrComponentJoin.PARAM_MTAS_JOIN,
  66 + // merge collection
  67 + if (rb.req.getParams().getBool(MtasSolrComponentCollection.PARAM_MTAS_COLLECTION,
69 68 false)) {
70   - ComponentFields componentFields = (ComponentFields) rb.req
71   - .getContext().get(ComponentFields.class);
72   - mtasResponse.add("join",
73   - new MtasSolrJoinResult(componentFields.join));
74   - mergeJoinResult(sreq, mtasResponse, "join", null);
75   -
  69 + mergeArrayList(sreq, mtasResponse, "collection", null, false);
76 70 }
77 71 // merge prefix
78 72 if (rb.req.getParams()
79 73 .getBool(MtasSolrComponentPrefix.PARAM_MTAS_PREFIX, false)) {
80 74 mergeArrayList(sreq, mtasResponse, "prefix", null, false);
81 75 }
  76 + } else if (rb.stage == MtasSolrSearchComponent.STAGE_COLLECTION_INIT) {
  77 + // merge collection
  78 + if (rb.req.getParams().getBool(
  79 + MtasSolrComponentCollection.PARAM_MTAS_COLLECTION, false)) {
  80 + mergeArrayList(sreq, mtasResponse, "collection", null, false);
  81 + }
82 82 } else if (rb.stage == MtasSolrSearchComponent.STAGE_TERMVECTOR_MISSING_KEY) {
83 83 // merge termvector
84 84 if (rb.req.getParams().getBool(
... ... @@ -109,57 +109,7 @@ public class MtasSolrResultMerge {
109 109 }
110 110 }
111 111 }
112   -
113   - /**
114   - * Merge join result.
115   - *
116   - * @param sreq the sreq
117   - * @param mtasResponse the mtas response
118   - * @param key the key
119   - * @param preferredPurpose the preferred purpose
120   - */
121   - @SuppressWarnings("unchecked")
122   - private void mergeJoinResult(ShardRequest sreq,
123   - NamedList<Object> mtasResponse, String key, Integer preferredPurpose) {
124   - Object o = mtasResponse.get(key);
125   - MtasSolrJoinResult mtasJoinResponse;
126   - if (o instanceof MtasSolrJoinResult) {
127   - mtasJoinResponse = (MtasSolrJoinResult) o;
128   - } else {
129   - mtasJoinResponse = null;
130   - }
131   - // collect responses for each shard
132   - HashMap<String, NamedList<Object>> mtasListShardResponses = new HashMap<>();
133   - for (ShardResponse response : sreq.responses) {
134   - // only continue if new shard or preferred purpose
135   - if (mtasListShardResponses.containsKey(response.getShard())
136   - && ((preferredPurpose == null)
137   - || (sreq.purpose != preferredPurpose))) {
138   - break;
139   - }
140   - // update
141   - try {
142   - NamedList<Object> result = response.getSolrResponse().getResponse();
143   - String data = (String) result.findRecursive("mtas", key);
144   - if (data != null) {
145   - MtasSolrJoinResult decodedData = (MtasSolrJoinResult) MtasSolrResultUtil
146   - .decode(data);
147   - if (mtasJoinResponse == null) {
148   - mtasJoinResponse = decodedData;
149   - } else {
150   - mtasJoinResponse.merge(decodedData);
151   - }
152   - }
153   - } catch (ClassCastException e) {
154   - log.debug(e);
155   - }
156   - }
157   - if (mtasJoinResponse != null) {
158   - mtasResponse.removeAll(key);
159   - mtasResponse.add(key, mtasJoinResponse);
160   - }
161   - }
162   -
  112 +
163 113 /**
164 114 * Merge named list.
165 115 *
... ... @@ -381,9 +331,9 @@ public class MtasSolrResultMerge {
381 331 } else if (original instanceof MtasSolrMtasResult) {
382 332 MtasSolrMtasResult originalComponentResult = (MtasSolrMtasResult) original;
383 333 originalComponentResult.merge((MtasSolrMtasResult) shardValue);
384   - } else if (original instanceof MtasSolrJoinResult) {
385   - MtasSolrJoinResult originalComponentResult = (MtasSolrJoinResult) original;
386   - originalComponentResult.merge((MtasSolrJoinResult) shardValue);
  334 + } else if (original instanceof MtasSolrCollectionResult) {
  335 + MtasSolrCollectionResult originalComponentResult = (MtasSolrCollectionResult) original;
  336 + originalComponentResult.merge((MtasSolrCollectionResult) shardValue);
387 337 } else if (original instanceof String) {
388 338 // ignore?
389 339 } else if (original instanceof Integer) {
... ...