Commit 466e52f3c655031b46451909622993868a59bc67

Authored by Matthijs Brouwer
1 parent bbd39cf9

thread safety

conf/parser/mtas.xml
... ... @@ -8,6 +8,7 @@
8 8 <configuration name="DDD" file="mtas/folia_ddd.xml" />
9 9 <configuration name="EDBO" file="mtas/folia_edbo.xml" />
10 10 <configuration name="SoNaR" file="mtas/folia_sonar.xml" />
  11 + <configuration name="isotei" file="mtas/iso_tei.xml" />
11 12 </configurations>
12 13 <configurations type="mtas.analysis.util.MtasCharFilterFactory">
13 14 <configuration name="test" type="file" />
... ... @@ -17,5 +18,6 @@
17 18 <configuration name="DDD" type="url" prefix="https://openskos.meertens.knaw.nl/nederlab/archief/get/" />
18 19 <configuration name="EDBO" type="url" prefix="https://openskos.meertens.knaw.nl/nederlab/archief/get/" />
19 20 <configuration name="SoNaR" type="url" prefix="https://openskos.meertens.knaw.nl/nederlab/archief/get/" />
  21 + <configuration name="isotei" type="file" prefix="/Users/matthijs/Software/Mtas/data/ISO_TEI/" postfix="" />
20 22 </configurations>
21 23 </mtas>
... ...
conf/parser/mtas/iso_tei.xml
... ... @@ -60,13 +60,13 @@
60 60 </post>
61 61 </token>
62 62 <token type="string" offset="false" realoffset="false" parent="true">
63   - <pre>
64   - <item type="string" value="w.id" />
65   - </pre>
66   - <post>
67   - <item type="attribute" name="#" />
68   - </post>
69   - </token>
  63 + <pre>
  64 + <item type="string" value="w.id" />
  65 + </pre>
  66 + <post>
  67 + <item type="attribute" name="#" />
  68 + </post>
  69 + </token>
70 70 </mapping>
71 71 <mapping type="word" name="anchor" start="synch" end="synch">
72 72 <token type="string" offset="false" realoffset="false" parent="false">
... ... @@ -80,30 +80,41 @@
80 80 </token>
81 81 </mapping>
82 82 <mapping type="word" name="pause">
83   - <token type="string" offset="false" realoffset="false" parent="false">
84   - <pre>
85   - <item type="name" />
86   - </pre>
87   - <post>
88   - <item type="attribute" name="type" />
89   - </post>
90   - </token>
91   - </mapping>
92   - <mapping type="word" name="vocal"></mapping>
93   - <mapping type="word" name="incident"></mapping>
  83 + <token type="string" offset="false" realoffset="false" parent="false">
  84 + <pre>
  85 + <item type="name" />
  86 + </pre>
  87 + <post>
  88 + <item type="attribute" name="type" />
  89 + </post>
  90 + </token>
  91 + </mapping>
  92 + <mapping type="word" name="pc">
  93 + <token type="string" offset="false" realoffset="false" parent="false">
  94 + <pre>
  95 + <item type="name" />
  96 + </pre>
  97 + <post>
  98 + <item type="text" />
  99 + </post>
  100 + </token>
  101 + </mapping>
  102 + <mapping type="word" name="vocal"></mapping>
  103 + <mapping type="word" name="incident"></mapping>
  104 + <mapping type="word" name="kinesic"></mapping>
94 105 <!-- END WORDS -->
95 106  
96 107 <!-- START WORD ANNOTATIONS -->
97 108 <mapping type="wordAnnotation" name="desc">
98   - <token type="string" offset="false" realoffset="false" parent="false">
99   - <pre>
100   - <item type="ancestorWordName" />
101   - </pre>
102   - <post>
103   - <item type="text" />
104   - </post>
105   - </token>
106   - </mapping>
  109 + <token type="string" offset="false" realoffset="false" parent="false">
  110 + <pre>
  111 + <item type="ancestorWordName" />
  112 + </pre>
  113 + <post>
  114 + <item type="text" />
  115 + </post>
  116 + </token>
  117 + </mapping>
107 118 <!-- END WORD ANNOTATIONS -->
108 119  
109 120 <!-- START RELATIONS -->
... ... @@ -133,30 +144,61 @@
133 144 <item type="variableFromAttribute" name="sex" value="who" />
134 145 </post>
135 146 </token>
  147 + <token type="string" offset="false" realoffset="false" parent="true">
  148 + <pre>
  149 + <item type="string" value="annotationBlock.id" />
  150 + </pre>
  151 + <post>
  152 + <item type="attribute" name="#" />
  153 + </post>
  154 + </token>
  155 + </mapping>
  156 + <mapping type="group" name="u">
  157 + <token type="string" offset="false" realoffset="false" parent="true">
  158 + <pre>
  159 + <item type="name" />
  160 + </pre>
  161 + </token>
  162 + <token type="string" offset="false" realoffset="false" parent="true">
  163 + <pre>
  164 + <item type="string" value="u.id" />
  165 + </pre>
  166 + <post>
  167 + <item type="attribute" name="#" />
  168 + </post>
  169 + </token>
  170 + </mapping>
  171 + <mapping type="group" name="seg">
  172 + <token type="string" offset="false" realoffset="false" parent="true">
  173 + <pre>
  174 + <item type="name" />
  175 + </pre>
  176 + </token>
  177 + </mapping>
  178 + <mapping type="group" name="choice">
  179 + <token type="string" offset="false" realoffset="false" parent="true">
  180 + <pre>
  181 + <item type="name" />
  182 + </pre>
  183 + </token>
  184 + </mapping>
  185 + <mapping type="group" name="unclear">
  186 + <token type="string" offset="false" realoffset="false" parent="true">
  187 + <pre>
  188 + <item type="name" />
  189 + </pre>
  190 + <post>
  191 + <item type="attribute" name="reason" />
  192 + </post>
  193 + </token>
136 194 </mapping>
137   - <mapping type="group" name="u" >
138   - <token type="string" offset="false" realoffset="false" parent="true">
139   - <pre>
140   - <item type="name" />
141   - </pre>
142   - </token>
143   - <token type="string" offset="false" realoffset="false" parent="true">
144   - <pre>
145   - <item type="name" />
146   - <item type="string" value=".id" />
147   - </pre>
148   - <post>
149   - <item type="attribute" name="#" />
150   - </post>
151   - </token>
152   - </mapping>
153 195 <!-- END GROUPS -->
154 196  
155 197 <!-- START GROUP ANNOTATIONS -->
156 198 <mapping type="groupAnnotation" name="spanGrp">
157 199 </mapping>
158 200  
159   - <mapping type="groupAnnotation" name="span" start="from" end="to">
  201 + <mapping type="groupAnnotation" name="span" start="from" end="to">
160 202 <token type="string" offset="false" realoffset="false" parent="false">
161 203 <pre>
162 204 <item type="name" />
... ... @@ -180,15 +222,24 @@
180 222 <post>
181 223 <item type="text" />
182 224 </post>
183   - </token>
  225 + </token>
  226 + </mapping>
  227 + <mapping type="groupAnnotation" name="gap" start="start" end="end">
  228 + <token type="string" offset="false" realoffset="false" parent="false">
  229 + <pre>
  230 + <item type="name" />
  231 + </pre>
  232 + <post>
  233 + <item type="attribute" name="reason" />
  234 + </post>
  235 + </token>
184 236 </mapping>
185   -
186 237  
187 238 <!-- END GROUP ANNOTATIONS -->
188 239  
189 240 </mappings>
190 241 <!-- END MAPPINGS -->
191   -
  242 +
192 243 </parser>
193 244 <!-- END CONFIGURATION MTAS ISO/TEI PARSER -->
194 245  
... ...
conf/parser/mtas/test_iso_tei.xml 0 → 100644
  1 +<?xml version="1.0" encoding="UTF-8" ?>
  2 +<mtas>
  3 +
  4 + <!-- START MTAS INDEX CONFIGURATION -->
  5 + <index>
  6 + <!-- START GENERAL SETTINGS MTAS INDEX PROCESS -->
  7 + <payload index="true" />
  8 + <offset index="true" />
  9 + <realoffset index="true" />
  10 + <parent index="true" />
  11 + <!-- END GENERAL SETTINGS MTAS INDEX PROCESS -->
  12 + </index>
  13 + <!-- END MTAS INDEX CONFIGURATION -->
  14 +
  15 + <!-- START CONFIGURATION MTAS ISO/TEI PARSER -->
  16 + <parser name="mtas.analysis.parser.MtasTEIParser">
  17 +
  18 + <!-- START GENERAL SETTINGS MTAS ISO/TEI PARSER -->
  19 + <autorepair value="true" />
  20 + <makeunique value="true" />
  21 + <!-- END GENERAL SETTINGS MTAS ISO/TEI PARSER -->
  22 +
  23 + <!-- START VARIABLES -->
  24 + <variables>
  25 + <variable name="when" value="interval">
  26 + <value>
  27 + <item type="attribute" name="interval" />
  28 + </value>
  29 + </variable>
  30 + <variable name="person" value="sex">
  31 + <value>
  32 + <item type="attribute" name="sex" />
  33 + </value>
  34 + </variable>
  35 + </variables>
  36 + <!-- END VARIABLES -->
  37 +
  38 + <!-- START REFERENCES -->
  39 + <!-- END REFERENCES -->
  40 +
  41 + <!-- START MAPPINGS -->
  42 + <mappings>
  43 +
  44 + <!-- START WORDS -->
  45 + <mapping type="word" name="w" start="#" end="#">
  46 + <token type="string" offset="false" realoffset="false" parent="true">
  47 + <pre>
  48 + <item type="string" value="w" />
  49 + </pre>
  50 + <post>
  51 + <item type="text" />
  52 + </post>
  53 + </token>
  54 + <token type="string" offset="false" realoffset="false" parent="false">
  55 + <pre>
  56 + <item type="string" value="w_lc" />
  57 + </pre>
  58 + <post>
  59 + <item type="text" filter="ascii,lowercase" />
  60 + </post>
  61 + </token>
  62 + <token type="string" offset="false" realoffset="false" parent="true">
  63 + <pre>
  64 + <item type="string" value="w.id" />
  65 + </pre>
  66 + <post>
  67 + <item type="attribute" name="#" />
  68 + </post>
  69 + </token>
  70 + </mapping>
  71 + <mapping type="word" name="anchor" start="synch" end="synch">
  72 + <token type="string" offset="false" realoffset="false" parent="false">
  73 + <pre>
  74 + <item type="name" />
  75 + <item type="string" value=".time" />
  76 + </pre>
  77 + <post>
  78 + <item type="variableFromAttribute" name="interval" value="synch" />
  79 + </post>
  80 + </token>
  81 + </mapping>
  82 + <mapping type="word" name="pause">
  83 + <token type="string" offset="false" realoffset="false" parent="false">
  84 + <pre>
  85 + <item type="name" />
  86 + </pre>
  87 + <post>
  88 + <item type="attribute" name="type" />
  89 + </post>
  90 + </token>
  91 + </mapping>
  92 + <mapping type="word" name="vocal"></mapping>
  93 + <mapping type="word" name="incident"></mapping>
  94 + <!-- END WORDS -->
  95 +
  96 + <!-- START WORD ANNOTATIONS -->
  97 + <mapping type="wordAnnotation" name="desc">
  98 + <token type="string" offset="false" realoffset="false" parent="false">
  99 + <pre>
  100 + <item type="ancestorWordName" />
  101 + </pre>
  102 + <post>
  103 + <item type="text" />
  104 + </post>
  105 + </token>
  106 + </mapping>
  107 + <!-- END WORD ANNOTATIONS -->
  108 +
  109 + <!-- START RELATIONS -->
  110 + <!-- END RELATIONS -->
  111 +
  112 +
  113 + <!-- START GROUPS -->
  114 + <mapping type="group" name="annotationBlock" start="start"
  115 + end="end">
  116 + <token type="string" offset="false" realoffset="false" parent="false">
  117 + <pre>
  118 + <item type="name" />
  119 + <item type="string" value=".interval" />
  120 + </pre>
  121 + <post>
  122 + <item type="variableFromAttribute" name="interval" value="start" />
  123 + <item type="variableFromAttribute" name="interval" value="end"
  124 + prefix=" - " />
  125 + </post>
  126 + </token>
  127 + <token type="string" offset="false" realoffset="false" parent="false">
  128 + <pre>
  129 + <item type="name" />
  130 + <item type="string" value=".sex" />
  131 + </pre>
  132 + <post>
  133 + <item type="variableFromAttribute" name="sex" value="who" />
  134 + </post>
  135 + </token>
  136 + </mapping>
  137 + <mapping type="group" name="u" >
  138 + <token type="string" offset="false" realoffset="false" parent="true">
  139 + <pre>
  140 + <item type="name" />
  141 + </pre>
  142 + </token>
  143 + <token type="string" offset="false" realoffset="false" parent="true">
  144 + <pre>
  145 + <item type="name" />
  146 + <item type="string" value=".id" />
  147 + </pre>
  148 + <post>
  149 + <item type="attribute" name="#" />
  150 + </post>
  151 + </token>
  152 + </mapping>
  153 + <!-- END GROUPS -->
  154 +
  155 + <!-- START GROUP ANNOTATIONS -->
  156 + <mapping type="groupAnnotation" name="spanGrp">
  157 + </mapping>
  158 +
  159 + <mapping type="groupAnnotation" name="span" start="from" end="to">
  160 + <token type="string" offset="false" realoffset="false" parent="false">
  161 + <pre>
  162 + <item type="name" />
  163 + <item type="ancestorAttribute" distance="0" name="type"
  164 + prefix="." />
  165 + <item type="string" value=".interval" />
  166 + </pre>
  167 + <post>
  168 + <item type="variableFromAttribute" name="interval" value="from" />
  169 + <item type="variableFromAttribute" name="interval" value="to"
  170 + prefix=" - " />
  171 + </post>
  172 + </token>
  173 + </mapping>
  174 +
  175 + <mapping type="groupAnnotation" name="span" start="from" end="to">
  176 + <token type="string" offset="false" realoffset="false" parent="false">
  177 + <pre>
  178 + <item type="ancestorAttribute" distance="0" name="type" />
  179 + </pre>
  180 + <post>
  181 + <item type="text" />
  182 + </post>
  183 + </token>
  184 + </mapping>
  185 +
  186 +
  187 + <!-- END GROUP ANNOTATIONS -->
  188 +
  189 + </mappings>
  190 + <!-- END MAPPINGS -->
  191 +
  192 + </parser>
  193 + <!-- END CONFIGURATION MTAS ISO/TEI PARSER -->
  194 +
  195 +</mtas>
... ...
docker/Dockerfile
1 1 # Automatically generated Dockerfile
2   -# - Build 2016-12-30 14:18
  2 +# - Build 2016-12-31 13:01
3 3 # - Lucene/Solr version 6.3.0
4 4 # - Mtas release 20161230
5 5 #
... ... @@ -54,7 +54,7 @@ RUN apt-get update &amp;&amp; apt-get install -y lsof software-properties-common python-
54 54 && chmod -R 755 /var/www/html \
55 55 && printf "echo\n" >> /start.sh \
56 56 && printf "echo \"================ Mtas -- Multi Tier Annotation Search =================\"\n" >> /start.sh \
57   -&& printf "echo \" Timestamp 2016-12-30 14:18\"\n" >> /start.sh \
  57 +&& printf "echo \" Timestamp 2016-12-31 13:01\"\n" >> /start.sh \
58 58 && printf "echo \" Lucene/Solr version 6.3.0\"\n" >> /start.sh \
59 59 && printf "echo \" Mtas release 20161230\"\n" >> /start.sh \
60 60 && printf "echo \" See https://meertensinstituut.github.io/mtas/ for more information\"\n" >> /start.sh \
... ...
src/mtas/analysis/parser/MtasBasicParser.java
1 1 package mtas.analysis.parser;
2 2  
3   -import java.io.IOException;
4 3 import java.io.UnsupportedEncodingException;
5 4 import java.util.ArrayList;
6 5 import java.util.Arrays;
... ... @@ -16,10 +15,8 @@ import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
16 15 import org.apache.lucene.analysis.payloads.PayloadHelper;
17 16 import org.apache.lucene.util.BytesRef;
18 17  
19   -import mtas.analysis.parser.MtasBasicParser.MtasParserType;
20   -import mtas.analysis.parser.MtasBasicParser.MtasParserVariable;
21   -import mtas.analysis.parser.MtasParser.MtasParserObject;
22 18 import mtas.analysis.token.MtasToken;
  19 +import mtas.analysis.token.MtasTokenIdFactory;
23 20 import mtas.analysis.token.MtasTokenString;
24 21 import mtas.analysis.util.MtasConfigException;
25 22 import mtas.analysis.util.MtasParserException;
... ... @@ -132,9 +129,13 @@ abstract public class MtasBasicParser extends MtasParser {
132 129 /** The Constant ITEM_TYPE_ANCESTOR_RELATION_ANNOTATION. */
133 130 protected final static String ITEM_TYPE_ANCESTOR_RELATION_ANNOTATION = "ancestorRelationAnnotation";
134 131  
  132 + /** The Constant ITEM_TYPE_VARIABLE_FROM_ATTRIBUTE. */
135 133 protected final static String ITEM_TYPE_VARIABLE_FROM_ATTRIBUTE = "variableFromAttribute";
136 134  
  135 + /** The Constant VARIABLE_SUBTYPE_VALUE. */
137 136 protected final static String VARIABLE_SUBTYPE_VALUE = "value";
  137 +
  138 + /** The Constant VARIABLE_SUBTYPE_VALUE_ITEM. */
138 139 protected final static String VARIABLE_SUBTYPE_VALUE_ITEM = "item";
139 140  
140 141 /** The Constant MAPPING_SUBTYPE_TOKEN. */
... ... @@ -170,14 +171,25 @@ abstract public class MtasBasicParser extends MtasParser {
170 171 /** The Constant UPDATE_TYPE_POSITION. */
171 172 protected final static String UPDATE_TYPE_POSITION = "positionUpdate";
172 173  
  174 + /** The Constant UPDATE_TYPE_VARIABLE. */
173 175 protected final static String UPDATE_TYPE_VARIABLE = "variableUpdate";
174 176  
  177 + /** The Constant UPDATE_TYPE_LOCAL_REF_OFFSET_START. */
175 178 protected final static String UPDATE_TYPE_LOCAL_REF_OFFSET_START = "localRefOffsetStartUpdate";
  179 +
  180 + /** The Constant UPDATE_TYPE_LOCAL_REF_OFFSET_END. */
176 181 protected final static String UPDATE_TYPE_LOCAL_REF_OFFSET_END = "localRefOffsetEndUpdate";
  182 +
  183 + /** The Constant UPDATE_TYPE_LOCAL_REF_POSITION_START. */
177 184 protected final static String UPDATE_TYPE_LOCAL_REF_POSITION_START = "localRefPositionStartUpdate";
  185 +
  186 + /** The Constant UPDATE_TYPE_LOCAL_REF_POSITION_END. */
178 187 protected final static String UPDATE_TYPE_LOCAL_REF_POSITION_END = "localRefPositionEndUpdate";
179 188  
  189 + /** The enc. */
180 190 private Base64.Encoder enc = Base64.getEncoder();
  191 +
  192 + /** The dec. */
181 193 private Base64.Decoder dec = Base64.getDecoder();
182 194  
183 195 /**
... ... @@ -189,13 +201,17 @@ abstract public class MtasBasicParser extends MtasParser {
189 201 /**
190 202 * Instantiates a new mtas basic parser.
191 203 *
192   - * @param config
193   - * the config
  204 + * @param config the config
194 205 */
195 206 public MtasBasicParser(MtasConfiguration config) {
196 207 this.config = config;
197 208 }
198 209  
  210 + /**
  211 + * Creates the current list.
  212 + *
  213 + * @return the hash map
  214 + */
199 215 protected HashMap<String, ArrayList<MtasParserObject>> createCurrentList() {
200 216 HashMap<String, ArrayList<MtasParserObject>> currentList = new HashMap<String, ArrayList<MtasParserObject>>();
201 217 currentList.put(MAPPING_TYPE_RELATION, new ArrayList<MtasParserObject>());
... ... @@ -211,6 +227,11 @@ abstract public class MtasBasicParser extends MtasParser {
211 227 return currentList;
212 228 }
213 229  
  230 + /**
  231 + * Creates the update list.
  232 + *
  233 + * @return the hash map
  234 + */
214 235 protected HashMap<String, HashMap<Integer, HashSet<String>>> createUpdateList() {
215 236 HashMap<String, HashMap<Integer, HashSet<String>>> updateList = new HashMap<String, HashMap<Integer, HashSet<String>>>();
216 237 updateList.put(UPDATE_TYPE_OFFSET, new HashMap<Integer, HashSet<String>>());
... ... @@ -228,7 +249,12 @@ abstract public class MtasBasicParser extends MtasParser {
228 249 new HashMap<Integer, HashSet<String>>());
229 250 return updateList;
230 251 }
231   -
  252 +
  253 + /**
  254 + * Creates the variables.
  255 + *
  256 + * @return the hash map
  257 + */
232 258 protected HashMap<String, HashMap<String, String>> createVariables() {
233 259 return new HashMap<String, HashMap<String, String>>();
234 260 }
... ... @@ -236,18 +262,15 @@ abstract public class MtasBasicParser extends MtasParser {
236 262 /**
237 263 * Compute mappings from object.
238 264 *
239   - * @param object
240   - * the object
241   - * @param currentList
242   - * the current list
243   - * @param updateList
244   - * the update list
245   - * @throws MtasParserException
246   - * the mtas parser exception
247   - * @throws MtasConfigException
248   - * the mtas config exception
  265 + * @param mtasTokenIdFactory the mtas token id factory
  266 + * @param object the object
  267 + * @param currentList the current list
  268 + * @param updateList the update list
  269 + * @throws MtasParserException the mtas parser exception
  270 + * @throws MtasConfigException the mtas config exception
249 271 */
250   - protected void computeMappingsFromObject(MtasParserObject object,
  272 + protected void computeMappingsFromObject(
  273 + MtasTokenIdFactory mtasTokenIdFactory, MtasParserObject object,
251 274 HashMap<String, ArrayList<MtasParserObject>> currentList,
252 275 HashMap<String, HashMap<Integer, HashSet<String>>> updateList)
253 276 throws MtasParserException, MtasConfigException {
... ... @@ -334,7 +357,8 @@ abstract public class MtasBasicParser extends MtasParser {
334 357 // create token and get id: from now on, we must continue, no
335 358 // exceptions allowed...
336 359 for (int k = 0; k < value.length; k++) {
337   - MtasTokenString token = new MtasTokenString(value[k]);
  360 + MtasTokenString token = new MtasTokenString(
  361 + mtasTokenIdFactory.createTokenId(), value[k]);
338 362 // store settings offset, realoffset and parent
339 363 token.setProvideOffset(mappingToken.offset);
340 364 token.setProvideRealOffset(mappingToken.realoffset);
... ... @@ -485,7 +509,7 @@ abstract public class MtasBasicParser extends MtasParser {
485 509 startAttribute = object.getId();
486 510 } else {
487 511 startAttribute = object.getAttribute(mapping.start);
488   - if (startAttribute!=null && startAttribute.startsWith("#")) {
  512 + if (startAttribute != null && startAttribute.startsWith("#")) {
489 513 startAttribute = startAttribute.substring(1);
490 514 }
491 515 }
... ... @@ -493,7 +517,7 @@ abstract public class MtasBasicParser extends MtasParser {
493 517 endAttribute = object.getId();
494 518 } else {
495 519 endAttribute = object.getAttribute(mapping.end);
496   - if (endAttribute!=null && endAttribute.startsWith("#")) {
  520 + if (endAttribute != null && endAttribute.startsWith("#")) {
497 521 endAttribute = endAttribute.substring(1);
498 522 }
499 523 }
... ... @@ -544,30 +568,43 @@ abstract public class MtasBasicParser extends MtasParser {
544 568 updateMappingsWithLocalReferences(object, currentList, updateList);
545 569 }
546 570  
  571 + /**
  572 + * Compute variables from object.
  573 + *
  574 + * @param object the object
  575 + * @param currentList the current list
  576 + * @param variables the variables
  577 + */
547 578 protected void computeVariablesFromObject(MtasParserObject object,
548 579 HashMap<String, ArrayList<MtasParserObject>> currentList,
549 580 HashMap<String, HashMap<String, String>> variables) {
550   - MtasParserType<MtasParserVariable> parserType = object.getType();
551   - String id = object.getId();
552   - if(id!=null) {
553   - for(MtasParserVariable variable : parserType.getItems()) {
554   - if(!variables.containsKey(variable.variable)) {
555   - variables.put(variable.variable, new HashMap<String,String>());
556   - }
557   - String value = "";
558   - for(MtasParserVariableValue variableValue : variable.values) {
559   - if(variableValue.type.equals("attribute")) {
560   - String subValue = object.getAttribute(variableValue.name);
561   - if(subValue!=null) {
562   - value+=subValue;
563   - }
  581 + MtasParserType<MtasParserVariable> parserType = object.getType();
  582 + String id = object.getId();
  583 + if (id != null) {
  584 + for (MtasParserVariable variable : parserType.getItems()) {
  585 + if (!variables.containsKey(variable.variable)) {
  586 + variables.put(variable.variable, new HashMap<String, String>());
  587 + }
  588 + String value = "";
  589 + for (MtasParserVariableValue variableValue : variable.values) {
  590 + if (variableValue.type.equals("attribute")) {
  591 + String subValue = object.getAttribute(variableValue.name);
  592 + if (subValue != null) {
  593 + value += subValue;
564 594 }
565 595 }
566   - variables.get(variable.variable).put(id, value);
567 596 }
  597 + variables.get(variable.variable).put(id, value);
568 598 }
  599 + }
569 600 }
570   -
  601 +
  602 + /**
  603 + * Check for variables.
  604 + *
  605 + * @param values the values
  606 + * @return true, if successful
  607 + */
571 608 private boolean checkForVariables(ArrayList<HashMap<String, String>> values) {
572 609 if (values == null || values.size() == 0) {
573 610 return false;
... ... @@ -583,6 +620,13 @@ abstract public class MtasBasicParser extends MtasParser {
583 620 return false;
584 621 }
585 622  
  623 + /**
  624 + * Update mappings with local references.
  625 + *
  626 + * @param currentObject the current object
  627 + * @param currentList the current list
  628 + * @param updateList the update list
  629 + */
586 630 private void updateMappingsWithLocalReferences(MtasParserObject currentObject,
587 631 HashMap<String, ArrayList<MtasParserObject>> currentList,
588 632 HashMap<String, HashMap<Integer, HashSet<String>>> updateList) {
... ... @@ -667,11 +711,9 @@ abstract public class MtasBasicParser extends MtasParser {
667 711 /**
668 712 * Compute type from mapping source.
669 713 *
670   - * @param source
671   - * the source
  714 + * @param source the source
672 715 * @return the string
673   - * @throws MtasParserException
674   - * the mtas parser exception
  716 + * @throws MtasParserException the mtas parser exception
675 717 */
676 718 private String computeTypeFromMappingSource(String source)
677 719 throws MtasParserException {
... ... @@ -700,15 +742,11 @@ abstract public class MtasBasicParser extends MtasParser {
700 742 /**
701 743 * Compute object from mapping value.
702 744 *
703   - * @param object
704   - * the object
705   - * @param mappingValue
706   - * the mapping value
707   - * @param currentList
708   - * the current list
  745 + * @param object the object
  746 + * @param mappingValue the mapping value
  747 + * @param currentList the current list
709 748 * @return the mtas parser object[]
710   - * @throws MtasParserException
711   - * the mtas parser exception
  749 + * @throws MtasParserException the mtas parser exception
712 750 */
713 751 private MtasParserObject[] computeObjectFromMappingValue(
714 752 MtasParserObject object, HashMap<String, String> mappingValue,
... ... @@ -750,17 +788,13 @@ abstract public class MtasBasicParser extends MtasParser {
750 788 /**
751 789 * Compute value from mapping values.
752 790 *
753   - * @param object
754   - * the object
755   - * @param mappingValues
756   - * the mapping values
757   - * @param currentList
758   - * the current list
  791 + * @param object the object
  792 + * @param mappingValues the mapping values
  793 + * @param currentList the current list
  794 + * @param containsVariables the contains variables
759 795 * @return the string[]
760   - * @throws MtasParserException
761   - * the mtas parser exception
762   - * @throws MtasConfigException
763   - * the mtas config exception
  796 + * @throws MtasParserException the mtas parser exception
  797 + * @throws MtasConfigException the mtas config exception
764 798 */
765 799 private String[] computeValueFromMappingValues(MtasParserObject object,
766 800 ArrayList<HashMap<String, String>> mappingValues,
... ... @@ -772,7 +806,7 @@ abstract public class MtasBasicParser extends MtasParser {
772 806 // directly
773 807 if (mappingValue.get("source").equals(MtasParserMapping.SOURCE_STRING)) {
774 808 if (mappingValue.get("type")
775   - .equals(MtasParserMapping.PARSER_TYPE_STRING)) {
  809 + .equals(MtasParserMapping.PARSER_TYPE_STRING)) {
776 810 String subvalue = computeFilteredPrefixedValue(
777 811 mappingValue.get("type"), mappingValue.get("text"), null, null);
778 812 if (subvalue != null) {
... ... @@ -806,15 +840,13 @@ abstract public class MtasBasicParser extends MtasParser {
806 840 } else if (mappingValue.get("type")
807 841 .equals(MtasParserMapping.PARSER_TYPE_ATTRIBUTE)) {
808 842 String tmpValue = null;
809   - if(mappingValue.get("name").equals("#")) {
  843 + if (mappingValue.get("name").equals("#")) {
810 844 tmpValue = checkObjects[0].getId();
811 845 } else {
812 846 tmpValue = checkObjects[0].getAttribute(mappingValue.get("name"));
813 847 }
814 848 String subvalue = computeFilteredPrefixedValue(
815   - mappingValue.get("type"),
816   - tmpValue,
817   - mappingValue.get("filter"),
  849 + mappingValue.get("type"), tmpValue, mappingValue.get("filter"),
818 850 value.equals("") ? null : mappingValue.get("prefix"));
819 851 if (subvalue != null) {
820 852 for (int i = 0; i < value.length; i++) {
... ... @@ -877,7 +909,7 @@ abstract public class MtasBasicParser extends MtasParser {
877 909 if (mappingValue.get("source")
878 910 .equals(MtasParserMapping.SOURCE_OWN)) {
879 911 String subvalue = object.getAttribute(variableValue);
880   - if(subvalue!=null && subvalue.startsWith("#")) {
  912 + if (subvalue != null && subvalue.startsWith("#")) {
881 913 subvalue = subvalue.substring(1);
882 914 }
883 915 if (subvalue != null) {
... ... @@ -909,16 +941,42 @@ abstract public class MtasBasicParser extends MtasParser {
909 941 }
910 942 }
911 943  
  944 + /**
  945 + * Adds the and encode variable.
  946 + *
  947 + * @param originalValue the original value
  948 + * @param newVariable the new variable
  949 + * @param newVariableName the new variable name
  950 + * @param encode the encode
  951 + * @return the string
  952 + */
912 953 private String addAndEncodeVariable(String originalValue, String newVariable,
913 954 String newVariableName, boolean encode) {
914 955 return addAndEncode(originalValue, newVariable, newVariableName, encode);
915 956 }
916 957  
  958 + /**
  959 + * Adds the and encode value.
  960 + *
  961 + * @param originalValue the original value
  962 + * @param newValue the new value
  963 + * @param encode the encode
  964 + * @return the string
  965 + */
917 966 private String addAndEncodeValue(String originalValue, String newValue,
918 967 boolean encode) {
919 968 return addAndEncode(originalValue, null, newValue, encode);
920 969 }
921 970  
  971 + /**
  972 + * Adds the and encode.
  973 + *
  974 + * @param originalValue the original value
  975 + * @param newType the new type
  976 + * @param newValue the new value
  977 + * @param encode the encode
  978 + * @return the string
  979 + */
922 980 private String addAndEncode(String originalValue, String newType,
923 981 String newValue, boolean encode) {
924 982 if (newValue == null) {
... ... @@ -949,8 +1007,19 @@ abstract public class MtasBasicParser extends MtasParser {
949 1007 }
950 1008 }
951 1009  
  1010 + /**
  1011 + * Decode and update with variables.
  1012 + *
  1013 + * @param encodedPrefix the encoded prefix
  1014 + * @param encodedPostfix the encoded postfix
  1015 + * @param variables the variables
  1016 + * @return the string
  1017 + */
952 1018 protected String decodeAndUpdateWithVariables(String encodedPrefix,
953   - String encodedPostfix, HashMap<String, HashMap<String, String>> variables) {
  1019 + String encodedPostfix,
  1020 + HashMap<String, HashMap<String, String>> variables) {
  1021 + // System.out.println("TEST "+encodedPrefix+" - "+encodedPostfix+"
  1022 + // "+variables);
954 1023 String[] prefixSplit, postfixSplit;
955 1024 if (encodedPrefix != null && !encodedPrefix.isEmpty()) {
956 1025 prefixSplit = encodedPrefix.split(" ");
... ... @@ -963,7 +1032,7 @@ abstract public class MtasBasicParser extends MtasParser {
963 1032 postfixSplit = new String[0];
964 1033 }
965 1034 try {
966   - String prefix = decodeAndUpdateWithVariables(prefixSplit, variables);
  1035 + String prefix = decodeAndUpdateWithVariables(prefixSplit, variables);
967 1036 String postfix = decodeAndUpdateWithVariables(postfixSplit, variables);
968 1037 return prefix + MtasToken.DELIMITER + postfix;
969 1038 } catch (MtasParserException e) {
... ... @@ -971,7 +1040,17 @@ abstract public class MtasBasicParser extends MtasParser {
971 1040 }
972 1041 }
973 1042  
974   - private String decodeAndUpdateWithVariables(String[] splitList, HashMap<String, HashMap<String, String>> variables) throws MtasParserException {
  1043 + /**
  1044 + * Decode and update with variables.
  1045 + *
  1046 + * @param splitList the split list
  1047 + * @param variables the variables
  1048 + * @return the string
  1049 + * @throws MtasParserException the mtas parser exception
  1050 + */
  1051 + private String decodeAndUpdateWithVariables(String[] splitList,
  1052 + HashMap<String, HashMap<String, String>> variables)
  1053 + throws MtasParserException {
975 1054 String decoded = "";
976 1055 for (String split : splitList) {
977 1056 if (split.contains(":")) {
... ... @@ -982,14 +1061,18 @@ abstract public class MtasBasicParser extends MtasParser {
982 1061 "UTF-8");
983 1062 String decodedVariableValue = new String(dec.decode(subSplit[1]),
984 1063 "UTF-8");
985   - if(variables.containsKey(decodedVariableName)) {
986   - if(variables.get(decodedVariableName).containsKey(decodedVariableValue)) {
987   - decoded = decoded + variables.get(decodedVariableName).get(decodedVariableValue);
  1064 + if (variables.containsKey(decodedVariableName)) {
  1065 + if (variables.get(decodedVariableName)
  1066 + .containsKey(decodedVariableValue)) {
  1067 + decoded = decoded + variables.get(decodedVariableName)
  1068 + .get(decodedVariableValue);
988 1069 } else {
989   - throw new MtasParserException("id "+decodedVariableValue+" not found in "+decodedVariableName);
  1070 + throw new MtasParserException("id " + decodedVariableValue
  1071 + + " not found in " + decodedVariableName);
990 1072 }
991 1073 } else {
992   - throw new MtasParserException("variable "+decodedVariableName+" unknown");
  1074 + throw new MtasParserException(
  1075 + "variable " + decodedVariableName + " unknown");
993 1076 }
994 1077 } catch (UnsupportedEncodingException e) {
995 1078 // do nothing
... ... @@ -1002,6 +1085,10 @@ abstract public class MtasBasicParser extends MtasParser {
1002 1085 decoded = decoded + new String(dec.decode(split), "UTF-8");
1003 1086 } catch (UnsupportedEncodingException e) {
1004 1087 // do nothing
  1088 + } catch (IllegalArgumentException e) {
  1089 + // do nothing
  1090 + // System.out.println("Probleem "+split+" - "+e.getMessage());
  1091 + e.printStackTrace();
1005 1092 }
1006 1093 }
1007 1094 }
... ... @@ -1011,15 +1098,11 @@ abstract public class MtasBasicParser extends MtasParser {
1011 1098 /**
1012 1099 * Compute payload from mapping payload.
1013 1100 *
1014   - * @param object
1015   - * the object
1016   - * @param mappingPayloads
1017   - * the mapping payloads
1018   - * @param currentList
1019   - * the current list
  1101 + * @param object the object
  1102 + * @param mappingPayloads the mapping payloads
  1103 + * @param currentList the current list
1020 1104 * @return the bytes ref
1021   - * @throws MtasParserException
1022   - * the mtas parser exception
  1105 + * @throws MtasParserException the mtas parser exception
1023 1106 */
1024 1107 private BytesRef computePayloadFromMappingPayload(MtasParserObject object,
1025 1108 ArrayList<HashMap<String, String>> mappingPayloads,
... ... @@ -1066,10 +1149,8 @@ abstract public class MtasBasicParser extends MtasParser {
1066 1149 /**
1067 1150 * Prevalidate object.
1068 1151 *
1069   - * @param object
1070   - * the object
1071   - * @param currentList
1072   - * the current list
  1152 + * @param object the object
  1153 + * @param currentList the current list
1073 1154 * @return the boolean
1074 1155 */
1075 1156 Boolean prevalidateObject(MtasParserObject object,
... ... @@ -1085,7 +1166,7 @@ abstract public class MtasBasicParser extends MtasParser {
1085 1166 return true;
1086 1167 } catch (MtasParserException e) {
1087 1168 // do nothing
1088   - // System.out.println(e.getMessage());
  1169 + System.out.println(e.getMessage());
1089 1170 }
1090 1171 }
1091 1172 return false;
... ... @@ -1094,14 +1175,10 @@ abstract public class MtasBasicParser extends MtasParser {
1094 1175 /**
1095 1176 * Precheck mapping conditions.
1096 1177 *
1097   - * @param object
1098   - * the object
1099   - * @param mappingConditions
1100   - * the mapping conditions
1101   - * @param currentList
1102   - * the current list
1103   - * @throws MtasParserException
1104   - * the mtas parser exception
  1178 + * @param object the object
  1179 + * @param mappingConditions the mapping conditions
  1180 + * @param currentList the current list
  1181 + * @throws MtasParserException the mtas parser exception
1105 1182 */
1106 1183 void precheckMappingConditions(MtasParserObject object,
1107 1184 ArrayList<HashMap<String, String>> mappingConditions,
... ... @@ -1244,14 +1321,10 @@ abstract public class MtasBasicParser extends MtasParser {
1244 1321 /**
1245 1322 * Postcheck mapping conditions.
1246 1323 *
1247   - * @param object
1248   - * the object
1249   - * @param mappingConditions
1250   - * the mapping conditions
1251   - * @param currentList
1252   - * the current list
1253   - * @throws MtasParserException
1254   - * the mtas parser exception
  1324 + * @param object the object
  1325 + * @param mappingConditions the mapping conditions
  1326 + * @param currentList the current list
  1327 + * @throws MtasParserException the mtas parser exception
1255 1328 */
1256 1329 private void postcheckMappingConditions(MtasParserObject object,
1257 1330 ArrayList<HashMap<String, String>> mappingConditions,
... ... @@ -1298,13 +1371,10 @@ abstract public class MtasBasicParser extends MtasParser {
1298 1371 /**
1299 1372 * Compute filtered split values.
1300 1373 *
1301   - * @param values
1302   - * the values
1303   - * @param filter
1304   - * the filter
  1374 + * @param values the values
  1375 + * @param filter the filter
1305 1376 * @return the string[]
1306   - * @throws MtasConfigException
1307   - * the mtas config exception
  1377 + * @throws MtasConfigException the mtas config exception
1308 1378 */
1309 1379 private String[] computeFilteredSplitValues(String[] values, String filter)
1310 1380 throws MtasConfigException {
... ... @@ -1365,17 +1435,12 @@ abstract public class MtasBasicParser extends MtasParser {
1365 1435 /**
1366 1436 * Compute filtered prefixed value.
1367 1437 *
1368   - * @param type
1369   - * the type
1370   - * @param value
1371   - * the value
1372   - * @param filter
1373   - * the filter
1374   - * @param prefix
1375   - * the prefix
  1438 + * @param type the type
  1439 + * @param value the value
  1440 + * @param filter the filter
  1441 + * @param prefix the prefix
1376 1442 * @return the string
1377   - * @throws MtasConfigException
1378   - * the mtas config exception
  1443 + * @throws MtasConfigException the mtas config exception
1379 1444 */
1380 1445 private String computeFilteredPrefixedValue(String type, String value,
1381 1446 String filter, String prefix) throws MtasConfigException {
... ... @@ -1421,12 +1486,9 @@ abstract public class MtasBasicParser extends MtasParser {
1421 1486 /**
1422 1487 * Compute maximum filtered payload.
1423 1488 *
1424   - * @param value
1425   - * the value
1426   - * @param payload
1427   - * the payload
1428   - * @param filter
1429   - * the filter
  1489 + * @param value the value
  1490 + * @param payload the payload
  1491 + * @param filter the filter
1430 1492 * @return the bytes ref
1431 1493 */
1432 1494 private BytesRef computeMaximumFilteredPayload(String value, BytesRef payload,
... ... @@ -1449,6 +1511,8 @@ abstract public class MtasBasicParser extends MtasParser {
1449 1511  
1450 1512 /**
1451 1513 * The Class MtasParserType.
  1514 + *
  1515 + * @param <T> the generic type
1452 1516 */
1453 1517 protected class MtasParserType<T> {
1454 1518  
... ... @@ -1464,18 +1528,15 @@ abstract public class MtasBasicParser extends MtasParser {
1464 1528 /** The ref attribute name. */
1465 1529 private String refAttributeName;
1466 1530  
1467   - /** The mappings. */
  1531 + /** The items. */
1468 1532 protected ArrayList<T> items = new ArrayList<T>();
1469 1533  
1470 1534 /**
1471 1535 * Instantiates a new mtas parser type.
1472 1536 *
1473   - * @param type
1474   - * the type
1475   - * @param name
1476   - * the name
1477   - * @param precheckText
1478   - * the precheck text
  1537 + * @param type the type
  1538 + * @param name the name
  1539 + * @param precheckText the precheck text
1479 1540 */
1480 1541 MtasParserType(String type, String name, boolean precheckText) {
1481 1542 this.type = type;
... ... @@ -1486,14 +1547,10 @@ abstract public class MtasBasicParser extends MtasParser {
1486 1547 /**
1487 1548 * Instantiates a new mtas parser type.
1488 1549 *
1489   - * @param type
1490   - * the type
1491   - * @param name
1492   - * the name
1493   - * @param precheckText
1494   - * the precheck text
1495   - * @param refAttributeName
1496   - * the ref attribute name
  1550 + * @param type the type
  1551 + * @param name the name
  1552 + * @param precheckText the precheck text
  1553 + * @param refAttributeName the ref attribute name
1497 1554 */
1498 1555 MtasParserType(String type, String name, boolean precheckText,
1499 1556 String refAttributeName) {
... ... @@ -1538,19 +1595,18 @@ abstract public class MtasBasicParser extends MtasParser {
1538 1595 }
1539 1596  
1540 1597 /**
1541   - * Adds the mapping.
  1598 + * Adds the item.
1542 1599 *
1543   - * @param item
1544   - * the mapping
  1600 + * @param item the item
1545 1601 */
1546 1602 public void addItem(T item) {
1547 1603 items.add(item);
1548 1604 }
1549 1605  
1550 1606 /**
1551   - * Gets the mappings.
  1607 + * Gets the items.
1552 1608 *
1553   - * @return the mappings
  1609 + * @return the items
1554 1610 */
1555 1611 public ArrayList<T> getItems() {
1556 1612 return items;
... ... @@ -1558,12 +1614,23 @@ abstract public class MtasBasicParser extends MtasParser {
1558 1614  
1559 1615 }
1560 1616  
  1617 + /**
  1618 + * The Class MtasParserVariableValue.
  1619 + */
1561 1620 protected class MtasParserVariableValue {
1562 1621  
  1622 + /** The type. */
1563 1623 public String type;
  1624 +
  1625 + /** The name. */
1564 1626 public String name;
1565   -
1566 1627  
  1628 + /**
  1629 + * Instantiates a new mtas parser variable value.
  1630 + *
  1631 + * @param type the type
  1632 + * @param name the name
  1633 + */
1567 1634 public MtasParserVariableValue(String type, String name) {
1568 1635 this.type = type;
1569 1636 this.name = name;
... ... @@ -1594,8 +1661,7 @@ abstract public class MtasBasicParser extends MtasParser {
1594 1661 /**
1595 1662 * Instantiates a new mtas parser mapping token.
1596 1663 *
1597   - * @param tokenType
1598   - * the token type
  1664 + * @param tokenType the token type
1599 1665 */
1600 1666 public MtasParserMappingToken(String tokenType) {
1601 1667 type = tokenType;
... ... @@ -1610,8 +1676,7 @@ abstract public class MtasBasicParser extends MtasParser {
1610 1676 /**
1611 1677 * Sets the offset.
1612 1678 *
1613   - * @param tokenOffset
1614   - * the new offset
  1679 + * @param tokenOffset the new offset
1615 1680 */
1616 1681 public void setOffset(Boolean tokenOffset) {
1617 1682 offset = tokenOffset;
... ... @@ -1620,8 +1685,7 @@ abstract public class MtasBasicParser extends MtasParser {
1620 1685 /**
1621 1686 * Sets the real offset.
1622 1687 *
1623   - * @param tokenRealOffset
1624   - * the new real offset
  1688 + * @param tokenRealOffset the new real offset
1625 1689 */
1626 1690 public void setRealOffset(Boolean tokenRealOffset) {
1627 1691 realoffset = tokenRealOffset;
... ... @@ -1630,8 +1694,7 @@ abstract public class MtasBasicParser extends MtasParser {
1630 1694 /**
1631 1695 * Sets the parent.
1632 1696 *
1633   - * @param tokenParent
1634   - * the new parent
  1697 + * @param tokenParent the new parent
1635 1698 */
1636 1699 public void setParent(Boolean tokenParent) {
1637 1700 parent = tokenParent;
... ... @@ -1639,22 +1702,38 @@ abstract public class MtasBasicParser extends MtasParser {
1639 1702  
1640 1703 }
1641 1704  
  1705 + /**
  1706 + * The Class MtasParserVariable.
  1707 + */
1642 1708 protected class MtasParserVariable {
1643 1709  
1644 1710 /** The name. */
1645 1711 public String name;
1646   -
  1712 +
  1713 + /** The variable. */
1647 1714 public String variable;
1648 1715  
1649   - /** The tokens. */
  1716 + /** The values. */
1650 1717 protected ArrayList<MtasParserVariableValue> values;
1651 1718  
  1719 + /**
  1720 + * Instantiates a new mtas parser variable.
  1721 + *
  1722 + * @param name the name
  1723 + * @param value the value
  1724 + */
1652 1725 public MtasParserVariable(String name, String value) {
1653 1726 this.name = name;
1654 1727 this.variable = value;
1655 1728 values = new ArrayList<MtasParserVariableValue>();
1656 1729 }
1657 1730  
  1731 + /**
  1732 + * Process config.
  1733 + *
  1734 + * @param config the config
  1735 + * @throws MtasConfigException the mtas config exception
  1736 + */
1658 1737 public void processConfig(MtasConfiguration config)
1659 1738 throws MtasConfigException {
1660 1739 for (int k = 0; k < config.children.size(); k++) {
... ... @@ -1664,13 +1743,16 @@ abstract public class MtasBasicParser extends MtasParser {
1664 1743 if (config.children.get(k).children.get(m).name
1665 1744 .equals(VARIABLE_SUBTYPE_VALUE_ITEM)) {
1666 1745 MtasConfiguration items = config.children.get(k).children.get(m);
1667   - String valueType = config.children.get(k).children.get(m).attributes.get("type");
1668   - String nameType = config.children.get(k).children.get(m).attributes.get("name");
1669   - if ((valueType != null) && valueType.equals("attribute") && nameType!=null) {
  1746 + String valueType = config.children.get(k).children
  1747 + .get(m).attributes.get("type");
  1748 + String nameType = config.children.get(k).children
  1749 + .get(m).attributes.get("name");
  1750 + if ((valueType != null) && valueType.equals("attribute")
  1751 + && nameType != null) {
1670 1752 MtasParserVariableValue variableValue = new MtasParserVariableValue(
1671 1753 valueType, nameType);
1672 1754 values.add(variableValue);
1673   - }
  1755 + }
1674 1756 }
1675 1757 }
1676 1758 } else {
... ... @@ -1681,9 +1763,14 @@ abstract public class MtasBasicParser extends MtasParser {
1681 1763 }
1682 1764 }
1683 1765  
  1766 + /*
  1767 + * (non-Javadoc)
  1768 + *
  1769 + * @see java.lang.Object#toString()
  1770 + */
1684 1771 @Override
1685 1772 public String toString() {
1686   - String text = "variable "+variable+" from "+name;
  1773 + String text = "variable " + variable + " from " + name;
1687 1774 for (int i = 0; i < values.size(); i++) {
1688 1775 text += "\n\tvalue " + i;
1689 1776 text += " - " + values.get(i).type;
... ... @@ -1696,8 +1783,7 @@ abstract public class MtasBasicParser extends MtasParser {
1696 1783 /**
1697 1784 * The Class MtasParserMapping.
1698 1785 *
1699   - * @param <T>
1700   - * the generic type
  1786 + * @param <T> the generic type
1701 1787 */
1702 1788 protected abstract class MtasParserMapping<T extends MtasParserMapping<T>> {
1703 1789  
... ... @@ -1735,6 +1821,7 @@ abstract public class MtasBasicParser extends MtasParser {
1735 1821 /** The Constant SOURCE_STRING. */
1736 1822 protected final static String SOURCE_STRING = "string";
1737 1823  
  1824 + /** The Constant PARSER_TYPE_VARIABLE. */
1738 1825 protected final static String PARSER_TYPE_VARIABLE = "variable";
1739 1826  
1740 1827 /** The Constant PARSER_TYPE_STRING. */
... ... @@ -1770,7 +1857,10 @@ abstract public class MtasBasicParser extends MtasParser {
1770 1857 /** The position. */
1771 1858 protected String position;
1772 1859  
  1860 + /** The start. */
1773 1861 protected String start;
  1862 +
  1863 + /** The end. */
1774 1864 protected String end;
1775 1865  
1776 1866 /** The tokens. */
... ... @@ -1796,10 +1886,8 @@ abstract public class MtasBasicParser extends MtasParser {
1796 1886 /**
1797 1887 * Process config.
1798 1888 *
1799   - * @param config
1800   - * the config
1801   - * @throws MtasConfigException
1802   - * the mtas config exception
  1889 + * @param config the config
  1890 + * @throws MtasConfigException the mtas config exception
1803 1891 */
1804 1892 public void processConfig(MtasConfiguration config)
1805 1893 throws MtasConfigException {
... ... @@ -2165,6 +2253,12 @@ abstract public class MtasBasicParser extends MtasParser {
2165 2253 }
2166 2254 }
2167 2255  
  2256 + /**
  2257 + * Sets the start end.
  2258 + *
  2259 + * @param start the start
  2260 + * @param end the end
  2261 + */
2168 2262 protected void setStartEnd(String start, String end) {
2169 2263 if (start != null && !start.isEmpty() && end != null && !end.isEmpty()) {
2170 2264 this.start = start;
... ... @@ -2175,8 +2269,7 @@ abstract public class MtasBasicParser extends MtasParser {
2175 2269 /**
2176 2270 * Condition unknown ancestor.
2177 2271 *
2178   - * @param number
2179   - * the number
  2272 + * @param number the number
2180 2273 */
2181 2274 private void conditionUnknownAncestor(String number) {
2182 2275 HashMap<String, String> mapConstructionItem = new HashMap<String, String>();
... ... @@ -2188,12 +2281,9 @@ abstract public class MtasBasicParser extends MtasParser {
2188 2281 /**
2189 2282 * Adds the string.
2190 2283 *
2191   - * @param mappingToken
2192   - * the mapping token
2193   - * @param type
2194   - * the type
2195   - * @param text
2196   - * the text
  2284 + * @param mappingToken the mapping token
  2285 + * @param type the type
  2286 + * @param text the text
2197 2287 */
2198 2288 private void addString(MtasParserMappingToken mappingToken, String type,
2199 2289 String text) {
... ... @@ -2211,10 +2301,8 @@ abstract public class MtasBasicParser extends MtasParser {
2211 2301 /**
2212 2302 * Payload string.
2213 2303 *
2214   - * @param mappingToken
2215   - * the mapping token
2216   - * @param text
2217   - * the text
  2304 + * @param mappingToken the mapping token
  2305 + * @param text the text
2218 2306 */
2219 2307 private void payloadString(MtasParserMappingToken mappingToken,
2220 2308 String text) {
... ... @@ -2228,14 +2316,10 @@ abstract public class MtasBasicParser extends MtasParser {
2228 2316 /**
2229 2317 * Adds the name.
2230 2318 *
2231   - * @param mappingToken
2232   - * the mapping token
2233   - * @param type
2234   - * the type
2235   - * @param prefix
2236   - * the prefix
2237   - * @param filter
2238   - * the filter
  2319 + * @param mappingToken the mapping token
  2320 + * @param type the type
  2321 + * @param prefix the prefix
  2322 + * @param filter the filter
2239 2323 */
2240 2324 private void addName(MtasParserMappingToken mappingToken, String type,
2241 2325 String prefix, String filter) {
... ... @@ -2254,10 +2338,8 @@ abstract public class MtasBasicParser extends MtasParser {
2254 2338 /**
2255 2339 * Condition name.
2256 2340 *
2257   - * @param condition
2258   - * the condition
2259   - * @param not
2260   - * the not
  2341 + * @param condition the condition
  2342 + * @param not the not
2261 2343 */
2262 2344 private void conditionName(String condition, String not) {
2263 2345 HashMap<String, String> mapConstructionItem = new HashMap<String, String>();
... ... @@ -2271,14 +2353,10 @@ abstract public class MtasBasicParser extends MtasParser {
2271 2353 /**
2272 2354 * Adds the text.
2273 2355 *
2274   - * @param mappingToken
2275   - * the mapping token
2276   - * @param type
2277   - * the type
2278   - * @param prefix
2279   - * the prefix
2280   - * @param filter
2281   - * the filter
  2356 + * @param mappingToken the mapping token
  2357 + * @param type the type
  2358 + * @param prefix the prefix
  2359 + * @param filter the filter
2282 2360 */
2283 2361 private void addText(MtasParserMappingToken mappingToken, String type,
2284 2362 String prefix, String filter) {
... ... @@ -2297,16 +2375,11 @@ abstract public class MtasBasicParser extends MtasParser {
2297 2375 /**
2298 2376 * Adds the text split.
2299 2377 *
2300   - * @param mappingToken
2301   - * the mapping token
2302   - * @param type
2303   - * the type
2304   - * @param split
2305   - * the split
2306   - * @param prefix
2307   - * the prefix
2308   - * @param filter
2309   - * the filter
  2378 + * @param mappingToken the mapping token
  2379 + * @param type the type
  2380 + * @param split the split
  2381 + * @param prefix the prefix
  2382 + * @param filter the filter
2310 2383 */
2311 2384 private void addTextSplit(MtasParserMappingToken mappingToken, String type,
2312 2385 String split, String prefix, String filter) {
... ... @@ -2326,12 +2399,9 @@ abstract public class MtasBasicParser extends MtasParser {
2326 2399 /**
2327 2400 * Condition text.
2328 2401 *
2329   - * @param condition
2330   - * the condition
2331   - * @param filter
2332   - * the filter
2333   - * @param not
2334   - * the not
  2402 + * @param condition the condition
  2403 + * @param filter the filter
  2404 + * @param not the not
2335 2405 */
2336 2406 private void conditionText(String condition, String filter, String not) {
2337 2407 HashMap<String, String> mapConstructionItem = new HashMap<String, String>();
... ... @@ -2346,10 +2416,8 @@ abstract public class MtasBasicParser extends MtasParser {
2346 2416 /**
2347 2417 * Payload text.
2348 2418 *
2349   - * @param mappingToken
2350   - * the mapping token
2351   - * @param filter
2352   - * the filter
  2419 + * @param mappingToken the mapping token
  2420 + * @param filter the filter
2353 2421 */
2354 2422 private void payloadText(MtasParserMappingToken mappingToken,
2355 2423 String filter) {
... ... @@ -2363,16 +2431,11 @@ abstract public class MtasBasicParser extends MtasParser {
2363 2431 /**
2364 2432 * Adds the attribute.
2365 2433 *
2366   - * @param mappingToken
2367   - * the mapping token
2368   - * @param type
2369   - * the type
2370   - * @param name
2371   - * the name
2372   - * @param prefix
2373   - * the prefix
2374   - * @param filter
2375   - * the filter
  2434 + * @param mappingToken the mapping token
  2435 + * @param type the type
  2436 + * @param name the name
  2437 + * @param prefix the prefix
  2438 + * @param filter the filter
2376 2439 */
2377 2440 private void addAttribute(MtasParserMappingToken mappingToken, String type,
2378 2441 String name, String prefix, String filter) {
... ... @@ -2391,6 +2454,15 @@ abstract public class MtasBasicParser extends MtasParser {
2391 2454 }
2392 2455 }
2393 2456  
  2457 + /**
  2458 + * Adds the variable from attribute.
  2459 + *
  2460 + * @param mappingToken the mapping token
  2461 + * @param type the type
  2462 + * @param name the name
  2463 + * @param prefix the prefix
  2464 + * @param value the value
  2465 + */
2394 2466 private void addVariableFromAttribute(MtasParserMappingToken mappingToken,
2395 2467 String type, String name, String prefix, String value) {
2396 2468 HashMap<String, String> mapConstructionItem = new HashMap<String, String>();
... ... @@ -2411,14 +2483,10 @@ abstract public class MtasBasicParser extends MtasParser {
2411 2483 /**
2412 2484 * Condition attribute.
2413 2485 *
2414   - * @param name
2415   - * the name
2416   - * @param condition
2417   - * the condition
2418   - * @param filter
2419   - * the filter
2420   - * @param not
2421   - * the not
  2486 + * @param name the name
  2487 + * @param condition the condition
  2488 + * @param filter the filter
  2489 + * @param not the not
2422 2490 */
2423 2491 private void conditionAttribute(String name, String condition,
2424 2492 String filter, String not) {
... ... @@ -2437,12 +2505,9 @@ abstract public class MtasBasicParser extends MtasParser {
2437 2505 /**
2438 2506 * Payload attribute.
2439 2507 *
2440   - * @param mappingToken
2441   - * the mapping token
2442   - * @param name
2443   - * the name
2444   - * @param filter
2445   - * the filter
  2508 + * @param mappingToken the mapping token
  2509 + * @param name the name
  2510 + * @param filter the filter
2446 2511 */
2447 2512 private void payloadAttribute(MtasParserMappingToken mappingToken,
2448 2513 String name, String filter) {
... ... @@ -2457,10 +2522,8 @@ abstract public class MtasBasicParser extends MtasParser {
2457 2522 /**
2458 2523 * Condition ancestor.
2459 2524 *
2460   - * @param ancestorType
2461   - * the ancestor type
2462   - * @param number
2463   - * the number
  2525 + * @param ancestorType the ancestor type
  2526 + * @param number the number
2464 2527 */
2465 2528 public void conditionAncestor(String ancestorType, String number) {
2466 2529 if (ancestorType.equals(SOURCE_ANCESTOR_GROUP)
... ... @@ -2480,18 +2543,12 @@ abstract public class MtasBasicParser extends MtasParser {
2480 2543 /**
2481 2544 * Adds the ancestor name.
2482 2545 *
2483   - * @param ancestorType
2484   - * the ancestor type
2485   - * @param mappingToken
2486   - * the mapping token
2487   - * @param type
2488   - * the type
2489   - * @param distance
2490   - * the distance
2491   - * @param prefix
2492   - * the prefix
2493   - * @param filter
2494   - * the filter
  2546 + * @param ancestorType the ancestor type
  2547 + * @param mappingToken the mapping token
  2548 + * @param type the type
  2549 + * @param distance the distance
  2550 + * @param prefix the prefix
  2551 + * @param filter the filter
2495 2552 */
2496 2553 private void addAncestorName(String ancestorType,
2497 2554 MtasParserMappingToken mappingToken, String type, String distance,
... ... @@ -2519,16 +2576,11 @@ abstract public class MtasBasicParser extends MtasParser {
2519 2576 /**
2520 2577 * Condition ancestor name.
2521 2578 *
2522   - * @param ancestorType
2523   - * the ancestor type
2524   - * @param distance
2525   - * the distance
2526   - * @param condition
2527   - * the condition
2528   - * @param filter
2529   - * the filter
2530   - * @param not
2531   - * the not
  2579 + * @param ancestorType the ancestor type
  2580 + * @param distance the distance
  2581 + * @param condition the condition
  2582 + * @param filter the filter
  2583 + * @param not the not
2532 2584 */
2533 2585 public void conditionAncestorName(String ancestorType, String distance,
2534 2586 String condition, String filter, String not) {
... ... @@ -2552,20 +2604,13 @@ abstract public class MtasBasicParser extends MtasParser {
2552 2604 /**
2553 2605 * Adds the ancestor attribute.
2554 2606 *
2555   - * @param ancestorType
2556   - * the ancestor type
2557   - * @param mappingToken
2558   - * the mapping token
2559   - * @param type
2560   - * the type
2561   - * @param distance
2562   - * the distance
2563   - * @param name
2564   - * the name
2565   - * @param prefix
2566   - * the prefix
2567   - * @param filter
2568   - * the filter
  2607 + * @param ancestorType the ancestor type
  2608 + * @param mappingToken the mapping token
  2609 + * @param type the type
  2610 + * @param distance the distance
  2611 + * @param name the name
  2612 + * @param prefix the prefix
  2613 + * @param filter the filter
2569 2614 */
2570 2615 public void addAncestorAttribute(String ancestorType,
2571 2616 MtasParserMappingToken mappingToken, String type, String distance,
... ... @@ -2596,18 +2641,12 @@ abstract public class MtasBasicParser extends MtasParser {
2596 2641 /**
2597 2642 * Condition ancestor attribute.
2598 2643 *
2599   - * @param ancestorType
2600   - * the ancestor type
2601   - * @param distance
2602   - * the distance
2603   - * @param name
2604   - * the name
2605   - * @param condition
2606   - * the condition
2607   - * @param filter
2608   - * the filter
2609   - * @param not
2610   - * the not
  2644 + * @param ancestorType the ancestor type
  2645 + * @param distance the distance
  2646 + * @param name the name
  2647 + * @param condition the condition
  2648 + * @param filter the filter
  2649 + * @param not the not
2611 2650 */
2612 2651 public void conditionAncestorAttribute(String ancestorType, String distance,
2613 2652 String name, String condition, String filter, String not) {
... ... @@ -2634,16 +2673,11 @@ abstract public class MtasBasicParser extends MtasParser {
2634 2673 /**
2635 2674 * Payload ancestor attribute.
2636 2675 *
2637   - * @param mappingToken
2638   - * the mapping token
2639   - * @param ancestorType
2640   - * the ancestor type
2641   - * @param distance
2642   - * the distance
2643   - * @param name
2644   - * the name
2645   - * @param filter
2646   - * the filter
  2676 + * @param mappingToken the mapping token
  2677 + * @param ancestorType the ancestor type
  2678 + * @param distance the distance
  2679 + * @param name the name
  2680 + * @param filter the filter
2647 2681 */
2648 2682 private void payloadAncestorAttribute(MtasParserMappingToken mappingToken,
2649 2683 String ancestorType, String distance, String name, String filter) {
... ... @@ -2668,11 +2702,9 @@ abstract public class MtasBasicParser extends MtasParser {
2668 2702 /**
2669 2703 * Compute ancestor source type.
2670 2704 *
2671   - * @param type
2672   - * the type
  2705 + * @param type the type
2673 2706 * @return the string
2674   - * @throws MtasConfigException
2675   - * the mtas config exception
  2707 + * @throws MtasConfigException the mtas config exception
2676 2708 */
2677 2709 private String computeAncestorSourceType(String type)
2678 2710 throws MtasConfigException {
... ... @@ -2696,8 +2728,7 @@ abstract public class MtasBasicParser extends MtasParser {
2696 2728 /**
2697 2729 * Compute distance.
2698 2730 *
2699   - * @param distance
2700   - * the distance
  2731 + * @param distance the distance
2701 2732 * @return the string
2702 2733 */
2703 2734 private String computeDistance(String distance) {
... ... @@ -2716,8 +2747,7 @@ abstract public class MtasBasicParser extends MtasParser {
2716 2747 /**
2717 2748 * Compute number.
2718 2749 *
2719   - * @param number
2720   - * the number
  2750 + * @param number the number
2721 2751 * @return the string
2722 2752 */
2723 2753 private String computeNumber(String number) {
... ...
src/mtas/analysis/parser/MtasCRMParser.java
... ... @@ -13,10 +13,8 @@ import java.util.concurrent.atomic.AtomicInteger;
13 13 import java.util.regex.Matcher;
14 14 import java.util.regex.Pattern;
15 15  
16   -import mtas.analysis.parser.MtasBasicParser.MtasParserMapping;
17   -import mtas.analysis.parser.MtasParser.MtasParserObject;
18   -import mtas.analysis.token.MtasToken;
19 16 import mtas.analysis.token.MtasTokenCollection;
  17 +import mtas.analysis.token.MtasTokenIdFactory;
20 18 import mtas.analysis.util.MtasBufferedReader;
21 19 import mtas.analysis.util.MtasConfigException;
22 20 import mtas.analysis.util.MtasConfiguration;
... ... @@ -64,8 +62,7 @@ public class MtasCRMParser extends MtasBasicParser {
64 62 /**
65 63 * Instantiates a new mtas crm parser.
66 64 *
67   - * @param config
68   - * the config
  65 + * @param config the config
69 66 */
70 67 public MtasCRMParser(MtasConfiguration config) {
71 68 super(config);
... ... @@ -88,7 +85,8 @@ public class MtasCRMParser extends MtasBasicParser {
88 85 super.initParser();
89 86 if (config != null) {
90 87 // always word, no mappings
91   - wordType = new MtasParserType<MtasParserMapping<?>>(MAPPING_TYPE_WORD, null, false);
  88 + wordType = new MtasParserType<MtasParserMapping<?>>(MAPPING_TYPE_WORD,
  89 + null, false);
92 90 for (int i = 0; i < config.children.size(); i++) {
93 91 MtasConfiguration current = config.children.get(i);
94 92 if (current.name.equals("mappings")) {
... ... @@ -109,8 +107,8 @@ public class MtasCRMParser extends MtasBasicParser {
109 107 if (wordAnnotationTypes.containsKey(nameMapping)) {
110 108 wordAnnotationTypes.get(nameMapping).addItem(m);
111 109 } else {
112   - MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(typeMapping,
113   - nameMapping, false);
  110 + MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(
  111 + typeMapping, nameMapping, false);
114 112 t.addItem(m);
115 113 wordAnnotationTypes.put(nameMapping, t);
116 114 }
... ... @@ -120,8 +118,8 @@ public class MtasCRMParser extends MtasBasicParser {
120 118 if (crmSentenceTypes.containsKey(nameMapping)) {
121 119 crmSentenceTypes.get(nameMapping).addItem(m);
122 120 } else {
123   - MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(MAPPING_TYPE_GROUP,
124   - nameMapping, true);
  121 + MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(
  122 + MAPPING_TYPE_GROUP, nameMapping, true);
125 123 t.addItem(m);
126 124 crmSentenceTypes.put(nameMapping, t);
127 125 }
... ... @@ -131,8 +129,8 @@ public class MtasCRMParser extends MtasBasicParser {
131 129 if (crmClauseTypes.containsKey(nameMapping)) {
132 130 crmClauseTypes.get(nameMapping).addItem(m);
133 131 } else {
134   - MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(MAPPING_TYPE_GROUP,
135   - nameMapping, true);
  132 + MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(
  133 + MAPPING_TYPE_GROUP, nameMapping, true);
136 134 t.addItem(m);
137 135 crmClauseTypes.put(nameMapping, t);
138 136 }
... ... @@ -142,8 +140,8 @@ public class MtasCRMParser extends MtasBasicParser {
142 140 if (crmPairTypes.containsKey(nameMapping)) {
143 141 crmPairTypes.get(nameMapping).addItem(m);
144 142 } else {
145   - MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(MAPPING_TYPE_RELATION,
146   - nameMapping, true);
  143 + MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(
  144 + MAPPING_TYPE_RELATION, nameMapping, true);
147 145 t.addItem(m);
148 146 crmPairTypes.put(nameMapping, t);
149 147 }
... ... @@ -234,10 +232,10 @@ public class MtasCRMParser extends MtasBasicParser {
234 232 HashMap<String, Integer[]> idOffsets = new HashMap<String, Integer[]>();
235 233  
236 234 HashMap<String, HashMap<Integer, HashSet<String>>> updateList = createUpdateList();
237   - HashMap<String, ArrayList<MtasParserObject>> currentList = createCurrentList();
  235 + HashMap<String, ArrayList<MtasParserObject>> currentList = createCurrentList();
238 236  
239 237 tokenCollection = new MtasTokenCollection();
240   - MtasToken.resetId();
  238 + MtasTokenIdFactory mtasTokenIdFactory = new MtasTokenIdFactory();
241 239 try (MtasBufferedReader br = new MtasBufferedReader(reader)) {
242 240 String line;
243 241 int currentOffset, previousOffset = br.getPosition();
... ... @@ -259,17 +257,18 @@ public class MtasCRMParser extends MtasBasicParser {
259 257 for (int i = 4; i < 8; i++) {
260 258 ArrayList<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<MtasCRMParserFunctionOutput>();
261 259 HashSet<MtasParserObject> tmpList = processCRMSentence(
262   - String.valueOf(i), matcherRegular.group((i + 1)), currentOffset,
  260 + mtasTokenIdFactory, String.valueOf(i),
  261 + matcherRegular.group((i + 1)), currentOffset,
263 262 functionOutputList, unknownAncestors, currentList, updateList,
264 263 idPositions, idOffsets, previousSentence, previousClause);
265 264 if (tmpList != null) {
266 265 newPreviousSentence.addAll(tmpList);
267 266 }
268 267 for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
269   - tmpList = processCRMSentence(functionOutput.name,
270   - functionOutput.value, currentOffset, functionOutputList,
271   - unknownAncestors, currentList, updateList, idPositions,
272   - idOffsets, previousSentence, previousClause);
  268 + tmpList = processCRMSentence(mtasTokenIdFactory,
  269 + functionOutput.name, functionOutput.value, currentOffset,
  270 + functionOutputList, unknownAncestors, currentList, updateList,
  271 + idPositions, idOffsets, previousSentence, previousClause);
273 272 if (tmpList != null) {
274 273 newPreviousSentence.addAll(tmpList);
275 274 }
... ... @@ -283,17 +282,18 @@ public class MtasCRMParser extends MtasBasicParser {
283 282 for (int i = 4; i < 8; i++) {
284 283 ArrayList<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<MtasCRMParserFunctionOutput>();
285 284 HashSet<MtasParserObject> tmpList = processCRMClause(
286   - String.valueOf(i), matcherRegular.group((i + 1)), currentOffset,
  285 + mtasTokenIdFactory, String.valueOf(i),
  286 + matcherRegular.group((i + 1)), currentOffset,
287 287 functionOutputList, unknownAncestors, currentList, updateList,
288 288 idPositions, idOffsets, previousClause);
289 289 if (tmpList != null) {
290 290 newPreviousClause.addAll(tmpList);
291 291 }
292 292 for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
293   - tmpList = processCRMClause(functionOutput.name,
294   - functionOutput.value, currentOffset, functionOutputList,
295   - unknownAncestors, currentList, updateList, idPositions,
296   - idOffsets, previousClause);
  293 + tmpList = processCRMClause(mtasTokenIdFactory,
  294 + functionOutput.name, functionOutput.value, currentOffset,
  295 + functionOutputList, unknownAncestors, currentList, updateList,
  296 + idPositions, idOffsets, previousClause);
297 297 if (tmpList != null) {
298 298 newPreviousClause.addAll(tmpList);
299 299 }
... ... @@ -324,25 +324,27 @@ public class MtasCRMParser extends MtasBasicParser {
324 324 // check for crmPair
325 325 for (int i = 0; i < 8; i++) {
326 326 ArrayList<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<MtasCRMParserFunctionOutput>();
327   - processCRMPair(p, String.valueOf(i),
  327 + processCRMPair(mtasTokenIdFactory, p, String.valueOf(i),
328 328 matcherRegular.group((i + 1)), currentOffset,
329 329 functionOutputList, unknownAncestors, currentList,
330 330 updateList, idPositions, idOffsets);
331 331 for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
332   - processCRMPair(p, functionOutput.name, functionOutput.value,
333   - currentOffset, functionOutputList, unknownAncestors,
334   - currentList, updateList, idPositions, idOffsets);
  332 + processCRMPair(mtasTokenIdFactory, p, functionOutput.name,
  333 + functionOutput.value, currentOffset, functionOutputList,
  334 + unknownAncestors, currentList, updateList, idPositions,
  335 + idOffsets);
335 336 }
336 337 }
337 338 // compute word annotations
338 339 for (int i = 0; i < 8; i++) {
339 340 ArrayList<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<MtasCRMParserFunctionOutput>();
340   - functionOutputList.addAll(processWordAnnotation(
341   - String.valueOf(i), matcherRegular.group((i + 1)),
342   - previousOffset, currentOffset, unknownAncestors,
343   - currentList, updateList, idPositions, idOffsets));
  341 + functionOutputList
  342 + .addAll(processWordAnnotation(mtasTokenIdFactory,
  343 + String.valueOf(i), matcherRegular.group((i + 1)),
  344 + previousOffset, currentOffset, unknownAncestors,
  345 + currentList, updateList, idPositions, idOffsets));
344 346 for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
345   - processWordAnnotation(functionOutput.name,
  347 + processWordAnnotation(mtasTokenIdFactory, functionOutput.name,
346 348 functionOutput.value, previousOffset, currentOffset,
347 349 unknownAncestors, currentList, updateList, idPositions,
348 350 idOffsets);
... ... @@ -372,7 +374,8 @@ public class MtasCRMParser extends MtasBasicParser {
372 374 idOffsets.put(currentObject.getId(), currentObject.getOffset());
373 375 currentObject.updateMappings(idPositions, idOffsets);
374 376 unknownAncestors = currentObject.getUnknownAncestorNumber();
375   - computeMappingsFromObject(currentObject, currentList, updateList);
  377 + computeMappingsFromObject(mtasTokenIdFactory, currentObject,
  378 + currentList, updateList);
376 379 }
377 380  
378 381 } else {
... ... @@ -381,10 +384,10 @@ public class MtasCRMParser extends MtasBasicParser {
381 384 }
382 385 previousOffset = br.getPosition();
383 386 }
384   - closePrevious(previousSentence, previousOffset, unknownAncestors,
385   - currentList, updateList, idPositions, idOffsets);
386   - closePrevious(previousClause, previousOffset, unknownAncestors,
387   - currentList, updateList, idPositions, idOffsets);
  387 + closePrevious(mtasTokenIdFactory, previousSentence, previousOffset,
  388 + unknownAncestors, currentList, updateList, idPositions, idOffsets);
  389 + closePrevious(mtasTokenIdFactory, previousClause, previousOffset,
  390 + unknownAncestors, currentList, updateList, idPositions, idOffsets);
388 391 } catch (IOException e) {
389 392 throw new MtasParserException(e.getMessage());
390 393 }
... ... @@ -397,33 +400,23 @@ public class MtasCRMParser extends MtasBasicParser {
397 400 /**
398 401 * Process word annotation.
399 402 *
400   - * @param name
401   - * the name
402   - * @param text
403   - * the text
404   - * @param previousOffset
405   - * the previous offset
406   - * @param currentOffset
407   - * the current offset
408   - * @param unknownAncestors
409   - * the unknown ancestors
410   - * @param currentList
411   - * the current list
412   - * @param updateList
413   - * the update list
414   - * @param idPositions
415   - * the id positions
416   - * @param idOffsets
417   - * the id offsets
  403 + * @param mtasTokenIdFactory the mtas token id factory
  404 + * @param name the name
  405 + * @param text the text
  406 + * @param previousOffset the previous offset
  407 + * @param currentOffset the current offset
  408 + * @param unknownAncestors the unknown ancestors
  409 + * @param currentList the current list
  410 + * @param updateList the update list
  411 + * @param idPositions the id positions
  412 + * @param idOffsets the id offsets
418 413 * @return the array list
419   - * @throws MtasParserException
420   - * the mtas parser exception
421   - * @throws MtasConfigException
422   - * the mtas config exception
  414 + * @throws MtasParserException the mtas parser exception
  415 + * @throws MtasConfigException the mtas config exception
423 416 */
424 417 private ArrayList<MtasCRMParserFunctionOutput> processWordAnnotation(
425   - String name, String text, Integer previousOffset, Integer currentOffset,
426   - Integer unknownAncestors,
  418 + MtasTokenIdFactory mtasTokenIdFactory, String name, String text,
  419 + Integer previousOffset, Integer currentOffset, Integer unknownAncestors,
427 420 HashMap<String, ArrayList<MtasParserObject>> currentList,
428 421 HashMap<String, HashMap<Integer, HashSet<String>>> updateList,
429 422 HashMap<String, TreeSet<Integer>> idPositions,
... ... @@ -483,7 +476,8 @@ public class MtasCRMParser extends MtasBasicParser {
483 476 }
484 477 currentObject.updateMappings(idPositions, idOffsets);
485 478 unknownAncestors = currentObject.getUnknownAncestorNumber();
486   - computeMappingsFromObject(currentObject, currentList, updateList);
  479 + computeMappingsFromObject(mtasTokenIdFactory, currentObject,
  480 + currentList, updateList);
487 481 }
488 482 }
489 483 return functionOutputList;
... ... @@ -492,35 +486,24 @@ public class MtasCRMParser extends MtasBasicParser {
492 486 /**
493 487 * Process crm sentence.
494 488 *
495   - * @param name
496   - * the name
497   - * @param text
498   - * the text
499   - * @param currentOffset
500   - * the current offset
501   - * @param functionOutputList
502   - * the function output list
503   - * @param unknownAncestors
504   - * the unknown ancestors
505   - * @param currentList
506   - * the current list
507   - * @param updateList
508   - * the update list
509   - * @param idPositions
510   - * the id positions
511   - * @param idOffsets
512   - * the id offsets
513   - * @param previous
514   - * the previous
515   - * @param previousClause
516   - * the previous clause
  489 + * @param mtasTokenIdFactory the mtas token id factory
  490 + * @param name the name
  491 + * @param text the text
  492 + * @param currentOffset the current offset
  493 + * @param functionOutputList the function output list
  494 + * @param unknownAncestors the unknown ancestors
  495 + * @param currentList the current list
  496 + * @param updateList the update list
  497 + * @param idPositions the id positions
  498 + * @param idOffsets the id offsets
  499 + * @param previous the previous
  500 + * @param previousClause the previous clause
517 501 * @return the hash set
518   - * @throws MtasParserException
519   - * the mtas parser exception
520   - * @throws MtasConfigException
521   - * the mtas config exception
  502 + * @throws MtasParserException the mtas parser exception
  503 + * @throws MtasConfigException the mtas config exception
522 504 */
523   - private HashSet<MtasParserObject> processCRMSentence(String name, String text,
  505 + private HashSet<MtasParserObject> processCRMSentence(
  506 + MtasTokenIdFactory mtasTokenIdFactory, String name, String text,
524 507 Integer currentOffset,
525 508 ArrayList<MtasCRMParserFunctionOutput> functionOutputList,
526 509 Integer unknownAncestors,
... ... @@ -540,10 +523,10 @@ public class MtasCRMParser extends MtasBasicParser {
540 523 if (!prevalidateObject(currentObject, currentList)) {
541 524 return null;
542 525 } else {
543   - closePrevious(previousClause, currentOffset, unknownAncestors,
544   - currentList, updateList, idPositions, idOffsets);
545   - closePrevious(previous, currentOffset, unknownAncestors, currentList,
546   - updateList, idPositions, idOffsets);
  526 + closePrevious(mtasTokenIdFactory, previousClause, currentOffset,
  527 + unknownAncestors, currentList, updateList, idPositions, idOffsets);
  528 + closePrevious(mtasTokenIdFactory, previous, currentOffset,
  529 + unknownAncestors, currentList, updateList, idPositions, idOffsets);
547 530 previous.clear();
548 531 currentList.get(MAPPING_TYPE_GROUP).add(currentObject);
549 532 unknownAncestors = 0;
... ... @@ -556,33 +539,23 @@ public class MtasCRMParser extends MtasBasicParser {
556 539 /**
557 540 * Process crm clause.
558 541 *
559   - * @param name
560   - * the name
561   - * @param text
562   - * the text
563   - * @param currentOffset
564   - * the current offset
565   - * @param functionOutputList
566   - * the function output list
567   - * @param unknownAncestors
568   - * the unknown ancestors
569   - * @param currentList
570   - * the current list
571   - * @param updateList
572   - * the update list
573   - * @param idPositions
574   - * the id positions
575   - * @param idOffsets
576   - * the id offsets
577   - * @param previous
578   - * the previous
  542 + * @param mtasTokenIdFactory the mtas token id factory
  543 + * @param name the name
  544 + * @param text the text
  545 + * @param currentOffset the current offset
  546 + * @param functionOutputList the function output list
  547 + * @param unknownAncestors the unknown ancestors
  548 + * @param currentList the current list
  549 + * @param updateList the update list
  550 + * @param idPositions the id positions
  551 + * @param idOffsets the id offsets
  552 + * @param previous the previous
579 553 * @return the hash set
580   - * @throws MtasParserException
581   - * the mtas parser exception
582   - * @throws MtasConfigException
583   - * the mtas config exception
  554 + * @throws MtasParserException the mtas parser exception
  555 + * @throws MtasConfigException the mtas config exception
584 556 */
585   - private HashSet<MtasParserObject> processCRMClause(String name, String text,
  557 + private HashSet<MtasParserObject> processCRMClause(
  558 + MtasTokenIdFactory mtasTokenIdFactory, String name, String text,
586 559 Integer currentOffset,
587 560 ArrayList<MtasCRMParserFunctionOutput> functionOutputList,
588 561 Integer unknownAncestors,
... ... @@ -601,8 +574,8 @@ public class MtasCRMParser extends MtasBasicParser {
601 574 if (!prevalidateObject(currentObject, currentList)) {
602 575 return null;
603 576 } else {
604   - closePrevious(previous, currentOffset, unknownAncestors, currentList,
605   - updateList, idPositions, idOffsets);
  577 + closePrevious(mtasTokenIdFactory, previous, currentOffset,
  578 + unknownAncestors, currentList, updateList, idPositions, idOffsets);
606 579 previous.clear();
607 580 currentList.get(MAPPING_TYPE_GROUP).add(currentObject);
608 581 unknownAncestors = 0;
... ... @@ -615,27 +588,20 @@ public class MtasCRMParser extends MtasBasicParser {
615 588 /**
616 589 * Close previous.
617 590 *
618   - * @param previous
619   - * the previous
620   - * @param currentOffset
621   - * the current offset
622   - * @param unknownAncestors
623   - * the unknown ancestors
624   - * @param currentList
625   - * the current list
626   - * @param updateList
627   - * the update list
628   - * @param idPositions
629   - * the id positions
630   - * @param idOffsets
631   - * the id offsets
632   - * @throws MtasParserException
633   - * the mtas parser exception
634   - * @throws MtasConfigException
635   - * the mtas config exception
  591 + * @param mtasTokenIdFactory the mtas token id factory
  592 + * @param previous the previous
  593 + * @param currentOffset the current offset
  594 + * @param unknownAncestors the unknown ancestors
  595 + * @param currentList the current list
  596 + * @param updateList the update list
  597 + * @param idPositions the id positions
  598 + * @param idOffsets the id offsets
  599 + * @throws MtasParserException the mtas parser exception
  600 + * @throws MtasConfigException the mtas config exception
636 601 */
637   - private void closePrevious(HashSet<MtasParserObject> previous,
638   - Integer currentOffset, Integer unknownAncestors,
  602 + private void closePrevious(MtasTokenIdFactory mtasTokenIdFactory,
  603 + HashSet<MtasParserObject> previous, Integer currentOffset,
  604 + Integer unknownAncestors,
639 605 HashMap<String, ArrayList<MtasParserObject>> currentList,
640 606 HashMap<String, HashMap<Integer, HashSet<String>>> updateList,
641 607 HashMap<String, TreeSet<Integer>> idPositions,
... ... @@ -647,7 +613,8 @@ public class MtasCRMParser extends MtasBasicParser {
647 613 idOffsets.put(previousObject.getId(), previousObject.getOffset());
648 614 previousObject.updateMappings(idPositions, idOffsets);
649 615 unknownAncestors = previousObject.getUnknownAncestorNumber();
650   - computeMappingsFromObject(previousObject, currentList, updateList);
  616 + computeMappingsFromObject(mtasTokenIdFactory, previousObject, currentList,
  617 + updateList);
651 618 currentList.get(MAPPING_TYPE_GROUP).remove(previousObject);
652 619 }
653 620 }
... ... @@ -655,33 +622,22 @@ public class MtasCRMParser extends MtasBasicParser {
655 622 /**
656 623 * Process crm pair.
657 624 *
658   - * @param position
659   - * the position
660   - * @param name
661   - * the name
662   - * @param text
663   - * the text
664   - * @param currentOffset
665   - * the current offset
666   - * @param functionOutputList
667   - * the function output list
668   - * @param unknownAncestors
669   - * the unknown ancestors
670   - * @param currentList
671   - * the current list
672   - * @param updateList
673   - * the update list
674   - * @param idPositions
675   - * the id positions
676   - * @param idOffsets
677   - * the id offsets
678   - * @throws MtasParserException
679   - * the mtas parser exception
680   - * @throws MtasConfigException
681   - * the mtas config exception
  625 + * @param mtasTokenIdFactory the mtas token id factory
  626 + * @param position the position
  627 + * @param name the name
  628 + * @param text the text
  629 + * @param currentOffset the current offset
  630 + * @param functionOutputList the function output list
  631 + * @param unknownAncestors the unknown ancestors
  632 + * @param currentList the current list
  633 + * @param updateList the update list
  634 + * @param idPositions the id positions
  635 + * @param idOffsets the id offsets
  636 + * @throws MtasParserException the mtas parser exception
  637 + * @throws MtasConfigException the mtas config exception
682 638 */
683   - private void processCRMPair(int position, String name, String text,
684   - Integer currentOffset,
  639 + private void processCRMPair(MtasTokenIdFactory mtasTokenIdFactory,
  640 + int position, String name, String text, Integer currentOffset,
685 641 ArrayList<MtasCRMParserFunctionOutput> functionOutputList,
686 642 Integer unknownAncestors,
687 643 HashMap<String, ArrayList<MtasParserObject>> currentList,
... ... @@ -720,7 +676,8 @@ public class MtasCRMParser extends MtasBasicParser {
720 676 idOffsets.put(currentObject.getId(), currentObject.getOffset());
721 677 currentObject.updateMappings(idPositions, idOffsets);
722 678 unknownAncestors = currentObject.getUnknownAncestorNumber();
723   - computeMappingsFromObject(currentObject, currentList, updateList);
  679 + computeMappingsFromObject(mtasTokenIdFactory, currentObject,
  680 + currentList, updateList);
724 681 } else {
725 682 currentObject = new MtasParserObject(tmpCurrentType);
726 683 currentObject.setUnknownAncestorNumber(unknownAncestors);
... ... @@ -759,14 +716,10 @@ public class MtasCRMParser extends MtasBasicParser {
759 716 /**
760 717 * Process functions.
761 718 *
762   - * @param name
763   - * the name
764   - * @param text
765   - * the text
766   - * @param type
767   - * the type
768   - * @param functionOutputList
769   - * the function output list
  719 + * @param name the name
  720 + * @param text the text
  721 + * @param type the type
  722 + * @param functionOutputList the function output list
770 723 */
771 724 private void processFunctions(String name, String text, String type,
772 725 ArrayList<MtasCRMParserFunctionOutput> functionOutputList) {
... ... @@ -822,13 +775,14 @@ public class MtasCRMParser extends MtasBasicParser {
822 775 /**
823 776 * Prints the config types.
824 777 *
825   - * @param types
826   - * the types
  778 + * @param types the types
827 779 * @return the string
828 780 */
829   - private String printConfigTypes(HashMap<?, MtasParserType<MtasParserMapping<?>>> types) {
  781 + private String printConfigTypes(
  782 + HashMap<?, MtasParserType<MtasParserMapping<?>>> types) {
830 783 String text = "";
831   - for (Entry<?, MtasParserType<MtasParserMapping<?>>> entry : types.entrySet()) {
  784 + for (Entry<?, MtasParserType<MtasParserMapping<?>>> entry : types
  785 + .entrySet()) {
832 786 text += "- " + entry.getKey() + ": " + entry.getValue().items.size()
833 787 + " mapping(s)\n";
834 788 for (int i = 0; i < entry.getValue().items.size(); i++) {
... ... @@ -852,10 +806,8 @@ public class MtasCRMParser extends MtasBasicParser {
852 806 /**
853 807 * Instantiates a new mtas crm parser function.
854 808 *
855   - * @param type
856   - * the type
857   - * @param split
858   - * the split
  809 + * @param type the type
  810 + * @param split the split
859 811 */
860 812 public MtasCRMParserFunction(String type, String split) {
861 813 this.split = split;
... ... @@ -878,10 +830,8 @@ public class MtasCRMParser extends MtasBasicParser {
878 830 /**
879 831 * Instantiates a new mtas crm parser function output.
880 832 *
881   - * @param name
882   - * the name
883   - * @param value
884   - * the value
  833 + * @param name the name
  834 + * @param value the value
885 835 */
886 836 public MtasCRMParserFunctionOutput(String name, String value) {
887 837 this.name = name;
... ... @@ -891,8 +841,7 @@ public class MtasCRMParser extends MtasBasicParser {
891 841 /**
892 842 * Creates the.
893 843 *
894   - * @param originalValue
895   - * the original value
  844 + * @param originalValue the original value
896 845 * @return the mtas crm parser function output
897 846 */
898 847 public MtasCRMParserFunctionOutput create(String originalValue) {
... ...
src/mtas/analysis/parser/MtasSketchParser.java
... ... @@ -11,10 +11,9 @@ import java.util.concurrent.atomic.AtomicInteger;
11 11 import java.util.regex.Matcher;
12 12 import java.util.regex.Pattern;
13 13  
14   -import mtas.analysis.parser.MtasBasicParser.MtasParserMapping;
15   -import mtas.analysis.parser.MtasParser.MtasParserObject;
16 14 import mtas.analysis.token.MtasToken;
17 15 import mtas.analysis.token.MtasTokenCollection;
  16 +import mtas.analysis.token.MtasTokenIdFactory;
18 17 import mtas.analysis.util.MtasBufferedReader;
19 18 import mtas.analysis.util.MtasConfigException;
20 19 import mtas.analysis.util.MtasConfiguration;
... ... @@ -43,8 +42,7 @@ final public class MtasSketchParser extends MtasBasicParser {
43 42 /**
44 43 * Instantiates a new mtas sketch parser.
45 44 *
46   - * @param config
47   - * the config
  45 + * @param config the config
48 46 */
49 47 public MtasSketchParser(MtasConfiguration config) {
50 48 super(config);
... ... @@ -67,7 +65,8 @@ final public class MtasSketchParser extends MtasBasicParser {
67 65 if (config != null) {
68 66  
69 67 // always word, no mappings
70   - wordType = new MtasParserType<MtasParserMapping<?>>(MAPPING_TYPE_WORD, null, false);
  68 + wordType = new MtasParserType<MtasParserMapping<?>>(MAPPING_TYPE_WORD,
  69 + null, false);
71 70  
72 71 for (int i = 0; i < config.children.size(); i++) {
73 72 MtasConfiguration current = config.children.get(i);
... ... @@ -89,8 +88,8 @@ final public class MtasSketchParser extends MtasBasicParser {
89 88 if (wordAnnotationTypes.containsKey(nameMapping)) {
90 89 wordAnnotationTypes.get(nameMapping).addItem(m);
91 90 } else {
92   - MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(typeMapping,
93   - nameMapping, false);
  91 + MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(
  92 + typeMapping, nameMapping, false);
94 93 t.addItem(m);
95 94 wordAnnotationTypes.put(Integer.parseInt(nameMapping), t);
96 95 }
... ... @@ -101,8 +100,8 @@ final public class MtasSketchParser extends MtasBasicParser {
101 100 if (groupTypes.containsKey(nameMapping)) {
102 101 groupTypes.get(nameMapping).addItem(m);
103 102 } else {
104   - MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(typeMapping,
105   - nameMapping, false);
  103 + MtasParserType<MtasParserMapping<?>> t = new MtasParserType<MtasParserMapping<?>>(
  104 + typeMapping, nameMapping, false);
106 105 t.addItem(m);
107 106 groupTypes.put(nameMapping, t);
108 107 }
... ... @@ -133,10 +132,10 @@ final public class MtasSketchParser extends MtasBasicParser {
133 132 HashMap<String, Integer[]> idOffsets = new HashMap<String, Integer[]>();
134 133  
135 134 HashMap<String, HashMap<Integer, HashSet<String>>> updateList = createUpdateList();
136   - HashMap<String, ArrayList<MtasParserObject>> currentList = createCurrentList();
  135 + HashMap<String, ArrayList<MtasParserObject>> currentList = createCurrentList();
137 136  
138 137 tokenCollection = new MtasTokenCollection();
139   - MtasToken.resetId();
  138 + MtasTokenIdFactory mtasTokenIdFactory = new MtasTokenIdFactory();
140 139 try (MtasBufferedReader br = new MtasBufferedReader(reader)) {
141 140 String line;
142 141 int currentOffset, previousOffset = br.getPosition();
... ... @@ -201,8 +200,8 @@ final public class MtasSketchParser extends MtasBasicParser {
201 200 idOffsets.put(currentObject.getId(), currentObject.getOffset());
202 201 currentObject.updateMappings(idPositions, idOffsets);
203 202 unknownAncestors = currentObject.getUnknownAncestorNumber();
204   - computeMappingsFromObject(currentObject, currentList,
205   - updateList);
  203 + computeMappingsFromObject(mtasTokenIdFactory, currentObject,
  204 + currentList, updateList);
206 205 }
207 206 }
208 207 }
... ... @@ -272,8 +271,8 @@ final public class MtasSketchParser extends MtasBasicParser {
272 271 }
273 272 currentObject.updateMappings(idPositions, idOffsets);
274 273 unknownAncestors = currentObject.getUnknownAncestorNumber();
275   - computeMappingsFromObject(currentObject, currentList,
276   - updateList);
  274 + computeMappingsFromObject(mtasTokenIdFactory, currentObject,
  275 + currentList, updateList);
277 276 }
278 277 }
279 278 }
... ... @@ -301,7 +300,8 @@ final public class MtasSketchParser extends MtasBasicParser {
301 300 idOffsets.put(currentObject.getId(), currentObject.getOffset());
302 301 currentObject.updateMappings(idPositions, idOffsets);
303 302 unknownAncestors = currentObject.getUnknownAncestorNumber();
304   - computeMappingsFromObject(currentObject, currentList, updateList);
  303 + computeMappingsFromObject(mtasTokenIdFactory, currentObject,
  304 + currentList, updateList);
305 305 }
306 306 }
307 307 }
... ... @@ -352,13 +352,14 @@ final public class MtasSketchParser extends MtasBasicParser {
352 352 /**
353 353 * Prints the config types.
354 354 *
355   - * @param types
356   - * the types
  355 + * @param types the types
357 356 * @return the string
358 357 */
359   - private String printConfigTypes(HashMap<?, MtasParserType<MtasParserMapping<?>>> types) {
  358 + private String printConfigTypes(
  359 + HashMap<?, MtasParserType<MtasParserMapping<?>>> types) {
360 360 String text = "";
361   - for (Entry<?, MtasParserType<MtasParserMapping<?>>> entry : types.entrySet()) {
  361 + for (Entry<?, MtasParserType<MtasParserMapping<?>>> entry : types
  362 + .entrySet()) {
362 363 text += "- " + entry.getKey() + ": " + entry.getValue().items.size()
363 364 + " mapping(s)\n";
364 365 for (int i = 0; i < entry.getValue().items.size(); i++) {
... ... @@ -368,8 +369,15 @@ final public class MtasSketchParser extends MtasBasicParser {
368 369 return text;
369 370 }
370 371  
  372 + /**
  373 + * The Class MtasSketchParserMappingWord.
  374 + */
371 375 private class MtasSketchParserMappingWord
372 376 extends MtasParserMapping<MtasSketchParserMappingWord> {
  377 +
  378 + /**
  379 + * Instantiates a new mtas sketch parser mapping word.
  380 + */
373 381 public MtasSketchParserMappingWord() {
374 382 super();
375 383 this.position = SOURCE_OWN;
... ... @@ -377,14 +385,16 @@ final public class MtasSketchParser extends MtasBasicParser {
377 385 this.offset = SOURCE_OWN;
378 386 this.type = MAPPING_TYPE_WORD;
379 387 }
380   -
  388 +
  389 + /* (non-Javadoc)
  390 + * @see mtas.analysis.parser.MtasBasicParser.MtasParserMapping#self()
  391 + */
381 392 @Override
382 393 protected MtasSketchParserMappingWord self() {
383 394 return this;
384 395 }
385 396 }
386 397  
387   -
388 398 /**
389 399 * The Class MtasSketchParserMappingWordAnnotation.
390 400 */
... ...
src/mtas/analysis/parser/MtasXMLParser.java
... ... @@ -17,6 +17,7 @@ import javax.xml.stream.XMLStreamReader;
17 17  
18 18 import mtas.analysis.token.MtasToken;
19 19 import mtas.analysis.token.MtasTokenCollection;
  20 +import mtas.analysis.token.MtasTokenIdFactory;
20 21 import mtas.analysis.util.MtasConfigException;
21 22 import mtas.analysis.util.MtasParserException;
22 23 import mtas.analysis.util.MtasConfiguration;
... ... @@ -354,9 +355,10 @@ abstract class MtasXMLParser extends MtasBasicParser {
354 355 HashMap<String, ArrayList<MtasParserObject>> currentList = createCurrentList();
355 356 HashMap<String, HashMap<String, String>> variables = createVariables();
356 357  
  358 +
357 359 tokenCollection = new MtasTokenCollection();
358   - MtasToken.resetId();
359   - XMLInputFactory factory = XMLInputFactory.newInstance();
  360 + MtasTokenIdFactory mtasTokenIdFactory = new MtasTokenIdFactory();
  361 + XMLInputFactory factory = XMLInputFactory.newInstance();
360 362 try {
361 363 XMLStreamReader streamReader = factory.createXMLStreamReader(reader);
362 364 QName qname;
... ... @@ -597,7 +599,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
597 599 currentObject.updateMappings(idPositions, idOffsets);
598 600 unknownAncestors = currentObject.getUnknownAncestorNumber();
599 601 // todo: necessary???
600   - computeMappingsFromObject(currentObject, currentList,
  602 + computeMappingsFromObject(mtasTokenIdFactory, currentObject, currentList,
601 603 updateList);
602 604 } else {
603 605 // this shouldn't happen
... ... @@ -632,7 +634,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
632 634 }
633 635 currentObject.updateMappings(idPositions, idOffsets);
634 636 unknownAncestors = currentObject.getUnknownAncestorNumber();
635   - computeMappingsFromObject(currentObject, currentList,
  637 + computeMappingsFromObject(mtasTokenIdFactory, currentObject, currentList,
636 638 updateList);
637 639 } else {
638 640 // this shouldn't happen
... ... @@ -664,7 +666,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
664 666 currentObject.getOffset());
665 667 currentObject.updateMappings(idPositions, idOffsets);
666 668 unknownAncestors = currentObject.getUnknownAncestorNumber();
667   - computeMappingsFromObject(currentObject, currentList,
  669 + computeMappingsFromObject(mtasTokenIdFactory, currentObject, currentList,
668 670 updateList);
669 671 } else {
670 672 // this shouldn't happen
... ... @@ -691,7 +693,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
691 693 currentObject.getOffset());
692 694 currentObject.updateMappings(idPositions, idOffsets);
693 695 unknownAncestors = currentObject.getUnknownAncestorNumber();
694   - computeMappingsFromObject(currentObject, currentList,
  696 + computeMappingsFromObject(mtasTokenIdFactory, currentObject, currentList,
695 697 updateList);
696 698 } else {
697 699 // this shouldn't happen
... ... @@ -717,7 +719,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
717 719 currentObject.getOffset());
718 720 currentObject.updateMappings(idPositions, idOffsets);
719 721 unknownAncestors = currentObject.getUnknownAncestorNumber();
720   - computeMappingsFromObject(currentObject, currentList,
  722 + computeMappingsFromObject(mtasTokenIdFactory, currentObject, currentList,
721 723 updateList);
722 724 } else {
723 725 // this shouldn't happen
... ... @@ -741,7 +743,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
741 743 currentObject.getOffset());
742 744 currentObject.updateMappings(idPositions, idOffsets);
743 745 unknownAncestors = currentObject.getUnknownAncestorNumber();
744   - computeMappingsFromObject(currentObject, currentList,
  746 + computeMappingsFromObject(mtasTokenIdFactory, currentObject, currentList,
745 747 updateList);
746 748 } else {
747 749 // this shouldn't happen
... ... @@ -765,7 +767,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
765 767 currentObject.getOffset());
766 768 currentObject.updateMappings(idPositions, idOffsets);
767 769 unknownAncestors = currentObject.getUnknownAncestorNumber();
768   - computeMappingsFromObject(currentObject, currentList,
  770 + computeMappingsFromObject(mtasTokenIdFactory, currentObject, currentList,
769 771 updateList);
770 772 } else {
771 773 unknownAncestors--;
... ... @@ -818,6 +820,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
818 820 } catch (XMLStreamException e) {
819 821 throw new MtasParserException("No valid XML: " + e.getMessage());
820 822 }
  823 +
821 824 // update tokens with variable
822 825 for (Entry<Integer, HashSet<String>> updateItem : updateList
823 826 .get(UPDATE_TYPE_VARIABLE).entrySet()) {
... ... @@ -831,7 +834,7 @@ abstract class MtasXMLParser extends MtasBasicParser {
831 834 for (Entry<Integer, HashSet<String>> updateItem : updateList
832 835 .get(UPDATE_TYPE_OFFSET).entrySet()) {
833 836 HashSet<String> refIdList = new HashSet<String>();
834   - for (String refId : updateItem.getValue()) {
  837 + for (String refId : updateItem.getValue()) {
835 838 if (idPositions.containsKey(refId)) {
836 839 refIdList.add(refId);
837 840 }
... ...
src/mtas/analysis/token/MtasOffset.java
... ... @@ -14,10 +14,8 @@ public class MtasOffset {
14 14 /**
15 15 * Instantiates a new mtas offset.
16 16 *
17   - * @param start
18   - * the start
19   - * @param end
20   - * the end
  17 + * @param start the start
  18 + * @param end the end
21 19 */
22 20 public MtasOffset(int start, int end) {
23 21 mtasOffsetStart = start;
... ... @@ -27,10 +25,8 @@ public class MtasOffset {
27 25 /**
28 26 * Adds the.
29 27 *
30   - * @param start
31   - * the start
32   - * @param end
33   - * the end
  28 + * @param start the start
  29 + * @param end the end
34 30 */
35 31 public void add(int start, int end) {
36 32 mtasOffsetStart = Math.min(mtasOffsetStart, start);
... ...
src/mtas/analysis/token/MtasPosition.java
... ... @@ -33,8 +33,7 @@ public class MtasPosition {
33 33 /**
34 34 * Instantiates a new mtas position.
35 35 *
36   - * @param position
37   - * the position
  36 + * @param position the position
38 37 */
39 38 public MtasPosition(int position) {
40 39 mtasPositionType = POSITION_SINGLE;
... ... @@ -44,10 +43,8 @@ public class MtasPosition {
44 43 /**
45 44 * Instantiates a new mtas position.
46 45 *
47   - * @param start
48   - * the start
49   - * @param end
50   - * the end
  46 + * @param start the start
  47 + * @param end the end
51 48 */
52 49 public MtasPosition(int start, int end) {
53 50 if (start == end) {
... ... @@ -63,8 +60,7 @@ public class MtasPosition {
63 60 /**
64 61 * Instantiates a new mtas position.
65 62 *
66   - * @param positions
67   - * the positions
  63 + * @param positions the positions
68 64 */
69 65 public MtasPosition(int[] positions) {
70 66 TreeSet<Integer> list = new TreeSet<Integer>();
... ... @@ -91,8 +87,7 @@ public class MtasPosition {
91 87 /**
92 88 * Check type.
93 89 *
94   - * @param type
95   - * the type
  90 + * @param type the type
96 91 * @return the boolean
97 92 */
98 93 public Boolean checkType(String type) {
... ... @@ -160,8 +155,7 @@ public class MtasPosition {
160 155 /**
161 156 * Adds the.
162 157 *
163   - * @param positions
164   - * the positions
  158 + * @param positions the positions
165 159 */
166 160 public void add(int[] positions) {
167 161 TreeSet<Integer> list = new TreeSet<Integer>();
... ... @@ -197,8 +191,7 @@ public class MtasPosition {
197 191 /**
198 192 * Adds the.
199 193 *
200   - * @param position
201   - * the position
  194 + * @param position the position
202 195 */
203 196 public void add(int position) {
204 197 if (mtasPositionType.equals(POSITION_SINGLE)) {
... ...
src/mtas/analysis/token/MtasToken.java
... ... @@ -6,7 +6,6 @@ import java.util.ArrayList;
6 6 import java.util.Arrays;
7 7 import java.util.List;
8 8 import java.util.TreeSet;
9   -import java.util.concurrent.atomic.AtomicInteger;
10 9 import java.util.regex.Matcher;
11 10 import java.util.regex.Pattern;
12 11  
... ... @@ -37,11 +36,8 @@ public abstract class MtasToken&lt;GenericType&gt; {
37 36 public static final Pattern patternPrePostFix = Pattern
38 37 .compile(regexpPrePostFix);
39 38  
40   - /** The Constant MTAS_TOKEN_ID. */
41   - static final AtomicInteger MTAS_TOKEN_ID = new AtomicInteger(0);
42   -
43 39 /** The token id. */
44   - private Integer tokenId = MTAS_TOKEN_ID.getAndIncrement();
  40 + private Integer tokenId;
45 41  
46 42 /** The token ref. */
47 43 private Long tokenRef = null;
... ... @@ -83,18 +79,13 @@ public abstract class MtasToken&lt;GenericType&gt; {
83 79 private Boolean provideParentId = true;
84 80  
85 81 /**
86   - * Reset id.
87   - */
88   - public static void resetId() {
89   - MTAS_TOKEN_ID.set(0);
90   - }
91   -
92   - /**
93 82 * Instantiates a new mtas token.
94 83 *
  84 + * @param tokenId the token id
95 85 * @param value the value
96 86 */
97   - protected MtasToken(String value) {
  87 + protected MtasToken(Integer tokenId, String value) {
  88 + this.tokenId = tokenId;
98 89 setType();
99 90 setValue(value);
100 91 }
... ... @@ -102,12 +93,12 @@ public abstract class MtasToken&lt;GenericType&gt; {
102 93 /**
103 94 * Instantiates a new mtas token.
104 95 *
  96 + * @param tokenId the token id
105 97 * @param value the value
106 98 * @param position the position
107 99 */
108   - protected MtasToken(String value, Integer position) {
109   - setType();
110   - setValue(value);
  100 + protected MtasToken(Integer tokenId, String value, Integer position) {
  101 + this(tokenId, value);
111 102 addPosition(position);
112 103 }
113 104  
... ... @@ -513,13 +504,18 @@ public abstract class MtasToken&lt;GenericType&gt; {
513 504 * @return the prefix from value
514 505 */
515 506 public static String getPrefixFromValue(String value) {
516   - String prefix = null;
517   - if (value.contains(DELIMITER)) {
518   - prefix = value.split(DELIMITER)[0];
  507 + if (value == null) {
  508 + return null;
  509 + } else if (value.contains(DELIMITER)) {
  510 + String[] list = value.split(DELIMITER);
  511 + if (list != null && list.length > 0) {
  512 + return list[0].replaceAll("\u0000", "");
  513 + } else {
  514 + return null;
  515 + }
519 516 } else {
520   - prefix = value;
  517 + return value.replaceAll("\u0000", "");
521 518 }
522   - return prefix.replaceAll("\u0000", "");
523 519 }
524 520  
525 521 /**
... ... @@ -694,17 +690,17 @@ public abstract class MtasToken&lt;GenericType&gt; {
694 690 * Creates the automata.
695 691 *
696 692 * @param prefix the prefix
  693 + * @param regexp the regexp
697 694 * @param valueList the value list
698 695 * @return the list
699 696 * @throws IOException Signals that an I/O exception has occurred.
700 697 */
701   - public static List<CompiledAutomaton> createAutomata(String prefix, String regexp,
702   - List<String> valueList) throws IOException {
  698 + public static List<CompiledAutomaton> createAutomata(String prefix,
  699 + String regexp, List<String> valueList) throws IOException {
703 700 List<CompiledAutomaton> list = new ArrayList<CompiledAutomaton>();
704 701 Automaton automatonRegexp = null;
705   - if(regexp!=null) {
706   - RegExp re = new RegExp(
707   - prefix + MtasToken.DELIMITER + regexp + "\u0000*");
  702 + if (regexp != null) {
  703 + RegExp re = new RegExp(prefix + MtasToken.DELIMITER + regexp + "\u0000*");
708 704 automatonRegexp = re.toAutomaton();
709 705 }
710 706 int step = 500;
... ... @@ -726,7 +722,7 @@ public abstract class MtasToken&lt;GenericType&gt; {
726 722 }
727 723 Automaton automatonList = Operations.union(listAutomaton);
728 724 Automaton automaton;
729   - if(automatonRegexp!=null) {
  725 + if (automatonRegexp != null) {
730 726 automaton = Operations.intersection(automatonList, automatonRegexp);
731 727 } else {
732 728 automaton = automatonList;
... ...
src/mtas/analysis/token/MtasTokenIdFactory.java 0 → 100644
  1 +package mtas.analysis.token;
  2 +
  3 +/**
  4 + * A factory for creating MtasTokenId objects.
  5 + */
  6 +public final class MtasTokenIdFactory {
  7 +
  8 + /** The token id. */
  9 + Integer tokenId;
  10 +
  11 + /**
  12 + * Instantiates a new mtas token id factory.
  13 + */
  14 + public MtasTokenIdFactory() {
  15 + tokenId = -1;
  16 + }
  17 +
  18 + /**
  19 + * Creates a new MtasTokenId object.
  20 + *
  21 + * @return the integer
  22 + */
  23 + public Integer createTokenId() {
  24 + return tokenId++;
  25 + }
  26 +
  27 +}
... ...
src/mtas/analysis/token/MtasTokenString.java
... ... @@ -11,23 +11,22 @@ public class MtasTokenString extends MtasToken&lt;String&gt; {
11 11 /**
12 12 * Instantiates a new mtas token string.
13 13 *
14   - * @param value
15   - * the value
  14 + * @param tokenId the token id
  15 + * @param value the value
16 16 */
17   - public MtasTokenString(String value) {
18   - super(value);
  17 + public MtasTokenString(Integer tokenId, String value) {
  18 + super(tokenId, value);
19 19 }
20 20  
21 21 /**
22 22 * Instantiates a new mtas token string.
23 23 *
24   - * @param value
25   - * the value
26   - * @param position
27   - * the position
  24 + * @param tokenId the token id
  25 + * @param value the value
  26 + * @param position the position
28 27 */
29   - public MtasTokenString(String value, Integer position) {
30   - super(value, position);
  28 + public MtasTokenString(Integer tokenId, String value, Integer position) {
  29 + super(tokenId, value, position);
31 30 }
32 31  
33 32 /*
... ...
src/mtas/codec/MtasCodecPostingsFormat.java
... ... @@ -223,8 +223,7 @@ public class MtasCodecPostingsFormat extends PostingsFormat {
223 223 MtasToken<String> token = null;
224 224 try {
225 225 inObject.seek(ref);
226   - token = new MtasTokenString("");
227   - token.setId(inObject.readVInt());
  226 + token = new MtasTokenString(inObject.readVInt(), "");
228 227 token.setTokenRef(ref);
229 228 int objectFlags = inObject.readVInt();
230 229 int[] positions = null;
... ...
src/mtas/solr/update/processor/MtasUpdateRequestProcessorFactory.java
... ... @@ -59,10 +59,8 @@ public class MtasUpdateRequestProcessorFactory
59 59 /**
60 60 * Inits the.
61 61 *
62   - * @param req
63   - * the req
64   - * @throws IOException
65   - * Signals that an I/O exception has occurred.
  62 + * @param req the req
  63 + * @throws IOException Signals that an I/O exception has occurred.
66 64 */
67 65 @SuppressWarnings("unchecked")
68 66 private void init(SolrQueryRequest req) throws IOException {
... ...