search_component_stats_tokens.html 18.6 KB

Edit Raw Blame History

<!DOCTYPE html>
<!--
 | Generated by Apache Maven Doxia Site Renderer 1.7.4 at 2017-10-16
 | Rendered using Apache Maven Fluido Skin 1.5
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  <head>
    <meta charset="UTF-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <meta name="Date-Revision-yyyymmdd" content="20171016" />
    <meta http-equiv="Content-Language" content="en" />
    <title>Multi Tier Annotation Search &#x2013; Statistics - tokens</title>
    <link rel="stylesheet" href="./css/apache-maven-fluido-1.5.min.css" />
    <link rel="stylesheet" href="./css/site.css" />
    <link rel="stylesheet" href="./css/print.css" media="print" />


    <script type="text/javascript" src="./js/apache-maven-fluido-1.5.min.js"></script>

                      </head>
        <body class="topBarDisabled">


        <div class="container-fluid">
          <div id="banner">
        <div class="pull-left">
                                <div id="bannerLeft">
                <h2>MTAS</h2>
                </div>
                      </div>
        <div class="pull-right">                  <a href="http://www.meertens.knaw.nl/" id="bannerRight">
                                                                                                <img src="images/meertens.png"  alt="Meertens Instituut" width="93" height="104"/>
                </a>
      </div>
        <div class="clear"><hr/></div>
      </div>

      <div id="breadcrumbs">
        <ul class="breadcrumb">

                  <li id="projectVersion">Version: 7.0.1
                          <span class="divider">|</span>
                    </li>
                              <li class="">
                    <a href="index.html" title="Mtas">
        Mtas</a>
                    <span class="divider">/</span>
      </li>
        <li class="active ">Statistics - tokens</li>


                  <li id="publishDate" class="pull-right">Last Published: 2017-10-16</li>

                            </ul>
      </div>


      <div class="row-fluid">
        <div id="leftColumn" class="span2">
          <div class="well sidebar-nav">

                <ul class="nav nav-list">
                    <li class="nav-header">Mtas</li>

      <li>

                          <a href="index.html" title="Introduction">
          <span class="none"></span>
        Introduction</a>
            </li>

      <li>

                          <a href="features.html" title="Features">
          <span class="none"></span>
        Features</a>
            </li>

      <li>

                          <a href="installation.html" title="Getting started">
          <span class="icon-chevron-right"></span>
        Getting started</a>
                  </li>

      <li>

                          <a href="indexing.html" title="Indexing">
          <span class="icon-chevron-right"></span>
        Indexing</a>
                  </li>

      <li>

                          <a href="search.html" title="Search">
          <span class="icon-chevron-down"></span>
        Search</a>
                    <ul class="nav nav-list">

      <li>

                          <a href="search_configuration.html" title="Configuration">
          <span class="none"></span>
        Configuration</a>
            </li>

      <li>

                          <a href="search_component.html" title="Component">
          <span class="icon-chevron-down"></span>
        Component</a>
                    <ul class="nav nav-list">

      <li>

                          <a href="search_component_stats.html" title="Statistics">
          <span class="icon-chevron-down"></span>
        Statistics</a>
                    <ul class="nav nav-list">

      <li>

                          <a href="search_component_stats_positions.html" title="Positions">
          <span class="none"></span>
        Positions</a>
            </li>

      <li class="active">

            <a href="#"><span class="none"></span>Tokens</a>
          </li>

      <li>

                          <a href="search_component_stats_spans.html" title="Spans">
          <span class="none"></span>
        Spans</a>
            </li>
              </ul>
        </li>

      <li>

                          <a href="search_component_kwic.html" title="Kwic">
          <span class="none"></span>
        Kwic</a>
            </li>

      <li>

                          <a href="search_component_list.html" title="List">
          <span class="none"></span>
        List</a>
            </li>

      <li>

                          <a href="search_component_document.html" title="Document">
          <span class="none"></span>
        Document</a>
            </li>

      <li>

                          <a href="search_component_termvector.html" title="Termvector">
          <span class="none"></span>
        Termvector</a>
            </li>

      <li>

                          <a href="search_component_facet.html" title="Facet">
          <span class="none"></span>
        Facet</a>
            </li>

      <li>

                          <a href="search_component_group.html" title="Group">
          <span class="none"></span>
        Group</a>
            </li>

      <li>

                          <a href="search_component_prefix.html" title="Prefix">
          <span class="none"></span>
        Prefix</a>
            </li>

      <li>

                          <a href="search_component_collection.html" title="Collection">
          <span class="none"></span>
        Collection</a>
            </li>
              </ul>
        </li>

      <li>

                          <a href="search_parser.html" title="Parser">
          <span class="icon-chevron-down"></span>
        Parser</a>
                    <ul class="nav nav-list">

      <li>

                          <a href="search_parser_cql.html" title="CQL query">
          <span class="none"></span>
        CQL query</a>
            </li>

      <li>

                          <a href="search_parser_join.html" title="Join query">
          <span class="none"></span>
        Join query</a>
            </li>
              </ul>
        </li>

      <li>

                          <a href="search_handler.html" title="Handler">
          <span class="none"></span>
        Handler</a>
            </li>

      <li>

                          <a href="search_stats.html" title="Type of statistics">
          <span class="none"></span>
        Type of statistics</a>
            </li>

      <li>

                          <a href="search_functions.html" title="Functions">
          <span class="none"></span>
        Functions</a>
            </li>

      <li>

                          <a href="search_cql.html" title="CQL">
          <span class="none"></span>
        CQL</a>
            </li>

      <li>

                          <a href="search_sharding.html" title="Sharding">
          <span class="none"></span>
        Sharding</a>
            </li>
              </ul>
        </li>

      <li>

                          <a href="download.html" title="Download">
          <span class="none"></span>
        Download</a>
            </li>
                              <li class="nav-header">Project Documentation</li>

      <li>

                          <a href="project-info.html" title="Project Information">
          <span class="icon-chevron-right"></span>
        Project Information</a>
                  </li>

      <li>

                          <a href="project-reports.html" title="Project Reports">
          <span class="icon-chevron-right"></span>
        Project Reports</a>
                  </li>
            </ul>


          <hr />

           <div id="poweredBy">
                            <div class="clear"></div>
                            <div class="clear"></div>
                            <div class="clear"></div>
                            <div class="clear"></div>
                             <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
        <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" />
      </a>
                  </div>
          </div>
        </div>


        <div id="bodyColumn"  class="span10" >

            <h1>Statistics - tokens</h1>
<p>To get statistics on the number of tokens within a set of documents in Solr requests, besides the parameter to enable <a href="search_component_stats.html">statistics</a>, the following parameter should be provided.</p>

<table border="0" class="table table-striped">
  <thead>

<tr class="a">

<th>Parameter </th>

<th>Value </th>

<th>Obligatory </th>
    </tr>
  </thead>
  <tbody>

<tr class="b">

<td>mtas.stats.tokens </td>

<td>true </td>

<td>yes </td>
    </tr>
  </tbody>
</table>
<p>Multiple statistics on tokens can be produced within the same request. To distinguish them, a unique identifier has to be provided for<br />each of the required statistics.</p>

<table border="0" class="table table-striped">
  <thead>

<tr class="a">

<th>Parameter </th>

<th>Value </th>

<th>Info </th>

<th>Obligatory </th>
    </tr>
  </thead>
  <tbody>

<tr class="b">

<td>mtas.stats.tokens.&lt;identifier&gt;.key </td>

<td>&lt;string&gt; </td>

<td>key used in response </td>

<td>no </td>
    </tr>

<tr class="a">

<td>mtas.stats.tokens.&lt;identifier&gt;.field </td>

<td>&lt;string&gt; </td>

<td>Mtas field </td>

<td>yes </td>
    </tr>

<tr class="b">

<td>mtas.stats.tokens.&lt;identifier&gt;.type </td>

<td>&lt;string&gt; </td>

<td>required <a href="search_stats.html">type of statistics</a> </td>

<td>no </td>
    </tr>

<tr class="a">

<td>mtas.stats.tokens.&lt;identifier&gt;.minimum </td>

<td>&lt;double&gt; </td>

<td>minimum number of tokens </td>

<td>no </td>
    </tr>

<tr class="b">

<td>mtas.stats.tokens.&lt;identifier&gt;.maximum </td>

<td>&lt;double&gt; </td>

<td>maximum number of tokens </td>

<td>no </td>
    </tr>
  </tbody>
</table>
<p>The <i>key</i> is added to the response and may be used to distinguish between multiple statistics on tokens, and should therefore be unique. The optional <i>minimum</i> and <i>maximum</i> can be used to focus only on documents satisfying a condition on the number of tokens.</p>
<hr />
<div class="section">
<h2><a name="Examples"></a>Examples</h2>

<ol style="list-style-type: decimal">

<li><a href="#basic">Basic</a> : basic statistics on the number of tokens.</li>

<li><a href="#minimum-and-maximum">Minimum and maximum</a> : statistics on the number of tokens with restrictions on this number.</li>

<li><a href="#subset">Subset</a> : statistics on the number of tokens within a subset of documents.</li>
</ol>
<hr />
<p><a name="basic"></a></p>
<div class="section">
<h3><a name="Basic"></a>Basic</h3>
<p><b>Example</b><br />Total and average number of tokens and the number of documents.</p>
<p><b>Request and response</b><br /><tt>q=*%3A*&amp;rows=0&amp;mtas=true&amp;mtas.stats=true&amp;mtas.stats.tokens=true&amp;mtas.stats.tokens.0.field=text&amp;mtas.stats.tokens.0.key=example - basic&amp;mtas.stats.tokens.0.type=sum,mean,n&amp;wt=json&amp;indent=true</tt></p>

<div class="source">
<div class="source"><pre class="prettyprint">&quot;mtas&quot;:{
    &quot;stats&quot;:{
      &quot;tokens&quot;:[{
          &quot;key&quot;:&quot;example - basic&quot;,
          &quot;mean&quot;:1949.101406523028,
          &quot;sum&quot;:4024520177,
          &quot;n&quot;:2064808}]}}
</pre></div></div>
<p><a name="minimum-and-maximum"></a></p></div>
<div class="section">
<h3><a name="Minimum_and_maximum"></a>Minimum and maximum</h3>
<p><b>Example</b><br />Full statistics on tokens for documents with a minimum of 100 tokens, for documents with a maximum of 200 tokens, and for documents with between 100 and 200 tokens.</p>
<p><b>Request and response</b><br /><tt>q=*%3A*&amp;rows=0&amp;mtas=true&amp;mtas.stats=true&amp;mtas.stats.tokens=true&amp;mtas.stats.tokens.0.field=text&amp;mtas.stats.tokens.0.key=example - minimum&amp;mtas.stats.tokens.0.type=all&amp;mtas.stats.tokens.0.minimum=500&amp;mtas.stats.tokens.1.field=text&amp;mtas.stats.tokens.1.key=example - maximum&amp;mtas.stats.tokens.1.type=all&amp;mtas.stats.tokens.1.maximum=1000&amp;mtas.stats.tokens.2.field=text&amp;mtas.stats.tokens.2.key=example - minimum and maximum&amp;mtas.stats.tokens.2.type=all&amp;mtas.stats.tokens.2.minimum=500&amp;mtas.stats.tokens.2.maximum=1000&amp;wt=json&amp;indent=true</tt></p>

<div class="source">
<div class="source"><pre class="prettyprint">&quot;mtas&quot;:{
    &quot;stats&quot;:{
      &quot;tokens&quot;:[{
          &quot;key&quot;:&quot;example - minimum&quot;,
          &quot;sumsq&quot;:2.91825668357275E14,
          &quot;populationvariance&quot;:2.022964435797023E8,
          &quot;max&quot;:3320612.0,
          &quot;sum&quot;:3.837278477E9,
          &quot;kurtosis&quot;:9580.99014557769,
          &quot;standarddeviation&quot;:14223.100544366072,
          &quot;n&quot;:1390207,
          &quot;quadraticmean&quot;:14488.452755067281,
          &quot;min&quot;:500.0,
          &quot;median&quot;:1359.0,
          &quot;variance&quot;:2.0229658909514648E8,
          &quot;mean&quot;:2760.2209433559033,
          &quot;geometricmean&quot;:1584.8982392362057,
          &quot;sumoflogs&quot;:1.0243428152831953E7,
          &quot;skewness&quot;:79.47215006871889},
        {
          &quot;key&quot;:&quot;example - maximum&quot;,
          &quot;sumsq&quot;:3.33432806009E11,
          &quot;populationvariance&quot;:65815.48228216589,
          &quot;max&quot;:1000.0,
          &quot;sum&quot;:5.49051031E8,
          &quot;kurtosis&quot;:-0.9495132030213522,
          &quot;standarddeviation&quot;:256.54539199058576,
          &quot;n&quot;:1178024,
          &quot;quadraticmean&quot;:532.0189410229931,
          &quot;min&quot;:0.0,
          &quot;median&quot;:441.0,
          &quot;variance&quot;:65815.53815160331,
          &quot;mean&quot;:466.07796700236827,
          &quot;geometricmean&quot;:0.0,
          &quot;sumoflogs&quot;:&quot;-Infinity&quot;,
          &quot;skewness&quot;:0.2518109944817064},
        {
          &quot;key&quot;:&quot;example - minimum and maximum&quot;,
          &quot;sumsq&quot;:2.70110872559E11,
          &quot;populationvariance&quot;:20021.06838039624,
          &quot;max&quot;:1000.0,
          &quot;sum&quot;:3.61809331E8,
          &quot;kurtosis&quot;:-1.0824803795579663,
          &quot;standarddeviation&quot;:141.49596513804715,
          &quot;n&quot;:503423,
          &quot;quadraticmean&quot;:732.4947329880449,
          &quot;min&quot;:500.0,
          &quot;median&quot;:704.0,
          &quot;variance&quot;:20021.108150347452,
          &quot;mean&quot;:718.6984523949043,
          &quot;geometricmean&quot;:704.889293672351,
          &quot;sumoflogs&quot;:3301468.553637138,
          &quot;skewness&quot;:0.2634725299866506}]}}
</pre></div></div>
<p><a name="subset"></a> </p></div>
<div class="section">
<h3><a name="Subset"></a>Subset</h3>
<p><b>Example</b><br />Total and average number of tokens and the number of documents for a subset of documents.</p>
<p><b>Request and response</b><br /><tt>q=text:koe&amp;rows=0&amp;mtas=true&amp;mtas.stats=true&amp;mtas.stats.tokens=true&amp;mtas.stats.tokens.0.field=text&amp;mtas.stats.tokens.0.key=example - subset&amp;mtas.stats.tokens.0.type=sum,mean,n&amp;wt=json&amp;indent=true</tt></p>

<div class="source">
<div class="source"><pre class="prettyprint">&quot;mtas&quot;:{
    &quot;stats&quot;:{
      &quot;tokens&quot;:[{
          &quot;key&quot;:&quot;example - subset&quot;,
          &quot;mean&quot;:42901.60996309963,
          &quot;sum&quot;:116263363,
          &quot;n&quot;:2710}]}}
</pre></div></div>
<hr /></div></div>
<div class="section">
<h2><a name="Lucene"></a>Lucene</h2>
<p>To use statistics on the number of tokens <a href="installation_lucene.html">directly in Lucene</a>, <i>ComponentToken</i> together with the provided <i>collect</i> method can be used. </p></div>
                  </div>
            </div>
          </div>

    <hr/>

    <footer>
            <div class="container-fluid">
                      <div class="row-fluid">
                                      <p >Copyright &copy;                    2017
                        <a href="http://www.meertens.knaw.nl/">Meertens Institute</a>.
            All rights reserved.
      </p>
                </div>


                </div>
    </footer>
        </body>
</html>