Update docker configuration.

Bartłomiej Nitoń
1 parent badd76e0
Showing 10 changed files with 87 additions and 356802 deletions
Dockerfile
Dockerfile.marcell
README.md
collector/collector/docker-settings.py
collector/pipeline/urls.py
collector/projects/marcell/utils.py
collector/terminology/eurovoc.py
entrypoint.sh
requirements.txt
resources/db/marcell-init.db
@@ -14,7 +14,8 @@ ENV PYTHONUNBUFFERED 1
 # install dependencies
 RUN apt-get update && \
-    apt-get install -y python3-pip
+    apt-get install -y python3-pip && \
+    pip3 install --upgrade pip
 COPY ./requirements.txt /usr/src/collector/requirements.txt
 RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /usr/src/collector/wheels -r requirements.txt
@@ -30,33 +31,35 @@ FROM ubuntu:18.04
 RUN apt-get update && \
     apt-get install -y locales && \
     locale-gen pl_PL.UTF-8
+
+# set envs
 ENV LANG pl_PL.UTF-8
 ENV LC_ALL pl_PL.UTF-8
+ENV HOME=/home/collector
+ENV APP_HOME=/home/collector/app
-# create directory for the collector user
-RUN mkdir -p /home/collector
-
-# create the collector user
-RUN addgroup --group collector && adduser collector --ingroup collector
+# create directory for the collector user and user itself
+RUN mkdir -p $HOME && \
+    addgroup --group collector && \
+    adduser collector --ingroup collector
 # create the appropriate directories
-ENV HOME=/home/collector
-ENV APP_HOME=/home/collector/app
 RUN mkdir $APP_HOME
 WORKDIR $APP_HOME
 # install dependencies
 COPY --from=builder /usr/src/collector/wheels /wheels
 COPY --from=builder /usr/src/collector/requirements.txt .
-RUN apt-get install -y netcat openjdk-8-jre python3-pip software-properties-common wget && \
+RUN apt-get update && apt-get install -y netcat openjdk-8-jre python3-pip software-properties-common wget && \
     wget -O - http://download.sgjp.pl/apt/sgjp.gpg.key|apt-key add - && \
     apt-add-repository http://download.sgjp.pl/apt/ubuntu && \
     wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
     echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |tee  /etc/apt/sources.list.d/pgdg.list && \
     apt-get update && \
-    apt-get install -y morfeusz2 python3-morfeusz2 && \
-    DEBIAN_FRONTEND="noninteractive" apt-get -y install postgresql-12
-RUN pip3 install --no-cache /wheels/*
+    apt-get install -y morfeusz2 python3-morfeusz2 libopenblas-dev libomp-dev && \
+    DEBIAN_FRONTEND="noninteractive" apt-get -y install postgresql-12 && \
+    pip3 install --upgrade pip && \
+    pip3 install --no-cache /wheels/*
 # copy project
 COPY . $APP_HOME
@@ -65,17 +68,29 @@ COPY . $APP_HOME
 WORKDIR ./tools/liner2/g419-external-dependencies
 RUN tar -xvf ./CRF++-0.57.tar.gz
 WORKDIR ./CRF++-0.57
-RUN ./configure
-RUN make
-RUN make install
-RUN ldconfig
+RUN ./configure && \
+    make && \
+    make install && \
+    make clean && \
+    ldconfig
+
+# install eurobert
+WORKDIR $APP_HOME/tools
+RUN wget https://manage.legis.nlp.ipipan.waw.pl/download/marcell/eurobert-model.tar.gz && \
+    tar -xvf ./eurobert-model.tar.gz && \
+    rm ./eurobert-model.tar.gz
+
+# install labse
+RUN wget https://manage.legis.nlp.ipipan.waw.pl/download/marcell/labse-model.tar.gz && \
+    tar -xvf ./labse-model.tar.gz && \
+    rm ./labse-model.tar.gz
 WORKDIR $APP_HOME
 # copy django settings
 COPY ./collector/collector/docker-settings.py $APP_HOME/collector/collector/settings.py
 # chown all the files to the collector user
-RUN chown -R collector:collector $APP_HOME
+RUN chown -R collector:collector $HOME
 # change to the collector user
 USER collector
@@ -14,7 +14,8 @@ ENV PYTHONUNBUFFERED 1
 # install dependencies
 RUN apt-get update && \
-    apt-get install -y python3-pip
+    apt-get install -y python3-pip && \
+    pip3 install --upgrade pip
 COPY ./requirements.txt /usr/src/collector/requirements.txt
 RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /usr/src/collector/wheels -r requirements.txt
@@ -38,8 +39,8 @@ ENV HOME=/home/collector
 ENV APP_HOME=/home/collector/app
 # create directory for the collector user and user itself
-RUN mkdir -p $HOME && \ 
-    addgroup --group collector && \ 
+RUN mkdir -p $HOME && \
+    addgroup --group collector && \
     adduser collector --ingroup collector
 # create the appropriate directories
@@ -55,8 +56,9 @@ RUN apt-get update &amp;&amp; apt-get install -y openjdk-8-jre python3-pip software-prop
     wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
     echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |tee  /etc/apt/sources.list.d/pgdg.list && \
     apt-get update && \
-    apt-get install -y morfeusz2 python3-morfeusz2 && \
+    apt-get install -y morfeusz2 python3-morfeusz2 libopenblas-dev libomp-dev && \
     DEBIAN_FRONTEND="noninteractive" apt-get -y install postgresql-12 && \
+    pip3 install --upgrade pip && \
     pip3 install --no-cache /wheels/*
 # copy project
@@ -66,25 +68,35 @@ COPY . $APP_HOME
 WORKDIR ./tools/liner2/g419-external-dependencies
 RUN tar -xvf ./CRF++-0.57.tar.gz
 WORKDIR ./CRF++-0.57
-RUN ./configure && \ 
-    make && \ 
-    make install && \ 
-    make clean && \ 
+RUN ./configure && \
+    make && \
+    make install && \
+    make clean && \
     ldconfig
+
+# install eurobert
+WORKDIR $APP_HOME/tools
+RUN wget https://manage.legis.nlp.ipipan.waw.pl/download/marcell/eurobert-model.tar.gz && \
+    tar -xvf ./eurobert-model.tar.gz && \
+    rm ./eurobert-model.tar.gz
 WORKDIR $APP_HOME
 # copy django settings
 COPY ./collector/collector/docker-settings.py $APP_HOME/collector/collector/settings.py
 # chown all the files to the collector user
-RUN chown -R collector:collector $APP_HOME
+RUN chown -R collector:collector $HOME
 # configure and init database
 USER postgres
 RUN /etc/init.d/postgresql start && \
     psql --command "CREATE USER collector WITH SUPERUSER PASSWORD 'collector';" && \
     createdb -O collector collector && \
-    psql collector < $APP_HOME/resources/db/marcell-init.db
+    wget https://manage.legis.nlp.ipipan.waw.pl/download/marcell/marcell-init.db.tar.gz && \
+    tar -xvf ./marcell-init.db.tar.gz && \
+    rm ./marcell-init.db.tar.gz && \
+    psql collector < ./marcell-init.db && \
+    rm ./marcell-init.db
 # change to the root user
 USER 0
@@ -2,13 +2,22 @@
 ## Running MARCELL annotate API using Docker
+#### Using docker-compose
+
 ```sh
 docker-compose up -d
 ```
-### Exemplary usage in Python
+#### Using docker
 ```sh
+docker build -t "marcell-pl" -f Dockerfile.marcell .
+docker run --name "marcell-pl-running" -p 8006:8000 -d marcell-pl
+```
+
+### Exemplary usage in Python
+
+```
 import requests
 url = 'http://<container_url>:<exposed_port>/annotate'
@@ -20,4 +29,3 @@ with open(&#39;/text/file/path.[txt/html/pdf]&#39;, &#39;rb&#39;) as text_file,  open(&#39;/meta/fil
 ## Conda based installation
 See INSTALL.md for installation instructions.
-
@@ -68,6 +68,9 @@ SOLR_URL = &#39;http://localhost:8983/solr/&#39;
 # Language-agnostic BERT sentence embedding model path
 LABSE_MODEL_PATH = os.path.join(TOOLS_DIR, 'labse', 'labse_bert_model')
+# expose REST API
+EXPOSE_API = True
+
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/
 from django.urls import path
 from . import views
+from collector import settings
-urlpatterns = [
-    path('annotate', views.AnnotateView.as_view(), name='annotate'),
-]
+
+urlpatterns = []
+if settings.EXPOSE_API:
+    urlpatterns = [
+        path('annotate', views.AnnotateView.as_view(), name='annotate'),
+    ]
@@ -100,6 +100,6 @@ def vectorize_eurovocs():
     labse = Labse(settings.LABSE_MODEL_PATH)
     for eurovoc_label in EuroVocLabel.objects.filter(vector__isnull=True):
         print('Creating EuroVoc vector representation for "{}".'.format(eurovoc_label.text))
-        laser_vector = [float(x) for x in list(labse.embed(eurovoc_label.text))]
-        eurovoc_label.vector = laser_vector
+        labse_vector = [float(x) for x in list(labse.embed(eurovoc_label.text))]
+        eurovoc_label.vector = labse_vector
         eurovoc_label.save()
@@ -20,10 +20,13 @@ def annotate(documents):
 def _add_document_level_domains(document, title_based_tld_preditor, min_score, k):
     print('Adding EuroVoc domains to: {}.'.format(document.id))
+    eurovoc_domains = []
+    tld_score_value = 'sim'
+
     if document.keywords.exists():
-        tld_score_value = 'sim'
         eurovoc_domains = _get_keyword_based_domains(document.keywords.all(), min_score, k)
-    else:
+
+    if not eurovoc_domains:
         tld_score_value = 'prob'
         eurovoc_domains = _get_title_based_domains(document.title, title_based_tld_preditor, min_score, k)
@@ -15,6 +15,8 @@ python3 collector/manage.py makemigrations --noinput
 python3 collector/manage.py migrate
 python3 collector/manage.py configure_marcell_pipelines
 python3 collector/manage.py load_eurovoc_terms -i resources/eurovoc -l pl
+python3 collector/manage.py load_eurovoc_terms -i resources/eurovoc -l en
 python3 collector/manage.py load_iate_terms -i resources/iate/iate.tbx
+python3 collector/manage.py map_eurovoc_terms
 exec "$@"
@@ -6,15 +6,14 @@ Django==2.2.5
 faiss
 gunicorn==20.0.4
 Keras==2.2.5
-laserembeddings[en,pl]
 natsort
 psycopg2-binary
 pysolr
 python-dateutil
-pytorch
 scikit-learn==0.22.1
 scrapy
-tensorflow==1.14
+tensorflow>=1.15,<2
 tika==1.19
-xmldiff
-
+torch
+transformers>=3.0.2
+xmldiff
 \ No newline at end of file