From 44cbbee54dc5c5b714b914d91509f922cd9e7bed Mon Sep 17 00:00:00 2001
From: Mateusz Kopeć <mkopec87@gmail.com>
Date: Wed, 23 Jan 2013 13:24:08 +0100
Subject: [PATCH] initial
---
README | 243 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
dfs/__init__.py | 0
dfs/config.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
dfs/database.py | 381 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
dfs/msg/__init__.py | 0
dfs/msg/message.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
dfs/msg/stream.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
dfs/repo.py | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
dfs/server.py | 401 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
dfs/utils.py | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
example_configuration/passwd | 6 ++++++
example_configuration/server.cfg | 23 +++++++++++++++++++++++
example_configuration/ssl.conf/cert.pem | 16 ++++++++++++++++
example_configuration/ssl.conf/key.pem | 16 ++++++++++++++++
example_configuration/users.cfg | 12 ++++++++++++
gui/CheckoutNumberDialog.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/ConfigDialog.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/DownloadNumberDialog.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/LogDialog.py | 42 ++++++++++++++++++++++++++++++++++++++++++
gui/LoginDialog.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/MainFrame.py | 227 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/ReturnReasonDialog.py | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/SortableListCtrl.py | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/client.py | 470 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/const.py | 36 ++++++++++++++++++++++++++++++++++++
gui/data/.dir.lst | 5 +++++
gui/data/cert.pem | 16 ++++++++++++++++
gui/data/global.cfg | 21 +++++++++++++++++++++
gui/dfs.wxg | 367 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/dfs/__init__.py | 0
gui/dfs/config.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
gui/dfs/database.py | 381 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/dfs/msg/__init__.py | 0
gui/dfs/msg/message.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/dfs/msg/stream.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/dfs/repo.py | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/dfs/server.py | 401 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/dfs/utils.py | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/dir_cache.py | 28 ++++++++++++++++++++++++++++
gui/footprints.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/i18n.py | 45 +++++++++++++++++++++++++++++++++++++++++++++
gui/img/down.png | Bin 0 -> 846 bytes
gui/img/ready.png | Bin 0 -> 849 bytes
gui/img/textimg_1.png | Bin 0 -> 522 bytes
gui/img/textimg_2.png | Bin 0 -> 648 bytes
gui/img/textimg_3.png | Bin 0 -> 782 bytes
gui/img/up.png | Bin 0 -> 837 bytes
gui/log_thread.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
gui/manager.py | 249 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/po/en/LC_MESSAGES/manager.po | 229 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/po/generate_pot.sh | 1 +
gui/po/manager.pot | 325 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/po/pl/LC_MESSAGES/manager.po | 354 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
gui/ready.py | 42 ++++++++++++++++++++++++++++++++++++++++++
gui/utils.py | 157 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
installation.sh | 43 +++++++++++++++++++++++++++++++++++++++++++
scripts/add_files.py | 105 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
scripts/clusters.py | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
scripts/db_averages_stats.py | 283 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
scripts/db_detailed_stats.py | 262 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
scripts/db_stats.py | 186 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
scripts/extract_files.py | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
scripts/fill_empty_repo.sh | 29 +++++++++++++++++++++++++++++
scripts/fix_files.py | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
scripts/quasi.py | 183 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
scripts/reject_files.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
scripts/remove_files.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
scripts/stats_atlas.py | 153 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
68 files changed, 7693 insertions(+), 0 deletions(-)
create mode 100644 README
create mode 100644 dfs/__init__.py
create mode 100644 dfs/config.py
create mode 100644 dfs/database.py
create mode 100644 dfs/msg/__init__.py
create mode 100644 dfs/msg/message.py
create mode 100644 dfs/msg/stream.py
create mode 100644 dfs/repo.py
create mode 100644 dfs/server.py
create mode 100644 dfs/utils.py
create mode 100644 example_configuration/passwd
create mode 100644 example_configuration/server.cfg
create mode 100644 example_configuration/ssl.conf/cert.pem
create mode 100644 example_configuration/ssl.conf/key.pem
create mode 100644 example_configuration/users.cfg
create mode 100644 gui/CheckoutNumberDialog.py
create mode 100644 gui/ConfigDialog.py
create mode 100644 gui/DownloadNumberDialog.py
create mode 100644 gui/LogDialog.py
create mode 100644 gui/LoginDialog.py
create mode 100644 gui/MainFrame.py
create mode 100644 gui/ReturnReasonDialog.py
create mode 100644 gui/SortableListCtrl.py
create mode 100644 gui/client.py
create mode 100644 gui/const.py
create mode 100644 gui/data/.dir.lst
create mode 100644 gui/data/cert.pem
create mode 100644 gui/data/global.cfg
create mode 100644 gui/dfs.wxg
create mode 100644 gui/dfs/__init__.py
create mode 100644 gui/dfs/config.py
create mode 100644 gui/dfs/database.py
create mode 100644 gui/dfs/msg/__init__.py
create mode 100644 gui/dfs/msg/message.py
create mode 100644 gui/dfs/msg/stream.py
create mode 100644 gui/dfs/repo.py
create mode 100644 gui/dfs/server.py
create mode 100644 gui/dfs/utils.py
create mode 100644 gui/dir_cache.py
create mode 100644 gui/footprints.py
create mode 100644 gui/i18n.py
create mode 100644 gui/img/down.png
create mode 100644 gui/img/ready.png
create mode 100644 gui/img/textimg_1.png
create mode 100644 gui/img/textimg_2.png
create mode 100644 gui/img/textimg_3.png
create mode 100644 gui/img/up.png
create mode 100644 gui/log_thread.py
create mode 100755 gui/manager.py
create mode 100644 gui/po/en/LC_MESSAGES/manager.po
create mode 100755 gui/po/generate_pot.sh
create mode 100644 gui/po/manager.pot
create mode 100644 gui/po/pl/LC_MESSAGES/manager.po
create mode 100644 gui/ready.py
create mode 100644 gui/utils.py
create mode 100644 installation.sh
create mode 100644 scripts/add_files.py
create mode 100755 scripts/clusters.py
create mode 100755 scripts/db_averages_stats.py
create mode 100755 scripts/db_detailed_stats.py
create mode 100644 scripts/db_stats.py
create mode 100755 scripts/extract_files.py
create mode 100755 scripts/fill_empty_repo.sh
create mode 100755 scripts/fix_files.py
create mode 100755 scripts/quasi.py
create mode 100755 scripts/reject_files.py
create mode 100755 scripts/remove_files.py
create mode 100755 scripts/stats_atlas.py
diff --git a/README b/README
new file mode 100644
index 0000000..2a094ca
--- /dev/null
+++ b/README
@@ -0,0 +1,243 @@
+Instrukcja instalacji systemu zarządzania plikami.
+
+
+W poniższej instrukcji zakładamy, że czytelnik posiada podstawową
+wiedzę na temat administracji serwera svn. Zakładamy, że
+po stronie serwera zainstalowane są następujące narzędzia:
+
+* subversion
+* python (2.6 lub 2.7)
+* pysvn
+
+GUI ułatwiające pobieranie i wysyłanie zaanotowanych plików
+zbudowane jest na bazie:
+
+* python
+* wxPython
+* wxGlade
+
+Przygotowana została binarna wersja GUI pod system Windows,
+dzięki czemu użytkownicy systemu nie muszą posiadać
+zainstalowanych powyższych narzedzi.
+
+
+------------------------------------------------------------------
+
+
+Razem z instrukcją README administrator systemu powinien otrzymać:
+
+* Moduł dfs -- implementację systemu zarządzania plikami,
+* Katalog scripts -- zestaw skryptów ułatwiających tworzenie i
+ zarządzanie systemem,
+* Moduł gui -- implementację GUI dla użytkowników systemu.
+* Katalog z przykładową konfiguracją serwera
+ (pliki $SERVER.cfg oraz $USERS.cfg),
+* Katalog z przykładową konfiguracją klienta GUI (plik $GUI.cfg).
+
+Ponadto, administrator powinien wygenerować klucze szyfrujące
+połączenie klient-serwer -- pliki key.pem i cert.pem.
+Odpowiednie informacje na temat generowania kluczy można znaleźć
+pod adresem:
+
+http://docs.python.org/dev/library/ssl.html#certificates
+
+
+==================================================================
+
+
+1) Zakładanie nowego repozytorium.
+
+
+Na serwerze, w katalogu, w którym ma znajdować się repozytorium,
+należy wykonać komendę:
+
+$ svnadmin create $REPO.PATH
+
+W katalogu konfiguracyjnym repozytorium ($REPO.PATH/conf/) należy
+zmodyfikować dwa pliki:
+
+i) svnserve.conf
+
+...
+### and "none". The sample settings below are the defaults.
+anon-access = none
+auth-access = write
+### The password-db option controls the location of the password
+...
+### Uncomment the line below to use the default password file.
+password-db = passwd
+### The authz-db option controls the location of the authorization
+...
+
+ii) passwd
+
+W pliku passwd należy wpisać login i hasło, które wykorzystywane
+będą po stronie serwera do zatwierdzania (commit-owania) zmian
+w repozytorium, np.:
+
+...
+machine = 95mnf83m-SDT%@vf
+...
+
+Ten sam login i hasło należy podać w konfiguracji serwera systemu
+zarządzania plikami, o czym w punkcie 2). W pliku passwd można
+podać również hasła dla innych użytkowników repozytorium -- więcej
+na ten temat można znaleźć w dokumentacji subversion.
+
+Hasła użytkowników systemu (anotatorów) również uzupełniane są w
+konfiguracji systemu zarządzania plikami opisanej w 2).
+
+
+------------------------------------------------------------------
+
+
+Na serwerze, w innym katalogu niż zostało założone repozytorium,
+należy utworzyć kopię roboczą repozytorium, z której korzystać
+będzie system zarządzania plikami. W tym celu należy wpisać:
+
+$ svn co file://$REPO.PATH $COPY.PATH
+Pobrano wersję 0.
+
+Gdzie $REPO.PATH to ścieżka repozytorium, a $COPY.PATH to
+docelowy katalog kopii roboczej.
+
+
+UWAGA: Zamiast ścieżki postaci file://$REPO.PATH, można również
+podać adres w postaci svn://$REPO.ADDR -- adres, który jest
+dostępny dla użytkowników repozytorium pracujących poza
+serwerem. Ponadto, kopię roboczą można założyć na innym
+serwerze niż repozytorium. Kopia robocza musi jednak znajdować
+się na tej samej maszynie, na której będzie później uruchamiany
+serwer systemu zarządzania plikami. Wadą tego rozwiązania
+może być wolniejsze działanie systemu.
+
+
+Pustą kopię roboczą należy na wstępie wypełnić strukturą katalogów
+dla systemu zarządzania plikami. Służy do tego skrypt
+fill_empty_repo.sh znajdujący się w katalogu scripts dostarczonym
+razem z tą instrukcją. Skrypt wywołuje się w następujący sposób:
+
+$ ./fill_empty_repo.sh $COPY.PATH
+
+Struktura katalogów powinna wyglądać następująco:
+
+$COPY.PATH/
+ new <- katalog z nowymi plikami
+ annotation/ <- wyniki anotacji plików
+ A
+ B
+ adjudication <- wyniki super-anotacji plików
+ db.xml <- baza danych z bierzącymi informacjami
+ dotyczącymi anotacji
+
+Na koniec należy watwierdzić zmiany wykonane w kopii roboczej:
+
+$ svn add $COPY.PATH/*
+$ svn commit --username=machine --password=95mnf83m-SDT%@vf -m "struktura"
+
+
+UWAGA: W katalogach annotation/A, annotation/B oraz adjudication
+znajdują się wysłane przez anotatorów na serwer pliki. Nie oznacza
+to jednak, że anotacja tych plików przez ich posiadaczy zakończyła
+się. Taką informację można uzyskać na podstawie bazy danych
+(pliki, których anotacja/super-anotacja zakończyła się, będą
+uzupełnione o element <checkinDate> w bazie danych).
+
+
+==================================================================
+
+
+2) Konfiguracja systemu zarządzania plikami.
+
+
+Konfiguracja systemu odbywa się w trzech plikach:
+* $SERVER.cfg
+* $USERS.cfg
+* $PASSWD
+
+Razem z instrukcją i implementacją serwera powinny zostać
+dostarczone również przykładowe pliki konfiguracyjne, w których
+można znaleźć dokładniejsze informacje na temat konfiguracji
+serwera.
+
+i) $SERVER.cfg
+
+Plik $SERVER.cfg jest głównym plikiem konfiguracyjnym i jedynym
+wczytywanym bezpośrednio przez serwer systemu (znajdują się
+w nim również ścieżki do pozostałych dwóch plików konfiguracyjnych
+-- $USERS.cfg i $PASSWD).
+
+ii) $USERS.cfg
+
+Plik zawiera informacje o uprawnieniach do super-anotacji oraz
+o limitach pobierania plików.
+
+ii) $PASSWD
+
+W pliku znajdują się hasła określone dla użytkowników systemu
+(anotatorów). Poszczególne linie tego pliku mają postać:
+login = hasło
+
+
+==================================================================
+
+
+3) Używanie serwera systemu zarzadznania plikami.
+
+
+Do uruchamiania serwera służy narzędzie server.py w module dfs.
+Serwer uruchamia się następującą komendą:
+
+$ python server.py $SERVER.cfg
+
+Ewentualne błędy będą wypisywane na standardowe wyjście.
+
+
+------------------------------------------------------------------
+
+
+Karmienie systemu nowymi plikami przeznaczonymi do anotacji.
+
+Do tego celu służy skrypt add_files.py w katalogu scripts.
+
+$ python add_files.py $SERVER.cfg PATH1 PATH2 ...
+
+gdzie PATH1, PATH2, ... to ścieżki do plików, które mają zostać
+dodane do repozytorium. Zmiany nie są automatycznie commit-owane,
+więc trzeba użyć komendy svn commit do ich zatwierdzenia:
+
+UWAGA: Dodawanie plików, jak również jakiekolwiek prace
+administracyje w repozytorium systemu powinny być przeprowadzane
+przy wyłączonym serwerze server.py. Serwer można w sposób
+bezpieczny wyłączyć wysyłając sygnał SIGINT (np. Control-C)
+(Sygnał zostanie zignorowany, jeśli odbywa się komunikacja
+z serwerem).
+
+
+==================================================================
+
+
+4) Graficzny interfejs systemu zarządzania plikami.
+
+
+Implementacja GUI znajduje się w module gui dostarczonym
+razem z instrukcją. GUI przeznaczone jest dla użytkowników
+systemu, czyli anotatorów.
+
+
+------------------------------------------------------------------
+
+
+Konfiguracja GUI znajduje się w katalogu gui/data w pliku
+global.cfg. W tym samym katalogu powinien znajdować się
+wygenerowany przez administratora klucz publiczny (cert.pem).
+Dokładniejsze informacje na temat konfiguracji znajdują się w
+przykładowym pliku konfiguracyjnym gui/data/global.cfg.
+
+
+------------------------------------------------------------------
+
+
+GUI uruchamia się komendą:
+
+$ python manager.py
diff --git a/dfs/__init__.py b/dfs/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/dfs/__init__.py
diff --git a/dfs/config.py b/dfs/config.py
new file mode 100644
index 0000000..5a53f9c
--- /dev/null
+++ b/dfs/config.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+import re
+import os
+
+SECTION = re.compile("^\[(.*)\]$")
+SUBSECTION = re.compile("^\{(.*)\}$")
+ENTRY = re.compile("^([^=\s]+)\s*=\s*([^=]+)\s*$")
+
+def cut_comment(s):
+ return s.split("#")[0]
+
+class Config:
+
+ def __init__(self, path):
+ section = None
+ subsection = None
+ self.d = {}
+ with open(path) as f:
+ for line in f:
+ line = cut_comment(line).strip()
+
+ m = SECTION.search(line)
+ if m is not None:
+ section = m.group(1)
+ subsection = None
+ continue
+
+ m = SUBSECTION.search(line)
+ if m is not None:
+ subsection = m.group(1)
+ continue
+
+ m = ENTRY.search(line)
+ if m is not None:
+ key = m.group(1)
+ val = m.group(2)
+ if subsection == "paths":
+ # print path, val, os.path.join(path, val)
+ val = self.relative_path(val, path)
+ if section is not None:
+ self.d[section + "." + key] = val
+ else:
+ self.d[key] = val
+
+ def relative_path(self, path, ref):
+ return os.path.join(os.path.dirname(ref), path)
+
+ def __getitem__(self, key):
+ return self.d[key]
diff --git a/dfs/database.py b/dfs/database.py
new file mode 100644
index 0000000..4f4458b
--- /dev/null
+++ b/dfs/database.py
@@ -0,0 +1,381 @@
+# -*- coding: utf-8 -*-
+import time
+from datetime import date, datetime
+from collections import defaultdict
+
+from xml.etree.ElementTree import Element, ElementTree, tostring
+import xml.etree.ElementTree as ET
+
+class Database:
+
+ def __init__(self, db, anno_per_file, from_file=True):
+ if from_file:
+ self.db_name = db
+ self.tree = ET.parse(db)
+ self.root = self.tree.getroot()
+ else:
+ self.db_name = None
+ self.tree = None
+ self.root = ET.fromstring(db)
+
+ self.anno_per_file = anno_per_file
+ self.make_index()
+
+ def owned(self, ann_elem):
+ return not self.finished(ann_elem) and not self.returned(ann_elem)
+
+ def finished(self, ann_elem):
+ return ann_elem.find("checkinDate") is not None
+
+ def returned(self, ann_elem):
+ return ann_elem.find("return") is not None
+
+ def get_reason(self, ann_elem):
+ reas = ann_elem.find("return").find("reason").text
+ if reas is None:
+ reas = ""
+ return reas
+
+ def rejected(self, file_elem):
+ return file_elem.find("rejected") is not None
+
+ def fixed(self, ann_elem):
+ ret = ann_elem.find("return")
+ if ret is not None and ret.find("fixed") is not None:
+ return True
+ return False
+
+ def ann_elem_for_user(self, file_elem, user):
+ idx = 0
+ for ann_elem in file_elem.findall("ann"):
+ if ann_elem.find("annName").text == user:
+ return ann_elem, idx
+ idx = idx + 1
+ return None, None
+
+ def sann_elem_for_user(self, file_elem, user):
+ for ann_elem in file_elem.findall("s_ann"):
+ if ann_elem.find("annName").text == user:
+ return ann_elem
+ return None
+
+ def fixed_for_user(self, file_elem, user, is_adj):
+ if is_adj:
+ ann = self.sann_elem_for_user(file_elem, user)
+ else:
+ ann, idx = self.ann_elem_for_user(file_elem, user)
+ if ann is not None:
+ return self.fixed(ann)
+ return False
+
+ def make_index(self):
+ self.file_index = {}
+ self.ann_owned_index = defaultdict(list)
+ self.adj_owned_index = defaultdict(list)
+
+ for file_elem in self.root:
+ name_elem = file_elem.find("name")
+ if name_elem == None:
+ raise Exception("No name assigned to a file element !")
+ self.file_index[name_elem.text] = file_elem
+
+ if self.rejected(file_elem):
+ continue
+
+ for ann_elem in file_elem.findall("ann"):
+ if self.owned(ann_elem):
+ ann_name = ann_elem.find("annName").text
+ self.ann_owned_index[ann_name].append(file_elem)
+
+ for ann_elem in file_elem.findall("s_ann"):
+ if self.owned(ann_elem):
+ ann_name = ann_elem.find("annName").text
+ self.adj_owned_index[ann_name].append(file_elem)
+
+ def fix(self, ann_elem):
+ if not self.returned(ann_elem):
+ raise Exception("File not returned in database!")
+ if self.fixed(ann_elem):
+ raise Exception("File already fixed in database!")
+
+ ret = ann_elem.find("return")
+ fixed = ET.SubElement(ret, "fixed")
+
+ def add(self, file_name):
+ if self.file_index.has_key(file_name):
+ raise Exception("File name already in database !")
+ file_elem = ET.SubElement(self.root, "file")
+ name_elem = ET.SubElement(file_elem, "name")
+ name_elem.text = file_name
+ date_elem = ET.SubElement(file_elem, "addDate")
+ date_elem.text = time.strftime("%c")
+ self.file_index[file_name] = file_elem
+
+ def remove(self, file_name):
+ if not self.file_index.has_key(file_name):
+ raise Exception("File name not in database !")
+ file_elem = self.file_index[file_name]
+ self.root.remove(file_elem)
+ self.make_index()
+
+ def reject(self, file_name, reason):
+ if not self.file_index.has_key(file_name):
+ raise Exception("File name not in database !")
+ file_elem = self.file_index[file_name]
+ if self.rejected(file_elem):
+ raise Exception("File already rejected !")
+ rej_elem = ET.SubElement(file_elem, "rejected")
+ rej_elem.text = reason
+ self.make_index()
+
+ # For prettyprint, used in 'save' method.
+ def indent(self, elem, level=0):
+ i = "\n" + level * " "
+ if len(elem):
+ if not elem.text or not elem.text.strip():
+ elem.text = i + " "
+ if not elem.tail or not elem.tail.strip():
+ elem.tail = i
+ for elem in elem:
+ self.indent(elem, level + 1)
+ if not elem.tail or not elem.tail.strip():
+ elem.tail = i
+ else:
+ if level and (not elem.tail or not elem.tail.strip()):
+ elem.tail = i
+
+ def save(self):
+ if not self.db_name:
+ raise Exception("XML not read from file !")
+
+ self.indent(self.root)
+ self.tree.write(self.db_name, encoding="utf-8")
+
+ def upload_prevention(self, file_name, user, is_super):
+ if not self.file_index.has_key(file_name):
+ return "Filename " + file_name.encode('utf-8') + " not known by server."
+ file_elem = self.file_index[file_name]
+ if self.rejected(file_elem):
+ return "File rejected: " + file_name.encode('utf-8')
+
+ if is_super:
+ ann_elem = file_elem.find("s_ann")
+ if ann_elem is None:
+ return "No superannotator assigned to file " + file_name.encode('utf-8')
+
+ if user != ann_elem.find("annName").text:
+ return "Different superannotator assigned to file " + file_name.encode('utf-8')
+
+ if ann_elem.find("checkinDate") is not None:
+ return "File " + file_name.encode('utf-8') + " already checked in!"
+
+ return None
+
+ else:
+ annotations = file_elem.findall("ann")
+ owners = map(lambda ann: ann.find("annName").text, annotations)
+ if not user in owners:
+ return "User " + user + " doesn't own the file " + file_name.encode('utf-8')
+
+ i = owners.index(user)
+ ann_elem = annotations[i]
+ if ann_elem.find("checkinDate") is not None:
+ return "User " + user + " already checked in the file " + file_name.encode('utf-8')
+
+ return None
+
+ def upload_dest(self, file_name, user):
+ """Upload destination (annName element and ID -- 1 or 2) directory."""
+ file_elem = self.file_index[file_name]
+ annotations = file_elem.findall("ann")
+
+ owners = map(lambda ann: ann.find("annName").text, annotations)
+ exc = Exception("Cannot upload " + file_name.encode('utf-8')
+ + " file by " + user + " annotator !")
+ if not user in owners:
+ raise exc
+
+ i = owners.index(user)
+ ann_elem = annotations[i]
+ if ann_elem.find("checkinDate") is not None:
+ raise exc
+
+ return (ann_elem, i)
+
+ def upload_id(self, file_name, user):
+ return self.upload_dest(file_name, user)[1]
+
+ def return_file(self, file_name, user, reason):
+ (ann_elem, i) = self.upload_dest(file_name, user)
+ ret_elem = ET.SubElement(ann_elem, "return")
+ date_elem = ET.SubElement(ret_elem, "date")
+ date_elem.text = time.strftime("%c")
+ date_elem = ET.SubElement(ret_elem, "reason")
+ if reason is None:
+ reason = ""
+ date_elem.text = reason
+ return i
+
+ def return_file_prim(self, file_name, user, reason):
+ file_elem = self.file_index[file_name]
+ ann_elem = file_elem.find("s_ann")
+
+ if (user != ann_elem.find("annName").text or
+ ann_elem.find("checkinDate") is not None):
+ raise Exception("Cannot upload " + file_name.encode('utf-8')
+ + " file by " + user + " annotator !")
+
+ ret_elem = ET.SubElement(ann_elem, "return")
+ date_elem = ET.SubElement(ret_elem, "date")
+ date_elem.text = time.strftime("%c")
+ date_elem = ET.SubElement(ret_elem, "reason")
+ if reason is None:
+ reason = ""
+ date_elem.text = reason
+
+ def upload(self, file_name, user):
+ (ann_elem, i) = self.upload_dest(file_name, user)
+ date_elem = ET.SubElement(ann_elem, "checkinDate")
+ date_elem.text = time.strftime("%c")
+ return i
+
+ def upload_prim(self, file_name, user):
+ file_elem = self.file_index[file_name]
+ ann_elem = file_elem.find("s_ann")
+
+ if (user != ann_elem.find("annName").text or self.finished(ann_elem)):
+ raise Exception("Cannot upload " + file_name.encode('utf-8')
+ + " file by " + user + " annotator !")
+
+ date_elem = ET.SubElement(ann_elem, "checkinDate")
+ date_elem.text = time.strftime("%c")
+
+ def download(self, file_name, user):
+ file_elem = self.file_index[file_name]
+
+ annotations = file_elem.findall("ann")
+ owners = map(lambda ann: ann.find("annName").text, annotations)
+
+ if self.fixed_for_user(file_elem, user, False):
+ ann, idx = self.ann_elem_for_user(file_elem, user)
+ date = ann.find("return").find("date").text
+ ann.remove(ann.find("return"))
+
+ date_elem = ET.SubElement(ann, "returnDate")
+ date_elem.text = date
+
+ date_elem = ET.SubElement(ann, "checkoutDate")
+ date_elem.text = time.strftime("%c")
+
+ return idx
+
+ else:
+ if user in owners or len(owners) + 1 > self.anno_per_file:
+ raise Exception("Cannot set " + user + " annotator to '"
+ + file_name.encode('utf-8') + "' file !")
+
+ ann_elem = ET.SubElement(file_elem, "ann")
+ ann_name_elem = ET.SubElement(ann_elem, "annName")
+ ann_name_elem.text = user
+
+ date_elem = ET.SubElement(ann_elem, "checkoutDate")
+ date_elem.text = time.strftime("%c")
+
+ return None
+
+ def download_prim(self, file_name, user):
+ file_elem = self.file_index[file_name]
+
+ if self.fixed_for_user(file_elem, user, True):
+ sann = self.sann_elem_for_user(file_elem, user)
+ date = sann.find("return").find("date").text
+ sann.remove(sann.find("return"))
+
+ date_elem = ET.SubElement(sann, "returnDate")
+ date_elem.text = date
+
+ date_elem = ET.SubElement(sann, "checkoutDate")
+ date_elem.text = time.strftime("%c")
+
+ return True
+
+ else:
+ if len(file_elem.findall("s_ann")) > 0:
+ raise Exception("Cannot set " + user + " adjudicator to '"
+ + file_name.encode('utf-8') + "' file !")
+
+ ann_elem = ET.SubElement(file_elem, "s_ann")
+ ann_name_elem = ET.SubElement(ann_elem, "annName")
+ ann_name_elem.text = user
+
+ date_elem = ET.SubElement(ann_elem, "checkoutDate")
+ date_elem.text = time.strftime("%c")
+
+ return False
+
+ def for_annotation(self, user):
+ priority = []
+ result = []
+ for file_elem in self.root:
+ if self.rejected(file_elem) or file_elem.find("s_ann") is not None:
+ continue
+
+ anns = file_elem.findall("ann")
+ owners = map(lambda ann: ann.find("annName").text, anns)
+ if len(owners) < self.anno_per_file and user not in owners:
+ result.append(file_elem.find("name").text)
+
+ if self.fixed_for_user(file_elem, user, False):
+ priority.append(file_elem.find("name").text)
+
+ return priority, result
+
+ def for_adjudication(self, user):
+ priority = []
+ result = []
+ for file_elem in self.root:
+ if self.rejected(file_elem):
+ continue
+
+ anns = file_elem.findall("ann")
+ owners = map(lambda ann: ann.find("annName").text, anns)
+ checked = map(lambda ann: ann.find("checkinDate") != None, anns)
+ if (len(owners) == self.anno_per_file
+ and file_elem.find("s_ann") is None
+ and user not in owners
+ and all(checked)):
+ result.append(file_elem.find("name").text)
+
+ if self.fixed_for_user(file_elem, user, True):
+ priority.append(file_elem.find("name").text)
+
+ return priority, result
+
+ def owns_normal(self, user):
+ return [ file_elem.find("name").text
+ for file_elem
+ in self.ann_owned_index[user] ]
+
+ def owns_super(self, user):
+ return [ file_elem.find("name").text
+ for file_elem
+ in self.adj_owned_index[user] ]
+
+ def owns(self, user):
+ return self.owns_normal(user) + self.owns_super(user)
+
+ def finished_count(self, user):
+ n = 0
+ for file_elem in self.root:
+
+ if self.rejected(file_elem):
+ continue
+
+ items = file_elem.findall("ann")
+ items.extend(file_elem.findall("s_ann"))
+ for ann_elem in items:
+ ann_name = ann_elem.find("annName").text
+ if ann_name == user and self.finished(ann_elem) and not self.returned(ann_elem):
+ n += 1
+
+ return n
diff --git a/dfs/msg/__init__.py b/dfs/msg/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/dfs/msg/__init__.py
diff --git a/dfs/msg/message.py b/dfs/msg/message.py
new file mode 100644
index 0000000..295fb11
--- /dev/null
+++ b/dfs/msg/message.py
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+__all__ = ["decode", "Message", "OkMessage", "KoMessage", "DownloadRequest"
+ , "UploadRequest", "DownloadPrimRequest", "ClientDone", "ServerDone"
+ , "NumMessage", "BoolMessage", "ReturnRequest", "CheckoutRequest",
+ "CheckinRequest", "StatsRequest"]
+
+
+class Message(object):
+
+ """
+ Base class for SSL communication messages.
+ """
+
+ def __init__(self, contents=""):
+ self.contents = contents
+
+ def get_contents(self):
+ return self.contents
+
+ @classmethod
+ def from_contents(cls, contents):
+ return cls(contents)
+
+ def encode(self):
+ """
+ Prepare message to be sent over network.
+ WARNING: Do not override this method.
+ """
+ # Add "-" on the beggining to prevent
+ # message contents from being void.
+ return self.msg_type, "-" + self.get_contents()
+
+
+class NumMessage(Message):
+
+ def __init__(self, n):
+ self.number = n
+
+ def get_contents(self):
+ return str(self.number)
+
+ @classmethod
+ def from_contents(cls, contents):
+ return cls(int(contents))
+
+ def get_number(self):
+ return self.number
+
+
+class BoolMessage(Message):
+
+ def __init__(self, b):
+ self.b = b
+
+ def get_contents(self):
+ return "1" if self.b else "0"
+
+ @classmethod
+ def from_contents(cls, contents):
+ return cls(bool(int(contents)))
+
+ def get_boolean(self):
+ return self.b
+
+class ReturnRequest(NumMessage):
+ pass
+
+class CheckoutRequest(Message):
+ pass
+
+class CheckinRequest(NumMessage):
+ pass
+
+class DownloadRequest(NumMessage):
+ pass
+
+class DownloadPrimRequest(NumMessage):
+ pass
+
+class UploadRequest(NumMessage):
+ pass
+
+class OkMessage(Message):
+ pass
+
+class KoMessage(Message):
+ pass
+
+class StatsRequest(Message):
+ pass
+
+class ServerDone(Message):
+ pass
+
+class ClientDone(Message):
+ pass
+
+
+def subclasses(cls):
+ yield cls
+ for child in cls.__subclasses__():
+ for desc in subclasses(child):
+ yield desc
+
+def decode(msg_type, contents):
+ for cls in subclasses(Message):
+ if cls.msg_type == msg_type:
+ # Discard first, dummy character;
+ # Confront Message.encode method.
+ return cls.from_contents(contents[1:])
+
+for i, cls in enumerate(subclasses(Message)):
+ cls.msg_type = i
diff --git a/dfs/msg/stream.py b/dfs/msg/stream.py
new file mode 100644
index 0000000..116bab0
--- /dev/null
+++ b/dfs/msg/stream.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+import struct
+
+from message import decode
+
+__all__ = ["read_msg", "write_msg", "EndOfStream"]
+
+class EndOfStream(Exception):
+ pass
+
+class BadMessage(Exception):
+ pass
+
+def read_msg(connstream):
+ encoded = read_encoded(connstream)
+ return decode(*encoded)
+
+def accept_msg(connstream, cls):
+ msg = read_msg(connstream)
+ if type(msg) != cls:
+ raise BadMessage("Expected %s, got %s" %
+ (cls.__name__, type(msg).__name__))
+ return msg
+
+def write_msg(connstream, msg):
+ encoded = msg.encode()
+ write_encoded(connstream, *encoded)
+
+def read_encoded(connstream):
+ _type = read_type(connstream)
+ length = read_length(connstream)
+ content = read_content(connstream, length)
+ return _type, content
+
+def write_encoded(connstream, _type, content):
+ write_type(connstream, _type)
+ write_length(connstream, len(content))
+ write_content(connstream, content)
+
+def read_type(connstream):
+ data = read_all(connstream, 1)
+ return struct.unpack("!B", data)[0]
+
+def write_type(connstream, _type):
+ data = struct.pack("!B", _type)
+ write_all(connstream, data)
+
+def read_length(connstream):
+ data = read_all(connstream, 4)
+ return struct.unpack("!L", data)[0]
+
+def write_length(connstream, n):
+ data = struct.pack("!L", n)
+ write_all(connstream, data)
+
+def read_content(connstream, length):
+ return read_all(connstream, length)
+
+def write_content(connstream, content):
+ write_all(connstream, content)
+
+def write_all(connstream, data):
+ k = connstream.write(data)
+ if k != len(data):
+ raise Exception("Did not send all data !")
+
+def read_all(connstream, k):
+ data = ""
+ while k > 0:
+ # part = connstream.read()
+ part = connstream.read(min(k, 1024))
+ if len(part) == 0:
+ raise EndOfStream
+ k -= len(part)
+ # if k < 0:
+ # raise Exception("Received more than expected !")
+ data += part
+ return data
+
diff --git a/dfs/repo.py b/dfs/repo.py
new file mode 100644
index 0000000..9729984
--- /dev/null
+++ b/dfs/repo.py
@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+import os
+import shutil
+import pysvn
+
+class Repo:
+
+ def __init__(self, path, login, passwd):
+ self.path = path
+ def get_login(_realm, _username, _may_save):
+ return True, login, passwd, False
+ self.svn = pysvn.Client()
+ self.svn.callback_get_login = get_login
+ self.svn.update(self.path)
+
+ def add(self, src_path):
+ part_name = os.path.basename(src_path)
+ dest_path = os.path.join(self.new_path(), part_name)
+
+ if (not self.svn.info(dest_path) is None) or (os.path.exists(dest_path)):
+ raise Exception(src_path+" already present in repository.")
+
+ shutil.copy(src_path, dest_path)
+ self.svn.add(dest_path)
+
+ def remove(self, filename, anno_per_file):
+ paths = set()
+ paths.add(os.path.join(self.new_path(), filename))
+ for i in range(anno_per_file):
+ paths.add(self.upload_path(filename, i))
+ paths.add(os.path.join(self.finito_path(), filename))
+
+ removed = []
+ for path in paths:
+ if os.path.exists(path):
+ self.svn.remove(path)
+ removed.append(path)
+
+ return removed
+
+ def db_path(self):
+ return os.path.join(self.path, "db.xml")
+
+ def new_path(self):
+ return os.path.join(self.path, "new")
+
+ def ann_path(self, idx):
+ ids = chr(idx+ord('A'))
+ tail = os.path.join("annotation", ids)
+ return os.path.join(self.path, tail)
+
+ def finito_path(self):
+ return os.path.join(self.path, "adjudication")
+
+ def upload_path(self, file_name, idx):
+ return os.path.join(self.ann_path(idx), file_name)
+
+ def upload_prim_path(self, file_name):
+ return os.path.join(self.finito_path(), file_name)
+
+ def read_data(self, path):
+ with open(path) as src:
+ return src.read()
+
+ def read_contents(self, path, exts):
+ contents = {}
+ for ext in exts:
+ contents[ext] = self.read_data(path + ext)
+ return contents
+
+ def write_data(self, path, data):
+ with open(path, "w") as dest:
+ dest.write(data)
+
+ def write_contents(self, path, contents):
+ for ext, data in contents.iteritems():
+ dest_path = path + ext
+ self.write_data(dest_path, data)
+ if self.svn.info(dest_path) is None:
+ self.svn.add(dest_path)
+
+ def upload(self, file_id, idx, contents):
+ self.write_contents(self.upload_path(file_id, idx), contents)
+
+ def upload_prim(self, file_id, contents):
+ self.write_contents(self.upload_prim_path(file_id), contents)
+
+ def download(self, file_id, exts):
+ return self.read_contents(os.path.join(self.new_path(), file_id), exts)
+
+ def download_prim(self, file_id, exts, anno_per_file):
+ result = []
+ for i in range(anno_per_file):
+ conts = self.read_contents(os.path.join(self.ann_path(i), file_id), exts)
+ result.append(conts)
+ return result
+
+ def checkout(self, file_id, idx, exts):
+ src_paths = [ self.upload_path(file_id, idx)
+ , os.path.join(self.new_path(), file_id) ]
+ for path in src_paths:
+ if all(os.path.exists(path + ext) for ext in exts):
+ return self.read_contents(path, exts)
+
+ def checkout_prim(self, file_id, exts):
+ path = os.path.join(self.finito_path(), file_id)
+ if all(os.path.exists(path + ext) for ext in exts):
+ return self.read_contents(path, exts)
+
+ def commit(self, message):
+ self.svn.checkin(self.path, message, recurse=True)
+
+ def revert(self):
+ self.svn.revert(self.path, recurse=True)
diff --git a/dfs/server.py b/dfs/server.py
new file mode 100644
index 0000000..639e5ec
--- /dev/null
+++ b/dfs/server.py
@@ -0,0 +1,401 @@
+# -*- coding: utf-8 -*-
+import sys
+import os
+import traceback
+import socket, ssl
+import random
+from optparse import OptionParser
+import signal
+
+from utils import validate_user, UserInvalid, send_contents, receive_contents
+from msg.stream import read_msg, write_msg, accept_msg, EndOfStream, BadMessage
+from msg.message import *
+from repo import Repo
+from database import Database
+from config import Config
+
+class SSLServer:
+
+ def __init__(self, host, port, backlog, cert_file, key_file,
+ repository, svn_login, svn_passwd, pass_file,
+ users_file, file_exts, anno_per_file, log=None):
+ """
+ Initialize SSL server.
+
+ params:
+ =======
+ host : str
+ Host name.
+ port : int
+ Port number.
+ backlog : int
+ Maximum number of waiting connections.
+ cert_file : path
+ Public PEM certificate.
+ key_file : path
+ Private key.
+ pass_file : path
+ File with client passwords.
+ users_file : path
+ File with additional users configuration.
+ repository : path
+ Subversion working copy.
+ svn_login : str
+ Subversion login.
+ svn_passwd : str
+ Subversion password.
+ file_exts : [str]
+ List of file extensions.
+ anno_per_file : int
+ Number of normal annotators per file (1 or 2)
+ """
+ self.cert_file = cert_file
+ self.key_file = key_file
+ self.pass_file = pass_file
+ self.users_file = users_file
+ self.wc = Repo(repository, svn_login, svn_passwd)
+ self.file_exts = file_exts
+ if log is not None:
+ self.log = open(log, "a")
+ else:
+ self.log = sys.stdout
+ self.bound = bind_socket(host, port, backlog=backlog)
+ self.serving = None
+ self.anno_per_file = anno_per_file
+
+ def run(self):
+ while True:
+ newsocket, fromaddr = self.bound.accept()
+ self.serving = fromaddr
+ connstream = None
+ try:
+ connstream = ssl.wrap_socket(newsocket,
+ server_side=True,
+ certfile=self.cert_file,
+ keyfile=self.key_file,
+ ssl_version=ssl.PROTOCOL_TLSv1)
+
+ login = validate_user(connstream, self.pass_file)
+ self.serve_client(connstream, login)
+ except UserInvalid as login:
+ print >> self.log, "AUTH ERROR:", login
+ except BadMessage as info:
+ print >> self.log, "BAD MESSAGE ERROR:", info
+ print >> self.log, traceback.format_exc().strip()
+ except EndOfStream:
+ print >> self.log, "UNEXPECTED END OF STREAM:"
+ print >> self.log, traceback.format_exc().strip()
+ except:
+ print >> self.log, "UNEXPECTED ERROR:"
+ print >> self.log, traceback.format_exc().strip()
+ finally:
+ try:
+ if not connstream is None:
+ connstream.shutdown(socket.SHUT_RDWR)
+ except:
+ print >> self.log, "UNEXPECTED ERROR:"
+ print >> self.log, traceback.format_exc().strip()
+ finally:
+ if not connstream is None:
+ connstream.close()
+ self.serving = None
+
+ def exit(self):
+ if self.serving is not None:
+ print ("Client from %s connected to the server"
+ % str(self.serving))
+ else:
+ sys.exit(0)
+
+ def process_msg(self, msg, stream, login):
+ if type(msg) == UploadRequest:
+ self.process_upload_msg(msg, stream, login, checkin=False)
+ elif type(msg) == CheckinRequest:
+ self.process_upload_msg(msg, stream, login, checkin=True)
+ elif type(msg) == CheckoutRequest:
+ self.process_checkout_msg(msg, stream, login)
+ elif type(msg) == DownloadRequest:
+ self.process_download_msg(msg, stream, login)
+ elif type(msg) == DownloadPrimRequest:
+ self.process_download_prim_msg(msg, stream, login)
+ elif type(msg) == ReturnRequest:
+ self.process_return_msg(msg, stream, login)
+ elif type(msg) == StatsRequest:
+ self.process_stats_msg(msg, stream, login)
+ else:
+ print >> self.log, login, "sent:", msg
+
+ def process_return_msg(self, msg, stream, login):
+ reason = accept_msg(stream, Message).get_contents().decode("utf-8")
+
+ usr_cfg = Config(self.users_file)
+ try:
+ user_adj = login in usr_cfg["auth.adjudicators"].split()
+ except KeyError:
+ user_adj = False
+
+ db = Database(self.wc.db_path(), self.anno_per_file)
+ n = msg.get_number()
+ for _ in range(n):
+ file_id = accept_msg(stream, Message).get_contents()
+ is_adj = accept_msg(stream, BoolMessage).get_boolean()
+ if is_adj and not user_adj:
+ write_msg(stream, KoMessage())
+ print >> self.log, "Upload error - user " + login + " tried to upload super annotated file " + file_id + " but is not a super annotator"
+ continue
+ else:
+ error = db.upload_prevention(file_id, login, is_adj)
+ if error is not None:
+ write_msg(stream, KoMessage())
+ print >> self.log, "Upload error by user " + login + ". Details: "+error
+ continue
+
+ write_msg(stream, OkMessage())
+ contents = receive_contents(stream)
+ for key in contents.keys():
+ if key not in self.file_exts:
+ print >> self.log, "Skipped file uploaded by user " + login + ". Filename:"+file_id+key
+ del contents[key]
+
+ if is_adj is False:
+ idx = db.return_file(file_id, login, reason)
+ self.wc.upload(file_id, idx, contents)
+ else:
+ db.return_file_prim(file_id, login, reason)
+ self.wc.upload_prim(file_id, contents)
+
+ db.save()
+ accept_msg(stream, ClientDone)
+ self.wc.commit("return request from %s"
+ % (login))
+ write_msg(stream, ServerDone())
+
+
+ def process_upload_msg(self, msg, stream, login, checkin=False):
+ usr_cfg = Config(self.users_file)
+ try:
+ user_adj = login in usr_cfg["auth.adjudicators"].split()
+ except KeyError:
+ user_adj = False
+
+ db = Database(self.wc.db_path(), self.anno_per_file)
+ n = msg.get_number()
+ for _ in range(n):
+ file_id = accept_msg(stream, Message).get_contents()
+ is_adj = accept_msg(stream, BoolMessage).get_boolean()
+ if is_adj and not user_adj:
+ write_msg(stream, KoMessage())
+ print >> self.log, "Upload error - user " + login + " tried to upload super annotated file " + file_id + " but is not a super annotator"
+ continue
+ else:
+ error = db.upload_prevention(file_id, login, is_adj)
+ if error is not None:
+ write_msg(stream, KoMessage())
+ print >> self.log, "Upload error by user " + login + ". Details: "+error
+ continue
+
+ write_msg(stream, OkMessage())
+ contents = receive_contents(stream)
+ for key in contents.keys():
+ if key not in self.file_exts:
+ print >> self.log, "Skipped file uploaded by user " + login + ". Filename:"+file_id+key
+ del contents[key]
+
+ if is_adj is False:
+ if checkin:
+ idx = db.upload(file_id, login)
+ else:
+ idx = db.upload_id(file_id, login)
+ self.wc.upload(file_id, idx, contents)
+ else:
+ if checkin:
+ db.upload_prim(file_id, login)
+ self.wc.upload_prim(file_id, contents)
+
+ if checkin:
+ db.save()
+ accept_msg(stream, ClientDone)
+ self.wc.commit("%s request from %s"
+ % ("checkin" if checkin else "upload", login))
+ write_msg(stream, ServerDone())
+
+ def process_checkout_msg(self, msg, stream, login):
+ db = Database(self.wc.db_path(), self.anno_per_file)
+
+ ann_files = db.owns_normal(login)
+ write_msg(stream, NumMessage(len(ann_files)))
+ for file_id in ann_files:
+ write_msg(stream, Message(file_id))
+ idx = db.upload_id(file_id, login)
+ contents = self.wc.checkout(file_id, idx, self.file_exts)
+ send_contents(stream, contents)
+
+ adj_files = db.owns_super(login)
+ write_msg(stream, NumMessage(len(adj_files)))
+ write_msg(stream, NumMessage(self.anno_per_file))
+ for file_id in adj_files:
+ write_msg(stream, Message(file_id))
+
+ contents = self.wc.download_prim(file_id, self.file_exts, self.anno_per_file)
+ for conts in contents:
+ send_contents(stream, conts)
+
+ conts3 = self.wc.checkout_prim(file_id, self.file_exts)
+ if conts3 is None:
+ write_msg(stream, BoolMessage(False))
+ else:
+ write_msg(stream, BoolMessage(True))
+ send_contents(stream, conts3)
+
+ accept_msg(stream, ClientDone)
+ #self.wc.commit("checkout request from %s" % login)
+ write_msg(stream, ServerDone())
+
+ def process_stats_msg(self, msg, stream, login):
+ usr_cfg = Config(self.users_file)
+ db = Database(self.wc.db_path(), self.anno_per_file)
+
+ write_msg(stream, NumMessage(db.finished_count(login)))
+
+ accept_msg(stream, ClientDone)
+ write_msg(stream, ServerDone())
+
+ def process_download_msg(self, msg, stream, login):
+ usr_cfg = Config(self.users_file)
+ db = Database(self.wc.db_path(), self.anno_per_file)
+
+ down_num = msg.get_number()
+ owns_num = len(db.owns_normal(login))
+ limit = float('inf')
+ try:
+ limit = int(usr_cfg["limits.%s.annotation" % login])
+ except KeyError:
+ limit = int(usr_cfg["limits.annotation"])
+ if down_num + owns_num > limit:
+ down_num = max(0, limit - owns_num)
+
+ for_ann_fixed, for_ann = db.for_annotation(login)
+ if len(for_ann) + len(for_ann_fixed) == 0:
+ print >> self.log, "User " + login + " has no more files to download for annotation."
+
+ down_files = sample(for_ann_fixed, down_num)
+ down_num = max(0, down_num - len(down_files))
+ down_files = down_files + sample(for_ann, down_num)
+
+ write_msg(stream, NumMessage(len(down_files)))
+
+ for file_id in down_files:
+ fixed_idx = db.download(file_id, login)
+ write_msg(stream, Message(file_id))
+ if fixed_idx is None:
+ contents = self.wc.download(file_id, self.file_exts)
+ else:
+ contents = self.wc.checkout(file_id, fixed_idx, self.file_exts)
+ send_contents(stream, contents)
+ db.save()
+ accept_msg(stream, ClientDone)
+ self.wc.commit("download request from %s" % login)
+ write_msg(stream, ServerDone())
+
+ def process_download_prim_msg(self, msg, stream, login):
+ usr_cfg = Config(self.users_file)
+ try:
+ adjudicators = usr_cfg["auth.adjudicators"].split()
+ except KeyError:
+ adjudicators = []
+ if login in adjudicators:
+ write_msg(stream, OkMessage())
+ else:
+ write_msg(stream, KoMessage())
+ return
+ db = Database(self.wc.db_path(), self.anno_per_file)
+
+ down_num = msg.get_number()
+ owns_num = len(db.owns_super(login))
+ limit = float('inf')
+ try:
+ limit = int(usr_cfg["limits.%s.adjudication" % login])
+ except KeyError:
+ limit = int(usr_cfg["limits.adjudication"])
+ if down_num + owns_num > limit:
+ down_num = max(0, limit - owns_num)
+
+ for_adj_fixed, for_adj = db.for_adjudication(login)
+ if len(for_adj) + len(for_adj_fixed) == 0:
+ print >> self.log, "User " + login + " has no more files to download for superannotation."
+
+ down_files = sample(for_adj_fixed, down_num)
+ down_num = max(0, down_num - len(down_files))
+ down_files = down_files + sample(for_adj, down_num)
+
+ write_msg(stream, NumMessage(len(down_files)))
+ write_msg(stream, NumMessage(self.anno_per_file))
+ for file_id in down_files:
+ write_msg(stream, Message(file_id))
+ fixed = db.download_prim(file_id, login)
+
+ contents = self.wc.download_prim(file_id, self.file_exts, self.anno_per_file)
+ for conts in contents:
+ send_contents(stream, conts)
+
+ if fixed:
+ write_msg(stream, BoolMessage(True))
+ contents = self.wc.checkout_prim(file_id, self.file_exts)
+ send_contents(stream, contents)
+ else:
+ write_msg(stream, BoolMessage(False))
+
+ db.save()
+ accept_msg(stream, ClientDone)
+ self.wc.commit("download' request from %s" % login)
+ write_msg(stream, ServerDone())
+
+ def serve_client(self, connstream, login):
+ msg = read_msg(connstream)
+ try:
+ self.process_msg(msg, connstream, login)
+ except:
+ self.wc.revert()
+ raise
+
+def bind_socket(host, port, backlog=5):
+ bound = socket.socket()
+ bound.bind((host, port))
+ bound.listen(backlog)
+ return bound
+
+def sample(population, n):
+ if len(population) > n:
+ return random.sample(population, n)
+ else:
+ return population
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog CONFIG""")
+ (options, args) = optparser.parse_args()
+ if len(args) != 1:
+ optparser.print_help()
+ sys.exit(0)
+
+ cfg = Config(args[0])
+ server = SSLServer(
+ host=cfg["connection.host"],
+ port=int(cfg["connection.port"]),
+ backlog=int(cfg["connection.backlog"]),
+ cert_file=cfg["connection.certfile"],
+ key_file=cfg["connection.keyfile"],
+ repository=cfg["svn.repository"],
+ svn_login=cfg["svn.login"],
+ svn_passwd=cfg["svn.passwd"],
+ pass_file=cfg["users.passfile"],
+ users_file=cfg["users.config"],
+ file_exts=cfg["file_extensions"].split(),
+ anno_per_file=int(cfg["anno_per_file"])
+ )
+
+ def handler(signum, frame):
+ server.exit()
+
+ signal.signal(signal.SIGINT, handler)
+
+ server.run()
diff --git a/dfs/utils.py b/dfs/utils.py
new file mode 100644
index 0000000..e8a6d88
--- /dev/null
+++ b/dfs/utils.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+import hashlib
+
+from msg.stream import accept_msg, write_msg
+from msg.message import Message, OkMessage, KoMessage, NumMessage
+
+class UserInvalid(Exception):
+ pass
+
+def validate_user(conns, pass_file):
+ pass_dict = parse_passwd(pass_file)
+ login = accept_msg(conns, Message).get_contents()
+ passwd = accept_msg(conns, Message).get_contents()
+ if pass_dict.has_key(login) and pass_dict[login] == passwd:
+ write_msg(conns, OkMessage())
+ return login
+ else:
+ write_msg(conns, KoMessage())
+ raise UserInvalid(login)
+
+def parse_passwd(pass_file):
+ result = {}
+ with open(pass_file) as f:
+ for line in f:
+ login, passwd = line.split("=")
+ login = login.strip()
+ passwd = passwd.strip()
+ result[login] = passwd
+ return result
+
+def _check_sum(*args):
+ m = hashlib.sha256()
+ for arg in args:
+ m.update(arg)
+ return m
+
+def check_sum(*args):
+ return _check_sum(*args).digest()
+
+def check_hexsum(*args):
+ return _check_sum(*args).hexdigest()
+
+def send_contents(stream, contents):
+ write_msg(stream, NumMessage(len(contents)))
+ for ext, data in contents.iteritems():
+ write_msg(stream, Message(ext))
+ write_msg(stream, Message(data))
+ write_msg(stream, Message(check_sum(ext, data)))
+ accept_msg(stream, OkMessage)
+
+def receive_contents(stream):
+ contents = {}
+ n = accept_msg(stream, NumMessage).get_number()
+ for _ in range(n):
+ ext = accept_msg(stream, Message).get_contents()
+ data = accept_msg(stream, Message).get_contents()
+ csum = accept_msg(stream, Message).get_contents()
+ if csum == check_sum(ext, data):
+ write_msg(stream, OkMessage())
+ else:
+ write_msg(stream, KoMessage())
+ contents[ext] = data
+ return contents
diff --git a/example_configuration/passwd b/example_configuration/passwd
new file mode 100644
index 0000000..c8a9d0f
--- /dev/null
+++ b/example_configuration/passwd
@@ -0,0 +1,6 @@
+adj1 = adj1
+adj2 = adj2
+ann1 = ann1
+ann2 = ann2
+ann3 = ann3
+ann4 = ann4
diff --git a/example_configuration/server.cfg b/example_configuration/server.cfg
new file mode 100644
index 0000000..6c274a7
--- /dev/null
+++ b/example_configuration/server.cfg
@@ -0,0 +1,23 @@
+file_extensions = .mmax _words.xml _mentions.xml
+anno_per_file = 2
+
+[connection]
+host = localhost
+port = 2225
+backlog = 5 # maximum number of waiting connections
+{paths} # paths relative to configuration file
+certfile = ssl.conf/cert.pem
+keyfile = ssl.conf/key.pem
+
+
+[svn]
+login = machine
+passwd = asdasdT7
+{paths}
+repository = ../data/repo_wc # path to working copy of repo
+
+
+[users]
+{paths}
+config = users.cfg
+passfile = passwd
diff --git a/example_configuration/ssl.conf/cert.pem b/example_configuration/ssl.conf/cert.pem
new file mode 100644
index 0000000..0dd2101
--- /dev/null
+++ b/example_configuration/ssl.conf/cert.pem
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICkDCCAfmgAwIBAgIJAJPcESSEV86nMA0GCSqGSIb3DQEBBQUAMGExCzAJBgNV
+BAYTAlBMMQ0wCwYDVQQIDARUZXN0MQ0wCwYDVQQHDARDaXR5MRAwDgYDVQQKDAdD
+b21wYW55MSIwIAYJKoZIhvcNAQkBFhNleGFtcGxlQGV4YW1wbGUuY29tMB4XDTEz
+MDEyMzEyMjIwN1oXDTE0MDEyMzEyMjIwN1owYTELMAkGA1UEBhMCUEwxDTALBgNV
+BAgMBFRlc3QxDTALBgNVBAcMBENpdHkxEDAOBgNVBAoMB0NvbXBhbnkxIjAgBgkq
+hkiG9w0BCQEWE2V4YW1wbGVAZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQAD
+gY0AMIGJAoGBAKUG9QHjUiCSjcvFQl1OqUgZ8c4Vzm7m7Ua7D33YDnGo3QGuhPML
+0JB+JPBG++MrPdQqMV8eeXe1dFlrm6SHnrEZIhwZg8iDnPhFKNjHiAkx0PCQdI93
+rAe4EcszkgbQCXXLGjQJNDikwGZ6ciylJ7C2XewWTlWjv7/LqMwhQotLAgMBAAGj
+UDBOMB0GA1UdDgQWBBRLRvOFIo9LIeiT2ohQalEzy687ijAfBgNVHSMEGDAWgBRL
+RvOFIo9LIeiT2ohQalEzy687ijAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUA
+A4GBAD1odtgH1kZ4a0TEbwjK2PaHTYyofkkgeXsGGnG3eMkwKQ7PHJarBl7GtxP5
+Ls91k/KvBuNqCM/lPmM94USDSXrKGe0Zzc9WZbDG9SXO23gGOwPl+Tg9v1ocRHd2
+dYYIgZM8/tLcsHyNNMd7c4BOMD4rzkjgbThFFMl/2NSkvrO7
+-----END CERTIFICATE-----
diff --git a/example_configuration/ssl.conf/key.pem b/example_configuration/ssl.conf/key.pem
new file mode 100644
index 0000000..a179014
--- /dev/null
+++ b/example_configuration/ssl.conf/key.pem
@@ -0,0 +1,16 @@
+-----BEGIN PRIVATE KEY-----
+MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAKUG9QHjUiCSjcvF
+Ql1OqUgZ8c4Vzm7m7Ua7D33YDnGo3QGuhPML0JB+JPBG++MrPdQqMV8eeXe1dFlr
+m6SHnrEZIhwZg8iDnPhFKNjHiAkx0PCQdI93rAe4EcszkgbQCXXLGjQJNDikwGZ6
+ciylJ7C2XewWTlWjv7/LqMwhQotLAgMBAAECgYEAjx9XggmqgoFX5K/xfIbqHZQS
+uE8FN/2jL0Kwjs3AySZQdlLrDvbiLU6ZrqGBd2VbPBRTuVDuu7ltbNf8plbQcxaa
+yaKfKHPRYYIh/EqdaqsZ2bHHEARi03iNLDnktYz49I0ebFqgz0BGZVR3xDsjCu40
+s9Tku7IQsDdO/zVvreECQQDXHW1vli7iuOmTYfwKqeIV7wFPj0vp8XrUVOKqjiG1
+Fdf2ZlLrzeR/HbPbd2SL6u1dMV1j8wH/yPNRzFqNwOXpAkEAxGR5DzqrEE2ihyKL
+WzQn6tTTA9FI8WPbgXdTm/R3dev2OgMLZycHlSgptCJ1f+fMfGNhWhotg8kZfrO0
+9qnDEwJBAIpKrNQT5Mh0wBSz5WillmcrY9hV8yPOznw6pg2nmhvkkxYg6iYWE1N1
+MO9ibg1VVouW9McQrrRV57cqfPysiiECQHP63d9Xh1R+dKAXj3LSD0zIWMGlY81i
+amw/uvmb2ryiF+xUhfRqATld2ZsOypM5ofJHgmOCmCR+q3a+y/zrbk8CQQC0jx/T
+TYw9ULoUe0bdx9RfJkrCEr7/+/DLs4MFwPiQ/e8foHcUL+CM9EY/FkMfbWbKjtuQ
+iKXS/h5qp6NFdNVj
+-----END PRIVATE KEY-----
diff --git a/example_configuration/users.cfg b/example_configuration/users.cfg
new file mode 100644
index 0000000..aef6ccd
--- /dev/null
+++ b/example_configuration/users.cfg
@@ -0,0 +1,12 @@
+[auth]
+adjudicators = adj1 adj2
+
+[limits]
+# number of files downloaded for annotation
+annotation = 10
+# number of file pairs download for adjudication
+adjudication = 5
+
+# limits per user
+# adj1.annotation = 2
+# adj2.adjudication = 5
diff --git a/gui/CheckoutNumberDialog.py b/gui/CheckoutNumberDialog.py
new file mode 100644
index 0000000..83d6a03
--- /dev/null
+++ b/gui/CheckoutNumberDialog.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+# generated by wxGlade 0.6.3 on Tue Sep 27 12:15:56 2011
+
+import wx
+
+# begin wxGlade: dependencies
+# end wxGlade
+
+# begin wxGlade: extracode
+
+# end wxGlade
+
+"""aktualnie nieużywane! """
+
+class CheckoutNumberDialog(wx.Dialog):
+ def __init__(self, *args, **kwds):
+ # begin wxGlade: CheckoutNumberDialog.__init__
+ kwds["style"] = wx.DEFAULT_DIALOG_STYLE
+ wx.Dialog.__init__(self, *args, **kwds)
+ self.label_4 = wx.StaticText(self, -1, "checkout files: ")
+ self.checkout_number = wx.SpinCtrl(self, -1, "1", min=1, max=300)
+ self.button_4 = wx.Button(self, wx.ID_YES, "OK")
+ self.button_5 = wx.Button(self, wx.ID_NO, "Cancel")
+
+ self.__set_properties()
+ self.__do_layout()
+
+ self.Bind(wx.EVT_BUTTON, self.on_ok, id=wx.ID_YES)
+ self.Bind(wx.EVT_BUTTON, self.on_cancel, id=wx.ID_NO)
+ # end wxGlade
+
+ def __set_properties(self):
+ # begin wxGlade: CheckoutNumberDialog.__set_properties
+ self.SetTitle("checkout")
+ self.SetSize((193, 79))
+ # end wxGlade
+
+ def __do_layout(self):
+ # begin wxGlade: CheckoutNumberDialog.__do_layout
+ sizer_8 = wx.BoxSizer(wx.VERTICAL)
+ grid_sizer_3 = wx.GridSizer(1, 2, 0, 0)
+ grid_sizer_2 = wx.GridSizer(1, 2, 0, 0)
+ grid_sizer_2.Add(self.label_4, 0, wx.ALIGN_CENTER_VERTICAL, 0)
+ grid_sizer_2.Add(self.checkout_number, 0, wx.ALIGN_CENTER_VERTICAL, 0)
+ sizer_8.Add(grid_sizer_2, 1, 0, 0)
+ grid_sizer_3.Add(self.button_4, 0, 0, 0)
+ grid_sizer_3.Add(self.button_5, 0, 0, 0)
+ sizer_8.Add(grid_sizer_3, 0, wx.BOTTOM, 0)
+ self.SetSizer(sizer_8)
+ self.Layout()
+ self.Centre()
+ # end wxGlade
+
+ def on_ok(self, event): # wxGlade: CheckoutNumberDialog.<event_handler>
+ print "Event handler `on_ok' not implemented!"
+ event.Skip()
+
+ def on_cancel(self, event): # wxGlade: CheckoutNumberDialog.<event_handler>
+ print "Event handler `on_cancel' not implemented!"
+ event.Skip()
+
+# end of class CheckoutNumberDialog
+
+
diff --git a/gui/ConfigDialog.py b/gui/ConfigDialog.py
new file mode 100644
index 0000000..ff4fab6
--- /dev/null
+++ b/gui/ConfigDialog.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# generated by wxGlade 0.6.3 on Tue Sep 27 14:54:14 2011
+
+import wx
+
+import i18n
+_ = i18n.language.ugettext #use ugettext instead of getttext to avoid unicode errors
+
+# begin wxGlade: dependencies
+# end wxGlade
+
+# begin wxGlade: extracode
+
+# end wxGlade
+
+class ConfigDialog(wx.Dialog):
+ def __init__(self, *args, **kwds):
+ # begin wxGlade: ConfigDialog.__init__
+ kwds["style"] = wx.DEFAULT_DIALOG_STYLE
+ wx.Dialog.__init__(self, *args, **kwds)
+ self.label_2 = wx.StaticText(self, -1, _("login")+":")
+ self.login_ctrl = wx.TextCtrl(self, -1, "")
+ self.label_6 = wx.StaticText(self, -1, _("password")+":")
+ self.passwd_ctrl = wx.TextCtrl(self, -1, "", style=wx.TE_PASSWORD)
+ self.label_5 = wx.StaticText(self, -1, _("extensions")+":")
+ self.exts_ctrl = wx.TextCtrl(self, -1, "", size=(250,30))
+ self.button_2 = wx.Button(self, wx.ID_OK, _("OK"))
+
+ self.__set_properties()
+ self.__do_layout()
+ # end wxGlade
+
+ def __set_properties(self):
+ # begin wxGlade: ConfigDialog.__set_properties
+ self.SetTitle(_("Directory configuration"))
+ self.label_2.SetToolTipString(_("Login supplied by system administrator"))
+ self.label_6.SetToolTipString(_("Password supplied by system administrator"))
+ self.label_5.SetToolTipString(_("List of space separated file extensions"))
+ self.button_2.SetMinSize((85, 29))
+ self.button_2.SetFocus()
+ # end wxGlade
+
+ def __do_layout(self):
+ # begin wxGlade: ConfigDialog.__do_layout
+ sizer_5 = wx.BoxSizer(wx.VERTICAL)
+ grid_sizer_4 = wx.GridSizer(3, 2, 0, 0)
+ grid_sizer_4.Add(self.label_2, 0, wx.ALIGN_CENTER_VERTICAL, 0)
+ grid_sizer_4.Add(self.login_ctrl, 0, wx.ALIGN_CENTER_VERTICAL, 0)
+ grid_sizer_4.Add(self.label_6, 0, wx.ALIGN_CENTER_VERTICAL, 0)
+ grid_sizer_4.Add(self.passwd_ctrl, 0, wx.ALIGN_CENTER_VERTICAL, 0)
+ grid_sizer_4.Add(self.label_5, 0, wx.ALIGN_CENTER_VERTICAL, 0)
+ grid_sizer_4.Add(self.exts_ctrl, 0, wx.ALIGN_CENTER_VERTICAL, 0)
+ sizer_5.Add(grid_sizer_4, 0, 0, 0)
+ sizer_5.Add(self.button_2, 0, wx.ALIGN_CENTER_HORIZONTAL, 0)
+ self.SetSizer(sizer_5)
+ sizer_5.Fit(self)
+ self.Layout()
+ # end wxGlade
+
+# end of class ConfigDialog
+
+
diff --git a/gui/DownloadNumberDialog.py b/gui/DownloadNumberDialog.py
new file mode 100644
index 0000000..d48e002
--- /dev/null
+++ b/gui/DownloadNumberDialog.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+# generated by wxGlade 0.6.3 on Wed Sep 28 10:10:39 2011
+
+import wx
+
+import i18n
+_ = i18n.language.ugettext #use ugettext instead of getttext to avoid unicode errors
+
+# begin wxGlade: dependencies
+# end wxGlade
+
+# begin wxGlade: extracode
+
+# end wxGlade
+
+class DownloadNumberDialog(wx.Dialog):
+ def __init__(self, *args, **kwds):
+ # begin wxGlade: DownloadNumberDialog.__init__
+ kwds["style"] = wx.DEFAULT_DIALOG_STYLE
+ wx.Dialog.__init__(self, *args, **kwds)
+ self.label_4 = wx.StaticText(self, -1, _("Download count")+":")
+ self.checkout_number = wx.SpinCtrl(self, -1, "1", min=1, max=300)
+ self.button_4 = wx.Button(self, wx.ID_YES, _("OK"))
+ self.button_5 = wx.Button(self, wx.ID_NO, _("Cancel"))
+
+ self.__set_properties()
+ self.__do_layout()
+
+ self.Bind(wx.EVT_BUTTON, self.on_ok, id=wx.ID_YES)
+ self.Bind(wx.EVT_BUTTON, self.on_cancel, id=wx.ID_NO)
+ # end wxGlade
+
+ def __set_properties(self):
+ # begin wxGlade: DownloadNumberDialog.__set_properties
+ self.SetTitle(_("Download"))
+ self.label_4.SetToolTipString(_("Number of files to download"))
+ # end wxGlade
+
+ def __do_layout(self):
+ # begin wxGlade: DownloadNumberDialog.__do_layout
+ grid_sizer_2 = wx.GridSizer(2, 2, 20, 20)
+ grid_sizer_2.Add(self.label_4, 0, wx.ALIGN_CENTER_HORIZONTAL|wx.ALIGN_CENTER_VERTICAL, 0)
+ grid_sizer_2.Add(self.checkout_number, 0, 0, 0)
+ grid_sizer_2.Add(self.button_4, 0, wx.ALIGN_CENTER_HORIZONTAL|wx.ALIGN_CENTER_VERTICAL, 0)
+ grid_sizer_2.Add(self.button_5, 0, wx.ALIGN_CENTER_HORIZONTAL|wx.ALIGN_CENTER_VERTICAL, 0)
+ self.SetSizer(grid_sizer_2)
+ grid_sizer_2.Fit(self)
+ self.Layout()
+ self.Centre()
+ # end wxGlade
+
+ def on_ok(self, event): # wxGlade: DownloadNumberDialog.<event_handler>
+ self.EndModal(wx.ID_OK)
+
+ def on_cancel(self, event): # wxGlade: DownloadNumberDialog.<event_handler>
+ self.EndModal(wx.ID_CANCEL)
+
+# end of class DownloadNumberDialog
+
+
diff --git a/gui/LogDialog.py b/gui/LogDialog.py
new file mode 100644
index 0000000..7863404
--- /dev/null
+++ b/gui/LogDialog.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+# generated by wxGlade 0.6.3 on Tue Sep 27 12:15:56 2011
+
+import wx
+
+# begin wxGlade: dependencies
+# end wxGlade
+
+# begin wxGlade: extracode
+
+# end wxGlade
+
+"""aktualnie nieużywane! """
+
+class LogDialog(wx.Dialog):
+ def __init__(self, *args, **kwds):
+ # begin wxGlade: LogDialog.__init__
+ kwds["style"] = wx.DEFAULT_DIALOG_STYLE|wx.RESIZE_BORDER|wx.THICK_FRAME
+ wx.Dialog.__init__(self, *args, **kwds)
+
+ self.__set_properties()
+ self.__do_layout()
+ # end wxGlade
+
+ def __set_properties(self):
+ # begin wxGlade: LogDialog.__set_properties
+ self.SetTitle("Log")
+ self.SetSize((400, 315))
+ # end wxGlade
+
+ def __do_layout(self):
+ # begin wxGlade: LogDialog.__do_layout
+ self.Layout()
+ # end wxGlade
+
+ def on_ok(self, event): # wxGlade: LogDialog.<event_handler>
+ print "Event handler `on_ok' not implemented!"
+ event.Skip()
+
+# end of class LogDialog
+
+
diff --git a/gui/LoginDialog.py b/gui/LoginDialog.py
new file mode 100644
index 0000000..9383d7e
--- /dev/null
+++ b/gui/LoginDialog.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+# generated by wxGlade 0.6.3 on Tue Sep 27 12:15:56 2011
+
+import wx
+
+# begin wxGlade: dependencies
+# end wxGlade
+
+# begin wxGlade: extracode
+
+# end wxGlade
+
+"""aktualnie nieużywane! """
+
+class LoginDialog(wx.Dialog):
+ def __init__(self, *args, **kwds):
+ # begin wxGlade: LoginDialog.__init__
+ kwds["style"] = wx.DEFAULT_DIALOG_STYLE|wx.RESIZE_BORDER|wx.THICK_FRAME
+ wx.Dialog.__init__(self, *args, **kwds)
+ self.label_1 = wx.StaticText(self, -1, "login: ")
+ self.login_ctrl = wx.TextCtrl(self, -1, "")
+ self.label_3 = wx.StaticText(self, -1, "password: ")
+ self.passwd_ctrl = wx.TextCtrl(self, -1, "", style=wx.TE_PASSWORD)
+ self.save_passwd = wx.CheckBox(self, -1, "save password")
+ self.button_1 = wx.Button(self, wx.ID_OK, "OK")
+
+ self.__set_properties()
+ self.__do_layout()
+
+ self.Bind(wx.EVT_BUTTON, self.on_ok, id=wx.ID_OK)
+ # end wxGlade
+
+ def __set_properties(self):
+ # begin wxGlade: LoginDialog.__set_properties
+ self.SetTitle("login")
+ self.button_1.SetMinSize((85, 32))
+ # end wxGlade
+
+ def __do_layout(self):
+ # begin wxGlade: LoginDialog.__do_layout
+ sizer_4 = wx.BoxSizer(wx.VERTICAL)
+ grid_sizer_1 = wx.GridSizer(2, 2, 0, 0)
+ grid_sizer_1.Add(self.label_1, 0, wx.ALIGN_CENTER_VERTICAL, 0)
+ grid_sizer_1.Add(self.login_ctrl, 0, 0, 0)
+ grid_sizer_1.Add(self.label_3, 0, wx.ALIGN_CENTER_VERTICAL, 0)
+ grid_sizer_1.Add(self.passwd_ctrl, 0, 0, 0)
+ sizer_4.Add(grid_sizer_1, 0, 0, 0)
+ sizer_4.Add(self.save_passwd, 0, 0, 0)
+ sizer_4.Add(self.button_1, 0, wx.ALIGN_CENTER_HORIZONTAL, 0)
+ self.SetSizer(sizer_4)
+ sizer_4.Fit(self)
+ self.Layout()
+ # end wxGlade
+
+ def on_ok(self, event): # wxGlade: LoginDialog.<event_handler>
+ print "Event handler `on_ok' not implemented!"
+ event.Skip()
+
+# end of class LoginDialog
+
+
diff --git a/gui/MainFrame.py b/gui/MainFrame.py
new file mode 100644
index 0000000..402945d
--- /dev/null
+++ b/gui/MainFrame.py
@@ -0,0 +1,227 @@
+# -*- coding: utf-8 -*-
+# generated by wxGlade 0.6.3 on Tue Sep 27 12:15:56 2011
+
+import wx
+
+import i18n
+_ = i18n.language.ugettext
+
+from DownloadNumberDialog import DownloadNumberDialog
+from ReturnReasonDialog import ReturnReasonDialog
+from SortableListCtrl import SortableListCtrl
+
+import const
+import utils
+
+# begin wxGlade: dependencies
+# end wxGlade
+
+# begin wxGlade: extracode
+
+# end wxGlade
+
+class MainFrame(wx.Frame):
+ def __init__(self, *args, **kwds):
+ # begin wxGlade: MainFrame.__init__
+ kwds["style"] = wx.DEFAULT_FRAME_STYLE
+ wx.Frame.__init__(self, *args, **kwds)
+ self.window_1 = wx.SplitterWindow(self, -1, style=wx.SP_3D|wx.SP_BORDER)
+ self.window_1_pane_2 = wx.Panel(self.window_1, -1)
+ self.window_1_pane_1 = wx.Panel(self.window_1, -1)
+ self.notebook_1 = wx.Notebook(self.window_1_pane_1, -1, style=wx.BK_DEFAULT)
+ self.notebook_1_pane_1 = wx.Panel(self.notebook_1, -1)
+ self.notebook_1_pane_2 = wx.Panel(self.notebook_1, -1)
+
+ self.log_ctrl = wx.TextCtrl(self.window_1_pane_2, -1, "", style=wx.TE_MULTILINE|wx.TE_READONLY)
+
+ self.files_list = SortableListCtrl(self.notebook_1_pane_1, self.log_ctrl, False, const.program, const.extension)
+ self.super_files_list = SortableListCtrl(self.notebook_1_pane_2, self.log_ctrl, True, const.program, const.extension)
+
+ # Menu Bar
+ self.frame_1_menubar = wx.MenuBar()
+ self.dir_menu = wx.Menu()
+ self.dir_menu_open = wx.MenuItem(self.dir_menu, 5, _("Open"), _("Open new directory"), wx.ITEM_NORMAL)
+ self.dir_menu.AppendItem(self.dir_menu_open)
+ self.dir_menu.Append(6, _("Reconfigure"), _("Reconfigure directory"), wx.ITEM_NORMAL)
+ self.dir_menu.AppendSeparator()
+ if not const.hide_dir_menu == "yes":
+ self.frame_1_menubar.Append(self.dir_menu, _("Directory"))
+ wxglade_tmp_menu = wx.Menu()
+ wxglade_tmp_menu.Append(1, _("Upload Selected"), _("Upload annotated or adjudicated files"), wx.ITEM_NORMAL)
+ wxglade_tmp_menu.Append(8, _("Checkin Selected"), _("Upload annotated or adjudicated; files will be deleted from local disk and you will not be able to modify them again"), wx.ITEM_NORMAL)
+ wxglade_tmp_menu.Append(10, _("Return Selected"), _("Return annotated or adjudicated because they are not suitable for annotation or contain errors"), wx.ITEM_NORMAL)
+ wxglade_tmp_menu.AppendSeparator()
+ wxglade_tmp_menu.Append(2, _("Download (annotation)"), _("Download files for annotation"), wx.ITEM_NORMAL)
+ wxglade_tmp_menu.Append(3, _("Download (adjudication)"), _("Download files for adjudication"), wx.ITEM_NORMAL)
+ wxglade_tmp_menu.AppendSeparator()
+ wxglade_tmp_menu.Append(7, _("Checkout"), _("Checkout files from server to your local, empty directory"), wx.ITEM_NORMAL)
+
+ self.frame_1_menubar.Append(wxglade_tmp_menu, _("Files"))
+ wxglade_tmp_menu = wx.Menu()
+ wxglade_tmp_menu.Append(4, _("Ready"), _("Select items which are marked for upload"), wx.ITEM_NORMAL)
+ wxglade_tmp_menu.Append(9, _("Modified"), _("Select items which are modified"), wx.ITEM_NORMAL)
+ self.frame_1_menubar.Append(wxglade_tmp_menu, _("Select"))
+
+ wxglade_tmp_menu = wx.Menu()
+ wxglade_tmp_menu.Append(11, _("Finished count"), _("Check number of finished texts"), wx.ITEM_NORMAL)
+ self.frame_1_menubar.Append(wxglade_tmp_menu, _("Annotation"))
+
+ self.SetMenuBar(self.frame_1_menubar)
+ # Menu Bar end
+
+ self.CreateStatusBar()
+
+ self.__set_properties()
+ self.__do_layout()
+
+ self.Bind(wx.EVT_MENU, self.open_directory, self.dir_menu_open)
+ self.Bind(wx.EVT_MENU, self.reconf_directory, id=6)
+ self.Bind(wx.EVT_MENU, self.upload, id=1)
+ self.Bind(wx.EVT_MENU, self.checkin, id=8)
+ self.Bind(wx.EVT_MENU, self.download, id=2)
+ self.Bind(wx.EVT_MENU, self.download_prim, id=3)
+ self.Bind(wx.EVT_MENU, self.checkout, id=7)
+ self.Bind(wx.EVT_MENU, self.select_ready, id=4)
+ self.Bind(wx.EVT_MENU, self.select_modified, id=9)
+ self.Bind(wx.EVT_MENU, self.return_files, id=10)
+ self.Bind(wx.EVT_MENU, self.check_stats, id=11)
+
+ self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, self.activate_item, self.files_list)
+ self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, self.activate_super_item, self.super_files_list)
+
+ def __set_properties(self):
+ # begin wxGlade: MainFrame.__set_properties
+ self.SetTitle("--")
+ self.SetSize((663, 556))
+ self.SetFocus()
+ # end wxGlade
+
+ def __do_layout(self):
+ # begin wxGlade: MainFrame.__do_layout
+ sizer_6 = wx.BoxSizer(wx.HORIZONTAL)
+ sizer_7 = wx.BoxSizer(wx.VERTICAL)
+ sizer_1 = wx.BoxSizer(wx.VERTICAL)
+ sizer_3 = wx.BoxSizer(wx.VERTICAL)
+ sizer_2 = wx.BoxSizer(wx.VERTICAL)
+ sizer_2.Add(self.files_list, 1, wx.EXPAND, 0)
+ self.notebook_1_pane_1.SetSizer(sizer_2)
+ sizer_3.Add(self.super_files_list, 1, wx.EXPAND, 0)
+ self.notebook_1_pane_2.SetSizer(sizer_3)
+ self.notebook_1.AddPage(self.notebook_1_pane_1, _("Annotation"))
+ self.notebook_1.AddPage(self.notebook_1_pane_2, _("Adjudication"))
+ sizer_1.Add(self.notebook_1, 1, wx.EXPAND, 0)
+ self.window_1_pane_1.SetSizer(sizer_1)
+ sizer_7.Add(self.log_ctrl, 1, wx.EXPAND, 0)
+ self.window_1_pane_2.SetSizer(sizer_7)
+ self.window_1.SplitHorizontally(self.window_1_pane_1, self.window_1_pane_2, -100)
+ sizer_6.Add(self.window_1, 1, wx.EXPAND, 0)
+ self.SetSizer(sizer_6)
+ self.Layout()
+ self.Centre()
+ # end wxGlade
+
+ def set_manager(self, manager):
+ self.manager = manager
+
+ def open_directory(self, event): # wxGlade: MainFrame.<event_handler>
+ if event.GetId() == self.dir_menu_open.GetId():
+ dlg = wx.DirDialog(self, message=_("Choose a directory"))
+ try:
+ if dlg.ShowModal() == wx.ID_OK:
+ self.manager.set_directory(dlg.GetPath())
+ finally:
+ dlg.Destroy()
+ else:
+ item = self.dir_menu.FindItemById(event.GetId())
+ self.manager.set_directory(item.GetLabel())
+
+ def upload(self, event): # wxGlade: MainFrame.<event_handler>
+ names = ( self.files_list.get_selected()
+ + self.super_files_list.get_selected() )
+ info = ( _("Do you want to upload the following files?")+"\n\n"
+ + "\n".join(names) )
+ dial = wx.MessageDialog( self, info, _('Question')
+ , wx.YES_NO | wx.NO_DEFAULT | wx.ICON_QUESTION )
+ try:
+ if dial.ShowModal() == wx.ID_YES:
+ self.manager.upload(names, checkin=False)
+ finally:
+ dial.Destroy()
+
+ def return_files(self, event): # wxGlade: MainFrame.<event_handler>
+ names = ( self.files_list.get_selected()
+ + self.super_files_list.get_selected() )
+ reason = None
+ dialog = ReturnReasonDialog(self, names)
+ try:
+ if dialog.ShowModal() == wx.ID_OK:
+ reason = dialog.reason.GetValue().encode("utf-8")
+ finally:
+ dialog.Destroy()
+ if reason is not None:
+ self.manager.return_files(names, reason)
+
+ def download(self, event): # wxGlade: MainFrame.<event_handler>
+ num = None
+ dialog = DownloadNumberDialog(self, -1)
+ try:
+ if dialog.ShowModal() == wx.ID_OK:
+ num = dialog.checkout_number.GetValue()
+ finally:
+ dialog.Destroy()
+ if num is not None:
+ self.manager.download(num)
+
+ def download_prim(self, event): # wxGlade: MainFrame.<event_handler>
+ num = None
+ dialog = DownloadNumberDialog(self, -1)
+ try:
+ if dialog.ShowModal() == wx.ID_OK:
+ num = dialog.checkout_number.GetValue()
+ finally:
+ dialog.Destroy()
+ if num is not None:
+ self.manager.download_prim(num)
+
+ def activate_item(self, event): # wxGlade: MainFrame.<event_handler>
+ utils.change_ready( self.manager.ready
+ , self.files_list
+ , event.GetIndex() )
+
+ def activate_super_item(self, event): # wxGlade: MainFrame.<event_handler>
+ utils.change_ready_prim( self.manager.ready
+ , self.super_files_list
+ , event.GetIndex() )
+
+ def reconf_directory(self, event): # wxGlade: MainFrame.<event_handler>
+ self.manager.reconf_directory()
+
+ def select_ready(self, event): # wxGlade: MainFrame.<event_handler>
+ self.files_list.select_ready()
+ self.super_files_list.select_ready()
+
+ def checkout(self, event): # wxGlade: MainFrame.<event_handler>
+ self.manager.checkout()
+
+ def checkin(self, event): # wxGlade: MainFrame.<event_handler>
+ names = ( self.files_list.get_selected()
+ + self.super_files_list.get_selected() )
+ info = ( _("Do you want to checkin the following files?")+"\n\n"
+ + "\n".join(names)
+ + "\n\n"+_("Files will be deleted from the local directory and you will not be able to read, write or modify them later."))
+ dial = wx.MessageDialog( self, info, _('Question')
+ , wx.YES_NO | wx.NO_DEFAULT | wx.ICON_QUESTION )
+ try:
+ if dial.ShowModal() == wx.ID_YES:
+ self.manager.upload(names, checkin=True)
+ finally:
+ dial.Destroy()
+
+ def select_modified(self, event): # wxGlade: MainFrame.<event_handler>
+ self.files_list.select_modified()
+ self.super_files_list.select_modified()
+
+ def check_stats(self, event): # wxGlade: MainFrame.<event_handler>
+ self.manager.check_stats()
+
+# end of class MainFrame
diff --git a/gui/ReturnReasonDialog.py b/gui/ReturnReasonDialog.py
new file mode 100644
index 0000000..2e579aa
--- /dev/null
+++ b/gui/ReturnReasonDialog.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# generated by wxGlade 0.6.3 on Wed Sep 28 10:10:39 2011
+
+import wx
+
+import i18n
+_ = i18n.language.ugettext #use ugettext instead of getttext to avoid unicode errors
+
+# begin wxGlade: dependencies
+# end wxGlade
+
+# begin wxGlade: extracode
+
+# end wxGlade
+
+class ReturnReasonDialog(wx.Dialog):
+ def __init__(self, parent, names):
+ # begin wxGlade: ReturnReasonDialog.__init__
+ wx.Dialog.__init__(self, parent, -1)
+
+ info = ( _("Why do you want to return the following files?")+"\n"
+ + "\n".join(names) )
+
+ self.label_1 = wx.StaticText(self, -1, info)
+ self.label_4 = wx.StaticText(self, -1, _("Reason")+":")
+ self.reason = wx.TextCtrl(self, -1, "", size=(250, 80), style=wx.TE_MULTILINE|wx.TE_PROCESS_ENTER)
+ self.button_4 = wx.Button(self, wx.ID_YES, _("OK"))
+ self.button_5 = wx.Button(self, wx.ID_NO, _("Cancel"))
+
+ self.__set_properties()
+ self.__do_layout()
+
+ self.Bind(wx.EVT_BUTTON, self.on_ok, id=wx.ID_YES)
+ self.Bind(wx.EVT_BUTTON, self.on_cancel, id=wx.ID_NO)
+ # end wxGlade
+
+ def __set_properties(self):
+ # begin wxGlade: ReturnReasonDialog.__set_properties
+ self.SetTitle(_("Return files"))
+ # end wxGlade
+
+ def __do_layout(self):
+ # begin wxGlade: ReturnReasonDialog.__do_layout
+ sizer_2 = wx.BoxSizer(wx.HORIZONTAL)
+ sizer_2.Add(self.label_4, 0, wx.ALIGN_CENTER_HORIZONTAL|wx.ALIGN_CENTER_VERTICAL, 0)
+ sizer_2.Add(self.reason, 0, 0, 0)
+
+ sizer_3 = wx.BoxSizer(wx.HORIZONTAL)
+ sizer_3.Add(self.button_4, 0, wx.ALIGN_CENTER_HORIZONTAL|wx.ALIGN_CENTER_VERTICAL, 0)
+ sizer_3.Add(self.button_5, 0, wx.ALIGN_CENTER_HORIZONTAL|wx.ALIGN_CENTER_VERTICAL, 0)
+
+ sizer_1 = wx.BoxSizer(wx.VERTICAL)
+ sizer_1.Add(self.label_1, 0, wx.ALIGN_CENTER_HORIZONTAL|wx.ALIGN_CENTER_VERTICAL, 1)
+ sizer_1.Add(sizer_2, 0, wx.ALIGN_CENTER_HORIZONTAL|wx.ALIGN_CENTER_VERTICAL, 1)
+ sizer_1.Add(sizer_3, 0, wx.ALIGN_CENTER_HORIZONTAL|wx.ALIGN_CENTER_VERTICAL, 1)
+ self.SetSizer(sizer_1)
+ sizer_1.Fit(self)
+ self.Layout()
+ # end wxGlade
+
+ def on_ok(self, event): # wxGlade: ReturnReasonDialog.<event_handler>
+ self.EndModal(wx.ID_OK)
+
+ def on_cancel(self, event): # wxGlade: ReturnReasonDialog.<event_handler>
+ self.EndModal(wx.ID_CANCEL)
+
+# end of class ReturnReasonDialog
+
+
diff --git a/gui/SortableListCtrl.py b/gui/SortableListCtrl.py
new file mode 100644
index 0000000..71e7516
--- /dev/null
+++ b/gui/SortableListCtrl.py
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+import threading
+from PIL import Image
+
+import i18n
+_ = i18n.language.ugettext #use ugettext instead of getttext to avoid unicode errors
+
+import os.path
+import wx
+import wx.lib.mixins.listctrl as listmix
+import subprocess
+
+try:
+ from agw import ultimatelistctrl as ULC
+except ImportError: # if it's not there locally, try the wxPython lib.
+ from wx.lib.agw import ultimatelistctrl as ULC
+
+class SortableListCtrl(ULC.UltimateListCtrl, listmix.ColumnSorterMixin, listmix.ListCtrlAutoWidthMixin):
+ def __init__(self, parent, log, superann, program, extension):
+
+ self._lock = threading.Lock()
+
+ self.program = program
+ self.extension = extension
+
+ ULC.UltimateListCtrl.__init__(self, parent, -1, agwStyle=ULC.ULC_REPORT | ULC.ULC_HAS_VARIABLE_ROW_HEIGHT | ULC.ULC_USER_ROW_HEIGHT)#| wx.LC_HRULES | wx.LC_VRULES )
+ listmix.ListCtrlAutoWidthMixin.__init__(self)
+
+ self.log_ctrl = log
+
+ self.il = wx.ImageList(16, 16)
+ up = wx.Image(os.path.join('img', 'up.png'), wx.BITMAP_TYPE_PNG).ConvertToBitmap()
+ down = wx.Image(os.path.join('img', 'down.png'), wx.BITMAP_TYPE_PNG).ConvertToBitmap()
+ if superann:
+ textimg_1 = wx.Image(os.path.join('img', 'textimg_2.png'), wx.BITMAP_TYPE_PNG).ConvertToBitmap()
+ else:
+ textimg_1 = wx.Image(os.path.join('img', 'textimg_1.png'), wx.BITMAP_TYPE_PNG).ConvertToBitmap()
+ ready = wx.Image(os.path.join('img', 'ready.png'), wx.BITMAP_TYPE_PNG).ConvertToBitmap()
+ self.sm_up = self.il.Add(up)
+ self.sm_dn = self.il.Add(down)
+ self.textimg_1 = self.il.Add(textimg_1)
+ self.ready = self.il.Add(ready)
+
+ self.AssignImageList(self.il, wx.IMAGE_LIST_SMALL)
+
+ self.SetUserLineHeight(23)
+ self.InsertColumn(0, _("File"))
+ self.InsertColumn(1, _("Status"))
+ self.InsertColumn(2, _("Ready"))
+ self.InsertColumn(3, _("Stage"))
+ self.InsertColumn(4, "")
+ self.SetColumnWidth(0, 80)
+ self.SetColumnWidth(1, 100)
+ self.SetColumnWidth(2, 120)
+ self.SetColumnWidth(3, 100)
+
+ self.itemDataMap = {}
+
+ listmix.ColumnSorterMixin.__init__(self, 5)
+
+ # required by ColumnSorterMixin
+ def GetListCtrl(self):
+ return self
+
+ def GetSortImages(self):
+ return (self.sm_dn, self.sm_up)
+ # required by ColumnSorterMixin - end
+
+ def show(self, data):
+ self._lock.acquire()
+ try:
+ # for some reason deleteAllItems() doesn't work with buttons inside ulc
+ for item in range(self.GetItemCount()):
+ self.DeleteItem(0)
+ self.itemDataMap.clear()
+ for i, (file_name, status, is_ready, stage) in enumerate(data):
+ img = self.textimg_1
+ if is_ready == _("Yes"):
+ img = self.ready
+ pos = self.InsertImageStringItem(i, file_name, img)
+ self.SetStringItem(pos, 1, status)
+ self.SetStringItem(pos, 2, is_ready)
+ self.SetStringItem(pos, 3, stage)
+
+ button = wx.Button(self, id=i, size=(80, 23), label=_("Open"))
+ self.SetItemWindow(pos, col=4, wnd=button, expand=False)
+ button.Bind(wx.EVT_BUTTON, self.open_button)
+
+ self.SetItemData(pos, i)
+
+ self.itemDataMap[i] = (file_name, status, is_ready, stage, None)
+ finally:
+ self._lock.release()
+
+ def shown_data(self):
+ result = []
+ for i in range(self.GetItemCount()):
+ part_result = []
+ for j in range(0, 4):
+ part_result.append(self.GetItem(i, j).GetText())
+ result.append(tuple(part_result))
+ return result
+
+ def SetReadyYes(self, pos):
+ self.SetItemImage(pos, self.ready)
+ return self._set_ready(pos, _("Yes"))
+
+ def SetReadyNo(self, pos):
+ self.SetItemImage(pos, self.textimg_1)
+ return self._set_ready(pos, _("No"))
+
+ def _set_ready(self, pos, val):
+ file_name = self.GetItem(pos, 0).GetText()
+ for key, (filename, status, ready, stage, sth) in self.itemDataMap.iteritems():
+ if filename == file_name:
+ self.itemDataMap[key] = (filename, status, val, stage, sth)
+ break
+ return self.SetStringItem(pos, 2, val)
+
+ def GetFilename(self, pos):
+ return self.GetItem(pos, 0).GetText()
+
+ def IsReady(self, pos):
+ return self.GetItem(pos, 2).GetText() == _("Yes")
+
+ def select_ready(self):
+ return self._select(lambda list, pos : list.GetItem(pos, 2).GetText() == _("Yes"))
+
+ def select_modified(self):
+ return self._select(lambda list, pos : list.GetItem(pos, 1).GetText() == _("Modified"))
+
+ def _select(self, cond):
+ for i in range(self.GetItemCount()):
+ if cond(self, i):
+ self.Select(i, 1)
+ else:
+ self.Select(i, 0)
+
+ def get_selected(self):
+ fnames = []
+ for i in range(self.GetItemCount()):
+ if self.IsSelected(i):
+ fnames.append(self.GetFilename(i))
+ return fnames
+
+ def open_button(self, event):
+ file_name = self.GetFilename(event.GetEventObject().GetId())
+ print event.GetEventObject().GetId()
+ if file_name:
+ self.log_ctrl.AppendText(_("Starting ") + self.program + _(" for text") + " " + file_name + "...\n")
+ app = os.path.join("..", self.program, self.program+".exe")
+ app_bin = os.path.join("..", self.program, self.program)
+ if not os.path.exists(app):
+ app = app_bin
+ text = os.path.join("..", "teksty", file_name + self.extension)
+ try:
+ subprocess.Popen([app, text])
+ except Exception as ex:
+ self.log_ctrl.AppendText(_("Error starting ") + self.program + ": " + unicode(str(ex), errors='replace') + "\n")
+
diff --git a/gui/client.py b/gui/client.py
new file mode 100644
index 0000000..db13f25
--- /dev/null
+++ b/gui/client.py
@@ -0,0 +1,470 @@
+# -*- coding: utf-8 -*-
+import sys
+import os
+import shutil
+import tempfile
+import socket
+import ssl
+import pprint
+from optparse import OptionParser
+import traceback
+import re
+
+import i18n
+_ = i18n.language.ugettext
+
+from dfs.msg.stream import write_msg, read_msg, accept_msg
+from dfs.msg.message import *
+from dfs.utils import check_sum, send_contents, receive_contents
+
+from utils import normal_files, super_files
+
+def init_stream(host, port, cert, log=None):
+ log_msg(_("Connecting server..."), log)
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ stream = ssl.wrap_socket(s, ca_certs=cert, cert_reqs=ssl.CERT_REQUIRED, ssl_version=ssl.PROTOCOL_TLSv1)
+ stream.connect((host, port))
+ return stream
+
+def upload(stream, files, src_dir, file_exts, checkin=False, log=None):
+ """Checkin / Upload files to the server."""
+
+ def is_adj(path):
+ return False if len(os.path.dirname(path)) == 0 else True
+
+ log_msg(_("Sending request..."), log)
+ if checkin:
+ write_msg(stream, CheckinRequest(len(files)))
+ else:
+ write_msg(stream, UploadRequest(len(files)))
+ done = []
+
+ failed = set()
+ for file_path in files:
+ log_msg(_("Uploading") + " " + file_path + "...", log)
+
+ if not is_adj(file_path):
+ file_id = file_path
+ contents = read_contents(src_dir, file_path, file_exts)
+ else:
+ file_id, choice = os.path.split(file_path)
+ contents = read_contents(os.path.join(src_dir, file_id), choice, file_exts)
+
+ write_msg(stream, Message(file_id))
+ write_msg(stream, BoolMessage(is_adj(file_path)))
+ msg = read_msg(stream)
+ if type(msg) != OkMessage:
+ log_msg(_("Upload failed: server doesn't accept this file from you"), log)
+ failed.add(file_path)
+ continue
+ send_contents(stream, contents)
+
+ write_msg(stream, ClientDone())
+ accept_msg(stream, ServerDone)
+
+ if not checkin:
+ log_msg(_("Done"), log)
+ return
+
+ # When checkin command
+ for file_path in files:
+ if file_path in failed:
+ continue
+ if not is_adj(file_path):
+ log_msg(_("Deleting") + " " + file_path + "...", log)
+ for ext in file_exts:
+ os.remove(os.path.join(src_dir, file_path + ext))
+ else:
+ file_id, ver = os.path.split(file_path)
+ log_msg(_("Deleting") + " " + file_id + "...", log)
+ shutil.rmtree(os.path.join(src_dir, file_id))
+
+ log_msg(_("Done"), log)
+
+def download(stream, n, dest_dir, tmp_dir, log=None):
+ """Download from server n files."""
+
+ log_msg(_("Sending request..."), log)
+ write_msg(stream, DownloadRequest(n))
+ n = accept_msg(stream, NumMessage).get_number()
+ if n == 0:
+ log_msg(_("No files to download"), log)
+
+ done = []
+ for i in range(n):
+ file_id = accept_msg(stream, Message).get_contents()
+ log_msg(_("Downloading") + " " + file_id + "...", log)
+ contents = receive_contents(stream)
+ save_contents(contents, tmp_dir, file_id)
+ for ext in contents.keys():
+ done.append(file_id + ext)
+
+ write_msg(stream, ClientDone())
+ accept_msg(stream, ServerDone)
+ for file_name in done:
+ from_path = os.path.join(tmp_dir, file_name)
+ to_path = os.path.join(dest_dir, file_name)
+ shutil.move(from_path, to_path)
+ log_msg(_("Done"), log)
+
+def download_prim(stream, n, dest_dir, tmp_dir, log=None):
+ """Download from server n pairs of files for adjudication."""
+
+ log_msg(_("Sending request..."), log)
+ write_msg(stream, DownloadPrimRequest(n))
+
+ msg = read_msg(stream)
+ if type(msg) != OkMessage:
+ log_msg(_("FAILED: you don't have adjudicator privileges"), log)
+ return
+
+ n = accept_msg(stream, NumMessage).get_number()
+ anno_per_file = accept_msg(stream, NumMessage).get_number()
+ if n == 0:
+ log_msg(_("No files to download"), log)
+
+ done = []
+ for i in range(n):
+ file_id = accept_msg(stream, Message).get_contents()
+ log_msg(_("Downloading") + " " + file_id + "...", log)
+
+ file_dir = os.path.join(tmp_dir, file_id)
+ os.mkdir(file_dir)
+ done.append(file_id)
+
+ for j in range(anno_per_file):
+ save_contents(receive_contents(stream), file_dir, chr(ord('A') + j))
+
+ if accept_msg(stream, BoolMessage).get_boolean() == True:
+ if anno_per_file == 1:
+ save_contents(receive_contents(stream), file_dir, "A")
+ else:
+ save_contents(receive_contents(stream), file_dir, "Super")
+
+ write_msg(stream, ClientDone())
+ accept_msg(stream, ServerDone)
+ for fild_id in done:
+ from_path = os.path.join(tmp_dir, fild_id)
+ to_path = os.path.join(dest_dir, fild_id)
+ shutil.move(from_path, to_path)
+ log_msg(_("Done"), log)
+
+def check_stats(stream, log=None):
+ """Check annotatator stats."""
+
+ log_msg(_("Sending request..."), log)
+ write_msg(stream, StatsRequest())
+ n = accept_msg(stream, NumMessage).get_number()
+ log_msg(_("Number of finished files: ") + str(n), log)
+ write_msg(stream, ClientDone())
+ accept_msg(stream, ServerDone)
+ log_msg(_("Done"), log)
+
+def checkout(stream, dest_dir, tmp_dir, exts, log=None):
+ log_msg(_("Sending request..."), log)
+ write_msg(stream, CheckoutRequest())
+
+ done = []
+ n = accept_msg(stream, NumMessage).get_number()
+ for i in range(n):
+ file_id = accept_msg(stream, Message).get_contents()
+ log_msg(_("Downloading") + " " + file_id + "...", log)
+
+ contents = receive_contents(stream)
+ save_contents(contents, tmp_dir, file_id)
+ for ext in contents.keys():
+ done.append(file_id + ext)
+
+ n = accept_msg(stream, NumMessage).get_number()
+ anno_per_file = accept_msg(stream, NumMessage).get_number()
+
+ for i in range(n):
+ file_id = accept_msg(stream, Message).get_contents()
+ log_msg(_("Downloading") + " " + file_id + "...", log)
+
+ file_dir = os.path.join(tmp_dir, file_id)
+ os.mkdir(file_dir)
+
+ for j in range(anno_per_file):
+ save_contents(receive_contents(stream), file_dir, chr(j + ord('A')))
+
+ if accept_msg(stream, BoolMessage).get_boolean() == True:
+ if anno_per_file == 1:
+ save_contents(receive_contents(stream), file_dir, "A")
+ else:
+ save_contents(receive_contents(stream), file_dir, "Super")
+
+ done.append(file_id)
+
+ write_msg(stream, ClientDone())
+ accept_msg(stream, ServerDone)
+
+ log_msg(_("Deleting local files..."), log)
+ try:
+ # normal texts
+ for f in normal_files(dest_dir, exts):
+ for ext in exts:
+ os.remove(os.path.join(dest_dir, f + ext))
+
+ # super annotated texts
+ dirs = set()
+ for f in super_files(dest_dir, exts):
+ dir = f.split(os.path.sep)[0]
+ dirs.add(dir)
+ for dir in dirs:
+ shutil.rmtree(os.path.join(dest_dir, dir))
+
+ except Exception as ex:
+ log_msg(_("Error occured:") + unicode(str(ex), errors='replace'), log)
+
+ log_msg(_("Saving dowloaded files..."), log)
+ for path in done:
+ from_path = os.path.join(tmp_dir, path)
+ to_path = os.path.join(dest_dir, path)
+ shutil.move(from_path, to_path)
+
+ log_msg(_("Done"), log)
+
+def return_files(stream, files, reason, src_dir, file_exts, log=None):
+ """Return files to the server."""
+
+ def is_adj(path):
+ return False if len(os.path.dirname(path)) == 0 else True
+
+ log_msg(_("Sending request..."), log)
+ write_msg(stream, ReturnRequest(len(files)))
+ write_msg(stream, Message(reason))
+
+ done = []
+
+ failed = set()
+ for file_path in files:
+ log_msg(_("Uploading") + " " + file_path + "...", log)
+
+ if not is_adj(file_path):
+ file_id = file_path
+ contents = read_contents(src_dir, file_path, file_exts)
+ else:
+ file_id, choice = os.path.split(file_path)
+ contents = read_contents(os.path.join(src_dir, file_id), choice, file_exts)
+
+ write_msg(stream, Message(file_id))
+ write_msg(stream, BoolMessage(is_adj(file_path)))
+ msg = read_msg(stream)
+ if type(msg) != OkMessage:
+ log_msg(_("Upload failed: server doesn't accept this file from you"), log)
+ failed.add(file_path)
+ continue
+ send_contents(stream, contents)
+
+ write_msg(stream, ClientDone())
+ accept_msg(stream, ServerDone)
+
+ for file_path in files:
+ if file_path in failed:
+ continue
+ if not is_adj(file_path):
+ log_msg(_("Deleting") + " " + file_path + "...", log)
+ for ext in file_exts:
+ os.remove(os.path.join(src_dir, file_path + ext))
+ else:
+ file_id, ver = os.path.split(file_path)
+ log_msg(_("Deleting") + " " + file_id + "...", log)
+ shutil.rmtree(os.path.join(src_dir, file_id))
+
+ log_msg(_("Done"), log)
+
+def save_contents(contents, dest_dir, file_id):
+ for ext, data in contents.iteritems():
+ dest_path = os.path.join(dest_dir, file_id + ext)
+ with open(dest_path, "w") as dest:
+ dest.write(data)
+
+def read_contents(src_dir, file_id, file_exts):
+ result = {}
+ regex = re.compile("%s(.*)$" % file_id)
+ for entry in os.listdir(src_dir):
+ match = regex.match(entry)
+ if not match:
+ continue
+ with open(os.path.join(src_dir, entry)) as src:
+ ext = match.group(1)
+ if ext in file_exts:
+ result[ext] = src.read()
+ else:
+ print "Skipping file " + entry
+ return result
+
+def run_auth(stream, login, passwd, log=None):
+ log_msg(_("Authentication..."), log)
+ write_msg(stream, Message(login))
+ write_msg(stream, Message(passwd))
+ msg = read_msg(stream)
+ if type(msg) != OkMessage:
+ log_msg(_("FAILED: incorrect login or password"), log)
+ return False
+ return True
+
+def run_upload(login, passwd, host, port, cert, files, src_dir, file_exts, checkin=False, log=None):
+ result = True
+ try:
+ stream = init_stream(host, port, cert, log=log)
+ try:
+ if run_auth(stream, login, passwd, log=log):
+ upload(stream, files, src_dir, file_exts, checkin=checkin, log=log)
+ finally:
+ stream.close()
+ except:
+ log_msg(_("FAILED: unable to connect to server"), log)
+ print traceback.format_exc()
+ result = False
+ if log:
+ log.put(None)
+ return result
+
+def run_download(login, passwd, host, port, cert, n, dest_dir, log=None):
+ result = True
+ try:
+ stream = init_stream(host, port, cert, log=log)
+ try:
+ tmp_dir = tempfile.mkdtemp()
+ try:
+ if run_auth(stream, login, passwd, log=log):
+ download(stream, n, dest_dir, tmp_dir, log=log)
+ finally:
+ shutil.rmtree(tmp_dir)
+ finally:
+ stream.close()
+ except:
+ log_msg(_("FAILED: unable to connect to server"), log)
+ print traceback.format_exc()
+ result = False
+ if log:
+ log.put(None)
+ return result
+
+def run_check_stats(login, passwd, host, port, cert, log=None):
+ result = True
+ try:
+ stream = init_stream(host, port, cert, log=log)
+ try:
+ if run_auth(stream, login, passwd, log=log):
+ check_stats(stream, log=log)
+ finally:
+ stream.close()
+ except:
+ log_msg(_("FAILED: unable to connect to server"), log)
+ print traceback.format_exc()
+ result = False
+ if log:
+ log.put(None)
+ return result
+
+def run_download_prim(login, passwd, host, port, cert, n, dest_dir, log=None):
+ result = True
+ try:
+ stream = init_stream(host, port, cert, log=log)
+ try:
+ tmp_dir = tempfile.mkdtemp()
+ try:
+ if run_auth(stream, login, passwd, log=log):
+ download_prim(stream, n, dest_dir, tmp_dir, log=log)
+ finally:
+ shutil.rmtree(tmp_dir)
+ finally:
+ stream.close()
+ except:
+ log_msg(_("FAILED: unable to connect to server"), log)
+ print traceback.format_exc()
+ result = False
+ if log:
+ log.put(None)
+ return result
+
+def run_checkout(login, passwd, host, port, cert, dest_dir, exts, log=None):
+ result = True
+ try:
+ stream = init_stream(host, port, cert, log=log)
+ try:
+ tmp_dir = tempfile.mkdtemp()
+ try:
+ if run_auth(stream, login, passwd, log=log):
+ checkout(stream, dest_dir, tmp_dir, exts, log=log)
+ finally:
+ shutil.rmtree(tmp_dir)
+ finally:
+ stream.close()
+ except:
+ log_msg(_("FAILED: unable to connect to server"), log)
+ print traceback.format_exc()
+ result = False
+ if log:
+ log.put(None)
+ return result
+
+def run_return(login, passwd, host, port, cert, files, reason, src_dir, file_exts, log=None):
+ result = True
+ try:
+ stream = init_stream(host, port, cert, log=log)
+ try:
+ if run_auth(stream, login, passwd, log=log):
+ return_files(stream, files, reason, src_dir, file_exts, log)
+ finally:
+ stream.close()
+ except:
+ log_msg(_("FAILED: unable to connect to server"), log)
+ print traceback.format_exc()
+ result = False
+ if log:
+ log.put(None)
+ return result
+
+def log_msg(msg, log, new_line=True):
+ if new_line:
+ msg = msg + "\n"
+ sys.stdout.write(msg.encode('utf-8'))
+ if log:
+ log.put(msg)
+
+if __name__ == "__main__":
+
+ optparser = OptionParser(usage="""usage: %prog [options] CMD CMD-ARGS
+
+ Command line client for files distribution system.""")
+
+ optparser.add_option("--login", dest="login")
+ optparser.add_option("--passwd", dest="passwd")
+ optparser.add_option("--home", metavar="DIR", dest="home",
+ help="Directory with users files.")
+
+ (options, args) = optparser.parse_args()
+
+ if options.login == None:
+ optparser.print_help()
+ print "\n--login option is mandatory"
+ sys.exit(0)
+ if options.passwd == None:
+ optparser.print_help()
+ print "\n--passwd option is mandatory"
+ sys.exit(0)
+ if options.home == None:
+ optparser.print_help()
+ print "\n--home option is mandatory"
+ sys.exit(0)
+
+ commands = ["upload", "download", "download2"]
+ if len(args) < 1 or args[0] not in commands:
+ optparser.print_help()
+ print "\nCMD should be one of the following:"
+ print commands
+ sys.exit(0)
+
+ cmd = args[0]
+ if cmd == "upload":
+ run_upload(options.login, options.passwd, args[1:], options.home)
+ elif cmd == "download":
+ run_download(options.login, options.passwd,
+ int(args[1]), options.home)
+ elif cmd == "download2":
+ run_download_prim(options.login, options.passwd,
+ int(args[1]), options.home)
diff --git a/gui/const.py b/gui/const.py
new file mode 100644
index 0000000..379be81
--- /dev/null
+++ b/gui/const.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+import os
+
+from dfs.config import Config
+
+# Global configuration file
+global_config_name = "data/global.cfg"
+_cfg = Config(global_config_name)
+
+# File with list of directories
+dir_cache_name = _cfg["general.dir_cache"]
+
+# File with directory configuration
+config_name = _cfg["general.config"]
+# File with file names marked as ready
+ready_name = _cfg["general.ready"]
+# File with files footprints
+footprints_name = _cfg["general.footprints"]
+# Default file extensions
+default_exts = _cfg["general.default_exts"].split()
+# Language (pl or en)
+lang = _cfg["general.lang"]
+# option to hide dir menu if precofigured
+hide_dir_menu = _cfg["general.hide_dir_menu"]
+
+program = _cfg["general.program"]
+
+extension = _cfg["general.extension"]
+
+conn_host = _cfg["connection.host"]
+conn_port = int(_cfg["connection.port"])
+conn_cert = _cfg["connection.cert"]
+
+# Path to directory-specific configuration files
+def path(p, name):
+ return os.path.join(p, name)
diff --git a/gui/data/.dir.lst b/gui/data/.dir.lst
new file mode 100644
index 0000000..2284183
--- /dev/null
+++ b/gui/data/.dir.lst
@@ -0,0 +1,5 @@
+/home/me2/workspace/dist-system/data/foldery_anotatorow/ann1
+/home/me2/workspace/dist-system/data/foldery_anotatorow
+/home/me2/workspace/core/dist-system/data/annotators_folders/ann2
+/home/me2/workspace/core/dist-system/data/annotators_folders/ann1
+/home/me2/workspace/core/dist-system/data/annotators_folders/adj1
diff --git a/gui/data/cert.pem b/gui/data/cert.pem
new file mode 100644
index 0000000..0dd2101
--- /dev/null
+++ b/gui/data/cert.pem
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICkDCCAfmgAwIBAgIJAJPcESSEV86nMA0GCSqGSIb3DQEBBQUAMGExCzAJBgNV
+BAYTAlBMMQ0wCwYDVQQIDARUZXN0MQ0wCwYDVQQHDARDaXR5MRAwDgYDVQQKDAdD
+b21wYW55MSIwIAYJKoZIhvcNAQkBFhNleGFtcGxlQGV4YW1wbGUuY29tMB4XDTEz
+MDEyMzEyMjIwN1oXDTE0MDEyMzEyMjIwN1owYTELMAkGA1UEBhMCUEwxDTALBgNV
+BAgMBFRlc3QxDTALBgNVBAcMBENpdHkxEDAOBgNVBAoMB0NvbXBhbnkxIjAgBgkq
+hkiG9w0BCQEWE2V4YW1wbGVAZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQAD
+gY0AMIGJAoGBAKUG9QHjUiCSjcvFQl1OqUgZ8c4Vzm7m7Ua7D33YDnGo3QGuhPML
+0JB+JPBG++MrPdQqMV8eeXe1dFlrm6SHnrEZIhwZg8iDnPhFKNjHiAkx0PCQdI93
+rAe4EcszkgbQCXXLGjQJNDikwGZ6ciylJ7C2XewWTlWjv7/LqMwhQotLAgMBAAGj
+UDBOMB0GA1UdDgQWBBRLRvOFIo9LIeiT2ohQalEzy687ijAfBgNVHSMEGDAWgBRL
+RvOFIo9LIeiT2ohQalEzy687ijAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUA
+A4GBAD1odtgH1kZ4a0TEbwjK2PaHTYyofkkgeXsGGnG3eMkwKQ7PHJarBl7GtxP5
+Ls91k/KvBuNqCM/lPmM94USDSXrKGe0Zzc9WZbDG9SXO23gGOwPl+Tg9v1ocRHd2
+dYYIgZM8/tLcsHyNNMd7c4BOMD4rzkjgbThFFMl/2NSkvrO7
+-----END CERTIFICATE-----
diff --git a/gui/data/global.cfg b/gui/data/global.cfg
new file mode 100644
index 0000000..e225ffc
--- /dev/null
+++ b/gui/data/global.cfg
@@ -0,0 +1,21 @@
+[general]
+
+config = .dir.cfg
+ready = .ready.lst
+footprints = .footprints.lst
+default_exts = .mmax _mentions.xml _words.xml
+lang = pl
+hide_dir_menu = no
+program = mmax
+extension = .mmax
+
+{paths}
+dir_cache = .dir.lst
+
+[connection]
+
+host = localhost
+port = 2225
+
+{paths}
+cert = cert.pem
diff --git a/gui/dfs.wxg b/gui/dfs.wxg
new file mode 100644
index 0000000..1f554a0
--- /dev/null
+++ b/gui/dfs.wxg
@@ -0,0 +1,367 @@
+<?xml version="1.0"?>
+<!-- generated by wxGlade 0.6.3 on Fri Sep 30 18:28:38 2011 -->
+
+<application path="/home/kuba/Pulpit/praca/nkjp/dist-system/gui" name="manager" class="FileManager" option="1" language="python" top_window="main_frame" encoding="UTF-8" use_gettext="0" overwrite="0" use_new_namespace="1" for_version="2.8" is_template="0">
+ <object class="MainFrame" name="main_frame" base="EditFrame">
+ <style>wxDEFAULT_FRAME_STYLE</style>
+ <title>--</title>
+ <menubar>1</menubar>
+ <centered>1</centered>
+ <focused>1</focused>
+ <size>663, 556</size>
+ <object class="wxMenuBar" name="frame_1_menubar" base="EditMenuBar">
+ <menus>
+ <menu name="dir_menu" label="Directory">
+ <item>
+ <label>Open</label>
+ <id>5</id>
+ <name>dir_menu_open</name>
+ <help_str>Open new directory</help_str>
+ <handler>open_directory</handler>
+ </item>
+ <item>
+ <label>Reconfigure</label>
+ <id>6</id>
+ <help_str>Reconfigure directory</help_str>
+ <handler>reconf_directory</handler>
+ </item>
+ <item>
+ <label>---</label>
+ <id>---</id>
+ <name>---</name>
+ </item>
+ </menu>
+ <menu name="" label="Files">
+ <item>
+ <label>Upload Selected</label>
+ <id>1</id>
+ <help_str>Upload annotated or adjudicated files</help_str>
+ <handler>upload</handler>
+ </item>
+ <item>
+ <label>Checkin Selected</label>
+ <id>8</id>
+ <help_str>Upload annotated or adjudicated; files will be deleted from local disk and you will not be able to modify them again</help_str>
+ <handler>checkin</handler>
+ </item>
+ <item>
+ <label>Download (annotation)</label>
+ <id>2</id>
+ <help_str>Download files for annotation</help_str>
+ <handler>download</handler>
+ </item>
+ <item>
+ <label>Download (adjudication)</label>
+ <id>3</id>
+ <help_str>Download files for adjudication</help_str>
+ <handler>download_prim</handler>
+ </item>
+ <item>
+ <label>Checkout</label>
+ <id>7</id>
+ <help_str>Checkout files from server to your local, empty directory</help_str>
+ <handler>checkout</handler>
+ </item>
+ </menu>
+ <menu name="" label="Select">
+ <item>
+ <label>Ready</label>
+ <id>4</id>
+ <help_str>Select items which are marked for upload</help_str>
+ <handler>select_ready</handler>
+ </item>
+ <item>
+ <label>Modified</label>
+ <id>9</id>
+ <help_str>Select items which are modified</help_str>
+ <handler>select_modified</handler>
+ </item>
+ </menu>
+ <menu name="" label="Help">
+ </menu>
+ </menus>
+ </object>
+ <object class="wxBoxSizer" name="sizer_6" base="EditBoxSizer">
+ <orient>wxHORIZONTAL</orient>
+ <object class="sizeritem">
+ <flag>wxEXPAND</flag>
+ <border>0</border>
+ <option>1</option>
+ <object class="wxSplitterWindow" name="window_1" base="EditSplitterWindow">
+ <style>wxSP_3D|wxSP_BORDER</style>
+ <orientation>wxSPLIT_HORIZONTAL</orientation>
+ <sash_pos>-100</sash_pos>
+ <window_2>window_1_pane_2</window_2>
+ <window_1>window_1_pane_1</window_1>
+ <object class="wxPanel" name="window_1_pane_1" base="EditPanel">
+ <style>wxTAB_TRAVERSAL</style>
+ <object class="wxBoxSizer" name="sizer_1" base="EditBoxSizer">
+ <orient>wxVERTICAL</orient>
+ <object class="sizeritem">
+ <flag>wxEXPAND</flag>
+ <border>0</border>
+ <option>1</option>
+ <object class="wxNotebook" name="notebook_1" base="EditNotebook">
+ <style>0</style>
+ <tabs>
+ <tab window="notebook_1_pane_1">annotation</tab>
+ <tab window="notebook_1_pane_2">adjudication</tab>
+ </tabs>
+ <object class="wxPanel" name="notebook_1_pane_1" base="EditPanel">
+ <style>wxTAB_TRAVERSAL</style>
+ <object class="wxBoxSizer" name="sizer_2" base="EditBoxSizer">
+ <orient>wxVERTICAL</orient>
+ <object class="sizeritem">
+ <flag>wxEXPAND</flag>
+ <border>0</border>
+ <option>1</option>
+ <object class="wxListCtrl" name="files_list" base="EditListCtrl">
+ <style>wxLC_REPORT|wxSIMPLE_BORDER</style>
+ <events>
+ <handler event="EVT_LIST_ITEM_ACTIVATED">activate_item</handler>
+ </events>
+ </object>
+ </object>
+ </object>
+ </object>
+ <object class="wxPanel" name="notebook_1_pane_2" base="EditPanel">
+ <style>wxTAB_TRAVERSAL</style>
+ <object class="wxBoxSizer" name="sizer_3" base="EditBoxSizer">
+ <orient>wxVERTICAL</orient>
+ <object class="sizeritem">
+ <flag>wxEXPAND</flag>
+ <border>0</border>
+ <option>1</option>
+ <object class="wxListCtrl" name="super_files_list" base="EditListCtrl">
+ <style>wxLC_REPORT|wxSIMPLE_BORDER</style>
+ <events>
+ <handler event="EVT_LIST_ITEM_ACTIVATED">activate_super_item</handler>
+ </events>
+ </object>
+ </object>
+ </object>
+ </object>
+ </object>
+ </object>
+ </object>
+ </object>
+ <object class="wxPanel" name="window_1_pane_2" base="EditPanel">
+ <style>wxTAB_TRAVERSAL</style>
+ <object class="wxBoxSizer" name="sizer_7" base="EditBoxSizer">
+ <orient>wxVERTICAL</orient>
+ <object class="sizeritem">
+ <flag>wxEXPAND</flag>
+ <border>0</border>
+ <option>1</option>
+ <object class="wxTextCtrl" name="log_ctrl" base="EditTextCtrl">
+ <style>wxTE_MULTILINE|wxTE_READONLY</style>
+ </object>
+ </object>
+ </object>
+ </object>
+ </object>
+ </object>
+ </object>
+ </object>
+ <object class="LoginDialog" name="login_dialog" base="EditDialog">
+ <style>wxDEFAULT_DIALOG_STYLE|wxRESIZE_BORDER|wxTHICK_FRAME</style>
+ <title>login</title>
+ <object class="wxBoxSizer" name="sizer_4" base="EditBoxSizer">
+ <orient>wxVERTICAL</orient>
+ <object class="sizeritem">
+ <border>0</border>
+ <option>0</option>
+ <object class="wxGridSizer" name="grid_sizer_1" base="EditGridSizer">
+ <hgap>0</hgap>
+ <rows>2</rows>
+ <cols>2</cols>
+ <vgap>0</vgap>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_VERTICAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxStaticText" name="label_1" base="EditStaticText">
+ <attribute>1</attribute>
+ <label>login: </label>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <border>0</border>
+ <option>0</option>
+ <object class="wxTextCtrl" name="login_ctrl" base="EditTextCtrl">
+ </object>
+ </object>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_VERTICAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxStaticText" name="label_3" base="EditStaticText">
+ <attribute>1</attribute>
+ <label>password: </label>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <border>0</border>
+ <option>0</option>
+ <object class="wxTextCtrl" name="passwd_ctrl" base="EditTextCtrl">
+ <style>wxTE_PASSWORD</style>
+ </object>
+ </object>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <border>0</border>
+ <option>0</option>
+ <object class="wxCheckBox" name="save_passwd" base="EditCheckBox">
+ <label>save password</label>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_HORIZONTAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxButton" name="button_1" base="EditButton">
+ <label>OK</label>
+ <id>wx.ID_OK</id>
+ <events>
+ <handler event="EVT_BUTTON">on_ok</handler>
+ </events>
+ <size>85, 32</size>
+ </object>
+ </object>
+ </object>
+ </object>
+ <object class="DownloadNumberDialog" name="download_number" base="EditDialog">
+ <style>wxDEFAULT_DIALOG_STYLE</style>
+ <title>checkout</title>
+ <centered>1</centered>
+ <object class="wxGridSizer" name="grid_sizer_2" base="EditGridSizer">
+ <hgap>0</hgap>
+ <rows>2</rows>
+ <cols>2</cols>
+ <vgap>0</vgap>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxStaticText" name="label_4" base="EditStaticText">
+ <attribute>1</attribute>
+ <tooltip>Number of files to download</tooltip>
+ <label>download:</label>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <border>0</border>
+ <option>0</option>
+ <object class="wxSpinCtrl" name="checkout_number" base="EditSpinCtrl">
+ <range>1, 300</range>
+ <value>1</value>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxButton" name="button_4" base="EditButton">
+ <label>OK</label>
+ <id>wx.ID_YES</id>
+ <events>
+ <handler event="EVT_BUTTON">on_ok</handler>
+ </events>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxButton" name="button_5" base="EditButton">
+ <label>Cancel</label>
+ <id>wx.ID_NO</id>
+ <events>
+ <handler event="EVT_BUTTON">on_cancel</handler>
+ </events>
+ </object>
+ </object>
+ </object>
+ </object>
+ <object class="ConfigDialog" name="config_dialog" base="EditDialog">
+ <style>wxDEFAULT_DIALOG_STYLE</style>
+ <title>Directory configuration</title>
+ <object class="wxBoxSizer" name="sizer_5" base="EditBoxSizer">
+ <orient>wxVERTICAL</orient>
+ <object class="sizeritem">
+ <border>0</border>
+ <option>0</option>
+ <object class="wxGridSizer" name="grid_sizer_4" base="EditGridSizer">
+ <hgap>0</hgap>
+ <rows>3</rows>
+ <cols>2</cols>
+ <vgap>0</vgap>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_VERTICAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxStaticText" name="label_2" base="EditStaticText">
+ <attribute>1</attribute>
+ <tooltip>Login supplied by system administrator</tooltip>
+ <label>login:</label>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_VERTICAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxTextCtrl" name="login_ctrl" base="EditTextCtrl">
+ </object>
+ </object>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_VERTICAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxStaticText" name="label_6" base="EditStaticText">
+ <attribute>1</attribute>
+ <tooltip>Password supplied by system administrator</tooltip>
+ <label>password:</label>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_VERTICAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxTextCtrl" name="passwd_ctrl" base="EditTextCtrl">
+ <style>wxTE_PASSWORD</style>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_VERTICAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxStaticText" name="label_5" base="EditStaticText">
+ <attribute>1</attribute>
+ <tooltip>List of space separated file extensions</tooltip>
+ <label>extensions:</label>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_VERTICAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxTextCtrl" name="exts_ctrl" base="EditTextCtrl">
+ </object>
+ </object>
+ </object>
+ </object>
+ <object class="sizeritem">
+ <flag>wxALIGN_CENTER_HORIZONTAL</flag>
+ <border>0</border>
+ <option>0</option>
+ <object class="wxButton" name="button_2" base="EditButton">
+ <label>OK</label>
+ <id>wx.ID_OK</id>
+ <focused>1</focused>
+ <size>85, 29</size>
+ </object>
+ </object>
+ </object>
+ </object>
+</application>
diff --git a/gui/dfs/__init__.py b/gui/dfs/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/gui/dfs/__init__.py
diff --git a/gui/dfs/config.py b/gui/dfs/config.py
new file mode 100644
index 0000000..5a53f9c
--- /dev/null
+++ b/gui/dfs/config.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+import re
+import os
+
+SECTION = re.compile("^\[(.*)\]$")
+SUBSECTION = re.compile("^\{(.*)\}$")
+ENTRY = re.compile("^([^=\s]+)\s*=\s*([^=]+)\s*$")
+
+def cut_comment(s):
+ return s.split("#")[0]
+
+class Config:
+
+ def __init__(self, path):
+ section = None
+ subsection = None
+ self.d = {}
+ with open(path) as f:
+ for line in f:
+ line = cut_comment(line).strip()
+
+ m = SECTION.search(line)
+ if m is not None:
+ section = m.group(1)
+ subsection = None
+ continue
+
+ m = SUBSECTION.search(line)
+ if m is not None:
+ subsection = m.group(1)
+ continue
+
+ m = ENTRY.search(line)
+ if m is not None:
+ key = m.group(1)
+ val = m.group(2)
+ if subsection == "paths":
+ # print path, val, os.path.join(path, val)
+ val = self.relative_path(val, path)
+ if section is not None:
+ self.d[section + "." + key] = val
+ else:
+ self.d[key] = val
+
+ def relative_path(self, path, ref):
+ return os.path.join(os.path.dirname(ref), path)
+
+ def __getitem__(self, key):
+ return self.d[key]
diff --git a/gui/dfs/database.py b/gui/dfs/database.py
new file mode 100644
index 0000000..4f4458b
--- /dev/null
+++ b/gui/dfs/database.py
@@ -0,0 +1,381 @@
+# -*- coding: utf-8 -*-
+import time
+from datetime import date, datetime
+from collections import defaultdict
+
+from xml.etree.ElementTree import Element, ElementTree, tostring
+import xml.etree.ElementTree as ET
+
+class Database:
+
+ def __init__(self, db, anno_per_file, from_file=True):
+ if from_file:
+ self.db_name = db
+ self.tree = ET.parse(db)
+ self.root = self.tree.getroot()
+ else:
+ self.db_name = None
+ self.tree = None
+ self.root = ET.fromstring(db)
+
+ self.anno_per_file = anno_per_file
+ self.make_index()
+
+ def owned(self, ann_elem):
+ return not self.finished(ann_elem) and not self.returned(ann_elem)
+
+ def finished(self, ann_elem):
+ return ann_elem.find("checkinDate") is not None
+
+ def returned(self, ann_elem):
+ return ann_elem.find("return") is not None
+
+ def get_reason(self, ann_elem):
+ reas = ann_elem.find("return").find("reason").text
+ if reas is None:
+ reas = ""
+ return reas
+
+ def rejected(self, file_elem):
+ return file_elem.find("rejected") is not None
+
+ def fixed(self, ann_elem):
+ ret = ann_elem.find("return")
+ if ret is not None and ret.find("fixed") is not None:
+ return True
+ return False
+
+ def ann_elem_for_user(self, file_elem, user):
+ idx = 0
+ for ann_elem in file_elem.findall("ann"):
+ if ann_elem.find("annName").text == user:
+ return ann_elem, idx
+ idx = idx + 1
+ return None, None
+
+ def sann_elem_for_user(self, file_elem, user):
+ for ann_elem in file_elem.findall("s_ann"):
+ if ann_elem.find("annName").text == user:
+ return ann_elem
+ return None
+
+ def fixed_for_user(self, file_elem, user, is_adj):
+ if is_adj:
+ ann = self.sann_elem_for_user(file_elem, user)
+ else:
+ ann, idx = self.ann_elem_for_user(file_elem, user)
+ if ann is not None:
+ return self.fixed(ann)
+ return False
+
+ def make_index(self):
+ self.file_index = {}
+ self.ann_owned_index = defaultdict(list)
+ self.adj_owned_index = defaultdict(list)
+
+ for file_elem in self.root:
+ name_elem = file_elem.find("name")
+ if name_elem == None:
+ raise Exception("No name assigned to a file element !")
+ self.file_index[name_elem.text] = file_elem
+
+ if self.rejected(file_elem):
+ continue
+
+ for ann_elem in file_elem.findall("ann"):
+ if self.owned(ann_elem):
+ ann_name = ann_elem.find("annName").text
+ self.ann_owned_index[ann_name].append(file_elem)
+
+ for ann_elem in file_elem.findall("s_ann"):
+ if self.owned(ann_elem):
+ ann_name = ann_elem.find("annName").text
+ self.adj_owned_index[ann_name].append(file_elem)
+
+ def fix(self, ann_elem):
+ if not self.returned(ann_elem):
+ raise Exception("File not returned in database!")
+ if self.fixed(ann_elem):
+ raise Exception("File already fixed in database!")
+
+ ret = ann_elem.find("return")
+ fixed = ET.SubElement(ret, "fixed")
+
+ def add(self, file_name):
+ if self.file_index.has_key(file_name):
+ raise Exception("File name already in database !")
+ file_elem = ET.SubElement(self.root, "file")
+ name_elem = ET.SubElement(file_elem, "name")
+ name_elem.text = file_name
+ date_elem = ET.SubElement(file_elem, "addDate")
+ date_elem.text = time.strftime("%c")
+ self.file_index[file_name] = file_elem
+
+ def remove(self, file_name):
+ if not self.file_index.has_key(file_name):
+ raise Exception("File name not in database !")
+ file_elem = self.file_index[file_name]
+ self.root.remove(file_elem)
+ self.make_index()
+
+ def reject(self, file_name, reason):
+ if not self.file_index.has_key(file_name):
+ raise Exception("File name not in database !")
+ file_elem = self.file_index[file_name]
+ if self.rejected(file_elem):
+ raise Exception("File already rejected !")
+ rej_elem = ET.SubElement(file_elem, "rejected")
+ rej_elem.text = reason
+ self.make_index()
+
+ # For prettyprint, used in 'save' method.
+ def indent(self, elem, level=0):
+ i = "\n" + level * " "
+ if len(elem):
+ if not elem.text or not elem.text.strip():
+ elem.text = i + " "
+ if not elem.tail or not elem.tail.strip():
+ elem.tail = i
+ for elem in elem:
+ self.indent(elem, level + 1)
+ if not elem.tail or not elem.tail.strip():
+ elem.tail = i
+ else:
+ if level and (not elem.tail or not elem.tail.strip()):
+ elem.tail = i
+
+ def save(self):
+ if not self.db_name:
+ raise Exception("XML not read from file !")
+
+ self.indent(self.root)
+ self.tree.write(self.db_name, encoding="utf-8")
+
+ def upload_prevention(self, file_name, user, is_super):
+ if not self.file_index.has_key(file_name):
+ return "Filename " + file_name.encode('utf-8') + " not known by server."
+ file_elem = self.file_index[file_name]
+ if self.rejected(file_elem):
+ return "File rejected: " + file_name.encode('utf-8')
+
+ if is_super:
+ ann_elem = file_elem.find("s_ann")
+ if ann_elem is None:
+ return "No superannotator assigned to file " + file_name.encode('utf-8')
+
+ if user != ann_elem.find("annName").text:
+ return "Different superannotator assigned to file " + file_name.encode('utf-8')
+
+ if ann_elem.find("checkinDate") is not None:
+ return "File " + file_name.encode('utf-8') + " already checked in!"
+
+ return None
+
+ else:
+ annotations = file_elem.findall("ann")
+ owners = map(lambda ann: ann.find("annName").text, annotations)
+ if not user in owners:
+ return "User " + user + " doesn't own the file " + file_name.encode('utf-8')
+
+ i = owners.index(user)
+ ann_elem = annotations[i]
+ if ann_elem.find("checkinDate") is not None:
+ return "User " + user + " already checked in the file " + file_name.encode('utf-8')
+
+ return None
+
+ def upload_dest(self, file_name, user):
+ """Upload destination (annName element and ID -- 1 or 2) directory."""
+ file_elem = self.file_index[file_name]
+ annotations = file_elem.findall("ann")
+
+ owners = map(lambda ann: ann.find("annName").text, annotations)
+ exc = Exception("Cannot upload " + file_name.encode('utf-8')
+ + " file by " + user + " annotator !")
+ if not user in owners:
+ raise exc
+
+ i = owners.index(user)
+ ann_elem = annotations[i]
+ if ann_elem.find("checkinDate") is not None:
+ raise exc
+
+ return (ann_elem, i)
+
+ def upload_id(self, file_name, user):
+ return self.upload_dest(file_name, user)[1]
+
+ def return_file(self, file_name, user, reason):
+ (ann_elem, i) = self.upload_dest(file_name, user)
+ ret_elem = ET.SubElement(ann_elem, "return")
+ date_elem = ET.SubElement(ret_elem, "date")
+ date_elem.text = time.strftime("%c")
+ date_elem = ET.SubElement(ret_elem, "reason")
+ if reason is None:
+ reason = ""
+ date_elem.text = reason
+ return i
+
+ def return_file_prim(self, file_name, user, reason):
+ file_elem = self.file_index[file_name]
+ ann_elem = file_elem.find("s_ann")
+
+ if (user != ann_elem.find("annName").text or
+ ann_elem.find("checkinDate") is not None):
+ raise Exception("Cannot upload " + file_name.encode('utf-8')
+ + " file by " + user + " annotator !")
+
+ ret_elem = ET.SubElement(ann_elem, "return")
+ date_elem = ET.SubElement(ret_elem, "date")
+ date_elem.text = time.strftime("%c")
+ date_elem = ET.SubElement(ret_elem, "reason")
+ if reason is None:
+ reason = ""
+ date_elem.text = reason
+
+ def upload(self, file_name, user):
+ (ann_elem, i) = self.upload_dest(file_name, user)
+ date_elem = ET.SubElement(ann_elem, "checkinDate")
+ date_elem.text = time.strftime("%c")
+ return i
+
+ def upload_prim(self, file_name, user):
+ file_elem = self.file_index[file_name]
+ ann_elem = file_elem.find("s_ann")
+
+ if (user != ann_elem.find("annName").text or self.finished(ann_elem)):
+ raise Exception("Cannot upload " + file_name.encode('utf-8')
+ + " file by " + user + " annotator !")
+
+ date_elem = ET.SubElement(ann_elem, "checkinDate")
+ date_elem.text = time.strftime("%c")
+
+ def download(self, file_name, user):
+ file_elem = self.file_index[file_name]
+
+ annotations = file_elem.findall("ann")
+ owners = map(lambda ann: ann.find("annName").text, annotations)
+
+ if self.fixed_for_user(file_elem, user, False):
+ ann, idx = self.ann_elem_for_user(file_elem, user)
+ date = ann.find("return").find("date").text
+ ann.remove(ann.find("return"))
+
+ date_elem = ET.SubElement(ann, "returnDate")
+ date_elem.text = date
+
+ date_elem = ET.SubElement(ann, "checkoutDate")
+ date_elem.text = time.strftime("%c")
+
+ return idx
+
+ else:
+ if user in owners or len(owners) + 1 > self.anno_per_file:
+ raise Exception("Cannot set " + user + " annotator to '"
+ + file_name.encode('utf-8') + "' file !")
+
+ ann_elem = ET.SubElement(file_elem, "ann")
+ ann_name_elem = ET.SubElement(ann_elem, "annName")
+ ann_name_elem.text = user
+
+ date_elem = ET.SubElement(ann_elem, "checkoutDate")
+ date_elem.text = time.strftime("%c")
+
+ return None
+
+ def download_prim(self, file_name, user):
+ file_elem = self.file_index[file_name]
+
+ if self.fixed_for_user(file_elem, user, True):
+ sann = self.sann_elem_for_user(file_elem, user)
+ date = sann.find("return").find("date").text
+ sann.remove(sann.find("return"))
+
+ date_elem = ET.SubElement(sann, "returnDate")
+ date_elem.text = date
+
+ date_elem = ET.SubElement(sann, "checkoutDate")
+ date_elem.text = time.strftime("%c")
+
+ return True
+
+ else:
+ if len(file_elem.findall("s_ann")) > 0:
+ raise Exception("Cannot set " + user + " adjudicator to '"
+ + file_name.encode('utf-8') + "' file !")
+
+ ann_elem = ET.SubElement(file_elem, "s_ann")
+ ann_name_elem = ET.SubElement(ann_elem, "annName")
+ ann_name_elem.text = user
+
+ date_elem = ET.SubElement(ann_elem, "checkoutDate")
+ date_elem.text = time.strftime("%c")
+
+ return False
+
+ def for_annotation(self, user):
+ priority = []
+ result = []
+ for file_elem in self.root:
+ if self.rejected(file_elem) or file_elem.find("s_ann") is not None:
+ continue
+
+ anns = file_elem.findall("ann")
+ owners = map(lambda ann: ann.find("annName").text, anns)
+ if len(owners) < self.anno_per_file and user not in owners:
+ result.append(file_elem.find("name").text)
+
+ if self.fixed_for_user(file_elem, user, False):
+ priority.append(file_elem.find("name").text)
+
+ return priority, result
+
+ def for_adjudication(self, user):
+ priority = []
+ result = []
+ for file_elem in self.root:
+ if self.rejected(file_elem):
+ continue
+
+ anns = file_elem.findall("ann")
+ owners = map(lambda ann: ann.find("annName").text, anns)
+ checked = map(lambda ann: ann.find("checkinDate") != None, anns)
+ if (len(owners) == self.anno_per_file
+ and file_elem.find("s_ann") is None
+ and user not in owners
+ and all(checked)):
+ result.append(file_elem.find("name").text)
+
+ if self.fixed_for_user(file_elem, user, True):
+ priority.append(file_elem.find("name").text)
+
+ return priority, result
+
+ def owns_normal(self, user):
+ return [ file_elem.find("name").text
+ for file_elem
+ in self.ann_owned_index[user] ]
+
+ def owns_super(self, user):
+ return [ file_elem.find("name").text
+ for file_elem
+ in self.adj_owned_index[user] ]
+
+ def owns(self, user):
+ return self.owns_normal(user) + self.owns_super(user)
+
+ def finished_count(self, user):
+ n = 0
+ for file_elem in self.root:
+
+ if self.rejected(file_elem):
+ continue
+
+ items = file_elem.findall("ann")
+ items.extend(file_elem.findall("s_ann"))
+ for ann_elem in items:
+ ann_name = ann_elem.find("annName").text
+ if ann_name == user and self.finished(ann_elem) and not self.returned(ann_elem):
+ n += 1
+
+ return n
diff --git a/gui/dfs/msg/__init__.py b/gui/dfs/msg/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/gui/dfs/msg/__init__.py
diff --git a/gui/dfs/msg/message.py b/gui/dfs/msg/message.py
new file mode 100644
index 0000000..295fb11
--- /dev/null
+++ b/gui/dfs/msg/message.py
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+__all__ = ["decode", "Message", "OkMessage", "KoMessage", "DownloadRequest"
+ , "UploadRequest", "DownloadPrimRequest", "ClientDone", "ServerDone"
+ , "NumMessage", "BoolMessage", "ReturnRequest", "CheckoutRequest",
+ "CheckinRequest", "StatsRequest"]
+
+
+class Message(object):
+
+ """
+ Base class for SSL communication messages.
+ """
+
+ def __init__(self, contents=""):
+ self.contents = contents
+
+ def get_contents(self):
+ return self.contents
+
+ @classmethod
+ def from_contents(cls, contents):
+ return cls(contents)
+
+ def encode(self):
+ """
+ Prepare message to be sent over network.
+ WARNING: Do not override this method.
+ """
+ # Add "-" on the beggining to prevent
+ # message contents from being void.
+ return self.msg_type, "-" + self.get_contents()
+
+
+class NumMessage(Message):
+
+ def __init__(self, n):
+ self.number = n
+
+ def get_contents(self):
+ return str(self.number)
+
+ @classmethod
+ def from_contents(cls, contents):
+ return cls(int(contents))
+
+ def get_number(self):
+ return self.number
+
+
+class BoolMessage(Message):
+
+ def __init__(self, b):
+ self.b = b
+
+ def get_contents(self):
+ return "1" if self.b else "0"
+
+ @classmethod
+ def from_contents(cls, contents):
+ return cls(bool(int(contents)))
+
+ def get_boolean(self):
+ return self.b
+
+class ReturnRequest(NumMessage):
+ pass
+
+class CheckoutRequest(Message):
+ pass
+
+class CheckinRequest(NumMessage):
+ pass
+
+class DownloadRequest(NumMessage):
+ pass
+
+class DownloadPrimRequest(NumMessage):
+ pass
+
+class UploadRequest(NumMessage):
+ pass
+
+class OkMessage(Message):
+ pass
+
+class KoMessage(Message):
+ pass
+
+class StatsRequest(Message):
+ pass
+
+class ServerDone(Message):
+ pass
+
+class ClientDone(Message):
+ pass
+
+
+def subclasses(cls):
+ yield cls
+ for child in cls.__subclasses__():
+ for desc in subclasses(child):
+ yield desc
+
+def decode(msg_type, contents):
+ for cls in subclasses(Message):
+ if cls.msg_type == msg_type:
+ # Discard first, dummy character;
+ # Confront Message.encode method.
+ return cls.from_contents(contents[1:])
+
+for i, cls in enumerate(subclasses(Message)):
+ cls.msg_type = i
diff --git a/gui/dfs/msg/stream.py b/gui/dfs/msg/stream.py
new file mode 100644
index 0000000..116bab0
--- /dev/null
+++ b/gui/dfs/msg/stream.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+import struct
+
+from message import decode
+
+__all__ = ["read_msg", "write_msg", "EndOfStream"]
+
+class EndOfStream(Exception):
+ pass
+
+class BadMessage(Exception):
+ pass
+
+def read_msg(connstream):
+ encoded = read_encoded(connstream)
+ return decode(*encoded)
+
+def accept_msg(connstream, cls):
+ msg = read_msg(connstream)
+ if type(msg) != cls:
+ raise BadMessage("Expected %s, got %s" %
+ (cls.__name__, type(msg).__name__))
+ return msg
+
+def write_msg(connstream, msg):
+ encoded = msg.encode()
+ write_encoded(connstream, *encoded)
+
+def read_encoded(connstream):
+ _type = read_type(connstream)
+ length = read_length(connstream)
+ content = read_content(connstream, length)
+ return _type, content
+
+def write_encoded(connstream, _type, content):
+ write_type(connstream, _type)
+ write_length(connstream, len(content))
+ write_content(connstream, content)
+
+def read_type(connstream):
+ data = read_all(connstream, 1)
+ return struct.unpack("!B", data)[0]
+
+def write_type(connstream, _type):
+ data = struct.pack("!B", _type)
+ write_all(connstream, data)
+
+def read_length(connstream):
+ data = read_all(connstream, 4)
+ return struct.unpack("!L", data)[0]
+
+def write_length(connstream, n):
+ data = struct.pack("!L", n)
+ write_all(connstream, data)
+
+def read_content(connstream, length):
+ return read_all(connstream, length)
+
+def write_content(connstream, content):
+ write_all(connstream, content)
+
+def write_all(connstream, data):
+ k = connstream.write(data)
+ if k != len(data):
+ raise Exception("Did not send all data !")
+
+def read_all(connstream, k):
+ data = ""
+ while k > 0:
+ # part = connstream.read()
+ part = connstream.read(min(k, 1024))
+ if len(part) == 0:
+ raise EndOfStream
+ k -= len(part)
+ # if k < 0:
+ # raise Exception("Received more than expected !")
+ data += part
+ return data
+
diff --git a/gui/dfs/repo.py b/gui/dfs/repo.py
new file mode 100644
index 0000000..9729984
--- /dev/null
+++ b/gui/dfs/repo.py
@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+import os
+import shutil
+import pysvn
+
+class Repo:
+
+ def __init__(self, path, login, passwd):
+ self.path = path
+ def get_login(_realm, _username, _may_save):
+ return True, login, passwd, False
+ self.svn = pysvn.Client()
+ self.svn.callback_get_login = get_login
+ self.svn.update(self.path)
+
+ def add(self, src_path):
+ part_name = os.path.basename(src_path)
+ dest_path = os.path.join(self.new_path(), part_name)
+
+ if (not self.svn.info(dest_path) is None) or (os.path.exists(dest_path)):
+ raise Exception(src_path+" already present in repository.")
+
+ shutil.copy(src_path, dest_path)
+ self.svn.add(dest_path)
+
+ def remove(self, filename, anno_per_file):
+ paths = set()
+ paths.add(os.path.join(self.new_path(), filename))
+ for i in range(anno_per_file):
+ paths.add(self.upload_path(filename, i))
+ paths.add(os.path.join(self.finito_path(), filename))
+
+ removed = []
+ for path in paths:
+ if os.path.exists(path):
+ self.svn.remove(path)
+ removed.append(path)
+
+ return removed
+
+ def db_path(self):
+ return os.path.join(self.path, "db.xml")
+
+ def new_path(self):
+ return os.path.join(self.path, "new")
+
+ def ann_path(self, idx):
+ ids = chr(idx+ord('A'))
+ tail = os.path.join("annotation", ids)
+ return os.path.join(self.path, tail)
+
+ def finito_path(self):
+ return os.path.join(self.path, "adjudication")
+
+ def upload_path(self, file_name, idx):
+ return os.path.join(self.ann_path(idx), file_name)
+
+ def upload_prim_path(self, file_name):
+ return os.path.join(self.finito_path(), file_name)
+
+ def read_data(self, path):
+ with open(path) as src:
+ return src.read()
+
+ def read_contents(self, path, exts):
+ contents = {}
+ for ext in exts:
+ contents[ext] = self.read_data(path + ext)
+ return contents
+
+ def write_data(self, path, data):
+ with open(path, "w") as dest:
+ dest.write(data)
+
+ def write_contents(self, path, contents):
+ for ext, data in contents.iteritems():
+ dest_path = path + ext
+ self.write_data(dest_path, data)
+ if self.svn.info(dest_path) is None:
+ self.svn.add(dest_path)
+
+ def upload(self, file_id, idx, contents):
+ self.write_contents(self.upload_path(file_id, idx), contents)
+
+ def upload_prim(self, file_id, contents):
+ self.write_contents(self.upload_prim_path(file_id), contents)
+
+ def download(self, file_id, exts):
+ return self.read_contents(os.path.join(self.new_path(), file_id), exts)
+
+ def download_prim(self, file_id, exts, anno_per_file):
+ result = []
+ for i in range(anno_per_file):
+ conts = self.read_contents(os.path.join(self.ann_path(i), file_id), exts)
+ result.append(conts)
+ return result
+
+ def checkout(self, file_id, idx, exts):
+ src_paths = [ self.upload_path(file_id, idx)
+ , os.path.join(self.new_path(), file_id) ]
+ for path in src_paths:
+ if all(os.path.exists(path + ext) for ext in exts):
+ return self.read_contents(path, exts)
+
+ def checkout_prim(self, file_id, exts):
+ path = os.path.join(self.finito_path(), file_id)
+ if all(os.path.exists(path + ext) for ext in exts):
+ return self.read_contents(path, exts)
+
+ def commit(self, message):
+ self.svn.checkin(self.path, message, recurse=True)
+
+ def revert(self):
+ self.svn.revert(self.path, recurse=True)
diff --git a/gui/dfs/server.py b/gui/dfs/server.py
new file mode 100644
index 0000000..639e5ec
--- /dev/null
+++ b/gui/dfs/server.py
@@ -0,0 +1,401 @@
+# -*- coding: utf-8 -*-
+import sys
+import os
+import traceback
+import socket, ssl
+import random
+from optparse import OptionParser
+import signal
+
+from utils import validate_user, UserInvalid, send_contents, receive_contents
+from msg.stream import read_msg, write_msg, accept_msg, EndOfStream, BadMessage
+from msg.message import *
+from repo import Repo
+from database import Database
+from config import Config
+
+class SSLServer:
+
+ def __init__(self, host, port, backlog, cert_file, key_file,
+ repository, svn_login, svn_passwd, pass_file,
+ users_file, file_exts, anno_per_file, log=None):
+ """
+ Initialize SSL server.
+
+ params:
+ =======
+ host : str
+ Host name.
+ port : int
+ Port number.
+ backlog : int
+ Maximum number of waiting connections.
+ cert_file : path
+ Public PEM certificate.
+ key_file : path
+ Private key.
+ pass_file : path
+ File with client passwords.
+ users_file : path
+ File with additional users configuration.
+ repository : path
+ Subversion working copy.
+ svn_login : str
+ Subversion login.
+ svn_passwd : str
+ Subversion password.
+ file_exts : [str]
+ List of file extensions.
+ anno_per_file : int
+ Number of normal annotators per file (1 or 2)
+ """
+ self.cert_file = cert_file
+ self.key_file = key_file
+ self.pass_file = pass_file
+ self.users_file = users_file
+ self.wc = Repo(repository, svn_login, svn_passwd)
+ self.file_exts = file_exts
+ if log is not None:
+ self.log = open(log, "a")
+ else:
+ self.log = sys.stdout
+ self.bound = bind_socket(host, port, backlog=backlog)
+ self.serving = None
+ self.anno_per_file = anno_per_file
+
+ def run(self):
+ while True:
+ newsocket, fromaddr = self.bound.accept()
+ self.serving = fromaddr
+ connstream = None
+ try:
+ connstream = ssl.wrap_socket(newsocket,
+ server_side=True,
+ certfile=self.cert_file,
+ keyfile=self.key_file,
+ ssl_version=ssl.PROTOCOL_TLSv1)
+
+ login = validate_user(connstream, self.pass_file)
+ self.serve_client(connstream, login)
+ except UserInvalid as login:
+ print >> self.log, "AUTH ERROR:", login
+ except BadMessage as info:
+ print >> self.log, "BAD MESSAGE ERROR:", info
+ print >> self.log, traceback.format_exc().strip()
+ except EndOfStream:
+ print >> self.log, "UNEXPECTED END OF STREAM:"
+ print >> self.log, traceback.format_exc().strip()
+ except:
+ print >> self.log, "UNEXPECTED ERROR:"
+ print >> self.log, traceback.format_exc().strip()
+ finally:
+ try:
+ if not connstream is None:
+ connstream.shutdown(socket.SHUT_RDWR)
+ except:
+ print >> self.log, "UNEXPECTED ERROR:"
+ print >> self.log, traceback.format_exc().strip()
+ finally:
+ if not connstream is None:
+ connstream.close()
+ self.serving = None
+
+ def exit(self):
+ if self.serving is not None:
+ print ("Client from %s connected to the server"
+ % str(self.serving))
+ else:
+ sys.exit(0)
+
+ def process_msg(self, msg, stream, login):
+ if type(msg) == UploadRequest:
+ self.process_upload_msg(msg, stream, login, checkin=False)
+ elif type(msg) == CheckinRequest:
+ self.process_upload_msg(msg, stream, login, checkin=True)
+ elif type(msg) == CheckoutRequest:
+ self.process_checkout_msg(msg, stream, login)
+ elif type(msg) == DownloadRequest:
+ self.process_download_msg(msg, stream, login)
+ elif type(msg) == DownloadPrimRequest:
+ self.process_download_prim_msg(msg, stream, login)
+ elif type(msg) == ReturnRequest:
+ self.process_return_msg(msg, stream, login)
+ elif type(msg) == StatsRequest:
+ self.process_stats_msg(msg, stream, login)
+ else:
+ print >> self.log, login, "sent:", msg
+
+ def process_return_msg(self, msg, stream, login):
+ reason = accept_msg(stream, Message).get_contents().decode("utf-8")
+
+ usr_cfg = Config(self.users_file)
+ try:
+ user_adj = login in usr_cfg["auth.adjudicators"].split()
+ except KeyError:
+ user_adj = False
+
+ db = Database(self.wc.db_path(), self.anno_per_file)
+ n = msg.get_number()
+ for _ in range(n):
+ file_id = accept_msg(stream, Message).get_contents()
+ is_adj = accept_msg(stream, BoolMessage).get_boolean()
+ if is_adj and not user_adj:
+ write_msg(stream, KoMessage())
+ print >> self.log, "Upload error - user " + login + " tried to upload super annotated file " + file_id + " but is not a super annotator"
+ continue
+ else:
+ error = db.upload_prevention(file_id, login, is_adj)
+ if error is not None:
+ write_msg(stream, KoMessage())
+ print >> self.log, "Upload error by user " + login + ". Details: "+error
+ continue
+
+ write_msg(stream, OkMessage())
+ contents = receive_contents(stream)
+ for key in contents.keys():
+ if key not in self.file_exts:
+ print >> self.log, "Skipped file uploaded by user " + login + ". Filename:"+file_id+key
+ del contents[key]
+
+ if is_adj is False:
+ idx = db.return_file(file_id, login, reason)
+ self.wc.upload(file_id, idx, contents)
+ else:
+ db.return_file_prim(file_id, login, reason)
+ self.wc.upload_prim(file_id, contents)
+
+ db.save()
+ accept_msg(stream, ClientDone)
+ self.wc.commit("return request from %s"
+ % (login))
+ write_msg(stream, ServerDone())
+
+
+ def process_upload_msg(self, msg, stream, login, checkin=False):
+ usr_cfg = Config(self.users_file)
+ try:
+ user_adj = login in usr_cfg["auth.adjudicators"].split()
+ except KeyError:
+ user_adj = False
+
+ db = Database(self.wc.db_path(), self.anno_per_file)
+ n = msg.get_number()
+ for _ in range(n):
+ file_id = accept_msg(stream, Message).get_contents()
+ is_adj = accept_msg(stream, BoolMessage).get_boolean()
+ if is_adj and not user_adj:
+ write_msg(stream, KoMessage())
+ print >> self.log, "Upload error - user " + login + " tried to upload super annotated file " + file_id + " but is not a super annotator"
+ continue
+ else:
+ error = db.upload_prevention(file_id, login, is_adj)
+ if error is not None:
+ write_msg(stream, KoMessage())
+ print >> self.log, "Upload error by user " + login + ". Details: "+error
+ continue
+
+ write_msg(stream, OkMessage())
+ contents = receive_contents(stream)
+ for key in contents.keys():
+ if key not in self.file_exts:
+ print >> self.log, "Skipped file uploaded by user " + login + ". Filename:"+file_id+key
+ del contents[key]
+
+ if is_adj is False:
+ if checkin:
+ idx = db.upload(file_id, login)
+ else:
+ idx = db.upload_id(file_id, login)
+ self.wc.upload(file_id, idx, contents)
+ else:
+ if checkin:
+ db.upload_prim(file_id, login)
+ self.wc.upload_prim(file_id, contents)
+
+ if checkin:
+ db.save()
+ accept_msg(stream, ClientDone)
+ self.wc.commit("%s request from %s"
+ % ("checkin" if checkin else "upload", login))
+ write_msg(stream, ServerDone())
+
+ def process_checkout_msg(self, msg, stream, login):
+ db = Database(self.wc.db_path(), self.anno_per_file)
+
+ ann_files = db.owns_normal(login)
+ write_msg(stream, NumMessage(len(ann_files)))
+ for file_id in ann_files:
+ write_msg(stream, Message(file_id))
+ idx = db.upload_id(file_id, login)
+ contents = self.wc.checkout(file_id, idx, self.file_exts)
+ send_contents(stream, contents)
+
+ adj_files = db.owns_super(login)
+ write_msg(stream, NumMessage(len(adj_files)))
+ write_msg(stream, NumMessage(self.anno_per_file))
+ for file_id in adj_files:
+ write_msg(stream, Message(file_id))
+
+ contents = self.wc.download_prim(file_id, self.file_exts, self.anno_per_file)
+ for conts in contents:
+ send_contents(stream, conts)
+
+ conts3 = self.wc.checkout_prim(file_id, self.file_exts)
+ if conts3 is None:
+ write_msg(stream, BoolMessage(False))
+ else:
+ write_msg(stream, BoolMessage(True))
+ send_contents(stream, conts3)
+
+ accept_msg(stream, ClientDone)
+ #self.wc.commit("checkout request from %s" % login)
+ write_msg(stream, ServerDone())
+
+ def process_stats_msg(self, msg, stream, login):
+ usr_cfg = Config(self.users_file)
+ db = Database(self.wc.db_path(), self.anno_per_file)
+
+ write_msg(stream, NumMessage(db.finished_count(login)))
+
+ accept_msg(stream, ClientDone)
+ write_msg(stream, ServerDone())
+
+ def process_download_msg(self, msg, stream, login):
+ usr_cfg = Config(self.users_file)
+ db = Database(self.wc.db_path(), self.anno_per_file)
+
+ down_num = msg.get_number()
+ owns_num = len(db.owns_normal(login))
+ limit = float('inf')
+ try:
+ limit = int(usr_cfg["limits.%s.annotation" % login])
+ except KeyError:
+ limit = int(usr_cfg["limits.annotation"])
+ if down_num + owns_num > limit:
+ down_num = max(0, limit - owns_num)
+
+ for_ann_fixed, for_ann = db.for_annotation(login)
+ if len(for_ann) + len(for_ann_fixed) == 0:
+ print >> self.log, "User " + login + " has no more files to download for annotation."
+
+ down_files = sample(for_ann_fixed, down_num)
+ down_num = max(0, down_num - len(down_files))
+ down_files = down_files + sample(for_ann, down_num)
+
+ write_msg(stream, NumMessage(len(down_files)))
+
+ for file_id in down_files:
+ fixed_idx = db.download(file_id, login)
+ write_msg(stream, Message(file_id))
+ if fixed_idx is None:
+ contents = self.wc.download(file_id, self.file_exts)
+ else:
+ contents = self.wc.checkout(file_id, fixed_idx, self.file_exts)
+ send_contents(stream, contents)
+ db.save()
+ accept_msg(stream, ClientDone)
+ self.wc.commit("download request from %s" % login)
+ write_msg(stream, ServerDone())
+
+ def process_download_prim_msg(self, msg, stream, login):
+ usr_cfg = Config(self.users_file)
+ try:
+ adjudicators = usr_cfg["auth.adjudicators"].split()
+ except KeyError:
+ adjudicators = []
+ if login in adjudicators:
+ write_msg(stream, OkMessage())
+ else:
+ write_msg(stream, KoMessage())
+ return
+ db = Database(self.wc.db_path(), self.anno_per_file)
+
+ down_num = msg.get_number()
+ owns_num = len(db.owns_super(login))
+ limit = float('inf')
+ try:
+ limit = int(usr_cfg["limits.%s.adjudication" % login])
+ except KeyError:
+ limit = int(usr_cfg["limits.adjudication"])
+ if down_num + owns_num > limit:
+ down_num = max(0, limit - owns_num)
+
+ for_adj_fixed, for_adj = db.for_adjudication(login)
+ if len(for_adj) + len(for_adj_fixed) == 0:
+ print >> self.log, "User " + login + " has no more files to download for superannotation."
+
+ down_files = sample(for_adj_fixed, down_num)
+ down_num = max(0, down_num - len(down_files))
+ down_files = down_files + sample(for_adj, down_num)
+
+ write_msg(stream, NumMessage(len(down_files)))
+ write_msg(stream, NumMessage(self.anno_per_file))
+ for file_id in down_files:
+ write_msg(stream, Message(file_id))
+ fixed = db.download_prim(file_id, login)
+
+ contents = self.wc.download_prim(file_id, self.file_exts, self.anno_per_file)
+ for conts in contents:
+ send_contents(stream, conts)
+
+ if fixed:
+ write_msg(stream, BoolMessage(True))
+ contents = self.wc.checkout_prim(file_id, self.file_exts)
+ send_contents(stream, contents)
+ else:
+ write_msg(stream, BoolMessage(False))
+
+ db.save()
+ accept_msg(stream, ClientDone)
+ self.wc.commit("download' request from %s" % login)
+ write_msg(stream, ServerDone())
+
+ def serve_client(self, connstream, login):
+ msg = read_msg(connstream)
+ try:
+ self.process_msg(msg, connstream, login)
+ except:
+ self.wc.revert()
+ raise
+
+def bind_socket(host, port, backlog=5):
+ bound = socket.socket()
+ bound.bind((host, port))
+ bound.listen(backlog)
+ return bound
+
+def sample(population, n):
+ if len(population) > n:
+ return random.sample(population, n)
+ else:
+ return population
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog CONFIG""")
+ (options, args) = optparser.parse_args()
+ if len(args) != 1:
+ optparser.print_help()
+ sys.exit(0)
+
+ cfg = Config(args[0])
+ server = SSLServer(
+ host=cfg["connection.host"],
+ port=int(cfg["connection.port"]),
+ backlog=int(cfg["connection.backlog"]),
+ cert_file=cfg["connection.certfile"],
+ key_file=cfg["connection.keyfile"],
+ repository=cfg["svn.repository"],
+ svn_login=cfg["svn.login"],
+ svn_passwd=cfg["svn.passwd"],
+ pass_file=cfg["users.passfile"],
+ users_file=cfg["users.config"],
+ file_exts=cfg["file_extensions"].split(),
+ anno_per_file=int(cfg["anno_per_file"])
+ )
+
+ def handler(signum, frame):
+ server.exit()
+
+ signal.signal(signal.SIGINT, handler)
+
+ server.run()
diff --git a/gui/dfs/utils.py b/gui/dfs/utils.py
new file mode 100644
index 0000000..e8a6d88
--- /dev/null
+++ b/gui/dfs/utils.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+import hashlib
+
+from msg.stream import accept_msg, write_msg
+from msg.message import Message, OkMessage, KoMessage, NumMessage
+
+class UserInvalid(Exception):
+ pass
+
+def validate_user(conns, pass_file):
+ pass_dict = parse_passwd(pass_file)
+ login = accept_msg(conns, Message).get_contents()
+ passwd = accept_msg(conns, Message).get_contents()
+ if pass_dict.has_key(login) and pass_dict[login] == passwd:
+ write_msg(conns, OkMessage())
+ return login
+ else:
+ write_msg(conns, KoMessage())
+ raise UserInvalid(login)
+
+def parse_passwd(pass_file):
+ result = {}
+ with open(pass_file) as f:
+ for line in f:
+ login, passwd = line.split("=")
+ login = login.strip()
+ passwd = passwd.strip()
+ result[login] = passwd
+ return result
+
+def _check_sum(*args):
+ m = hashlib.sha256()
+ for arg in args:
+ m.update(arg)
+ return m
+
+def check_sum(*args):
+ return _check_sum(*args).digest()
+
+def check_hexsum(*args):
+ return _check_sum(*args).hexdigest()
+
+def send_contents(stream, contents):
+ write_msg(stream, NumMessage(len(contents)))
+ for ext, data in contents.iteritems():
+ write_msg(stream, Message(ext))
+ write_msg(stream, Message(data))
+ write_msg(stream, Message(check_sum(ext, data)))
+ accept_msg(stream, OkMessage)
+
+def receive_contents(stream):
+ contents = {}
+ n = accept_msg(stream, NumMessage).get_number()
+ for _ in range(n):
+ ext = accept_msg(stream, Message).get_contents()
+ data = accept_msg(stream, Message).get_contents()
+ csum = accept_msg(stream, Message).get_contents()
+ if csum == check_sum(ext, data):
+ write_msg(stream, OkMessage())
+ else:
+ write_msg(stream, KoMessage())
+ contents[ext] = data
+ return contents
diff --git a/gui/dir_cache.py b/gui/dir_cache.py
new file mode 100644
index 0000000..02ee649
--- /dev/null
+++ b/gui/dir_cache.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+"""Directory cache manipulation"""
+
+import os
+
+class DirCache:
+
+ def __init__(self, path, size):
+ if not os.path.exists(path):
+ with open(path, "w") as f: pass
+ self.path = path
+ self.size = size
+
+ def add(self, dir_name):
+ cache = self.get()
+ if dir_name in cache:
+ cache.remove(dir_name)
+ cache.insert(0, dir_name)
+ self.write(cache[:self.size])
+
+ def get(self):
+ with open(self.path) as f:
+ return [x.strip() for x in f.readlines()]
+
+ def write(self, cache):
+ with open(self.path, "w") as f:
+ for entry in cache:
+ print >> f, entry
diff --git a/gui/footprints.py b/gui/footprints.py
new file mode 100644
index 0000000..773900b
--- /dev/null
+++ b/gui/footprints.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+import os
+import copy
+
+# from dfs.utils import check_hexsum as check_sum
+
+def getmtime(path):
+ return round(os.path.getmtime(path))
+
+from utils import all_files
+
+class FootPrints:
+
+ """Class for footprints file manipulation."""
+
+ def __init__(self, prints_path, dir_path, exts):
+ self.prints_path = prints_path
+ self.dir_path = dir_path
+ self.exts = exts
+ if not os.path.exists(prints_path):
+ with open(prints_path, "w") as f: pass
+ self.prints = self.read()
+ self.backup = copy.deepcopy(self.prints)
+
+ def modified(self, name):
+ try:
+ # (mtime, csum) = self.get_print(name)
+ mtime = self.get_print(name)
+ except KeyError:
+ return False
+
+ path = os.path.join(self.dir_path, name)
+ if mtime != self.print_now(path):
+ return True
+ return False
+
+ """ sprawdza czy nie pojawily sie jakies nowe pliki, ewentualnie dodajac dla nich wpisy """
+ def refresh(self):
+ old_prints = self.prints
+ new_prints = {}
+ for name in all_files(self.dir_path, self.exts):
+ if old_prints.has_key(name):
+ new_prints[name] = old_prints[name]
+ else:
+ path = os.path.join(self.dir_path, name)
+ new_prints[name] = self.print_now(path)
+ self.prints = new_prints
+
+ def renew(self, name):
+ path = os.path.join(self.dir_path, name)
+ try:
+ self.prints[name] = self.print_now(path)
+ except:
+ del self.prints[name]
+ print "Footprints: "+ path + " file not found."
+
+ def renew_all(self):
+ for key in self.prints.keys():
+ self.renew(key)
+
+ def get_print(self, name):
+ path = os.path.join(self.dir_path, name)
+ return self.prints[name]
+
+ def print_now(self, path):
+ return max(getmtime(path + ext) for ext in self.exts)
+
+ def read(self):
+ prints = {}
+ with open(self.prints_path) as f:
+ for line in f:
+ key, mtime = line.split()
+ prints[key] = float(mtime)
+ return prints
+
+ def write(self):
+ with open(self.prints_path, "w") as f:
+ # for key, (mtime, csum) in sorted(self.prints.items()):
+ # print >> f, key, "=", mtime, csum
+ for key, mtime in sorted(self.prints.items()):
+ print >> f, key, mtime
+
+ def save(self):
+ if self.prints != self.backup:
+ self.write()
diff --git a/gui/i18n.py b/gui/i18n.py
new file mode 100644
index 0000000..cda2011
--- /dev/null
+++ b/gui/i18n.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+import os, sys
+import locale
+import gettext
+import const
+
+def get_path():
+ '''Get the path to this script no matter how it's run.'''
+ #Determine if the application is a py/pyw or a frozen exe.
+ if hasattr(sys, 'frozen'):
+ # If run from exe
+ dir_path = os.path.dirname(sys.executable)
+ elif '__file__' in locals():
+ # If run from py
+ dir_path = os.path.dirname(__file__)
+ else:
+ # If run from command line
+ dir_path = sys.path[0]
+ return dir_path
+
+
+# Change this variable to your app name!
+# The translation files will be under
+# @LOCALE_DIR@/@LANGUAGE@/LC_MESSAGES/@APP_NAME@.mo
+#
+APP_NAME = "manager"
+# This is ok for maemo. Not sure in a regular desktop:
+APP_DIR = get_path()
+LOCALE_DIR = os.path.join(APP_DIR, 'po')
+
+langs = ["pl"]
+if (const.lang == "en"):
+ langs = ["en"]
+
+# Lets tell those details to gettext
+gettext.install (True)
+gettext.bindtextdomain (APP_NAME,
+ LOCALE_DIR)
+gettext.textdomain (APP_NAME)
+language = gettext.translation (APP_NAME,
+ LOCALE_DIR,
+ languages = langs,
+ fallback = False)
+
+
\ No newline at end of file
diff --git a/gui/img/down.png b/gui/img/down.png
new file mode 100644
index 0000000..e54d773
Binary files /dev/null and b/gui/img/down.png differ
diff --git a/gui/img/ready.png b/gui/img/ready.png
new file mode 100644
index 0000000..4e60989
Binary files /dev/null and b/gui/img/ready.png differ
diff --git a/gui/img/textimg_1.png b/gui/img/textimg_1.png
new file mode 100644
index 0000000..b77bd57
Binary files /dev/null and b/gui/img/textimg_1.png differ
diff --git a/gui/img/textimg_2.png b/gui/img/textimg_2.png
new file mode 100644
index 0000000..6bfcc34
Binary files /dev/null and b/gui/img/textimg_2.png differ
diff --git a/gui/img/textimg_3.png b/gui/img/textimg_3.png
new file mode 100644
index 0000000..1e64793
Binary files /dev/null and b/gui/img/textimg_3.png differ
diff --git a/gui/img/up.png b/gui/img/up.png
new file mode 100644
index 0000000..7e20039
Binary files /dev/null and b/gui/img/up.png differ
diff --git a/gui/log_thread.py b/gui/log_thread.py
new file mode 100644
index 0000000..7cd6767
--- /dev/null
+++ b/gui/log_thread.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+import wx
+from Queue import Queue
+from threading import Thread
+import copy
+
+class RThread(Thread):
+
+ def __init__(self, target, args, kwargs):
+ self.target = target
+ self.args = args
+ self.kwargs = kwargs
+ self.result = None
+ Thread.__init__(self)
+
+ def run(self):
+ self.result = self.target(*self.args, **self.kwargs)
+
+class LogThread(Thread):
+
+ def __init__(self, text_ctrl, target, on_end=None, args=(), kwargs={}):
+ self.text_ctrl = text_ctrl
+ self.target = target
+ self.on_end = on_end
+ self.args = args
+ self.kwargs = kwargs
+ Thread.__init__(self)
+
+ def run(self):
+ queue = Queue()
+ kwargs = copy.copy(self.kwargs)
+ kwargs["log"] = queue
+ worker = RThread( target=self.target
+ , args=self.args
+ , kwargs=kwargs )
+ worker.start()
+ while True:
+ msg = queue.get()
+ if msg is None:
+ break
+ wx.CallAfter(self.print_msg, msg)
+ worker.join()
+
+ if self.on_end:
+ wx.CallAfter(self.on_end, worker.result)
+
+ def print_msg(self, msg):
+ self.text_ctrl.AppendText(msg)
diff --git a/gui/manager.py b/gui/manager.py
new file mode 100755
index 0000000..8b699dc
--- /dev/null
+++ b/gui/manager.py
@@ -0,0 +1,249 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# generated by wxGlade 0.6.3 on Tue Sep 27 12:15:56 2011
+
+import os
+import wx
+
+import i18n
+_ = i18n.language.ugettext #use ugettext instead of getttext to avoid unicode errors
+
+from dfs.config import Config
+from client import run_upload, run_download, run_download_prim, run_checkout, run_return, run_check_stats
+
+from MainFrame import MainFrame
+import utils
+import const
+from ready import Ready
+from dir_cache import DirCache
+from log_thread import LogThread
+from footprints import FootPrints
+
+class FileManager(wx.App):
+
+ def OnInit(self):
+ wx.InitAllImageHandlers()
+ self.main_frame = MainFrame(None, -1, "")
+ self.SetTopWindow(self.main_frame)
+ self.main_frame.Show()
+
+ # Custom code
+ self.main_frame.set_manager(self)
+
+ self.curr_dir = None
+ self.config = None
+ self.selection = None
+ self.connected = False
+ self.dir_cache = DirCache(const.dir_cache_name, size=5)
+ self.refresh_thread = utils.RefreshThread(self, delay=2)
+ self.refresh_thread.start()
+
+ utils.fill_dir_menu(self.main_frame, self.dir_cache.get())
+ # End custom code
+
+ return 1
+
+ def reconf_directory(self):
+ if not self.curr_dir:
+ return
+ config_path = const.path(self.curr_dir, const.config_name)
+ if utils.create_dir_config( config_path, self.main_frame
+ , config=self.config ):
+ self.config = Config(config_path)
+ self.refresh()
+
+ def set_directory(self, path):
+ config_path = const.path(path, const.config_name)
+ if not os.path.exists(config_path):
+ if not utils.create_dir_config(config_path, self.main_frame):
+ return
+ self.config = Config(config_path)
+ self.ready = Ready(const.path(path, const.ready_name))
+ self.curr_dir = path
+ self.dir_cache.add(path)
+ self.main_frame.SetTitle(path)
+ self.refresh()
+
+ def begin_conn(self):
+ if not self.curr_dir:
+ wx.MessageBox(_('Directory not set'), _('Info'))
+ return False
+ if self.connected:
+ wx.MessageBox(_('Another operation running'), _('Info'))
+ return False
+ self.connected = True
+ return True
+
+ def end_conn(self, op_result):
+ self.connected = False
+
+ def upload(self, files, checkin=False):
+ if not self.begin_conn():
+ return
+
+ def on_end(result):
+ self.end_conn(result)
+ if checkin:
+ return
+ if result == True:
+ prints = self.footprints()
+ for file_name in files:
+ prints.renew(file_name)
+ prints.save()
+
+ logger = LogThread( text_ctrl=self.main_frame.log_ctrl
+ , target=run_upload
+ , args=( self.config["login"]
+ , self.config["passwd"]
+ , const.conn_host
+ , const.conn_port
+ , const.conn_cert
+ , files
+ , self.curr_dir
+ , self.config["extensions"].split()
+ , checkin )
+ , on_end=on_end )
+ logger.start()
+
+ def download(self, n):
+ if not self.begin_conn():
+ return
+ logger = LogThread( text_ctrl=self.main_frame.log_ctrl
+ , target=run_download
+ , args=( self.config["login"]
+ , self.config["passwd"]
+ , const.conn_host
+ , const.conn_port
+ , const.conn_cert
+ , n
+ , self.curr_dir )
+ , on_end=self.end_conn )
+ logger.start()
+
+ def download_prim(self, n):
+ if not self.begin_conn():
+ return
+ logger = LogThread( text_ctrl=self.main_frame.log_ctrl
+ , target=run_download_prim
+ , args=( self.config["login"]
+ , self.config["passwd"]
+ , const.conn_host
+ , const.conn_port
+ , const.conn_cert
+ , n
+ , self.curr_dir )
+ , on_end=self.end_conn )
+ logger.start()
+
+ def checkout(self):
+ if not self.begin_conn():
+ return
+
+ def on_end(result):
+ self.end_conn(result)
+ if result == True:
+ prints = self.footprints()
+ files_before = prints.prints.keys()
+ prints.renew_all()
+ prints.save()
+
+
+ logger = LogThread( text_ctrl=self.main_frame.log_ctrl
+ , target=run_checkout
+ , args=( self.config["login"]
+ , self.config["passwd"]
+ , const.conn_host
+ , const.conn_port
+ , const.conn_cert
+ , self.curr_dir
+ , self.config["extensions"].split() )
+ , on_end=on_end )
+ logger.start()
+
+ def return_files(self, files, reason):
+ if not self.begin_conn():
+ return
+
+ def on_end(result):
+ self.end_conn(result)
+
+ logger = LogThread( text_ctrl=self.main_frame.log_ctrl
+ , target=run_return
+ , args=( self.config["login"]
+ , self.config["passwd"]
+ , const.conn_host
+ , const.conn_port
+ , const.conn_cert
+ , files
+ , reason
+ , self.curr_dir
+ , self.config["extensions"].split()
+ )
+ , on_end=on_end )
+ logger.start()
+
+ def footprints(self):
+ return FootPrints( const.path(self.curr_dir, const.footprints_name)
+ , self.curr_dir
+ , self.config["extensions"].split() )
+
+ def refresh(self):
+ if self.curr_dir is None:
+ return
+
+ ready = self.ready.get_entries()
+ extensions = self.config["extensions"].split()
+
+ footprints = self.footprints()
+ footprints.refresh()
+
+ data = utils.normal_entries(
+ self.curr_dir,
+ extensions,
+ ready,
+ footprints)
+
+ shown = self.main_frame.files_list.shown_data()
+
+ if utils.differ(data, shown):
+ print "Data changed !"
+ self.main_frame.files_list.show(data)
+ footprints.save()
+
+ data = utils.super_entries(
+ self.curr_dir,
+ extensions,
+ ready,
+ footprints)
+
+ shown = self.main_frame.super_files_list.shown_data()
+
+ if utils.differ(data, shown):
+ print "Data changed !"
+ self.main_frame.super_files_list.show(data)
+
+ footprints.save()
+
+ def check_stats(self):
+ if not self.begin_conn():
+ return
+
+ def on_end(result):
+ self.end_conn(result)
+
+ logger = LogThread( text_ctrl=self.main_frame.log_ctrl
+ , target=run_check_stats
+ , args=( self.config["login"]
+ , self.config["passwd"]
+ , const.conn_host
+ , const.conn_port
+ , const.conn_cert )
+ , on_end=on_end )
+ logger.start()
+
+# end of class FileManager
+
+if __name__ == "__main__":
+ manager = FileManager(0)
+ manager.MainLoop()
+ manager.refresh_thread.exitFlag = True
diff --git a/gui/po/en/LC_MESSAGES/manager.po b/gui/po/en/LC_MESSAGES/manager.po
new file mode 100644
index 0000000..8407c9a
--- /dev/null
+++ b/gui/po/en/LC_MESSAGES/manager.po
@@ -0,0 +1,229 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: manager\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2011-12-14 15:46+0100\n"
+"PO-Revision-Date: \n"
+"Last-Translator: Mateusz Kopeć <m.kopec@ipipan.waw.pl>\n"
+"Language-Team: IPI PAN\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"X-Poedit-Language: Polish\n"
+"X-Poedit-Country: POLAND\n"
+"X-Poedit-SourceCharset: utf-8\n"
+
+#: .././MainFrame.py:40
+msgid "Open"
+msgstr ""
+
+#: .././MainFrame.py:40
+msgid "Open new directory"
+msgstr ""
+
+#: .././MainFrame.py:42
+msgid "Reconfigure"
+msgstr ""
+
+#: .././MainFrame.py:42
+msgid "Reconfigure directory"
+msgstr ""
+
+#: .././MainFrame.py:45
+msgid "Directory"
+msgstr ""
+
+#: .././MainFrame.py:47
+msgid "Upload Selected"
+msgstr ""
+
+#: .././MainFrame.py:47
+msgid "Upload annotated or adjudicated files"
+msgstr ""
+
+#: .././MainFrame.py:48
+msgid "Checkin Selected"
+msgstr ""
+
+#: .././MainFrame.py:48
+msgid "Upload annotated or adjudicated; files will be deleted from local disk and you will not be able to modify them again"
+msgstr ""
+
+#: .././MainFrame.py:50
+msgid "Download (annotation)"
+msgstr ""
+
+#: .././MainFrame.py:50
+msgid "Download files for annotation"
+msgstr ""
+
+#: .././MainFrame.py:51
+msgid "Download (adjudication)"
+msgstr ""
+
+#: .././MainFrame.py:51
+msgid "Download files for adjudication"
+msgstr ""
+
+#: .././MainFrame.py:53
+msgid "Checkout"
+msgstr ""
+
+#: .././MainFrame.py:53
+msgid "Checkout files from server to your local, empty directory"
+msgstr ""
+
+#: .././MainFrame.py:54
+msgid "Files"
+msgstr ""
+
+#: .././MainFrame.py:56
+#: .././SortableListCtrl.py:27
+msgid "Ready"
+msgstr ""
+
+#: .././MainFrame.py:56
+msgid "Select items which are marked for upload"
+msgstr ""
+
+#: .././MainFrame.py:57
+#: .././utils.py:45
+#: .././utils.py:94
+msgid "Modified"
+msgstr ""
+
+#: .././MainFrame.py:57
+msgid "Select items which are modified"
+msgstr ""
+
+#: .././MainFrame.py:58
+msgid "Select"
+msgstr ""
+
+#: .././MainFrame.py:106
+msgid "Annotation"
+msgstr ""
+
+#: .././MainFrame.py:107
+msgid "Adjudication"
+msgstr ""
+
+#: .././MainFrame.py:124
+msgid "Choose a directory"
+msgstr ""
+
+#: .././MainFrame.py:137
+msgid "Do you want to upload the following files?"
+msgstr ""
+
+#: .././MainFrame.py:139
+msgid "Question"
+msgstr ""
+
+#: .././MainFrame.py:192
+msgid "Do you want to checkin the following files?"
+msgstr ""
+
+#: .././MainFrame.py:194
+msgid "Files will be deleted from the local directory and you will not be able to read, write or modify them later."
+msgstr ""
+
+#: .././MainFrame.py:210
+#: .././MainFrame.py:221
+msgid "Starting mmax for text"
+msgstr ""
+
+#: .././MainFrame.py:228
+msgid "Error starting mmax:"
+msgstr ""
+
+#: .././utils.py:46
+#: .././utils.py:82
+#: .././utils.py:90
+#: .././utils.py:122
+msgid "Yes"
+msgstr ""
+
+#: .././utils.py:46
+#: .././utils.py:81
+#: .././utils.py:85
+#: .././utils.py:115
+#: .././utils.py:120
+#: .././utils.py:125
+msgid "No"
+msgstr ""
+
+#: .././manager.py:70
+msgid "Directory not set"
+msgstr ""
+
+#: .././manager.py:70
+#: .././manager.py:73
+msgid "Info"
+msgstr ""
+
+#: .././manager.py:73
+msgid "Another operation running"
+msgstr ""
+
+#: .././ConfigDialog.py:21
+msgid "login"
+msgstr ""
+
+#: .././ConfigDialog.py:23
+msgid "password"
+msgstr ""
+
+#: .././ConfigDialog.py:25
+msgid "extensions"
+msgstr ""
+
+#: .././ConfigDialog.py:27
+#: .././DownloadNumberDialog.py:23
+msgid "OK"
+msgstr ""
+
+#: .././ConfigDialog.py:35
+msgid "Directory configuration"
+msgstr ""
+
+#: .././ConfigDialog.py:36
+msgid "Login supplied by system administrator"
+msgstr ""
+
+#: .././ConfigDialog.py:37
+msgid "Password supplied by system administrator"
+msgstr ""
+
+#: .././ConfigDialog.py:38
+msgid "List of space separated file extensions"
+msgstr ""
+
+#: .././DownloadNumberDialog.py:21
+msgid "Download count"
+msgstr ""
+
+#: .././DownloadNumberDialog.py:24
+msgid "Cancel"
+msgstr ""
+
+#: .././DownloadNumberDialog.py:35
+msgid "Download"
+msgstr ""
+
+#: .././DownloadNumberDialog.py:36
+msgid "Number of files to download"
+msgstr ""
+
+#: .././SortableListCtrl.py:25
+msgid "File"
+msgstr ""
+
+#: .././SortableListCtrl.py:26
+msgid "Status"
+msgstr ""
+
+#: .././SortableListCtrl.py:28
+msgid "Stage"
+msgstr ""
+
diff --git a/gui/po/generate_pot.sh b/gui/po/generate_pot.sh
new file mode 100755
index 0000000..8091007
--- /dev/null
+++ b/gui/po/generate_pot.sh
@@ -0,0 +1 @@
+xgettext --language=Python --keyword=_ --output=manager.pot `find ../. -name "*.py"`
diff --git a/gui/po/manager.pot b/gui/po/manager.pot
new file mode 100644
index 0000000..2df6a9e
--- /dev/null
+++ b/gui/po/manager.pot
@@ -0,0 +1,325 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2012-06-27 16:25+0200\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=CHARSET\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#: .././utils.py:46 .././MainFrame.py:62 .././SortableListCtrl.py:130
+msgid "Modified"
+msgstr ""
+
+#: .././utils.py:47 .././SortableListCtrl.py:78 .././SortableListCtrl.py:106
+#: .././SortableListCtrl.py:124 .././SortableListCtrl.py:127
+msgid "Yes"
+msgstr ""
+
+#: .././utils.py:47 .././SortableListCtrl.py:110
+msgid "No"
+msgstr ""
+
+#: .././DownloadNumberDialog.py:21
+msgid "Download count"
+msgstr ""
+
+#: .././DownloadNumberDialog.py:23 .././ReturnReasonDialog.py:27
+#: .././ConfigDialog.py:27
+msgid "OK"
+msgstr ""
+
+#: .././DownloadNumberDialog.py:24 .././ReturnReasonDialog.py:28
+msgid "Cancel"
+msgstr ""
+
+#: .././DownloadNumberDialog.py:35
+msgid "Download"
+msgstr ""
+
+#: .././DownloadNumberDialog.py:36
+msgid "Number of files to download"
+msgstr ""
+
+#: .././ReturnReasonDialog.py:21
+msgid "Why do you want to return the following files?"
+msgstr ""
+
+#: .././ReturnReasonDialog.py:25
+msgid "Reason"
+msgstr ""
+
+#: .././ReturnReasonDialog.py:39
+msgid "Return files"
+msgstr ""
+
+#: .././ConfigDialog.py:21
+msgid "login"
+msgstr ""
+
+#: .././ConfigDialog.py:23
+msgid "password"
+msgstr ""
+
+#: .././ConfigDialog.py:25
+msgid "extensions"
+msgstr ""
+
+#: .././ConfigDialog.py:35
+msgid "Directory configuration"
+msgstr ""
+
+#: .././ConfigDialog.py:36
+msgid "Login supplied by system administrator"
+msgstr ""
+
+#: .././ConfigDialog.py:37
+msgid "Password supplied by system administrator"
+msgstr ""
+
+#: .././ConfigDialog.py:38
+msgid "List of space separated file extensions"
+msgstr ""
+
+#: .././MainFrame.py:43 .././SortableListCtrl.py:85
+msgid "Open"
+msgstr ""
+
+#: .././MainFrame.py:43
+msgid "Open new directory"
+msgstr ""
+
+#: .././MainFrame.py:45
+msgid "Reconfigure"
+msgstr ""
+
+#: .././MainFrame.py:45
+msgid "Reconfigure directory"
+msgstr ""
+
+#: .././MainFrame.py:48
+msgid "Directory"
+msgstr ""
+
+#: .././MainFrame.py:50
+msgid "Upload Selected"
+msgstr ""
+
+#: .././MainFrame.py:50
+msgid "Upload annotated or adjudicated files"
+msgstr ""
+
+#: .././MainFrame.py:51
+msgid "Checkin Selected"
+msgstr ""
+
+#: .././MainFrame.py:51
+msgid ""
+"Upload annotated or adjudicated; files will be deleted from local disk and "
+"you will not be able to modify them again"
+msgstr ""
+
+#: .././MainFrame.py:52
+msgid "Return Selected"
+msgstr ""
+
+#: .././MainFrame.py:52
+msgid ""
+"Return annotated or adjudicated because they are not suitable for annotation "
+"or contain errors"
+msgstr ""
+
+#: .././MainFrame.py:54
+msgid "Download (annotation)"
+msgstr ""
+
+#: .././MainFrame.py:54
+msgid "Download files for annotation"
+msgstr ""
+
+#: .././MainFrame.py:55
+msgid "Download (adjudication)"
+msgstr ""
+
+#: .././MainFrame.py:55
+msgid "Download files for adjudication"
+msgstr ""
+
+#: .././MainFrame.py:57
+msgid "Checkout"
+msgstr ""
+
+#: .././MainFrame.py:57
+msgid "Checkout files from server to your local, empty directory"
+msgstr ""
+
+#: .././MainFrame.py:59
+msgid "Files"
+msgstr ""
+
+#: .././MainFrame.py:61 .././SortableListCtrl.py:49
+msgid "Ready"
+msgstr ""
+
+#: .././MainFrame.py:61
+msgid "Select items which are marked for upload"
+msgstr ""
+
+#: .././MainFrame.py:62
+msgid "Select items which are modified"
+msgstr ""
+
+#: .././MainFrame.py:63
+msgid "Select"
+msgstr ""
+
+#: .././MainFrame.py:66
+msgid "Finished count"
+msgstr ""
+
+#: .././MainFrame.py:66
+msgid "Check number of finished texts"
+msgstr ""
+
+#: .././MainFrame.py:67 .././MainFrame.py:110
+msgid "Annotation"
+msgstr ""
+
+#: .././MainFrame.py:111
+msgid "Adjudication"
+msgstr ""
+
+#: .././MainFrame.py:128
+msgid "Choose a directory"
+msgstr ""
+
+#: .././MainFrame.py:141
+msgid "Do you want to upload the following files?"
+msgstr ""
+
+#: .././MainFrame.py:143 .././MainFrame.py:212
+msgid "Question"
+msgstr ""
+
+#: .././MainFrame.py:209
+msgid "Do you want to checkin the following files?"
+msgstr ""
+
+#: .././MainFrame.py:211
+msgid ""
+"Files will be deleted from the local directory and you will not be able to "
+"read, write or modify them later."
+msgstr ""
+
+#: .././manager.py:69
+msgid "Directory not set"
+msgstr ""
+
+#: .././manager.py:69 .././manager.py:72
+msgid "Info"
+msgstr ""
+
+#: .././manager.py:72
+msgid "Another operation running"
+msgstr ""
+
+#: .././client.py:23
+msgid "Connecting server..."
+msgstr ""
+
+#: .././client.py:35 .././client.py:87 .././client.py:113 .././client.py:155
+#: .././client.py:164 .././client.py:234
+msgid "Sending request..."
+msgstr ""
+
+#: .././client.py:44 .././client.py:242
+msgid "Uploading"
+msgstr ""
+
+#: .././client.py:57 .././client.py:255
+msgid "Upload failed: server doesn't accept this file from you"
+msgstr ""
+
+#: .././client.py:66 .././client.py:82 .././client.py:108 .././client.py:150
+#: .././client.py:161 .././client.py:226 .././client.py:275
+msgid "Done"
+msgstr ""
+
+#: .././client.py:74 .././client.py:79 .././client.py:267 .././client.py:272
+msgid "Deleting"
+msgstr ""
+
+#: .././client.py:91 .././client.py:124
+msgid "No files to download"
+msgstr ""
+
+#: .././client.py:96 .././client.py:129 .././client.py:171 .././client.py:183
+msgid "Downloading"
+msgstr ""
+
+#: .././client.py:118
+msgid "FAILED: you don't have adjudicator privileges"
+msgstr ""
+
+#: .././client.py:158
+msgid "Number of finished files: "
+msgstr ""
+
+#: .././client.py:202
+msgid "Deleting local files..."
+msgstr ""
+
+#: .././client.py:218
+msgid "Error occured:"
+msgstr ""
+
+#: .././client.py:220
+msgid "Saving dowloaded files..."
+msgstr ""
+
+#: .././client.py:299
+msgid "Authentication..."
+msgstr ""
+
+#: .././client.py:304
+msgid "FAILED: incorrect login or password"
+msgstr ""
+
+#: .././client.py:318 .././client.py:339 .././client.py:356 .././client.py:377
+#: .././client.py:398 .././client.py:415
+msgid "FAILED: unable to connect to server"
+msgstr ""
+
+#: .././SortableListCtrl.py:47
+msgid "File"
+msgstr ""
+
+#: .././SortableListCtrl.py:48
+msgid "Status"
+msgstr ""
+
+#: .././SortableListCtrl.py:50
+msgid "Stage"
+msgstr ""
+
+#: .././SortableListCtrl.py:150
+msgid "Starting "
+msgstr ""
+
+#: .././SortableListCtrl.py:150
+msgid " for text"
+msgstr ""
+
+#: .././SortableListCtrl.py:159
+msgid "Error starting "
+msgstr ""
diff --git a/gui/po/pl/LC_MESSAGES/manager.po b/gui/po/pl/LC_MESSAGES/manager.po
new file mode 100644
index 0000000..51ecbc6
--- /dev/null
+++ b/gui/po/pl/LC_MESSAGES/manager.po
@@ -0,0 +1,354 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: manager\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2012-06-27 16:25+0200\n"
+"PO-Revision-Date: \n"
+"Last-Translator: Mateusz Kopeć <m.kopec@ipipan.waw.pl>\n"
+"Language-Team: IPI PAN\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"X-Poedit-Language: Polish\n"
+"X-Poedit-Country: POLAND\n"
+"X-Poedit-SourceCharset: utf-8\n"
+
+#: .././utils.py:46
+#: .././MainFrame.py:62
+#: .././SortableListCtrl.py:130
+msgid "Modified"
+msgstr "Zmienione"
+
+#: .././utils.py:47
+#: .././SortableListCtrl.py:78
+#: .././SortableListCtrl.py:106
+#: .././SortableListCtrl.py:124
+#: .././SortableListCtrl.py:127
+msgid "Yes"
+msgstr "Tak"
+
+#: .././utils.py:47
+#: .././SortableListCtrl.py:110
+msgid "No"
+msgstr "Nie"
+
+#: .././DownloadNumberDialog.py:21
+msgid "Download count"
+msgstr "Liczba tekstów do pobrania"
+
+#: .././DownloadNumberDialog.py:23
+#: .././ReturnReasonDialog.py:27
+#: .././ConfigDialog.py:27
+msgid "OK"
+msgstr "OK"
+
+#: .././DownloadNumberDialog.py:24
+#: .././ReturnReasonDialog.py:28
+msgid "Cancel"
+msgstr "Anuluj"
+
+#: .././DownloadNumberDialog.py:35
+msgid "Download"
+msgstr "Pobieranie"
+
+#: .././DownloadNumberDialog.py:36
+msgid "Number of files to download"
+msgstr "Liczba nowych tekstów do pobrania z serwera"
+
+#: .././ReturnReasonDialog.py:21
+msgid "Why do you want to return the following files?"
+msgstr "Dlaczego chcesz odesłać poniższe teksty?"
+
+#: .././ReturnReasonDialog.py:25
+msgid "Reason"
+msgstr "Przyczyna"
+
+#: .././ReturnReasonDialog.py:39
+msgid "Return files"
+msgstr "Odeślij teksty"
+
+#: .././ConfigDialog.py:21
+msgid "login"
+msgstr "Login"
+
+#: .././ConfigDialog.py:23
+msgid "password"
+msgstr "Hasło"
+
+#: .././ConfigDialog.py:25
+msgid "extensions"
+msgstr "Rozszerzenia"
+
+#: .././ConfigDialog.py:35
+msgid "Directory configuration"
+msgstr "Konfiguracja katalogu"
+
+#: .././ConfigDialog.py:36
+msgid "Login supplied by system administrator"
+msgstr "Login dostarczony przez administratora systemu"
+
+#: .././ConfigDialog.py:37
+msgid "Password supplied by system administrator"
+msgstr "Hasło dostarczone przez administratora systemu"
+
+#: .././ConfigDialog.py:38
+msgid "List of space separated file extensions"
+msgstr "Lista rozszerzeń plików oddzielona spacjami"
+
+#: .././MainFrame.py:43
+#: .././SortableListCtrl.py:85
+msgid "Open"
+msgstr "Otwórz"
+
+#: .././MainFrame.py:43
+msgid "Open new directory"
+msgstr "Otwórz nowy katalog."
+
+#: .././MainFrame.py:45
+msgid "Reconfigure"
+msgstr "Zmień konfigurację"
+
+#: .././MainFrame.py:45
+msgid "Reconfigure directory"
+msgstr "Zmień konfigurację aktualnego katalogu - hasło i login anotatora oraz rozszerzenia plików."
+
+#: .././MainFrame.py:48
+msgid "Directory"
+msgstr "Katalog tekstów"
+
+#: .././MainFrame.py:50
+msgid "Upload Selected"
+msgstr "Zapisz zaznaczone teksty na serwerze"
+
+#: .././MainFrame.py:50
+msgid "Upload annotated or adjudicated files"
+msgstr "Zapisz zaznaczone teksty na serwerze. Nie zostaną usunięte z Twojego komputera."
+
+#: .././MainFrame.py:51
+msgid "Checkin Selected"
+msgstr "Zakończ pracę nad zaznaczonymi tekstami"
+
+#: .././MainFrame.py:51
+msgid "Upload annotated or adjudicated; files will be deleted from local disk and you will not be able to modify them again"
+msgstr "Zakończ pracę nad zaznaczonymi tekstami. Zostaną one usunięte z Twojego komputera."
+
+#: .././MainFrame.py:52
+msgid "Return Selected"
+msgstr "Odeślij zaznaczone teksty"
+
+#: .././MainFrame.py:52
+msgid "Return annotated or adjudicated because they are not suitable for annotation or contain errors"
+msgstr "Odeślij anotowane lub superanotowane teksty ponieważ nie nadają się do anotacji lub zawierają błędy"
+
+#: .././MainFrame.py:54
+msgid "Download (annotation)"
+msgstr "Pobierz nowe teksty (anotacja)"
+
+#: .././MainFrame.py:54
+msgid "Download files for annotation"
+msgstr "Pobierz nowe teksty do anotacji."
+
+#: .././MainFrame.py:55
+msgid "Download (adjudication)"
+msgstr "Pobierz nowe teksty (superanotacja)"
+
+#: .././MainFrame.py:55
+msgid "Download files for adjudication"
+msgstr "Pobierz nowe teksty do superanotacji."
+
+#: .././MainFrame.py:57
+msgid "Checkout"
+msgstr "Odtwórz stan z serwera lokalnie"
+
+#: .././MainFrame.py:57
+msgid "Checkout files from server to your local, empty directory"
+msgstr "Pobiera wszystkie pliki przypisane do Ciebie z serwera. Nadpisuje lokalne zmiany."
+
+#: .././MainFrame.py:59
+msgid "Files"
+msgstr "Teksty"
+
+#: .././MainFrame.py:61
+#: .././SortableListCtrl.py:49
+msgid "Ready"
+msgstr "Zakończone"
+
+#: .././MainFrame.py:61
+msgid "Select items which are marked for upload"
+msgstr "Zaznacz teksty, które są oznaczone jako \"Zakończone\"."
+
+#: .././MainFrame.py:62
+msgid "Select items which are modified"
+msgstr "Zaznacz teksty, które są oznaczone jako \"Zmienione\"."
+
+#: .././MainFrame.py:63
+msgid "Select"
+msgstr "Zaznacz teksty"
+
+#: .././MainFrame.py:66
+msgid "Finished count"
+msgstr "Liczba zakończonych tekstów"
+
+#: .././MainFrame.py:66
+msgid "Check number of finished texts"
+msgstr "Sprawdź liczbę zakończonych tekstów"
+
+#: .././MainFrame.py:67
+#: .././MainFrame.py:110
+msgid "Annotation"
+msgstr "Anotacja"
+
+#: .././MainFrame.py:111
+msgid "Adjudication"
+msgstr "Superanotacja"
+
+#: .././MainFrame.py:128
+msgid "Choose a directory"
+msgstr "Wybierz katalog"
+
+#: .././MainFrame.py:141
+msgid "Do you want to upload the following files?"
+msgstr "Czy chcesz zapisać na serwerze poniższe teksty?"
+
+#: .././MainFrame.py:143
+#: .././MainFrame.py:212
+msgid "Question"
+msgstr "Pytanie"
+
+#: .././MainFrame.py:209
+msgid "Do you want to checkin the following files?"
+msgstr "Czy chcesz zakończyć pracę nad poniższymi tekstami?"
+
+#: .././MainFrame.py:211
+msgid "Files will be deleted from the local directory and you will not be able to read, write or modify them later."
+msgstr "Teksty zostaną skasowane z tego komputera i nie będziesz mógł/mogła ich odczytać ani edytować kiedykolwiek później."
+
+#: .././manager.py:69
+msgid "Directory not set"
+msgstr "Katalog nie został wybrany"
+
+#: .././manager.py:69
+#: .././manager.py:72
+msgid "Info"
+msgstr "Informacja"
+
+#: .././manager.py:72
+msgid "Another operation running"
+msgstr "Inna operacja w toku"
+
+#: .././client.py:23
+msgid "Connecting server..."
+msgstr "Łączenie z serwerem..."
+
+#: .././client.py:35
+#: .././client.py:87
+#: .././client.py:113
+#: .././client.py:155
+#: .././client.py:164
+#: .././client.py:234
+msgid "Sending request..."
+msgstr "Wysyłanie żądania..."
+
+#: .././client.py:44
+#: .././client.py:242
+msgid "Uploading"
+msgstr "Wysyłanie"
+
+#: .././client.py:57
+#: .././client.py:255
+msgid "Upload failed: server doesn't accept this file from you"
+msgstr "Wysyłanie nie powiodło się: serwer nie akceptuje tego tekstu od Ciebie."
+
+#: .././client.py:66
+#: .././client.py:82
+#: .././client.py:108
+#: .././client.py:150
+#: .././client.py:161
+#: .././client.py:226
+#: .././client.py:275
+msgid "Done"
+msgstr "Gotowe."
+
+#: .././client.py:74
+#: .././client.py:79
+#: .././client.py:267
+#: .././client.py:272
+msgid "Deleting"
+msgstr "Usuwanie"
+
+#: .././client.py:91
+#: .././client.py:124
+msgid "No files to download"
+msgstr "Brak nowych tekstów do pobrania z serwera (lub pobrano już maksymalną ilość)."
+
+#: .././client.py:96
+#: .././client.py:129
+#: .././client.py:171
+#: .././client.py:183
+msgid "Downloading"
+msgstr "Pobieranie"
+
+#: .././client.py:118
+msgid "FAILED: you don't have adjudicator privileges"
+msgstr "Nie powiodło się: nie masz uprawnień superanotatora."
+
+#: .././client.py:158
+msgid "Number of finished files: "
+msgstr "Liczba zakończonych tekstów:"
+
+#: .././client.py:202
+msgid "Deleting local files..."
+msgstr "Usuwanie lokalnych tekstów..."
+
+#: .././client.py:218
+msgid "Error occured:"
+msgstr "Wystąpił błąd:"
+
+#: .././client.py:220
+msgid "Saving dowloaded files..."
+msgstr "Zapisywanie pobranych tekstów..."
+
+#: .././client.py:299
+msgid "Authentication..."
+msgstr "Uwierzytelnianie..."
+
+#: .././client.py:304
+msgid "FAILED: incorrect login or password"
+msgstr "Nie powiodło się: nieprawidłowy login lub hasło."
+
+#: .././client.py:318
+#: .././client.py:339
+#: .././client.py:356
+#: .././client.py:377
+#: .././client.py:398
+#: .././client.py:415
+msgid "FAILED: unable to connect to server"
+msgstr "Nie powiodło się: problem z łącznością z serwerem."
+
+#: .././SortableListCtrl.py:47
+msgid "File"
+msgstr "Tekst"
+
+#: .././SortableListCtrl.py:48
+msgid "Status"
+msgstr "Status"
+
+#: .././SortableListCtrl.py:50
+msgid "Stage"
+msgstr "Etap"
+
+#: .././SortableListCtrl.py:150
+msgid "Starting "
+msgstr "Uruchamianie "
+
+#: .././SortableListCtrl.py:150
+msgid " for text"
+msgstr " dla tekstu "
+
+#: .././SortableListCtrl.py:159
+msgid "Error starting "
+msgstr "Błąd podczas uruchamiania "
+
+#~ msgid "Starting mmax for text"
+#~ msgstr "Uruchamianie mmax dla tekstu"
diff --git a/gui/ready.py b/gui/ready.py
new file mode 100644
index 0000000..3273dfa
--- /dev/null
+++ b/gui/ready.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+import os
+
+class Ready:
+
+ """Class for ready file manipulation."""
+
+ def __init__(self, ready_path):
+ if not os.path.exists(ready_path):
+ with open(ready_path, "w") as f: pass
+ self.ready_path = ready_path
+
+ def _check_file(self, file_name, is_ready):
+ entries = self.get_entries()
+ if is_ready == True:
+ entries.append(file_name)
+ else:
+ entries.remove(file_name)
+ self.write_entries(entries)
+
+ def remove_file(self, file_name):
+ self._check_file(file_name, False)
+
+ def save_file(self, file_name):
+ self._check_file(file_name, True)
+
+# def remove_super_file(self, file_name):
+# self._check_file(file_name, False, type="s_checkin")
+#
+# def save_super_file(self, file_name):
+# self._check_file(file_name, True, type="s_checkin")
+
+ def get_entries(self):
+ result = []
+ with open(self.ready_path) as inp:
+ result = [x.strip() for x in inp.readlines()]
+ return result
+
+ def write_entries(self, entries):
+ with open(self.ready_path, "w") as out:
+ for entry in entries:
+ print >> out, entry
diff --git a/gui/utils.py b/gui/utils.py
new file mode 100644
index 0000000..1c4f738
--- /dev/null
+++ b/gui/utils.py
@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+import os
+import re
+import wx
+import threading
+import time
+
+import i18n
+_ = i18n.language.ugettext #use ugettext instead of getttext to avoid unicode errors
+
+import wx.lib.mixins.listctrl as listmix
+
+from ConfigDialog import ConfigDialog
+import const
+
+def match_ext(name, exts):
+ return any(name.endswith(ext) for ext in exts)
+
+def cut_ext(name, exts):
+ return re.sub("(%s)$" % "|".join(exts), "", name)
+
+def normal_files(work_dir, exts):
+ names = set(cut_ext(name, exts)
+ for name in os.listdir(work_dir)
+ if match_ext(name, exts))
+ return sorted(names)
+
+def super_files(work_dir, exts):
+ names = []
+ for name in os.listdir(work_dir):
+ path = os.path.join(work_dir, name)
+ if os.path.isdir(path):
+ local = set(cut_ext(name, exts)
+ for name in os.listdir(path)
+ if match_ext(name, exts))
+ names.extend(os.path.join(name, tail) for tail in local)
+ return sorted(names)
+
+def all_files(work_dir, exts):
+ return normal_files(work_dir, exts) + super_files(work_dir, exts)
+
+def get_entries(get_names, work_dir, exts, ready, footprints):
+ names = get_names(work_dir, exts)
+ result = []
+ for file_name in names:
+ status = _("Modified") if footprints.modified(file_name) else "--"
+ is_ready = _("Yes") if file_name in ready else _("No")
+ stage = "--"
+ result.append((file_name, status, is_ready, stage))
+ return result
+
+def normal_entries(work_dir, exts, ready, footprints):
+ return get_entries(normal_files, work_dir, exts, ready, footprints)
+
+def super_entries(work_dir, exts, ready, footprints):
+ return get_entries(super_files, work_dir, exts, ready, footprints)
+
+def differ(data1, data2):
+ if len(data1) != len(data2):
+ return True
+ d1={}
+ for i in range(len(data1)):
+ d1[data1[i][0]]=data1[i][1:]
+ for i in range(len(data2)):
+ if not d1.has_key(data2[i][0]):
+ return True
+ if d1[data2[i][0]] != data2[i][1:]:
+ return True
+ return False
+
+def change_ready(ready, file_list, pos):
+ file_name = file_list.GetFilename(pos)
+ if file_list.IsReady(pos):
+ file_list.SetReadyNo(pos)
+ ready.remove_file(file_name)
+ else:
+ file_list.SetReadyYes(pos)
+ ready.save_file(file_name)
+
+def change_ready_prim(ready, super_file_list, pos):
+ def set_no(dirname):
+ for i in range(super_file_list.GetItemCount()):
+ other_name = super_file_list.GetFilename(i)
+ if os.path.dirname(other_name) == dirname:
+ super_file_list.SetReadyNo(i)
+ if other_name in ready.get_entries():
+ ready.remove_file(other_name)
+
+ file_name = super_file_list.GetFilename(pos)
+ if super_file_list.IsReady(pos):
+ super_file_list.SetReadyNo(pos)
+ ready.remove_file(file_name)
+ else:
+ set_no(os.path.dirname(file_name))
+ super_file_list.SetReadyYes(pos)
+ ready.save_file(file_name)
+
+def create_dir_config(path, parent, config=None):
+ dialog = ConfigDialog(parent, -1)
+ if config:
+ dialog.login_ctrl.SetValue(config["login"])
+ dialog.passwd_ctrl.SetValue(config["passwd"])
+ dialog.exts_ctrl.SetValue(config["extensions"])
+ else:
+ dialog.exts_ctrl.SetValue(" ".join(const.default_exts))
+ try:
+ if dialog.ShowModal() == wx.ID_OK:
+ login = dialog.login_ctrl.GetValue()
+ password = dialog.passwd_ctrl.GetValue()
+ file_exts = dialog.exts_ctrl.GetValue()
+ with open(path, "w") as conf:
+ print >> conf, "login =", login
+ print >> conf, "passwd =", password
+ print >> conf, "extensions =", file_exts
+ return True
+ finally:
+ dialog.Destroy()
+ return False
+
+def fill_dir_menu(main_frame, cache):
+ dir_menu = main_frame.dir_menu
+ last_dir_set=False
+ for entry in cache:
+ if not os.path.isdir(entry):
+ continue
+ wx_id = wx.NewId()
+ dir_menu.Append(wx_id, entry, kind=wx.ITEM_NORMAL)
+ main_frame.Bind(wx.EVT_MENU, main_frame.open_directory, id=wx_id)
+ if not last_dir_set:
+ cmd = wx.CommandEvent(wx.EVT_MENU.evtType[0])
+ cmd.SetEventObject(main_frame)
+ cmd.SetId(wx_id)
+ main_frame.GetEventHandler().ProcessEvent(cmd)
+ last_dir_set = True
+
+# Watek odswieza informacje o zawartosci pliku to-checkin.txt
+# (moze sie zmienic za posrednictwem TrEda lub wyedytowany recznie).
+class RefreshThread (threading.Thread):
+ def __init__(self, manager, delay=2):
+ self.exitFlag = False
+ self.delay = delay
+ self.manager = manager
+ threading.Thread.__init__(self)
+
+ def run(self):
+ while self.exitFlag != True:
+ wx.CallAfter(self.manager.refresh)
+ time.sleep(self.delay)
+
+# self.mtime = None # message file 'modified time'
+# while self.exitFlag != True:
+# new_mtime = os.stat(self.ready_path).st_mtime
+# if new_mtime != self.mtime:
+# wx.CallAfter(self.manager.refresh)
+# self.mtime = new_mtime
+# time.sleep(self.delay)
+
diff --git a/installation.sh b/installation.sh
new file mode 100644
index 0000000..2027734
--- /dev/null
+++ b/installation.sh
@@ -0,0 +1,43 @@
+#wersja instalacji dla lokalnego pythona
+
+#subversion
+wget http://subversion.tigris.org/downloads/subversion-1.6.15.tar.gz
+wget http://subversion.tigris.org/downloads/subversion-deps-1.6.15.tar.gz
+tar -xzf subversion-1.6.15.tar.gz
+tar -xzf subversion-deps-1.6.15.tar.gz
+cd subversion-1.6.15
+./configure --prefix=$HOME/subversion --without-apxs --without-berkeley-db --with-ssl LDFLAGS="-L/lib64"
+make
+make install
+
+#python
+wget -c http://python.org/ftp/python/2.7.2/Python-2.7.2.tar.bz2
+tar -jxvf Python-2.7.2.tar.bz2
+cd Python-2.7.2
+./configure --prefix=$HOME/.local
+make
+make install
+echo 'PATH=$PATH:$HOME/.local/bin' >> ~/.bashrc
+source ~/.bashrc
+
+#setuptools ???????????????????
+wget http://pypi.python.org/packages/source/s/setuptools/setuptools-0.6c11.tar.gz#md5=7df2a529a074f613b509fb44feefe74e
+tar -xzf setuptools-0.6c11.tar.gz
+cd setuptools-0.6c11
+$HOME/.local/bin/python2.7 setup.py install
+
+#pysvn
+wget http://pysvn.barrys-emacs.org/source_kits/pysvn-1.7.5.tar.gz
+tar -xzf pysvn-1.7.5.tar.gz
+cd pysvn-1.7.5
+cd Source
+$HOME/.local/bin/python2.7 setup.py configure --svn-inc-dir=$HOME/subversion/include/subversion-1/ --svn-lib-dir=$HOME/subversion/lib/
+cd ..
+$HOME/.local/bin/python2.7 setup.py install
+
+
+
+
+
+
+
diff --git a/scripts/add_files.py b/scripts/add_files.py
new file mode 100644
index 0000000..0f82ea6
--- /dev/null
+++ b/scripts/add_files.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import shutil
+from collections import defaultdict
+from optparse import OptionParser
+
+# Solution with no hard coded path would be welcome
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), ".."))
+
+from dfs.database import Database
+from dfs.repo import Repo
+from dfs.config import Config
+
+def add_file(wc, db, file_id, src_paths, anno_per_file):
+ try:
+ db.add(file_id)
+ except Exception as ex:
+ print str(ex)
+ return False
+
+ print "Adding file "+file_id+":"
+ added = []
+ for src_path in src_paths:
+ try:
+ wc.add(src_path)
+ print "\t - "+src_path
+ added.append(src_path)
+ except Exception as ex:
+ print "\t error: "+str(ex)
+ for a in added:
+ print "\t\t removing already added "+a
+ for b in wc.remove(added, anno_per_file):
+ print "\t\t\t "+b
+
+ db.remove(file_id)
+ return False
+
+ return True
+
+def match_ext(path, exts):
+ for ext in exts:
+ if path.endswith(ext):
+ return ext
+
+def path_id(path, ext):
+ _, filename = os.path.split(path)
+ return re.sub("%s$" % ext, "", filename)
+
+def group_paths(paths, exts):
+ result = defaultdict(list)
+ for path in paths:
+ ext = match_ext(path, exts)
+ if (ext != None):
+ file_id = path_id(path, ext)
+ result[file_id].append(path)
+ return result
+
+def get_rec_paths(paths):
+ result = []
+ for path in paths:
+ if os.path.isdir(path):
+ for dirname, dirnames, filenames in os.walk(path):
+ for filename in filenames:
+ result.append(os.path.join(dirname, filename))
+ else:
+ result.append(path)
+ return result
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog [options] CONFIG FILES""")
+ optparser.add_option("--extensions", dest="exts", default=".mmax,_mentions.xml,_words.xml",
+ help="List of comma-separated file extensions")
+ (options, args) = optparser.parse_args()
+ if len(args) < 2:
+ optparser.print_help()
+ sys.exit(0)
+
+ conf_path = args[0]
+ cfg = Config(conf_path)
+ anno_per_file = int(cfg["anno_per_file"])
+ paths = get_rec_paths(args[1:])
+ files = group_paths(paths, options.exts.split(","))
+ wc = Repo(cfg["svn.repository"], cfg["svn.login"], cfg["svn.passwd"])
+ db = Database(wc.db_path(), anno_per_file)
+
+ success = []
+ fail = []
+ for file_id, paths in files.iteritems():
+ if add_file(wc, db, file_id, paths, anno_per_file):
+ success.append(file_id)
+ else:
+ fail.append(file_id)
+
+ db.save()
+ wc.commit("Added files: "+str(success))
+
+ print ""
+ if len(success) > 0:
+ print "Added files: "+str(success)
+ if len(fail) > 0:
+ print "Failed to add files: "+str(fail)
+
\ No newline at end of file
diff --git a/scripts/clusters.py b/scripts/clusters.py
new file mode 100755
index 0000000..aecc1d2
--- /dev/null
+++ b/scripts/clusters.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+import re
+import sys
+import os
+import re
+import shutil
+from collections import defaultdict
+from optparse import OptionParser
+
+# Solution with no hard coded path would be welcome
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), ".."))
+
+from dfs.database import Database
+from dfs.repo import Repo
+from dfs.config import Config
+
+def fill(text):
+ return fill_custom(text, 20, " ")
+
+def fill_custom(text, l, sym):
+ to_add = max(0, l - len(text))
+ spaces = ""
+ for i in range(to_add):
+ spaces = spaces + sym
+ return text + spaces
+
+def parse_span(span, words_list):
+ words = []
+
+ w = span.split(",")
+ for fragment in w:
+ f = fragment.split("..")
+ id1 = words_list.index(f[0])
+ id2 = words_list.index(f[-1])
+ for i in range(id1, id2+1):
+ words.append(words_list[i])
+
+ return words
+
+def get_context(ids, words_list, size):
+ first = -1
+ last = -1
+
+ i = 0
+ for i in range(len(words_list)):
+ w = words_list[i]
+ if w in ids:
+ if first == -1:
+ first = i
+ last = i
+
+ first = max(0, first - size)
+ last = min(len(words_list), last + size)
+
+ return words_list[first:last]
+
+def print_file(path):
+ word_id_2_orth = {}
+ words_list = []
+ w = re.compile("<word.* id=\"(.*?)\".*>(.*?)</word>.*")
+ with open(path + "_words.xml", "r") as f:
+ for line in f.readlines():
+ groups = w.findall(line)
+ if len(groups) == 1:
+ group = groups[0]
+ id = group[0]
+ orth = group[1]
+ word_id_2_orth[id] = orth
+ words_list.append(id)
+
+ me = re.compile("<markable.*id=\"(.*?)\".*")
+ sp = re.compile(".*span=\"(.*?)\".*")
+ co = re.compile(".*comment=\"(.*?)\".*")
+ mg = re.compile(".*mention_group=\"(.*?)\".*")
+
+ mention_id_2_span = {}
+ mention_id_2_comment = {}
+ clusters = {}
+ with open(path + "_mentions.xml", "r") as f:
+ for line in f.readlines():
+ groups1 = me.findall(line)
+ groups2 = sp.findall(line)
+ groups3 = co.findall(line)
+
+ if len(groups1) == 1 and len(groups2) == 1:
+ id = groups1[0]
+ span = groups2[0]
+ mention_id_2_span[id] = parse_span(span, words_list)
+
+ if len(groups3) == 1:
+ mention_id_2_comment[id] = groups3[0]
+
+ clu = mg.findall(line)
+ if len(clu) == 1 and clu[0] != "empty" and clu[0] != "":
+ if clu[0] not in clusters:
+ clusters[clu[0]] = []
+ clusters[clu[0]].append(id)
+
+ all_clusters = {}
+ for list in clusters.values():
+ l = ["["+" ".join(word_id_2_orth[wid] for wid in mention_id_2_span[id])+"]" for id in list]
+ length = len(l)
+ if length not in all_clusters:
+ all_clusters[length] = []
+ all_clusters[length].append(l)
+
+ return all_clusters
+
+def merge_dicts(dict1, dict2):
+ for key, val in dict2.iteritems():
+ if key in dict1:
+ dict1[key].extend(val)
+ else:
+ dict1[key] = val
+
+def print_clusters(db, min_size):
+ all = {}
+ for filename, file in db.file_index.iteritems():
+
+ if db.rejected(file):
+ continue
+
+ sann = file.find("s_ann")
+
+ if sann is not None and db.finished(sann):
+ path = wc.upload_prim_path(filename)
+ clus = print_file(path)
+ merge_dicts(all, clus)
+
+ for key, val in sorted(all.iteritems()):
+ if key >= min_size:
+ print
+ print "#######", "Dlugosc klastra:", key, "#######"
+ for cl in val:
+ print
+ print ", ".join(cl)
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog CONFIG MINCLUSTERSIZE""")
+ (options, args) = optparser.parse_args()
+ if len(args) < 2:
+ optparser.print_help()
+ sys.exit(0)
+
+ conf_path = args[0]
+ min_size = int(args[1])
+ cfg = Config(conf_path)
+ wc = Repo(cfg["svn.repository"], cfg["svn.login"], cfg["svn.passwd"])
+ db = Database(wc.db_path(), int(cfg["anno_per_file"]))
+
+ print_clusters(db, min_size)
+
diff --git a/scripts/db_averages_stats.py b/scripts/db_averages_stats.py
new file mode 100755
index 0000000..e06df51
--- /dev/null
+++ b/scripts/db_averages_stats.py
@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+import re
+import sys
+import os
+import re
+import shutil
+from collections import defaultdict
+from optparse import OptionParser
+
+# Solution with no hard coded path would be welcome
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), ".."))
+
+from dfs.database import Database
+from dfs.repo import Repo
+from dfs.config import Config
+
+def fill(text):
+ return fill_custom(text, 20, " ")
+
+def fill_custom(text, l, sym):
+ text = text.decode("utf-8")
+ to_add = max(0, l - len(text))
+ spaces = ""
+ for i in range(to_add):
+ spaces = spaces + sym
+ return (text + spaces).encode("utf-8")
+
+def get_span_size(span, words):
+ w = span.split(",")
+ size = 0
+ for fragment in w:
+ f = fragment.split("..")
+ id1 = words.index(f[0])
+ id2 = words.index(f[-1])
+ size = size + id2 - id1 + 1
+
+ return size
+
+def count_file_stats(path):
+ words = []
+ id = re.compile("<word.* id=\"(.*?)\".*")
+ with open(path + "_words.xml", "r") as f:
+ for line in f.readlines():
+ groups = id.findall(line)
+ if len(groups) == 1:
+ ident = groups[0]
+ words.append(ident)
+
+ sp = re.compile("<markable.*span=\"(.*?)\".*")
+ mg = re.compile(".*mention_group=\"(.*?)\".*")
+ ni = re.compile(".*near_identity=\"(.*?)\".*")
+
+ sets = {}
+ near_id = 0
+ mentions = 0
+ mention_sizes = {}
+ with open(path + "_mentions.xml", "r") as f:
+ for line in f.readlines():
+ groups = sp.findall(line)
+ if len(groups) == 1:
+ mentions = mentions + 1
+ span = groups[0]
+
+ mention_size = get_span_size(span, words)
+ if mention_size in mention_sizes:
+ mention_sizes[mention_size] = mention_sizes[mention_size] + 1
+ else:
+ mention_sizes[mention_size] = 1
+
+ group = mg.findall(line)[0]
+ near = ni.findall(line)[0]
+
+ if near != "empty" and near != "":
+ near_id = near_id + 1
+
+ if group != "empty" and group != "":
+ if group in sets:
+ sets[group] = sets[group] + 1
+ else:
+ sets[group] = 1
+
+ mg_sizes = {}
+ for key, val in sets.iteritems():
+ if val in mg_sizes:
+ mg_sizes[val] = mg_sizes[val] + 1
+ else:
+ mg_sizes[val] = 1
+
+ return len(words), mention_sizes, mg_sizes, near_id
+
+def merge_dicts(dict1, dict2):
+
+ for key, val in dict2.iteritems():
+ if key in dict1:
+ dict1[key] = dict1[key] + val
+ else:
+ dict1[key] = val
+
+def get_text_types(db, used_path, mapping_path):
+ names = []
+ for filename, file in db.file_index.iteritems():
+ if db.rejected(file):
+ continue
+ names.append(filename)
+
+ id2nrs = {}
+ with open(used_path, "r") as f:
+ for line in f.readlines():
+ spl = line.split(";")
+ id = spl[1]
+ nr = spl[0]
+ if id not in id2nrs:
+ id2nrs[id] = []
+ id2nrs[id].append(nr)
+
+ types = {}
+ with open(mapping_path, "r") as f:
+ for line in f.readlines():
+ spl = line.strip().split(";")
+ id = spl[2]
+ if id in id2nrs.keys():
+ for nr in id2nrs[id]:
+ types[nr] = spl[0].decode("utf-8")
+
+ return types
+
+def print_stats(db, ann, types):
+
+ gms = {}
+ gmgs = {}
+ wbalance = {}
+ tbalance = {}
+ nibalance = {}
+
+ for filename, file in db.file_index.iteritems():
+
+ type = types[filename]
+ idx = 0
+
+ if ann == "rejected":
+ if db.rejected(file):
+ path = os.path.join(wc.new_path(), filename)
+ wcnt, ms, mgs, nidcnt = count_file_stats(path)
+
+ merge_dicts(wbalance, {type : wcnt})
+ merge_dicts(tbalance, {type : 1})
+ merge_dicts(nibalance, {type : nidcnt})
+
+ if type not in gms:
+ gms[type] = {}
+ if type not in gmgs:
+ gmgs[type] = {}
+
+ merge_dicts(gms[type], ms)
+ merge_dicts(gmgs[type], mgs)
+
+ else:
+ if db.rejected(file):
+ continue
+
+ for annotation in file.findall(ann):
+ if db.finished(annotation):
+ if ann == "ann":
+ path = wc.upload_path(filename, idx)
+ else:
+ path = wc.upload_prim_path(filename)
+
+ wcnt, ms, mgs, nidcnt = count_file_stats(path)
+
+ merge_dicts(wbalance, {type : wcnt})
+ merge_dicts(tbalance, {type : 1})
+ merge_dicts(nibalance, {type : nidcnt})
+
+ if type not in gms:
+ gms[type] = {}
+ if type not in gmgs:
+ gmgs[type] = {}
+
+ merge_dicts(gms[type], ms)
+ merge_dicts(gmgs[type], mgs)
+
+ idx = idx + 1
+
+
+ gwcnt = reduce((lambda l, a : l + a), wbalance.values(), 0)
+ tc = reduce((lambda l, a : l + a), tbalance.values(), 0)
+ print
+ print "######## Statystyki ilosciowe"
+ print fill_custom("Typ tekstow", 60, " "), fill("Liczba tekstow"), fill("Liczba slow"), fill("Procent slow")
+ print fill_custom("", 120, "-")
+ for type in sorted(set(wbalance.keys()) | set(tbalance.keys())):
+ percent = round(1.0 * wbalance[type] / gwcnt * 100, 2)
+ print fill_custom(type.encode("utf-8"), 60, " "), fill(str(tbalance[type])), fill(str(wbalance[type])), fill(str(percent) + "%")
+ print fill_custom("", 120, "-")
+ print fill_custom("dowolny", 60, " "), fill(str(tc)), fill(str(gwcnt)), fill("100.0%")
+
+ if ann != "rejected":
+ print
+ print "######## Statystyki wystapien"
+ print fill_custom("Typ tekstow", 60, " "), fill("Wystapien/tekst"), fill("Dl. wystapienia")
+ print fill_custom("", 100, "-")
+ tmc = 0
+ tsc = 0
+ for type in sorted(set(gms.keys())):
+ mention_count = reduce((lambda l, a : l + a), gms[type].values(), 0)
+ segment_count = reduce((lambda a, (k, v) : k * v + a), gms[type].iteritems(), 0)
+ avg_ms = round(1.0 * segment_count / mention_count, 2)
+ avg_mpt = round(1.0 * mention_count / tbalance[type], 2)
+ print fill_custom(type.encode("utf-8"), 60, " "), fill(str(avg_mpt)), fill(str(avg_ms))
+ tmc = tmc + mention_count
+ tsc = tsc + segment_count
+ print fill_custom("", 100, "-")
+
+ if tmc == 0:
+ tavg_ms = 0
+ else:
+ tavg_ms = round(1.0 * tsc / tmc, 2)
+
+ if tc == 0:
+ tavg_mpt = 0
+ else:
+ tavg_mpt = round(1.0 * tmc / tc, 2)
+
+ print fill_custom("dowolny", 60, " "), fill(str(tavg_mpt)), fill(str(tavg_ms))
+
+ print
+ print "######## Statystyki klastrow i linkow"
+ print fill_custom("Typ tekstow", 60, " "), fill("Klastrow/tekst"), fill("Wielkosc klastra"), fill("Linkow/tekst")
+ print fill_custom("", 120, "-")
+ tmc = 0
+ tsc = 0
+ tnic = 0
+ for type in sorted(set(gmgs.keys())):
+ group_count = reduce((lambda l, a : l + a), gmgs[type].values(), 0)
+ mention_count = reduce((lambda a, (k, v) : k * v + a), gmgs[type].iteritems(), 0)
+ avg_ms = round(1.0 * mention_count / group_count, 2)
+ avg_mpt = round(1.0 * group_count / tbalance[type], 2)
+ avg_ni = round(1.0 * nibalance[type] / tbalance[type], 2)
+ print fill_custom(type.encode("utf-8"), 60, " "), fill(str(avg_mpt)), fill(str(avg_ms)), fill(str(avg_ni))
+ tmc = tmc + group_count
+ tsc = tsc + mention_count
+ tnic = tnic + nibalance[type]
+ print fill_custom("", 120, "-")
+
+ if tmc == 0:
+ tavg_ms = 0
+ else:
+ tavg_ms = round(1.0 * tsc / tmc, 2)
+
+ if tc == 0:
+ tavg_mpt = 0
+ tavg_ni = 0
+ else:
+ tavg_mpt = round(1.0 * tmc / tc, 2)
+ tavg_ni = round(1.0 * tnic / tc, 2)
+
+ print fill_custom("dowolny", 60, " "), fill(str(tavg_mpt)), fill(str(tavg_ms)), fill(str(tavg_ni))
+
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog CONFIG USED_LIST MAPPING""")
+ (options, args) = optparser.parse_args()
+ if len(args) < 3:
+ optparser.print_help()
+ sys.exit(0)
+
+ conf_path = args[0]
+ used_path = args[1]
+ mapping_path = args[2]
+ cfg = Config(conf_path)
+ wc = Repo(cfg["svn.repository"], cfg["svn.login"], cfg["svn.passwd"])
+ db = Database(wc.db_path(), int(cfg["anno_per_file"]))
+
+ types = get_text_types(db, used_path, mapping_path)
+
+ print "################ Statystyki tekstow anotowanych #############################"
+ print_stats(db, "ann", types)
+ print
+ print "################ Statystyki tekstow superanotowanych ########################"
+ print_stats(db, "s_ann", types)
+ print
+# print "################ Statystyki tekstow odrzuconych #############################"
+# print_stats(db, "rejected", types)
diff --git a/scripts/db_detailed_stats.py b/scripts/db_detailed_stats.py
new file mode 100755
index 0000000..33f826e
--- /dev/null
+++ b/scripts/db_detailed_stats.py
@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+
+#!/usr/bin/env python
+import re
+import sys
+import os
+import re
+import shutil
+from collections import defaultdict
+from optparse import OptionParser
+
+# Solution with no hard coded path would be welcome
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), ".."))
+
+from dfs.database import Database
+from dfs.repo import Repo
+from dfs.config import Config
+
+def get_desired():
+ result = {}
+ result[u"Dzienniki"] = 25.5
+ result[u"Pozostałe periodyki"] = 23.5
+ result[u"Książki publicystyczne"] = 1.0
+ result[u"Literatura piękna"] = 16.0
+ result[u"Literatura faktu"] = 5.5
+ result[u"Typ informacyjno-poradnikowy"] = 5.5
+ result[u"Typ naukowo-dydaktyczny"] = 2.0
+ result[u"Internetowe interaktywne (blogi, fora, usenet)"] = 3.5
+ result[u"Internetowe nieinteraktywne (statyczne strony, Wikipedia)"] = 3.5
+ result[u"Quasi-mówione (protokoły sesji parlamentu)"] = 2.5
+ result[u"Mówione medialne"] = 2.5
+ result[u"Mówione konwersacyjne"] = 5.0
+ result[u"Inne teksty pisane"] = 3.0
+ result[u"Książka niebeletrystyczna nieklasyfikowana"] = 1.0
+
+ return result
+
+def fill(text):
+ return fill_custom(text, 20, " ")
+
+def fill_custom(text, l, sym):
+ text = text.decode("utf-8")
+ to_add = max(0, l - len(text))
+ spaces = ""
+ for i in range(to_add):
+ spaces = spaces + sym
+ return (text + spaces).encode("utf-8")
+
+def get_span_size(span, words):
+ w = span.split(",")
+ size = 0
+ for fragment in w:
+ f = fragment.split("..")
+ id1 = words.index(f[0])
+ id2 = words.index(f[-1])
+ size = size + id2 - id1 + 1
+
+ return size
+
+def count_file_stats(path):
+ words = []
+ id = re.compile("<word.* id=\"(.*?)\".*")
+ with open(path + "_words.xml", "r") as f:
+ for line in f.readlines():
+ groups = id.findall(line)
+ if len(groups) == 1:
+ ident = groups[0]
+ words.append(ident)
+
+ sp = re.compile("<markable.*span=\"(.*?)\".*")
+ mg = re.compile(".*mention_group=\"(.*?)\".*")
+ ni = re.compile(".*near_identity=\"(.*?)\".*")
+
+ sets = {}
+ near_id = 0
+ mentions = 0
+ mention_sizes = {}
+ with open(path + "_mentions.xml", "r") as f:
+ for line in f.readlines():
+ groups = sp.findall(line)
+ if len(groups) == 1:
+ mentions = mentions + 1
+ span = groups[0]
+
+ mention_size = get_span_size(span, words)
+ if mention_size in mention_sizes:
+ mention_sizes[mention_size] = mention_sizes[mention_size] + 1
+ else:
+ mention_sizes[mention_size] = 1
+
+ group = mg.findall(line)[0]
+ near = ni.findall(line)[0]
+
+ if near != "empty" and near != "":
+ near_id = near_id + 1
+
+ if group != "empty" and group != "":
+ if group in sets:
+ sets[group] = sets[group] + 1
+ else:
+ sets[group] = 1
+
+ mg_sizes = {}
+ for key, val in sets.iteritems():
+ if val in mg_sizes:
+ mg_sizes[val] = mg_sizes[val] + 1
+ else:
+ mg_sizes[val] = 1
+
+ return len(words), mention_sizes, mg_sizes, near_id
+
+def merge_dicts(dict1, dict2):
+
+ for key, val in dict2.iteritems():
+ if key in dict1:
+ dict1[key] = dict1[key] + val
+ else:
+ dict1[key] = val
+
+def get_text_types(db, used_path, mapping_path):
+ names = []
+ for filename, file in db.file_index.iteritems():
+ if db.rejected(file):
+ continue
+ names.append(filename)
+
+ id2nrs = {}
+ with open(used_path, "r") as f:
+ for line in f.readlines():
+ spl = line.split(";")
+ id = spl[1]
+ nr = spl[0]
+ if id not in id2nrs:
+ id2nrs[id] = []
+ id2nrs[id].append(nr)
+
+ types = {}
+ with open(mapping_path, "r") as f:
+ for line in f.readlines():
+ spl = line.strip().split(";")
+ id = spl[2]
+ if id in id2nrs.keys():
+ for nr in id2nrs[id]:
+ types[nr] = spl[0].decode("utf-8")
+
+ return types
+
+def print_stats(db, ann, types):
+ gnidcnt = 0
+ gwcnt = 0
+ gms = {}
+ gmgs = {}
+ wbalance = {}
+ tbalance = {}
+
+ tc = 0
+
+ for filename, file in db.file_index.iteritems():
+
+ type = types[filename]
+ idx = 0
+
+ if ann == "rejected":
+ if db.rejected(file):
+ path = os.path.join(wc.new_path(), filename)
+ wcnt, ms, mgs, nidcnt = count_file_stats(path)
+
+ merge_dicts(wbalance, {type : wcnt})
+ merge_dicts(tbalance, {type : 1})
+
+ gwcnt = gwcnt + wcnt
+ gnidcnt = gnidcnt + nidcnt
+ merge_dicts(gms, ms)
+ merge_dicts(gmgs, mgs)
+
+ tc = tc + 1
+
+ else:
+ if db.rejected(file):
+ continue
+
+ for annotation in file.findall(ann):
+ if db.finished(annotation):
+ if ann == "ann":
+ path = wc.upload_path(filename, idx)
+ else:
+ path = wc.upload_prim_path(filename)
+
+ wcnt, ms, mgs, nidcnt = count_file_stats(path)
+
+ merge_dicts(wbalance, {type : wcnt})
+ merge_dicts(tbalance, {type : 1})
+
+ gwcnt = gwcnt + wcnt
+ gnidcnt = gnidcnt + nidcnt
+ merge_dicts(gms, ms)
+ merge_dicts(gmgs, mgs)
+
+ tc = tc + 1
+
+ idx = idx + 1
+
+ desired = get_desired()
+ print
+ print fill_custom("Typ tekstow", 60, " "), fill("Liczba tekstow"), fill("Liczba slow"), fill("Procent slow"), fill("Docelowy procent")
+ print fill_custom("", 150, "-")
+ for type in sorted(set(wbalance.keys()) | set(tbalance.keys())):
+ percent = round(1.0 * wbalance[type] / gwcnt * 100, 2)
+ print fill_custom(type.encode("utf-8"), 60, " "), fill(str(tbalance[type])), fill(str(wbalance[type])), fill(str(percent) + "%"), fill(str(desired[type])+"%")
+ print fill_custom("", 150, "-")
+ print fill_custom("dowolny", 60, " "), fill(str(tc)), fill(str(gwcnt)), fill("100.0%"), fill("100.0%")
+
+ print
+
+ if ann != "rejected":
+ print "Wielkosci wystapien:"
+ suma = 0
+ for key, val in sorted(gms.iteritems()):
+ print " ", val, "wystapien o wielkosci", key
+ suma = suma + val
+ print " ", "--------------------------"
+ print " ", suma, "wystapien o dowolnej wielkosci"
+
+ print
+
+ print "Wielkosci klastrow:"
+ suma = 0
+ for key, val in sorted(gmgs.iteritems()):
+ print " ", val, "klastrow o wielkosci", key
+ suma = suma + val
+ print " ", "--------------------------"
+ print " ", suma, "klastrow o dowolnej wielkosci"
+
+ print
+
+ print "Liczba linkow:", gnidcnt
+
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog CONFIG USED_LIST MAPPING""")
+ (options, args) = optparser.parse_args()
+ if len(args) < 3:
+ optparser.print_help()
+ sys.exit(0)
+
+ conf_path = args[0]
+ used_path = args[1]
+ mapping_path = args[2]
+ cfg = Config(conf_path)
+ wc = Repo(cfg["svn.repository"], cfg["svn.login"], cfg["svn.passwd"])
+ db = Database(wc.db_path(), int(cfg["anno_per_file"]))
+
+ types = get_text_types(db, used_path, mapping_path)
+
+ print "################ Statystyki anotacji #########################"
+ print_stats(db, "ann", types)
+ print
+ print "################ Statystyki superanotacji ####################"
+ print_stats(db, "s_ann", types)
+ print
+# print "################ rejected files stats####################"
+# print_stats(db, "rejected", types)
diff --git a/scripts/db_stats.py b/scripts/db_stats.py
new file mode 100644
index 0000000..b0e473f
--- /dev/null
+++ b/scripts/db_stats.py
@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import shutil
+from collections import defaultdict
+from optparse import OptionParser
+
+# Solution with no hard coded path would be welcome
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), ".."))
+
+from dfs.database import Database
+from dfs.repo import Repo
+from dfs.config import Config
+
+def fill(text):
+ return fill_custom(text, 20, " ")
+
+def fill_custom(text, l, sym):
+ to_add = max(0, l - len(text))
+ spaces = ""
+ for i in range(to_add):
+ spaces = spaces + sym
+ return text + spaces
+
+def count_rejected(db):
+ rejected = 0
+ for filename, file in db.file_index.iteritems():
+ if db.rejected(file):
+ rejected = rejected + 1
+ return rejected
+
+def count_anno_stats(db, all, anno_per_file):
+ finished = {}
+ current = {}
+ returned = {}
+ returned_fixed = {}
+ finished_by_all = 0
+ fetched_by_all = 0
+
+ for filename, file in db.file_index.iteritems():
+ if db.rejected(file):
+ continue
+
+ fin = True
+ for annotation in file.findall("ann"):
+ owner = annotation.find("annName").text
+
+ if not owner in finished:
+ finished[owner] = 0
+ if not owner in current:
+ current[owner] = 0
+ if not owner in returned:
+ returned[owner] = 0
+ if not owner in returned_fixed:
+ returned_fixed[owner] = 0
+
+ if db.owned(annotation):
+ current[owner] = current[owner] + 1
+ elif db.returned(annotation):
+ returned[owner] = returned[owner] + 1
+ if db.fixed(annotation):
+ returned_fixed[owner] = returned_fixed[owner] + 1
+ elif db.finished(annotation):
+ finished[owner] = finished[owner] + 1
+ else:
+ print "Strange state! ", annotation.text
+
+ if not db.finished(annotation):
+ fin = False
+
+ if len(file.findall("ann")) == anno_per_file:
+ if fin:
+ finished_by_all = finished_by_all + 1
+ else:
+ fetched_by_all = fetched_by_all + 1
+
+
+ print fill("Login"), fill("Akt. pobr."), fill("Zwrocone (Napr.)"), fill("Zakonczone"), fill("Do pobrania")
+ print fill_custom("", 100, "-")
+ for user in set(current.keys()) | set(finished.keys()) | set(returned_fixed.keys()) | set(returned.keys()):
+ print fill(user), fill(str(current[user])), fill(str(returned[user]) + " (" + str(returned_fixed[user]) + ")"), fill(str(finished[user])),
+ priority, normal = db.for_annotation(user)
+ print fill(str(len(normal)))
+
+ print fill_custom("", 100, "-")
+ all_curr = sum([i for i in current.values()])
+ all_ret = sum([i for i in returned.values()])
+ all_ret_fixed = sum([i for i in returned_fixed.values()])
+ all_fin = sum([i for i in finished.values()])
+ print fill("Suma"), fill(str(all_curr)), fill(str(all_ret) + " (" + str(all_ret_fixed) + ")"), fill(str(all_fin)), fill(str(all - all_curr - all_fin - all_ret))
+ print ""
+ print "Pobranych (nie zakonczonych) jednoczesnie przez", anno_per_file, "tekstow:", fetched_by_all
+ print "Zakonczonych jednoczesnie przez", anno_per_file, "tekstow:", finished_by_all
+ print "Pozostalo do pobrania naprawionych:", all_ret_fixed
+ print "Pozostalo do obejrzenia zwroconych:", all_ret - all_ret_fixed
+ print
+
+ return finished_by_all
+
+
+def count_super_stats(db, all, anno_per_file):
+ finished = {}
+ current = {}
+ returned = {}
+ returned_fixed = {}
+
+ for filename, file in db.file_index.iteritems():
+ if db.rejected(file):
+ continue
+
+ for annotation in file.findall("s_ann"):
+ owner = annotation.find("annName").text
+
+ if not owner in finished:
+ finished[owner] = 0
+ if not owner in current:
+ current[owner] = 0
+ if not owner in returned:
+ returned[owner] = 0
+ if not owner in returned_fixed:
+ returned_fixed[owner] = 0
+
+ if db.owned(annotation):
+ current[owner] = current[owner] + 1
+ elif db.returned(annotation):
+ returned[owner] = returned[owner] + 1
+ if db.fixed(annotation):
+ returned_fixed[owner] = returned_fixed[owner] + 1
+ elif db.finished(annotation):
+ finished[owner] = finished[owner] + 1
+ else:
+ print "Strange state! ", annotation.text
+
+ print fill("Login"), fill("Akt. pobr."), fill("Zwrocone (Napr.)"), fill("Zakonczone"), fill("Do pobrania")
+ print fill_custom("", 100, "-")
+ for user in set(current.keys()) | set(finished.keys()) | set(returned_fixed.keys()) | set(returned.keys()):
+ print fill(user), fill(str(current[user])), fill(str(returned[user]) + " (" + str(returned_fixed[user]) + ")"), fill(str(finished[user])),
+ priority, normal = db.for_adjudication(user)
+ print fill(str(len(normal)))
+
+ print fill_custom("", 100, "-")
+ all_curr = sum([i for i in current.values()])
+ all_ret = sum([i for i in returned.values()])
+ all_ret_fixed = sum([i for i in returned_fixed.values()])
+ all_fin = sum([i for i in finished.values()])
+ print fill("Suma"), fill(str(all_curr)), fill(str(all_ret) + " (" + str(all_ret_fixed) + ")"), fill(str(all_fin)), fill(str(all - all_curr - all_fin - all_ret))
+ print ""
+ print "Pozostalo do pobrania naprawionych:", all_ret_fixed
+ print "Pozostalo do obejrzenia zwroconych:", all_ret - all_ret_fixed
+ print
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog CONFIG""")
+ (options, args) = optparser.parse_args()
+ if len(args) < 1:
+ optparser.print_help()
+ sys.exit(0)
+
+ conf_path = args[0]
+ cfg = Config(conf_path)
+ wc = Repo(cfg["svn.repository"], cfg["svn.login"], cfg["svn.passwd"])
+ db = Database(wc.db_path(), int(cfg["anno_per_file"]))
+ anno_per_file = int(cfg["anno_per_file"])
+
+ rejected = count_rejected(db)
+ print
+ print fill_custom("", 100, "-")
+ all = len(db.file_index.keys())
+ print "Wszystkich tekstow w bazie: " + str(all)
+ print "Odrzuconych: " + str(rejected)
+ print "Wszystkich tekstow w bazie bez odrzuconych: " + str(all - rejected)
+ all = all * anno_per_file - rejected
+ print "Anotacji do wykonania: " + str(all)
+
+ print fill_custom("", 100, "-")
+ print
+
+ print fill_custom("----Anotacja", 100, "-")
+ anno_count = count_anno_stats(db, all, anno_per_file)
+
+ print fill_custom("----SuperAnotacja", 100, "-")
+ count_super_stats(db, anno_count, anno_per_file)
+
+
diff --git a/scripts/extract_files.py b/scripts/extract_files.py
new file mode 100755
index 0000000..4d59d64
--- /dev/null
+++ b/scripts/extract_files.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import shutil
+from collections import defaultdict
+from optparse import OptionParser
+
+# Solution with no hard coded path would be welcome
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), ".."))
+
+from dfs.database import Database
+from dfs.repo import Repo
+from dfs.config import Config
+
+def extract_file(file, exts, target_dir):
+ print "Extracting file", file
+ for ext in exts:
+ filename = file + ext
+ print "\t-", os.path.basename(filename)
+ shutil.copy(filename, target_dir)
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog [options] CONFIG TYPE TARGET_DIR""")
+ optparser.add_option("--extensions", dest="exts", default=".mmax,_mentions.xml,_words.xml",
+ help="List of comma-separated file extensions")
+ (options, args) = optparser.parse_args()
+ if len(args) != 3:
+ optparser.print_help()
+ sys.exit(0)
+
+ conf_path = args[0]
+ cfg = Config(conf_path)
+ type = args[1]
+ target_dir = args[2]
+ anno_per_file = int(cfg["anno_per_file"])
+ exts = options.exts.split(",")
+ wc = Repo(cfg["svn.repository"], cfg["svn.login"], cfg["svn.passwd"])
+ db = Database(wc.db_path(), anno_per_file)
+
+ poss_types = ["finished", "returned", "annotated"]
+ if type not in poss_types:
+ print "Possible types: ", poss_types
+ sys.exit(0)
+
+ filepaths = {}
+ for filename, file_elem in db.file_index.iteritems():
+ if db.rejected(file_elem):
+ continue
+
+ if type == "finished":
+ sann = file_elem.find("s_ann")
+ if sann is not None and db.finished(sann):
+ filepaths[wc.upload_prim_path(filename)] = None
+
+ elif type == "annotated":
+ idx = 0
+ for ann in file_elem.findall("ann"):
+ if db.finished(ann):
+ filepaths[wc.upload_path(filename, idx)] = None
+ idx = idx + 1
+
+ elif type == "returned":
+ reason = None
+ idx = 0
+ for ann in file_elem.findall("ann"):
+ if db.returned(ann):
+ reason = db.get_reason(ann)
+ break
+ idx = idx + 1
+
+ if reason is not None:
+ filepaths[wc.upload_path(filename, idx)] = reason
+ continue
+
+ reason = None
+ for sann in file_elem.findall("s_ann"):
+ if db.returned(sann):
+ reason = db.get_reason(sann)
+ break
+
+ if reason is not None:
+ filepaths[wc.upload_prim_path(filename)] = reason
+
+ for k, v in filepaths.iteritems():
+ extract_file(k, exts, target_dir)
+
+ if type == "returned":
+ filename = os.path.join(target_dir, "reasons.txt")
+ with open(filename, 'w') as f:
+ for k, v in sorted(filepaths.iteritems()):
+ name = os.path.basename(k)
+ f.write(name)
+ f.write(" - ")
+ f.write(v.encode("utf-8"))
+ f.write("\n")
+
\ No newline at end of file
diff --git a/scripts/fill_empty_repo.sh b/scripts/fill_empty_repo.sh
new file mode 100755
index 0000000..921a637
--- /dev/null
+++ b/scripts/fill_empty_repo.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+if [ $# == 2 ]; then
+ echo "Filling empty repository in: $1 with $2 annos per file"
+else
+ echo "Usage: $0 DIR ANNO_PER_FILE"
+ exit
+fi
+
+path=$1
+annos=$2
+
+mkdir $path/new
+mkdir $path/annotation
+
+c=0
+for x in {A..Z}
+do
+ mkdir $path/annotation/$x
+ c=$((c+1))
+ if [ $c -ge $annos ]
+ then
+ break
+ fi
+done
+
+mkdir $path/adjudication
+
+echo "<files></files>" > $path/db.xml
diff --git a/scripts/fix_files.py b/scripts/fix_files.py
new file mode 100755
index 0000000..82a5001
--- /dev/null
+++ b/scripts/fix_files.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import shutil
+from collections import defaultdict
+from optparse import OptionParser
+
+# Solution with no hard coded path would be welcome
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), ".."))
+
+from dfs.database import Database
+from dfs.repo import Repo
+from dfs.config import Config
+
+def fix_file(wc, db, file_id, contents, anno_per_file):
+ file_elem = db.file_index[file_id]
+ if file_elem is None:
+ print "File", file_id, "not found in database."
+ return False
+
+ ann_elem = None
+ idx = -1
+ for ann in file_elem.findall("ann") + file_elem.findall("s_ann"):
+ idx = idx + 1
+ if db.returned(ann):
+ ann_elem = ann
+ break
+
+ if ann_elem is None:
+ print "File", file_id, "not returned in database."
+ return False
+
+ print "Fixing file " + file_id
+
+ try:
+ db.fix(ann_elem)
+
+ if ann_elem.tag == "ann":
+ wc.upload(file_id, idx, contents)
+ elif ann_elem.tag == "s_ann":
+ wc.upload_prim(file_id, contents)
+
+ except Exception as ex:
+ print "\t error: " + str(ex)
+ wc.revert()
+ return False
+
+ return True
+
+def match_ext(path, exts):
+ for ext in exts:
+ if path.endswith(ext):
+ return ext
+
+def path_id(path, ext):
+ _, filename = os.path.split(path)
+ return re.sub("%s$" % ext, "", filename)
+
+def group_paths(paths, exts):
+ result = defaultdict(list)
+ for path in paths:
+ ext = match_ext(path, exts)
+ if (ext != None):
+ file_id = path_id(path, ext)
+ if file_id not in result:
+ result[file_id] = {}
+ with open(path, "r") as f:
+ result[file_id][ext]=f.read()
+ return result
+
+def get_rec_paths(paths):
+ result = []
+ for path in paths:
+ if os.path.isdir(path):
+ for dirname, dirnames, filenames in os.walk(path):
+ for filename in filenames:
+ result.append(os.path.join(dirname, filename))
+ else:
+ result.append(path)
+ return result
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog [options] CONFIG FILES""")
+ optparser.add_option("--extensions", dest="exts", default=".mmax,_mentions.xml,_words.xml",
+ help="List of comma-separated file extensions")
+ (options, args) = optparser.parse_args()
+ if len(args) < 2:
+ optparser.print_help()
+ sys.exit(0)
+
+ conf_path = args[0]
+ cfg = Config(conf_path)
+ anno_per_file = int(cfg["anno_per_file"])
+ paths = get_rec_paths(args[1:])
+ exts = options.exts.split(",")
+ files = group_paths(paths, exts)
+ wc = Repo(cfg["svn.repository"], cfg["svn.login"], cfg["svn.passwd"])
+ db = Database(wc.db_path(), anno_per_file)
+
+ success = []
+ fail = []
+ for file_id, contents in files.iteritems():
+ if fix_file(wc, db, file_id, contents, anno_per_file):
+ success.append(file_id)
+ else:
+ fail.append(file_id)
+
+ db.save()
+ wc.commit("Fixed files: " + str(success))
+
+ print ""
+ if len(success) > 0:
+ print "Fixed files: " + str(success)
+ if len(fail) > 0:
+ print "Failed to fix files: " + str(fail)
+
diff --git a/scripts/quasi.py b/scripts/quasi.py
new file mode 100755
index 0000000..c042e11
--- /dev/null
+++ b/scripts/quasi.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python
+import re
+import sys
+import os
+import re
+import shutil
+from collections import defaultdict
+from optparse import OptionParser
+
+# Solution with no hard coded path would be welcome
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), ".."))
+
+from dfs.database import Database
+from dfs.repo import Repo
+from dfs.config import Config
+
+def fill(text):
+ return fill_custom(text, 20, " ")
+
+def fill_custom(text, l, sym):
+ to_add = max(0, l - len(text))
+ spaces = ""
+ for i in range(to_add):
+ spaces = spaces + sym
+ return text + spaces
+
+def parse_span(span, words_list):
+ words = []
+
+ w = span.split(",")
+ for fragment in w:
+ f = fragment.split("..")
+ id1 = words_list.index(f[0])
+ id2 = words_list.index(f[-1])
+ for i in range(id1, id2+1):
+ words.append(words_list[i])
+
+ return words
+
+def get_context(ids, words_list, size):
+ first = -1
+ last = -1
+
+ i = 0
+ for i in range(len(words_list)):
+ w = words_list[i]
+ if w in ids:
+ if first == -1:
+ first = i
+ last = i
+
+ first = max(0, first - size)
+ last = min(len(words_list), last + size)
+
+ return words_list[first:last]
+
+def print_file(path):
+ word_id_2_orth = {}
+ words_list = []
+ w = re.compile("<word.* id=\"(.*?)\".*>(.*?)</word>.*")
+ with open(path + "_words.xml", "r") as f:
+ for line in f.readlines():
+ groups = w.findall(line)
+ if len(groups) == 1:
+ group = groups[0]
+ id = group[0]
+ orth = group[1]
+ word_id_2_orth[id] = orth
+ words_list.append(id)
+
+ me = re.compile("<markable.*id=\"(.*?)\".*")
+ sp = re.compile(".*span=\"(.*?)\".*")
+ co = re.compile(".*comment=\"(.*?)\".*")
+ ni = re.compile(".*near_identity=\"(.*?)\".*")
+
+ mention_id_2_span = {}
+ mention_id_2_comment = {}
+ near_links = []
+ with open(path + "_mentions.xml", "r") as f:
+ for line in f.readlines():
+ groups1 = me.findall(line)
+ groups2 = sp.findall(line)
+ groups3 = co.findall(line)
+
+ if len(groups1) == 1 and len(groups2) == 1:
+ id = groups1[0]
+ span = groups2[0]
+ mention_id_2_span[id] = parse_span(span, words_list)
+
+ if len(groups3) == 1:
+ mention_id_2_comment[id] = groups3[0]
+
+ near = ni.findall(line)
+ if len(near) == 1 and near[0] != "empty" and near[0] != "":
+ near_links.append((id, near[0]))
+
+ if len(near_links) > 0:
+ print "###", "Tekst", path.split("/")[-1], "###"
+ print
+
+ c = 0
+ for m1, m2 in near_links:
+ comments = []
+ if m1 in mention_id_2_comment:
+ comments.append(mention_id_2_comment[m1])
+ if m2 in mention_id_2_comment:
+ comments.append(mention_id_2_comment[m2])
+
+ if m1 not in mention_id_2_span or m2 not in mention_id_2_span:
+ print "ERROR", m1, m2
+ continue
+
+ span1 = mention_id_2_span[m1]
+ span2 = mention_id_2_span[m2]
+
+ spans = set(span1)
+ spans = spans.union(set(span2))
+ ctx = get_context(spans, words_list, 3)
+
+ result = ""
+ for wid in ctx:
+ result = result + " "
+ if wid == span1[0]:
+ result = result + "["
+ if wid == span2[0]:
+ result = result + "["
+
+ result = result + word_id_2_orth[wid]
+
+ if wid == span1[-1]:
+ result = result + "]"
+ if wid == span2[-1]:
+ result = result + "]"
+
+ m1orth = "[" + reduce(lambda a, i: a + " " + i, map(lambda i : word_id_2_orth[i], span1)) + "]"
+ m2orth = "[" + reduce(lambda a, i: a + " " + i, map(lambda i : word_id_2_orth[i], span2)) + "]"
+
+ if words_list.index(span1[0]) <= words_list.index(span2[0]):
+ print str(c)+".", m1orth, "<-->", m2orth
+ else:
+ print str(c)+".", m2orth, "<-->", m1orth
+
+ for comm in comments:
+ print "Komentarz:", comm
+ print "...", result, "..."
+ print
+
+ c = c + 1
+
+ return c
+
+def print_quasi(db):
+ links = 0
+ for filename, file in db.file_index.iteritems():
+
+ if db.rejected(file):
+ continue
+
+ sann = file.find("s_ann")
+
+ if sann is not None and db.finished(sann):
+ path = wc.upload_prim_path(filename)
+ links = links + print_file(path)
+
+ return links
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog CONFIG """)
+ (options, args) = optparser.parse_args()
+ if len(args) < 1:
+ optparser.print_help()
+ sys.exit(0)
+
+ conf_path = args[0]
+ cfg = Config(conf_path)
+ wc = Repo(cfg["svn.repository"], cfg["svn.login"], cfg["svn.passwd"])
+ db = Database(wc.db_path(), int(cfg["anno_per_file"]))
+
+ l = print_quasi(db)
+ print "##################"
+ print
+ print "Wszystkich linkow:", l
+
diff --git a/scripts/reject_files.py b/scripts/reject_files.py
new file mode 100755
index 0000000..d599392
--- /dev/null
+++ b/scripts/reject_files.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import shutil
+from collections import defaultdict
+from optparse import OptionParser
+
+# Solution with no hard coded path would be welcome
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), ".."))
+
+from dfs.database import Database
+from dfs.repo import Repo
+from dfs.config import Config
+
+def get_reason(db, file_id):
+ reason = ""
+ file_elem = db.file_index[file_id]
+ for ann in file_elem.findall("ann"):
+ if db.returned(ann):
+ reason = db.get_reason(ann)
+ break
+
+ for ann in file_elem.findall("s_ann"):
+ if db.returned(ann):
+ reason = db.get_reason(ann)
+ break
+
+ return reason
+
+
+def reject_file(db, file_id, reason):
+ try:
+ if reason == "":
+ reason = get_reason(db, file_id)
+
+ print "Rejecting", file_id, "because", reason
+ db.reject(file_id, reason)
+ except Exception as ex:
+ print file_id, "-", str(ex)
+ return False
+
+ return True
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog -r REASON CONFIG FILES""")
+ optparser.add_option("-r", dest="reason", default="", help="Reason to reject files")
+ (options, args) = optparser.parse_args()
+ if len(args) < 2:
+ optparser.print_help()
+ sys.exit(0)
+
+ conf_path = args[0]
+ cfg = Config(conf_path)
+ ids = args[1:]
+ reason = options.reason
+ anno_per_file = int(cfg["anno_per_file"])
+ wc = Repo(cfg["svn.repository"], cfg["svn.login"], cfg["svn.passwd"])
+ db = Database(wc.db_path(), anno_per_file)
+
+ success = []
+ fail = []
+ for file_id in ids:
+ if reject_file(db, file_id, reason):
+ success.append(file_id)
+ else:
+ fail.append(file_id)
+
+ db.save()
+ wc.commit("Rejected files: "+str(ids))
+
+ print ""
+ if len(success) > 0:
+ print "Rejected files: "+str(success)
+ if len(fail) > 0:
+ print "Failed to reject files: "+str(fail)
+
\ No newline at end of file
diff --git a/scripts/remove_files.py b/scripts/remove_files.py
new file mode 100755
index 0000000..4075481
--- /dev/null
+++ b/scripts/remove_files.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import shutil
+from collections import defaultdict
+from optparse import OptionParser
+
+# Solution with no hard coded path would be welcome
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), ".."))
+
+from dfs.database import Database
+from dfs.repo import Repo
+from dfs.config import Config
+
+def remove_file(wc, db, file_id, extensions, anno_per_file):
+ try:
+ db.remove(file_id)
+ except Exception as ex:
+ print str(ex)
+ return False
+
+ print "Removing "+file_id+" :"
+ for ext in extensions:
+ file = file_id+ext
+ for r in wc.remove(file, anno_per_file):
+ print "\t - "+r
+
+ return True
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog [options] CONFIG FILES""")
+ optparser.add_option("--extensions", dest="exts", default=".mmax,_mentions.xml,_words.xml",
+ help="List of comma-separated file extensions")
+ (options, args) = optparser.parse_args()
+ if len(args) < 2:
+ optparser.print_help()
+ sys.exit(0)
+
+ conf_path = args[0]
+ cfg = Config(conf_path)
+ ids = args[1:]
+ extensions = options.exts.split(",")
+ anno_per_file = int(cfg["anno_per_file"])
+ wc = Repo(cfg["svn.repository"], cfg["svn.login"], cfg["svn.passwd"])
+ db = Database(wc.db_path(), anno_per_file)
+
+ success = []
+ fail = []
+ for file_id in ids:
+ if remove_file(wc, db, file_id, extensions, anno_per_file):
+ success.append(file_id)
+ else:
+ fail.append(file_id)
+
+ db.save()
+ wc.commit("Removed files: "+str(ids))
+
+ print ""
+ if len(success) > 0:
+ print "Removed files: "+str(success)
+ if len(fail) > 0:
+ print "Failed to remove files: "+str(fail)
\ No newline at end of file
diff --git a/scripts/stats_atlas.py b/scripts/stats_atlas.py
new file mode 100755
index 0000000..6a5a54a
--- /dev/null
+++ b/scripts/stats_atlas.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import shutil
+from collections import defaultdict
+from optparse import OptionParser
+
+# Solution with no hard coded path would be welcome
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), ".."))
+
+from dfs.database import Database
+from dfs.repo import Repo
+from dfs.config import Config
+
+def fill(text):
+ return fill_custom(text, 20, " ")
+
+def fill_custom(text, l, sym):
+ to_add = max(0, l - len(text))
+ spaces = ""
+ for i in range(to_add):
+ spaces = spaces + sym
+ return text + spaces
+
+def count_rejected(db):
+ rejected = 0
+ for filename, file in db.file_index.iteritems():
+ if db.rejected(file):
+ rejected = rejected + 1
+ return rejected
+
+
+def count_file_stats(path):
+ text = ""
+ with open(path + ".txt", "r") as f:
+ for line in f.readlines():
+ if line.startswith("#### SUMMARIES ####"):
+ break;
+ text += line +"\n"
+
+ return len(text)
+
+def count_anno_stats(db, all, anno_per_file):
+ finished = {}
+ current = {}
+ returned = {}
+ returned_fixed = {}
+ finished_by_all = 0
+ fetched_by_all = 0
+ chars = {}
+
+ for filename, file in db.file_index.iteritems():
+ if db.rejected(file):
+ continue
+
+ fin = True
+ idx = 0
+ for annotation in file.findall("ann"):
+ owner = annotation.find("annName").text
+
+ if not owner in finished:
+ finished[owner] = 0
+ if not owner in current:
+ current[owner] = 0
+ if not owner in returned:
+ returned[owner] = 0
+ if not owner in returned_fixed:
+ returned_fixed[owner] = 0
+
+ if db.owned(annotation):
+ current[owner] = current[owner] + 1
+ elif db.returned(annotation):
+ returned[owner] = returned[owner] + 1
+ if db.fixed(annotation):
+ returned_fixed[owner] = returned_fixed[owner] + 1
+ elif db.finished(annotation):
+ finished[owner] = finished[owner] + 1
+ else:
+ print "Strange state! ", annotation.text
+
+ if not db.finished(annotation):
+ fin = False
+ else:
+ path = wc.upload_path(filename, idx)
+ ccnt = count_file_stats(path)
+ if owner not in chars:
+ chars[owner] = 0
+ chars[owner] = chars[owner] + ccnt
+
+ if len(file.findall("ann")) == anno_per_file:
+ if fin:
+ finished_by_all = finished_by_all + 1
+ else:
+ fetched_by_all = fetched_by_all + 1
+
+ idx += 1
+
+
+ print fill("Login"), fill("Akt. pobr."), fill("Zwrocone (Napr.)"), fill("Zakonczone"), fill("Zak. znaki"), fill("Do pobrania")
+ print fill_custom("", 120, "-")
+ for user in set(current.keys()) | set(finished.keys()) | set(returned_fixed.keys()) | set(returned.keys()):
+ print fill(user), fill(str(current[user])), fill(str(returned[user]) + " (" + str(returned_fixed[user]) + ")"), fill(str(finished[user])),
+ print fill(str(chars[user])),
+ priority, normal = db.for_annotation(user)
+ print fill(str(len(normal)))
+
+ print fill_custom("", 120, "-")
+ all_curr = sum([i for i in current.values()])
+ all_ret = sum([i for i in returned.values()])
+ all_ret_fixed = sum([i for i in returned_fixed.values()])
+ all_fin = sum([i for i in finished.values()])
+ all_chars = sum([i for i in chars.values()])
+ print fill("Suma"), fill(str(all_curr)), fill(str(all_ret) + " (" + str(all_ret_fixed) + ")"), fill(str(all_fin)), fill(str(all_chars)), fill(str(all - all_curr - all_fin - all_ret))
+ print ""
+ print "Pobranych (nie zakonczonych) jednoczesnie przez", anno_per_file, "tekstow:", fetched_by_all
+ print "Zakonczonych jednoczesnie przez", anno_per_file, "tekstow:", finished_by_all
+ print "Pozostalo do pobrania naprawionych:", all_ret_fixed
+ print "Pozostalo do obejrzenia zwroconych:", all_ret - all_ret_fixed
+ print
+
+ return finished_by_all
+
+if __name__ == "__main__":
+ optparser = OptionParser(usage="""usage: %prog CONFIG""")
+ (options, args) = optparser.parse_args()
+ if len(args) < 1:
+ optparser.print_help()
+ sys.exit(0)
+
+ conf_path = args[0]
+ cfg = Config(conf_path)
+ wc = Repo(cfg["svn.repository"], cfg["svn.login"], cfg["svn.passwd"])
+ db = Database(wc.db_path(), int(cfg["anno_per_file"]))
+ anno_per_file = int(cfg["anno_per_file"])
+
+ rejected = count_rejected(db)
+ print
+ print fill_custom("", 100, "-")
+ all = len(db.file_index.keys())
+ print "Wszystkich tekstow w bazie: " + str(all)
+ print "Odrzuconych: " + str(rejected)
+ print "Wszystkich tekstow w bazie bez odrzuconych: " + str(all - rejected)
+ all = all * anno_per_file - rejected
+ print "Anotacji do wykonania: " + str(all)
+
+ print fill_custom("", 100, "-")
+ print
+
+ print fill_custom("----Anotacja", 100, "-")
+ anno_count = count_anno_stats(db, all, anno_per_file)
+
\ No newline at end of file
--
libgit2 0.22.2