create_walenty.py
7.22 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
#-*- coding:utf-8 -*-
#Copyright (c) 2015, Bartłomiej Nitoń
#All rights reserved.
#Redistribution and use in source and binary forms, with or without modification, are permitted provided
#that the following conditions are met:
# Redistributions of source code must retain the above copyright notice, this list of conditions and
# the following disclaimer.
# Redistributions in binary form must reproduce the above copyright notice, this list of conditions
# and the following disclaimer in the documentation and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import datetime
import os
import tarfile
from collections import Counter
from django.core.management.base import BaseCommand
from accounts.models import User
from dictionary.ajax_vocabulary_management import create_text_walenty, \
get_stats, write_stats, \
update_walenty_stats
from dictionary.ajax_argument_realizations import create_realizations_file
from dictionary.models import Frame_Opinion, Lemma, Vocabulary, POS, \
get_checked_statuses, get_ready_statuses
from settings import WALENTY_PATH
class Command(BaseCommand):
args = 'none'
def handle(self, *args, **options):
now = datetime.datetime.now().strftime('%Y%m%d')
filename_base = '%s_%s' % ('walenty', now)
realizations_path = os.path.join(WALENTY_PATH,
' %s_%s.txt' % ('phrase_types_expand', now))
checked_stats_path = os.path.join(WALENTY_PATH, u'%s_%s.txt' % (filename_base.replace('walenty', 'stats'),
'verified'))
ready_stats_path = os.path.join(WALENTY_PATH, u'%s_%s.txt' % (filename_base.replace('walenty', 'stats'),
'all'))
create_realizations_file(realizations_path)
try:
all_stats = Counter({})
verified_stats = Counter({})
base_path = os.path.join(WALENTY_PATH, filename_base)
archive = tarfile.open(base_path + '.tar.gz', 'w:gz')
os.chdir(WALENTY_PATH)
archive.add(os.path.basename(realizations_path))
for pos in POS.objects.exclude(tag=u'unk').order_by('priority'):
pos_stats = create_pos_archive_and_get_stats(archive, pos, filename_base)
all_stats = all_stats + Counter(pos_stats['all'])
verified_stats = verified_stats + Counter(pos_stats['verified'])
write_stats(checked_stats_path, verified_stats)
archive.add(os.path.basename(checked_stats_path))
write_stats(ready_stats_path, all_stats)
archive.add(os.path.basename(ready_stats_path))
finally:
archive.close()
os.remove(realizations_path)
os.remove(checked_stats_path)
os.remove(ready_stats_path)
def create_pos_archive_and_get_stats(archive, pos, filename_base):
all_stats = {}
checked_stats = {}
try:
checked_statuses = get_checked_statuses()
ready_statuses = get_ready_statuses()
lemmas = Lemma.objects.filter(old=False, entry_obj__pos=pos).order_by('entry')
checked_lemmas = lemmas.filter(status__in=checked_statuses)
ready_lemmas = lemmas.filter(status__in=ready_statuses)
all_path = os.path.join(WALENTY_PATH, u'%s_%ss_%s' % (filename_base, pos.tag, 'all'))
walenty_path_ready = create_text_walenty(file_name=all_path,
lemmas=ready_lemmas,
vocabularies=Vocabulary.objects.none(),
frame_opinions=Frame_Opinion.objects.none(),
lemma_statuses=ready_statuses,
owners=User.objects.none(),
poss=POS.objects.filter(pk=pos.pk),
add_frame_opinions=True)
all_filename = os.path.basename(walenty_path_ready)
archive.add(name=all_filename, arcname=os.path.join(u'%ss' % pos.tag, all_filename))
checked_path = os.path.join(WALENTY_PATH, u'%s_%ss_%s' % (filename_base, pos.tag, 'verified'))
walenty_path_checked = create_text_walenty(file_name=checked_path,
lemmas=checked_lemmas,
vocabularies=Vocabulary.objects.none(),
frame_opinions=Frame_Opinion.objects.none(),
lemma_statuses=checked_statuses,
owners=User.objects.none(),
poss=POS.objects.filter(pk=pos.pk),
add_frame_opinions=True)
checked_filename = os.path.basename(walenty_path_checked)
archive.add(name=checked_filename, arcname=os.path.join(u'%ss' % pos.tag, checked_filename))
all_stats = get_stats(ready_statuses, pos.tag)
all_stats_path = os.path.join(WALENTY_PATH, u'%s_%ss_%s' % (filename_base.replace('walenty', 'stats'),
pos.tag, 'all'))
write_stats(all_stats_path, all_stats)
all_stats_filename = os.path.basename(all_stats_path)
archive.add(name=all_stats_filename, arcname=os.path.join(u'%ss' % pos.tag, all_stats_filename))
checked_stats = get_stats(checked_statuses, pos.tag)
checked_stats_path = os.path.join(WALENTY_PATH, u'%s_%ss_%s' % (filename_base.replace('walenty', 'stats'),
pos.tag, 'verified'))
write_stats(checked_stats_path, checked_stats)
checked_stats_filename = os.path.basename(checked_stats_path)
archive.add(name=checked_stats_filename, arcname=os.path.join(u'%ss' % pos.tag, checked_stats_filename))
update_walenty_stats(all_stats)
finally:
os.remove(walenty_path_ready)
os.remove(walenty_path_checked)
os.remove(all_stats_path)
os.remove(checked_stats_path)
return {'all': all_stats,
'verified': checked_stats}