Commit 291f42a66d14c900901ee06be4b6f55367938a06

Authored by janek37
1 parent b15716a6

zmodyfikowany import SGJP

common/util.py
... ... @@ -2,9 +2,7 @@
2 2  
3 3 import re
4 4 import sys
5   -from htmlentitydefs import name2codepoint
6   -from django.utils.encoding import smart_unicode, force_unicode
7   -from django.conf.urls import defaults
  5 +from django.conf import urls
8 6 from django.http import HttpResponseRedirect, Http404
9 7  
10 8  
... ... @@ -40,7 +38,7 @@ def generator_slice(generator, count):
40 38 def url(regex, view, **kwargs):
41 39 if 'name' not in kwargs:
42 40 kwargs['name'] = view.rsplit('.', 1)[-1]
43   - return defaults.url(regex, view, **kwargs)
  41 + return urls.url(regex, view, **kwargs)
44 42  
45 43 def stringify_keys(dictionary):
46 44 return dict((keyword.encode('ascii'), value)
... ...
dictionary/management/commands/import_data.py
... ... @@ -10,76 +10,19 @@ from common.util import no_history
10 10 from dictionary.models import *
11 11  
12 12 DEFAULT_DATABASE = 'data/sgjp.db'
13   -MINI_LEXEME_COUNT = 500
14   -MINI_MODE = False # do debugowania
15   -SQL_MODE = True
16   -cursor = connection.cursor()
17   -db = None
18   -sqlite_cursor = None
19   -no_history()
20   -
21   -class Command(BaseCommand):
22   - args = '<input db filename>'
23   - help = 'Imports initial data'
24   -
25   - def handle(self, db_name=DEFAULT_DATABASE, **options):
26   - global db
27   - db = db_name
28   - delete_and_import()
29   -
30   -def get_cursor(db):
31   - conn = sqlite3.connect(db)
32   - conn.row_factory = sqlite3.Row
33   - return conn.cursor()
34   -
35 13  
36   -def import_lexical_classes():
37   - lc = LexicalClass(symbol='inne')
38   - lc.save()
39   - for row in sqlite_cursor.execute('select distinct pos from wzory'):
40   - if not SQL_MODE:
41   - lc = LexicalClass(symbol=row['pos'])
42   - lc.save()
43   - else:
44   - cursor.execute("INSERT INTO czescimowy (czm) VALUES (%s)",
45   - [row['pos']])
46   -
47   -
48   -def import_parts_of_speech():
49   - other = 'inne'
50   - lcs = {}
51   - for row in sqlite_cursor.execute('select distinct wzory.pos, leksemy.pos from wzory '
52   - 'natural join odmieniasie join leksemy on '
53   - 'leksemy.nr = odmieniasie.nr'):
54   - lcs[row[1]] = row[0]
55   - #print row
56   - for row in sqlite_cursor.execute('SELECT pos FROM klasygramatyczne'):
57   - lc = lcs.get(row['pos'], other)
58   - if not SQL_MODE:
59   - pos = PartOfSpeech(symbol=row['pos'])
60   - pos.lexical_class = LexicalClass.objects.get(symbol=lc)
61   - pos.save()
62   - else:
63   - cursor.execute("INSERT INTO klasygramatyczne (pos, czm) VALUES (%s, %s)",
64   - [row['pos'], lc])
  14 +MINI_MODE = False # do debugowania
  15 +MINI_LEXEME_COUNT = 500
  16 +MINI_LEXEME_QUERY = 'SELECT %s FROM leksemy LIMIT ?'
65 17  
  18 +# UWAGA: aktualnie ustawienie SQL_MODE = False jest niekompletne
  19 +SQL_MODE = True
66 20  
67   -def import_base_form_labels():
68   - query_result = sqlite_cursor.execute("""
69   - SELECT efobaz FROM paradygmaty
70   - UNION
71   - SELECT efobaz FROM zakonczenia
72   - """)
73   - for row in query_result:
74   - if not SQL_MODE:
75   - bfl = BaseFormLabel(entry=row[0])
76   - bfl.save()
77   - else:
78   - cursor.execute("INSERT INTO efobazy (efobaz) VALUES (%s)", [row[0]])
  21 +OTHER = 'inne'
  22 +DEFAULT_VOCAB = 'SGJP'
79 23  
80   -# tymczasowa tabelka - powinna przychodziฤ‡ z bazฤ…
81   -# (albo zrobiฤ‡ w kodzie wyjฤ…tek na p123... jak juลผ i tak sฤ… wyjฤ…tki)
82   -basic_form_labels = {
  24 +# tymczasowa tabelka
  25 +BASIC_FORM_LABELS = {
83 26 '0-': '1',
84 27 '3+': '1',
85 28 'f': 'sg:nom',
... ... @@ -96,7 +39,7 @@ basic_form_labels = {
96 39 }
97 40  
98 41 # to chyba nie jest najlepsze rozwiฤ…zanie...
99   -basic_form_labels_pos = {
  42 +BASIC_FORM_LABELS_POS = {
100 43 'v': '5',
101 44 'ger': '11',
102 45 'pact': '3',
... ... @@ -105,437 +48,461 @@ basic_form_labels_pos = {
105 48 'pred': '5',
106 49 }
107 50  
108   -ics = {}
  51 +class Command(BaseCommand):
  52 + args = '<input db filename>'
  53 + help = 'Imports initial data'
109 54  
110   -def import_inflection_characteristics():
111   - for row in sqlite_cursor.execute('SELECT DISTINCT charfl, pos FROM paradygmaty'):
112   - if row['charfl'] == '':
113   - bfl_entry = '1' if row['pos'] in ('adj', 'adjcom') else ''
114   - else:
115   - bfl_entry = basic_form_labels.get(row['charfl'], '')
116   - if row['pos'] in basic_form_labels_pos:
117   - bfl_entry = basic_form_labels_pos[row['pos']]
118   - bfl = BaseFormLabel.objects.get(entry=bfl_entry)
119   - if not SQL_MODE:
120   - ic = InflectionCharacteristic(
  55 + def handle(self, db_name=DEFAULT_DATABASE, **options):
  56 + ImportData(db_name).delete_and_import()
  57 +
  58 +def get_cursor(db):
  59 + conn = sqlite3.connect(db)
  60 + conn.row_factory = sqlite3.Row
  61 + return conn.cursor()
  62 +
  63 +METHOD_NAMES = {
  64 + CrossReference: 'import_cross_references',
  65 + Ending: 'import_endings',
  66 + LexemeInflectionPattern: 'import_lexeme_inflection_patterns',
  67 + Lexeme: 'import_lexemes',
  68 + PatternType: 'import_pattern_types',
  69 + TableTemplate: 'import_tables',
  70 + BaseFormLabel: 'new_base_form_labels',
  71 + CrossReferenceType: 'new_cross_reference_types',
  72 + InflectionCharacteristic: 'new_inflection_characteristics',
  73 + LexemeAssociation: 'new_lexeme_associations',
  74 + LexicalClass: 'new_lexical_classes',
  75 + PartOfSpeech: 'new_parts_of_speech',
  76 + Pattern: 'new_patterns',
  77 + Qualifier: 'new_qualifiers',
  78 + TableHeader: 'new_table_headers',
  79 + Vocabulary: 'new_vocabularies',
  80 +}
  81 +
  82 +class ImportData(object):
  83 + def __init__(self, db):
  84 + self.cursor = connection.cursor()
  85 + self.sqlite_cursor = get_cursor(db)
  86 + no_history()
  87 +
  88 + def close(self):
  89 + self.cursor.close()
  90 + self.sqlite_cursor.close()
  91 +
  92 + def new_lexical_classes(self):
  93 + yield LexicalClass(symbol=OTHER)
  94 + for row in self.sqlite_cursor.execute('select distinct pos from wzory'):
  95 + yield LexicalClass(symbol=row['pos'])
  96 +
  97 + def new_parts_of_speech(self):
  98 + lcs = {}
  99 + for row in self.sqlite_cursor.execute(
  100 + 'select distinct wzory.pos, leksemy.pos from wzory '
  101 + 'natural join odmieniasie join leksemy on leksemy.nr = odmieniasie.nr'):
  102 + lcs[row[1]] = row[0]
  103 +
  104 + for row in self.sqlite_cursor.execute('SELECT pos FROM klasygramatyczne'):
  105 + lc = lcs.get(row['pos'], OTHER)
  106 + yield PartOfSpeech(
  107 + symbol=row['pos'], lexical_class = LexicalClass.objects.get(symbol=lc))
  108 +
  109 + def new_base_form_labels(self):
  110 + query_result = self.sqlite_cursor.execute("""
  111 + SELECT efobaz FROM paradygmaty
  112 + UNION
  113 + SELECT efobaz FROM zakonczenia
  114 + """)
  115 + for row in query_result:
  116 + yield BaseFormLabel(entry=row[0])
  117 +
  118 + def new_inflection_characteristics(self):
  119 + for row in self.sqlite_cursor.execute(
  120 + 'SELECT DISTINCT charfl, pos FROM paradygmaty'):
  121 + if row['charfl'] == '':
  122 + bfl_entry = '1' if row['pos'] in ('adj', 'adjcom') else ''
  123 + else:
  124 + bfl_entry = BASIC_FORM_LABELS.get(row['charfl'], '')
  125 + if row['pos'] in BASIC_FORM_LABELS_POS:
  126 + bfl_entry = BASIC_FORM_LABELS_POS[row['pos']]
  127 + bfl = BaseFormLabel.objects.get(entry=bfl_entry)
  128 + yield InflectionCharacteristic(
121 129 entry=row['charfl'], basic_form_label=bfl,
122 130 part_of_speech=PartOfSpeech.objects.get(pk=row['pos']))
123   - ic.save()
124   - else:
125   - cursor.execute("INSERT INTO charfle (charfl, pos, efobaz) "
126   - "VALUES (%s, %s, %s)", [row['charfl'], row['pos'], bfl.pk])
127   - for ic in InflectionCharacteristic.objects.all():
128   - ics[(ic.entry, ic.part_of_speech.symbol)] = ic
129   -
130   -
131   -sgjp_domain = ('SGJP', 'WSJP', 'SJPDor', 'zmiotki')
132   -
133   -
134   -def import_vocabularies():
135   - result = sqlite_cursor.execute("""
136   - SELECT slownik FROM leksemy
137   - UNION
138   - SELECT slownik_uz FROM slowniki_uzywajace
139   - """)
140   - for row in result:
141   - v = Vocabulary()
142   - v.id = row[0]
143   - v.save()
144   -
145   -
146   -def import_qualifiers():
147   - sgjp = Vocabulary.objects.get(id='SGJP')
148   - query_result = sqlite_cursor.execute("""
149   - SELECT okwal FROM odmieniasie
150   - UNION
151   - SELECT zkwal FROM zakonczenia
152   - UNION
153   - SELECT lkwal FROM leksemy
  131 +
  132 + def cache_ics(self):
  133 + self.ics = {}
  134 + for ic in InflectionCharacteristic.objects.all():
  135 + self.ics[(ic.basic_form_label.entry, ic.part_of_speech.symbol)] = ic
  136 +
  137 + def new_vocabularies(self):
  138 + result = self.sqlite_cursor.execute("""
  139 + SELECT slownik FROM leksemy
  140 + UNION
  141 + SELECT slownik_uz FROM slowniki_uzywajace
154 142 """)
155   - added = set()
156   - for row in query_result:
157   - if row[0]:
158   - for qualifier in row[0].split('|'):
159   - if qualifier not in added:
160   - added.add(qualifier)
161   - if not SQL_MODE:
162   - q = Qualifier(label=qualifier, vocabulary=sgjp)
163   - q.save()
164   - else:
165   - cursor.execute("INSERT INTO kwalifikatory (kwal, slownik, usuniety) "
166   - "VALUES (%s, %s, %s)", [qualifier, sgjp.pk, False])
167   -
168   -
169   -mini_lexeme_query = 'SELECT %s FROM leksemy LIMIT ?'
170   -
171   -def import_lexemes():
172   - if MINI_MODE:
173   - result = sqlite_cursor.execute(mini_lexeme_query % '*', (MINI_LEXEME_COUNT,))
174   - else:
175   - result = sqlite_cursor.execute('SELECT * FROM leksemy')
176   - date = datetime.datetime.now()
177   - cv_table = dict(ClassificationValue.objects.values_list('label', 'pk'))
178   - for row in result:
179   - slownik = row['slownik']
180   - status = 'conf' if slownik != 'zmiotki' else 'cand'
181   - if not SQL_MODE:
182   - l = Lexeme()
183   - l.id = row['nr']
184   - l.entry = row['haslo']
185   - l.part_of_speech = PartOfSpeech.objects.get(pk=row['pos'])
186   - l.comment = row['komentarz']
187   - l.source = 'SGJP' # potrzebne to w ogรณle?
188   - l.status = status
189   - l.gloss = row['glosa'] or ''
190   - l.entry_suffix = row['haslosuf'] or ''
191   - l.note = row['nota'] or ''
192   - l.pronunciation = row['wymowa'] or ''
193   - cv = ClassificationValue.objects.get(label=row['pospolitosc'])
194   - cv.lexemes.add(l) #add
195   - l.fix_homonym_number()
196   -
197   - l.owner_vocabulary = Vocabulary.objects.get(pk=slownik)
198   - l.save()
199   - LexemeAssociation.objects.create(
200   - vocabulary=Vocabulary.objects.get(pk=slownik), lexeme=l)
201   - # brak kwalifikatorรณw - nieauktualne
  143 + for row in result:
  144 + yield Vocabulary(id = row[0])
  145 +
  146 + def new_qualifiers(self):
  147 + sgjp = Vocabulary.objects.get(id=DEFAULT_VOCAB)
  148 + query_result = self.sqlite_cursor.execute("""
  149 + SELECT okwal FROM odmieniasie
  150 + UNION
  151 + SELECT zkwal FROM zakonczenia
  152 + UNION
  153 + SELECT lkwal FROM leksemy
  154 + """)
  155 + added = set()
  156 + for row in query_result:
  157 + if row[0]:
  158 + for qualifier_label in row[0].split('|'):
  159 + if qualifier_label not in added:
  160 + added.add(qualifier_label)
  161 + yield Qualifier(label=qualifier_label, vocabulary=sgjp)
  162 +
  163 + def import_lexemes(self):
  164 + if MINI_MODE:
  165 + result = self.sqlite_cursor.execute(
  166 + MINI_LEXEME_QUERY % '*',(MINI_LEXEME_COUNT,))
202 167 else:
  168 + result = self.sqlite_cursor.execute('SELECT * FROM leksemy')
  169 + date = datetime.datetime.now()
  170 + cv_table = dict(ClassificationValue.objects.values_list('label', 'pk'))
  171 + for row in result:
  172 + slownik = row['slownik']
  173 + status = 'conf' if slownik != 'zmiotki' else 'cand'
203 174 cv_pk = cv_table[row['pospolitosc']]
204   - cursor.execute(
  175 + self.cursor.execute(
205 176 "INSERT INTO leksemy (id, haslo, haslosuf, glosa, nota, wymowa, hom, "
206 177 "pos, zrodlo, status, komentarz, data_modyfikacji, slownik, usuniety) "
207 178 "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
208 179 [row['nr'], row['haslo'], row['haslosuf'] or '', row['glosa'] or '',
209 180 row['nota'] or '', row['wymowa'] or '', 1, row['pos'], 'SGJP',
210 181 status, row['komentarz'], date, row['slownik'], False])
211   - cursor.execute(
  182 + self.cursor.execute(
212 183 "INSERT INTO leksemy_w_slownikach (l_id, slownik) "
213 184 "VALUES (%s, %s)", [row['nr'], slownik])
214   - cursor.execute(
  185 + self.cursor.execute(
215 186 "INSERT INTO wartosci_klasyfikacji_lexemes (classificationvalue_id, "
216 187 "lexeme_id) VALUES (%s, %s)", [cv_pk, row['nr']])
217 188 if row['lkwal']:
218 189 for qual in row['lkwal'].split('|'):
219 190 q_id = Qualifier.objects.get(label=qual).pk
220   - cursor.execute(
  191 + self.cursor.execute(
221 192 "INSERT INTO kwalifikatory_leksemow (lexeme_id, "
222 193 "qualifier_id) VALUES (%s, %s)", [row['nr'], q_id])
223 194  
224   -
225   -def import_lexeme_associations():
226   - if MINI_MODE:
227   - result = sqlite_cursor.execute('SELECT * FROM slowniki_uzywajace WHERE nr in (%s)' %
228   - (mini_lexeme_query % 'nr'), [MINI_LEXEME_COUNT])
229   - else:
230   - result = sqlite_cursor.execute('SELECT * FROM slowniki_uzywajace')
231   - for row in result:
232   - if not SQL_MODE:
233   - v = Vocabulary.objects.get(pk=row['slownik_uz'])
234   - l = Lexeme.objects.get(pk=row['nr'])
235   - LexemeAssociation.objects.create(vocabulary=v, lexeme=l)
  195 + def new_lexeme_associations(self):
  196 + if MINI_MODE:
  197 + result = self.sqlite_cursor.execute(
  198 + 'SELECT * FROM slowniki_uzywajace WHERE nr in (%s)'
  199 + % (MINI_LEXEME_QUERY % 'nr'), [MINI_LEXEME_COUNT])
236 200 else:
237   - cursor.execute(
238   - "INSERT INTO leksemy_w_slownikach (l_id, slownik) "
239   - "VALUES (%s, %s)", [row['nr'], row['slownik_uz']])
240   -
241   -
242   -def import_cross_reference_types():
243   - result = sqlite_cursor.execute(
244   - 'select distinct l1.pos pos1, l2.pos pos2, t.* '
245   - 'from odsylacze o join leksemy l1 on nrod=l1.nr '
246   - 'join leksemy l2 on nrdo=l2.nr '
247   - 'join typyodsylaczy t on t.typods=o.typods')
248   - for row in result:
249   - t = CrossReferenceType()
250   - t.symbol = row['typods']
251   - t.desc = row['naglowek']
252   - t.index = row['kolejnosc']
253   - t.from_pos = PartOfSpeech.objects.get(symbol=row['pos1'])
254   - t.to_pos = PartOfSpeech.objects.get(symbol=row['pos2'])
255   - t.save()
256   -
257   -
258   -def import_cross_references():
259   - if MINI_MODE:
260   - result = sqlite_cursor.execute(
261   - 'SELECT * FROM odsylacze WHERE nrod in (%(subq)s) and nrdo in (%(subq)s)'
262   - % {'subq': mini_lexeme_query % 'nr'},
263   - [MINI_LEXEME_COUNT, MINI_LEXEME_COUNT])
264   - else:
265   - result = sqlite_cursor.execute('SELECT * FROM odsylacze')
266   - for row in result:
267   - if row['nrod'] and row['nrdo']: # no bo nie wiem jak to interpretowaฤ‡
268   - l_from = Lexeme.objects.get(pk=row['nrod'])
269   - l_to = Lexeme.objects.get(pk=row['nrdo'])
270   - try:
271   - cr_type = CrossReferenceType.objects.get(
272   - symbol=row['typods'], from_pos=l_from.part_of_speech,
273   - to_pos=l_to.part_of_speech)
274   - except CrossReferenceType.DoesNotExist:
275   - print row['typods'], l_from.part_of_speech, l_to.part_of_speech
276   - if not SQL_MODE:
277   - CrossReference.objects.create(
278   - from_lexeme=l_from, to_lexeme=l_to, type=cr_type)
279   - else:
280   - cursor.execute(
281   - "INSERT INTO odsylacze (l_id_od, l_id_do, typods_id) "
282   - "VALUES (%s, %s, %s)", [row['nrod'], row['nrdo'], cr_type.pk])
283   -
284   -
285   -def import_pattern_types():
286   - for row in sqlite_cursor.execute('SELECT DISTINCT typr, pos FROM paradygmaty'):
287   - lc = PartOfSpeech.objects.get(symbol=row['pos']).lexical_class
288   - PatternType.objects.get_or_create(lexical_class=lc, entry=row['typr'])
289   - # prowizorka z powodu pustej klasy 'skr'
290   - for row in sqlite_cursor.execute('SELECT DISTINCT typr, pos FROM wzory'):
291   - lc = LexicalClass.objects.get(symbol=row['pos'])
292   - PatternType.objects.get_or_create(lexical_class=lc, entry=row['typr'])
293   -
294   -
295   -def import_patterns():
296   - for row in sqlite_cursor.execute('SELECT * FROM wzory'):
297   - pt = PatternType.objects.get(
298   - lexical_class__symbol=row['pos'], entry=row['typr'])
299   - status = 'temp'
300   - if not SQL_MODE:
301   - p = Pattern()
302   - p.name = row['wzor']
303   - p.type = pt
304   - p.basic_form_ending = row['zakp']
305   - p.example = row['przyklad'] or ''
306   - p.comment = row['wkomentarz'] or ''
307   -
308   - p.status = status
309   - p.save()
  201 + result = self.sqlite_cursor.execute('SELECT * FROM slowniki_uzywajace')
  202 + vocab_table = dict(
  203 + (v.id, v) for v in Vocabulary.objects.all()
  204 + )
  205 + for row in result:
  206 + yield LexemeAssociation(
  207 + vocabulary=vocab_table[row['slownik_uz']], lexeme__id=row['nr'])
  208 +
  209 + def new_cross_reference_types(self):
  210 + result = self.sqlite_cursor.execute(
  211 + 'select distinct l1.pos pos1, l2.pos pos2, t.* '
  212 + 'from odsylacze o join leksemy l1 on nrod=l1.nr '
  213 + 'join leksemy l2 on nrdo=l2.nr '
  214 + 'join typyodsylaczy t on t.typods=o.typods')
  215 + for row in result:
  216 + yield CrossReferenceType(
  217 + symbol=row['typods'],
  218 + desc=row['naglowek'],
  219 + index=row['kolejnosc'],
  220 + from_pos=PartOfSpeech.objects.get(symbol=row['pos1']),
  221 + to_pos=PartOfSpeech.objects.get(symbol=row['pos2']),
  222 + )
  223 +
  224 + def import_cross_references(self):
  225 + if MINI_MODE:
  226 + result = self.sqlite_cursor.execute(
  227 + 'SELECT o.*, l1.pos pos1, l2.pos pos2 FROM odsylacze o '
  228 + 'JOIN leksemy l1 on nrod=l1.nr '
  229 + 'JOIN leksemy l2 on nrdo=l2.nr '
  230 + 'WHERE nrod in (%(subq)s) and nrdo in (%(subq)s)'
  231 + % {'subq': MINI_LEXEME_QUERY % 'nr'},
  232 + [MINI_LEXEME_COUNT, MINI_LEXEME_COUNT])
310 233 else:
311   - cursor.execute(
312   - "INSERT INTO wzory (w_id, typ, przyklad, zakp, status, komentarz) "
313   - "VALUES (%s, %s, %s, %s, %s, %s)",
314   - [row['wzor'], pt.pk, row['przyklad'] or '', row['zakp'], status,
315   - row['wkomentarz'] or ''])
316   -
317   -
318   -def import_lexeme_inflection_patterns():
319   - if MINI_MODE:
320   - result = sqlite_cursor.execute(
321   - 'SELECT * FROM odmieniasie WHERE nr IN (%s)' % (mini_lexeme_query % 'nr'),
322   - (MINI_LEXEME_COUNT,))
323   - else:
324   - result = sqlite_cursor.execute('SELECT * FROM odmieniasie')
325   - pos_table = dict(Lexeme.objects.values_list('pk', 'part_of_speech'))
326   - pattern_pk_table = dict(Pattern.objects.values_list('name', 'pk'))
327   - for row in result:
328   - if not SQL_MODE:
329   - lip = LexemeInflectionPattern()
330   - lip.lexeme = Lexeme.objects.get(id=row['nr'])
331   - lip.index = row['oskl']
332   - lip.pattern = Pattern.objects.get(name=row['wzor'])
333   - lip.inflection_characteristic = ics[(row['charfl'],
334   - lip.lexeme.part_of_speech)]
335   - lip.root = row['rdzen']
336   - lip.save()
337   - # nieaktualne
338   - if row['okwal']:
339   - lip.qualifiers.add(Qualifier.objects.get(label=row['okwal'])) #add
  234 + result = self.sqlite_cursor.execute(
  235 + 'SELECT o.*, l1.pos pos1, l2.pos pos2 FROM odsylacze o '
  236 + 'JOIN leksemy l1 on nrod=l1.nr '
  237 + 'JOIN leksemy l2 on nrdo=l2.nr'
  238 + )
  239 + cr_type_table = dict(
  240 + ((crt.symbol, crt.from_pos, crt.to_pos), crt)
  241 + for crt in CrossReferenceType.objects.all()
  242 + )
  243 + for row in result:
  244 + # niekompletne odsyล‚acze zdarzajฤ… siฤ™ dla 'asp'
  245 + if row['nrod'] and row['nrdo']:
  246 + cr_type = cr_type_table[(row['typods'], row['pos1'], row['pos2'])]
  247 + yield CrossReference(
  248 + from_lexeme__id=row['nrod'], to_lexeme__id=row['nrdo'],
  249 + type=cr_type)
  250 +
  251 + def import_pattern_types(self):
  252 + result = self.sqlite_cursor.execute(
  253 + 'SELECT DISTINCT typr, pos FROM paradygmaty')
  254 + lc_pos_table = dict(
  255 + (pos.symbol, pos.lexical_class) for pos in PartOfSpeech.objects.all()
  256 + )
  257 + for row in result:
  258 + lc = lc_pos_table[row['pos']]
  259 + PatternType.objects.get_or_create(lexical_class=lc, entry=row['typr'])
  260 + # prowizorka z powodu pustej klasy 'skr'
  261 + lc_table = dict(
  262 + (lc.symbol, lc) for lc in LexicalClass.objects.all()
  263 + )
  264 + result = self.sqlite_cursor.execute('SELECT DISTINCT typr, pos FROM wzory')
  265 + for row in result:
  266 + lc = lc_table[row['pos']]
  267 + PatternType.objects.get_or_create(lexical_class=lc, entry=row['typr'])
  268 +
  269 + def new_patterns(self):
  270 + pt_table = dict(
  271 + ((pt.lexical_class.symbol, pt.entry), pt)
  272 + for pt in PatternType.objects.all()
  273 + )
  274 + for row in self.sqlite_cursor.execute('SELECT * FROM wzory'):
  275 + pt = pt_table[(row['pos'], row['typr'])]
  276 + status = 'temp'
  277 + yield Pattern(
  278 + name=row['wzor'],
  279 + type=pt,
  280 + basic_form_ending=row['zakp'],
  281 + example=row['przyklad'] or '',
  282 + comment=row['wkomentarz'] or '',
  283 + status = status,
  284 + )
  285 +
  286 + def import_endings(self):
  287 + if SQL_MODE:
  288 + pattern_pk_table = dict(Pattern.objects.values_list('name', 'pk'))
  289 + bfl_table = dict(BaseFormLabel.objects.values_list('entry', 'pk'))
  290 + for row in self.sqlite_cursor.execute('SELECT * FROM zakonczenia'):
  291 + if row['zak'] is not None:
  292 + if not SQL_MODE:
  293 + e = Ending(
  294 + pattern=Pattern.objects.get(name=row['wzor']),
  295 + base_form_label = BaseFormLabel.objects.get(entry=row['efobaz']),
  296 + string = row['zak'],
  297 + index = row['nrskl'],
  298 + )
  299 + e.save()
  300 + for qual in row['zkwal'].split('|'):
  301 + e.qualifiers.add(Qualifier.objects.get(label=qual)) #add
  302 + else:
  303 + pattern_pk = pattern_pk_table[row['wzor']]
  304 + if pattern_pk:
  305 + efobaz_id = bfl_table[row['efobaz']]
  306 + self.cursor.execute(
  307 + "INSERT INTO zakonczenia (w_id, efobaz, zind, zak) VALUES "
  308 + "(%s, %s, %s, %s)",
  309 + [pattern_pk, efobaz_id, row['nrskl'], row['zak']])
  310 + if row['zkwal']:
  311 + self.cursor.execute("select currval('zakonczenia_id_seq')")
  312 + last_id = self.cursor.fetchone()[0]
  313 + for qual in row['zkwal'].split('|'):
  314 + q_id = Qualifier.objects.get(label=qual).pk
  315 + self.cursor.execute(
  316 + "INSERT INTO kwalifikatory_zakonczen (ending_id, qualifier_id) "
  317 + "VALUES (%s, %s)", [last_id, q_id])
  318 +
  319 + def import_lexeme_inflection_patterns(self):
  320 + if MINI_MODE:
  321 + result = self.sqlite_cursor.execute(
  322 + 'SELECT * FROM odmieniasie WHERE nr IN (%s)' % (MINI_LEXEME_QUERY % 'nr'),
  323 + (MINI_LEXEME_COUNT,))
340 324 else:
341   - pos = pos_table[row['nr']]
342   - pattern_pk = pattern_pk_table[row['wzor']]
343   - charfl_id = ics[(row['charfl'], pos)].pk
344   - cursor.execute(
345   - "INSERT INTO odmieniasie (l_id, oind, w_id, charfl, rdzen) "
346   - "VALUES (%s, %s, %s, %s, %s) ", [row['nr'], row['oskl'], pattern_pk,
347   - charfl_id, row['rdzen']])
348   - if row['okwal']:
349   - cursor.execute("select currval('odmieniasie_id_seq')")
350   - last_id = cursor.fetchone()[0]
351   - for qual in row['okwal'].split('|'):
352   - q_id = Qualifier.objects.get(label=qual).pk
353   - cursor.execute(
354   - "INSERT INTO kwalifikatory_odmieniasiow (lexemeinflectionpattern_id, "
355   - "qualifier_id) VALUES (%s, %s)", [last_id, q_id])
356   -
357   -def import_endings():
358   - if SQL_MODE:
  325 + result = self.sqlite_cursor.execute('SELECT * FROM odmieniasie')
  326 + pos_table = dict(Lexeme.objects.values_list('pk', 'part_of_speech'))
359 327 pattern_pk_table = dict(Pattern.objects.values_list('name', 'pk'))
360   - bfl_table = dict(BaseFormLabel.objects.values_list('entry', 'pk'))
361   - for row in sqlite_cursor.execute('SELECT * FROM zakonczenia'):
362   - if row['zak'] is not None:
  328 + for row in result:
363 329 if not SQL_MODE:
364   - e = Ending()
365   - e.pattern = Pattern.objects.get(name=row['wzor'])
366   - e.base_form_label = BaseFormLabel.objects.get(entry=row['efobaz'])
367   - e.string = row['zak']
368   - e.index = row['nrskl']
369   - e.save()
370   - for qual in row['zkwal'].split('|'):
371   - e.qualifiers.add(Qualifier.objects.get(label=qual)) #add
  330 + lip = LexemeInflectionPattern(
  331 + lexeme__id=row['nr'],
  332 + index=row['oskl'],
  333 + pattern=Pattern.objects.get(name=row['wzor']),
  334 + inflection_characteristic=self.ics[
  335 + (row['charfl'], lip.lexeme.part_of_speech)],
  336 + root=row['rdzen'],
  337 + )
  338 + lip.save()
  339 + # nieaktualne
  340 + if row['okwal']:
  341 + lip.qualifiers.add(Qualifier.objects.get(label=row['okwal'])) #add
372 342 else:
  343 + pos = pos_table[row['nr']]
373 344 pattern_pk = pattern_pk_table[row['wzor']]
374   - if pattern_pk:
375   - efobaz_id = bfl_table[row['efobaz']]
376   - cursor.execute(
377   - "INSERT INTO zakonczenia (w_id, efobaz, zind, zak) VALUES "
378   - "(%s, %s, %s, %s)",
379   - [pattern_pk, efobaz_id, row['nrskl'], row['zak']])
380   - if row['zkwal']:
381   - cursor.execute("select currval('zakonczenia_id_seq')")
382   - last_id = cursor.fetchone()[0]
383   - for qual in row['zkwal'].split('|'):
384   - q_id = Qualifier.objects.get(label=qual).pk
385   - cursor.execute(
386   - "INSERT INTO kwalifikatory_zakonczen (ending_id, qualifier_id) "
387   - "VALUES (%s, %s)", [last_id, q_id])
388   -
389   -
390   -def import_tables():
391   - bfl_table = dict(BaseFormLabel.objects.values_list('entry', 'pk'))
392   - for row in sqlite_cursor.execute('SELECT * FROM paradygmaty'):
393   - lc = PartOfSpeech.objects.get(symbol=row['pos']).lexical_class
394   - variant, _created = Variant.objects.get_or_create(id=row['wariant'])
395   - tt_data = {
396   - 'variant': variant,
397   - 'pattern_type': PatternType.objects.get(
398   - entry=row['typr'], lexical_class=lc),
399   - 'inflection_characteristic': InflectionCharacteristic.objects.get(
400   - entry=row['charfl'], part_of_speech__symbol=row['pos']),
401   - }
402   - tt, _created = TableTemplate.objects.get_or_create(**tt_data)
403   - if not SQL_MODE:
404   - c = Cell()
405   - c.table_template = tt
406   - c.base_form_label = BaseFormLabel.objects.get(entry=row['efobaz'])
407   - c.tag = row['morf']
408   - c.prefix = row['pref']
409   - c.suffix = row['suf']
410   - c.index = row['kskl']
411   - c.save()
412   - if row['row']:
413   - tc = TableCell()
414   - tc.cell = c
415   - tc.row = row['row']
416   - tc.col = row['col']
417   - tc.rowspan = row['rowspan']
418   - tc.colspan = row['colspan']
419   - tc.save()
420   - else:
421   - efobaz_id = bfl_table[row['efobaz']]
422   - cursor.execute(
423   - "INSERT INTO klatki (st_id, efobaz, tag, prefiks, sufiks, kind) "
424   - "VALUES (%s, %s, %s, %s, %s, %s)", [tt.pk, efobaz_id, row['morf'],
425   - row['pref'], row['suf'], row['kskl']])
426   - if row['row']:
427   - cursor.execute("select currval('klatki_id_seq')")
428   - last_id = cursor.fetchone()[0]
429   - cursor.execute(
430   - "INSERT INTO komorki_tabel (k_id, row, col, rowspan, colspan) "
431   - "VALUES (%s, %s, %s, %s, %s)", [last_id, row['row'],
432   - row['col'], row['rowspan'], row['colspan']])
433   -
434   -
435   -def import_table_headers():
436   - for row in sqlite_cursor.execute('SELECT * FROM naglowkiwierszy'):
437   - if row['styl'] != 'b' and row['nagl']:
438   - tts = TableTemplate.objects.filter(
439   - variant__id=row['wariant'], pattern_type__entry=row['typr'],
440   - inflection_characteristic__entry=row['charfl'],
441   - inflection_characteristic__part_of_speech__symbol=row['pos'])
442   - if tts:
443   - tt = tts[0]
444   - if not SQL_MODE:
445   - th = TableHeader()
446   - th.table_template = tt
447   - th.row = row['row']
448   - th.col = row['col']
449   - th.rowspan = row['rowspan']
450   - th.colspan = row['colspan']
451   - th.label = row['nagl']
452   - th.horizontal = row['styl'] == 'h'
453   - th.save()
  345 + charfl_id = self.ics[(row['charfl'], pos)].pk
  346 + self.cursor.execute(
  347 + "INSERT INTO odmieniasie (l_id, oind, w_id, charfl, rdzen) "
  348 + "VALUES (%s, %s, %s, %s, %s) ", [row['nr'], row['oskl'], pattern_pk,
  349 + charfl_id, row['rdzen']])
  350 + if row['okwal']:
  351 + self.cursor.execute("select currval('odmieniasie_id_seq')")
  352 + last_id = self.cursor.fetchone()[0]
  353 + for qual in row['okwal'].split('|'):
  354 + q_id = Qualifier.objects.get(label=qual).pk
  355 + self.cursor.execute(
  356 + "INSERT INTO kwalifikatory_odmieniasiow (lexemeinflectionpattern_id, "
  357 + "qualifier_id) VALUES (%s, %s)", [last_id, q_id])
  358 +
  359 + def import_tables(self):
  360 + bfl_table = dict(BaseFormLabel.objects.values_list('entry', 'pk'))
  361 + lc_pos_table = dict(
  362 + (pos.symbol, pos.lexical_class) for pos in PartOfSpeech.objects.all()
  363 + )
  364 + for row in self.sqlite_cursor.execute('SELECT * FROM paradygmaty'):
  365 + lc = lc_pos_table[row['pos']]
  366 + variant, _created = Variant.objects.get_or_create(id=row['wariant'])
  367 + tt_data = {
  368 + 'variant': variant,
  369 + 'pattern_type': PatternType.objects.get(
  370 + entry=row['typr'], lexical_class=lc),
  371 + 'inflection_characteristic': InflectionCharacteristic.objects.get(
  372 + entry=row['charfl'], part_of_speech__symbol=row['pos']),
  373 + }
  374 + tt, _created = TableTemplate.objects.get_or_create(**tt_data)
  375 + if not SQL_MODE:
  376 + c = Cell(
  377 + table_template=tt,
  378 + base_form_label=BaseFormLabel.objects.get(entry=row['efobaz']),
  379 + tag=row['morf'],
  380 + prefix=row['pref'],
  381 + suffix=row['suf'],
  382 + index=row['kskl'],
  383 + )
  384 + c.save()
  385 + if row['row']:
  386 + tc = TableCell(
  387 + cell=c,
  388 + row=row['row'],
  389 + col=row['col'],
  390 + rowspan=row['rowspan'],
  391 + colspan=row['colspan'],
  392 + )
  393 + tc.save()
  394 + else:
  395 + efobaz_id = bfl_table[row['efobaz']]
  396 + self.cursor.execute(
  397 + "INSERT INTO klatki (st_id, efobaz, tag, prefiks, sufiks, kind) "
  398 + "VALUES (%s, %s, %s, %s, %s, %s)", [tt.pk, efobaz_id, row['morf'],
  399 + row['pref'], row['suf'], row['kskl']])
  400 + if row['row']:
  401 + self.cursor.execute("select currval('klatki_id_seq')")
  402 + last_id = self.cursor.fetchone()[0]
  403 + self.cursor.execute(
  404 + "INSERT INTO komorki_tabel (k_id, row, col, rowspan, colspan) "
  405 + "VALUES (%s, %s, %s, %s, %s)", [last_id, row['row'],
  406 + row['col'], row['rowspan'], row['colspan']])
  407 +
  408 + def new_table_headers(self):
  409 + for row in self.sqlite_cursor.execute('SELECT * FROM naglowkiwierszy'):
  410 + if row['styl'] != 'b' and row['nagl']:
  411 + tts = TableTemplate.objects.filter(
  412 + variant__id=row['wariant'], pattern_type__entry=row['typr'],
  413 + inflection_characteristic__entry=row['charfl'],
  414 + inflection_characteristic__part_of_speech__symbol=row['pos'])
  415 + if tts:
  416 + tt = tts.get()
  417 + yield TableHeader(
  418 + table_template=tt,
  419 + row=row['row'],
  420 + col=row['col'],
  421 + rowspan=row['rowspan'],
  422 + colspan=row['colspan'],
  423 + label=row['nagl'],
  424 + horizontal=row['styl'] == 'h',
  425 + )
454 426 else:
455   - cursor.execute(
456   - "INSERT INTO naglowki_tabel (st_id, row, col, rowspan, colspan, "
457   - "nagl, wierszowy) VALUES (%s, %s, %s, %s, %s, %s, %s)",
458   - [tt.pk, row['row'], row['col'], row['rowspan'], row['colspan'],
459   - row['nagl'], row['styl'] == 'h'])
460   -
461   -
462   -def single_import(fun, db):
463   - global sqlite_cursor, cursor
464   - transaction.commit_unless_managed()
465   - transaction.enter_transaction_management()
466   - transaction.managed()
467   - sqlite_cursor = get_cursor(db)
468   - cursor = connection.cursor()
469   - fun()
470   - sqlite_cursor.close()
471   - cursor.close()
472   - transaction.commit()
473   - transaction.leave_transaction_management()
474   -
475   -
476   -def delete_and_import():
477   - transaction.commit_unless_managed()
478   - transaction.enter_transaction_management()
479   - transaction.managed()
480   - models = (
481   - Qualifier,
482   - LexicalClass,
483   - PartOfSpeech,
484   - BaseFormLabel,
485   - InflectionCharacteristic,
486   - Lexeme,
487   - PatternType,
488   - Pattern,
489   - LexemeInflectionPattern,
490   - Ending,
491   - TableTemplate,
492   - Cell,
493   - TableCell,
494   - Vocabulary,
495   - LexemeAssociation,
496   - )
497   - print 'deleting old data...'
498   - for model in models:
499   - model.objects.all().delete()
500   -
501   - global sqlite_cursor
502   - sqlite_cursor = get_cursor(db)
503   - print 'importing lexical classes...'
504   - import_lexical_classes()
505   - print 'importing parts of speech'
506   - import_parts_of_speech()
507   - print 'importing base form labels'
508   - import_base_form_labels()
509   - print 'importing inflection characteristics'
510   - import_inflection_characteristics()
511   - print 'importing vocabularies...'
512   - import_vocabularies()
513   - print 'importing qualifiers...'
514   - import_qualifiers()
515   - print 'importing lexemes...'
516   - import_lexemes()
517   - print 'importing lexeme associations...'
518   - import_lexeme_associations()
519   - print 'importing cross-reference types...'
520   - import_cross_reference_types()
521   - print 'importing cross-references...'
522   - import_cross_references()
523   - print 'importing pattern types...'
524   - import_pattern_types()
525   - print 'importing patterns...'
526   - import_patterns()
527   - print 'importing lexeme inflection patterns...'
528   - import_lexeme_inflection_patterns()
529   - print 'importing endings...'
530   - import_endings()
531   - print 'importing table templates...'
532   - import_tables()
533   - print 'importing table headers...'
534   - import_table_headers()
535   - sqlite_cursor.close()
536   - cursor.close()
537   - transaction.commit()
538   - transaction.leave_transaction_management()
  427 + raise Exception('Brak szablonu dla nagล‚รณwka: %s', dict(row))
  428 +
  429 + def delete_and_import(self):
  430 + transaction.commit_unless_managed()
  431 + transaction.enter_transaction_management()
  432 + transaction.managed()
  433 + models = (
  434 + TableCell,
  435 + Cell,
  436 + TableTemplate,
  437 + CrossReference,
  438 + CrossReferenceType,
  439 + LexemeAssociation,
  440 + LexemeInflectionPattern,
  441 + Lexeme,
  442 + Ending,
  443 + Pattern,
  444 + PatternType,
  445 + Qualifier,
  446 + Vocabulary,
  447 + InflectionCharacteristic,
  448 + BaseFormLabel,
  449 + PartOfSpeech,
  450 + LexicalClass,
  451 + )
  452 + print 'deleting old data...'
  453 + for model in models:
  454 + model.objects.all().delete()
  455 +
  456 + print 'importing lexical classes...'
  457 + LexicalClass.objects.bulk_create(self.new_lexical_classes())
  458 + print 'importing parts of speech'
  459 + PartOfSpeech.objects.bulk_create(self.new_parts_of_speech())
  460 + print 'importing base form labels'
  461 + BaseFormLabel.objects.bulk_create(self.new_base_form_labels())
  462 + print 'importing inflection characteristics'
  463 + InflectionCharacteristic.objects.bulk_create(
  464 + self.new_inflection_characteristics())
  465 + print 'importing vocabularies...'
  466 + Vocabulary.objects.bulk_create(self.new_vocabularies())
  467 + print 'importing qualifiers...'
  468 + Qualifier.objects.bulk_create(self.new_qualifiers())
  469 + print 'importing lexemes...'
  470 + self.import_lexemes()
  471 + print 'importing lexeme associations...'
  472 + LexemeAssociation.objects.bulk_create(self.new_lexeme_associations())
  473 + print 'importing cross-reference types...'
  474 + CrossReferenceType.objects.bulk_create(
  475 + self.new_cross_reference_types())
  476 + print 'importing cross-references...'
  477 + self.import_cross_references()
  478 + print 'importing pattern types...'
  479 + self.import_pattern_types()
  480 + print 'importing patterns...'
  481 + Pattern.objects.bulk_create(self.new_patterns())
  482 + print 'importing lexeme inflection patterns...'
  483 + self.import_lexeme_inflection_patterns()
  484 + print 'importing endings...'
  485 + self.import_endings()
  486 + print 'importing table templates...'
  487 + self.import_tables()
  488 + print 'importing table headers...'
  489 + TableHeader.objects.bulk_create(self.new_table_headers())
  490 + self.close()
  491 + transaction.commit()
  492 + transaction.leave_transaction_management()
  493 +
  494 + def single_import(self, model):
  495 + transaction.commit_unless_managed()
  496 + transaction.enter_transaction_management()
  497 + transaction.managed()
  498 + method_name = METHOD_NAMES[model]
  499 + if method_name.startswith('new'):
  500 + model.objects.bulk_create(self.__getattribute__(method_name)())
  501 + elif method_name.startswith('import'):
  502 + self.__getattribute__(method_name)()
  503 + self.close()
  504 + transaction.commit()
  505 + transaction.leave_transaction_management()
539 506  
540 507 import sys
541 508 if __name__ == '__main__':
... ... @@ -546,4 +513,4 @@ if __name__ == &#39;__main__&#39;:
546 513 db = sys.argv[1]
547 514 else:
548 515 db = DEFAULT_DATABASE
549   - delete_and_import()
  516 + ImportData(db).delete_and_import()
... ...
1 1 # -*- coding: utf-8 -*-
2 2  
3   -from django.conf.urls.defaults import *
  3 +from django.conf.urls import *
4 4 from django.conf import settings
5 5 from common.util import url
6 6  
... ... @@ -14,7 +14,7 @@ urlpatterns = patterns(&#39;&#39;,
14 14 url(r'^accounts/register/$', 'accounts.views.register'),
15 15 url(r'^accounts/settings/$', 'accounts.views.settings'),
16 16 url(r'^accounts/manage-groups/$', 'accounts.views.manage_groups'),
17   - (r'^accounts/', include('registration.urls')),
  17 + (r'^accounts/', include('registration.backends.default.urls')),
18 18  
19 19 url(r'^ajax/sort_rules/$', 'dictionary.ajax_jqgrid.sort_rules'),
20 20 url(r'^ajax/history-table/$', 'dictionary.ajax_history.history_table'),
... ...
urls_main.py
1   -from django.conf.urls.defaults import *
  1 +from django.conf.urls import *
2 2  
3 3 from django.conf import settings
4 4  
... ...