SGJP / Kuznia

#-*- coding:utf-8 -*-

from django.core.management.base import BaseCommand
import sys
import json
from common.util import suffixes, cut_end, debug
from dictionary.models import Pattern, Lexeme, InflectionCharacteristic, \
  Ending, LexicalClass, CrossReference, BaseFormLabel, Vocabulary
from dictionary.util import expand_tag, compare_patterns
from dictionary.pattern_blacklist import blacklist

class Command(BaseCommand):
  args = '<symbol części mowy> <nazwa pliku wejściowego>'
  help = 'Creates scripts for importing data from Morfologik'

  def handle(self, lc_sym, input_file, **options):
    parse_file(lc_sym, input_file)

DEBUG = False

#sgjp = Vocabulary.objects.get(id='SGJP').owned_lexemes_pk()
morf = Vocabulary.objects.get(id='Morfologik').owned_lexemes_pk()
sgjp = Lexeme.objects.filter(deleted=False, source='SGJP')

# może być nierozpoznane pltant
ending_genders = {
  'm1': ['m1', 'p1'],
  'm': ['m2', 'm3', 'p2', 'p3'],
  'f': ['f', 'p2', 'p3'],
  'n': ['n1', 'n2', 'p2', 'p3'],
  'p1': ['p1'],
  'p3': ['p2', 'p3'],
  'i': [], # wszystkie
}

genders = {
  'm1': ['m1'],
  'm': ['m2', 'm3'], #, 'p1', 'p2', 'p3'],
  'f': ['f'], #, 'p2', 'p3'],
  'n': ['n1', 'n2'], #, 'p2', 'p3'],
  'p1': ['p1'],
  'p3': ['p2', 'p3'],
  'i': [], # wszystkie
}

# piękne nazwy...
genders2 = {
  'm1': ['m1', 'p1'],
  'm': ['m2', 'm3', 'p2', 'p3'],
  'f': ['f', 'p2', 'p3'],
  'n': ['n1', 'n2', 'p2', 'p3'],
  'p1': ['p1', 'm1'],
  'p3': ['p2', 'p3', 'f', 'm2', 'm3', 'n1', 'n2'],
  'i': [], # wszystkie
}

def get_basic_endings(lexical_class, parts_of_speech, gender=None):
  if lexical_class.symbol == 'v':
    return Ending.objects.filter(base_form_label__entry='5',
                                 pattern__type__lexical_class=lexical_class)
  ics = InflectionCharacteristic.objects.filter(
    part_of_speech__in=parts_of_speech)
  if gender and ending_genders[gender]:
    ics = ics.filter(entry__in=ending_genders[gender])
  basic_form_labels = ics.values_list('basic_form_label', flat=True).distinct()
  return Ending.objects.filter(base_form_label__pk__in=basic_form_labels,
                               pattern__type__lexical_class=lexical_class)

basic_form_endings_dict = {}
for lexical_class in LexicalClass.objects.all():
  parts_of_speech = lexical_class.partofspeech_set.all()
  if lexical_class.symbol == 'subst':
    for gender in ending_genders:
      basic_form_endings_dict[(lexical_class, gender)] = get_basic_endings(
        lexical_class, parts_of_speech, gender)
  else:
    basic_form_endings_dict[lexical_class] = get_basic_endings(
      lexical_class, parts_of_speech)

def join(patterns):
  return ', '.join(sorted(p[0].name for p in patterns))

def join_many(patterns):
  return ', '.join('[%s]' % join(set) for set in patterns)

def prepare_endings(forms):
  root = forms[0][0]
  for form in forms[1:]:
    new_root = ''
    for char1, char2 in zip(root, form[0]):
      if char1 == char2:
        new_root += char1
      else:
        break
    root = new_root
  endings = set()
  for form in forms:
    endings.add((form[0][len(root):], form[1]))
  return endings, root

def get_form_set(forms):
  form_set = set(form[0] for form in forms)
  return form_set - set('nie' + form for form in form_set)

def expand_forms(forms):
  expanded = []
  for form in forms:
    tags = expand_tag(form[1])
    expanded += [(form[0], tag) for tag in tags]
  return expanded

def get_tantum(forms):
  sg = set()
  pl = set()
  for form in forms:
    if form[1][1].startswith('pl'):
      pl.add(form[0])
    elif form[1][1].startswith('sg'): # ignorujemy irreg
      sg.add(form[0])
  if sg == pl:
    return
  # za dużo fałszywych pozytywów
  #elif pl.issubset(sg):
  #  return 'sg'
  elif sg.issubset(pl): # or len(sg) == 1: -- czasem psuje
    return 'pl'
  else:
    return

def tantum_a_posteriori(form_set, patterns):
  tantum = None
  for pattern, root in patterns:
    tantum_forms = {
      'sg': set(root + e for e in
                pattern.endings.filter(base_form_label__entry__startswith='sg')
                .values_list('string', flat=True)),
      'pl': set(root + e for e in
                pattern.endings.filter(base_form_label__entry__startswith='pl')
                .values_list('string', flat=True)),
    }
    for num in ('sg', 'pl'):
      if form_set.issubset(tantum_forms[num]):
        tantum = num
    if tantum:
      return tantum
  if not patterns:
    return 'sg'
  return None

v_base_forms = [
  ('1', 'verb:fin:sg:ter:', u''),
  ('2', 'verb:fin:sg:pri:', u''),
  ('3', 'verb:fin:pl:ter:', u''),
  ('4', 'verb:impt:sg:sec:', u''),
  ('5', 'verb:inf:', u''),
  ('6', 'verb:praet:sg:ter:m:', u''),
  ("6'", 'pant:perf', u'szy'),
  ('7', 'verb:praet:sg:pri:m:', u'em'),
  ('8', 'verb:praet:sg:pri:f:', u'am'),
  ('9', 'verb:praet:pl:ter:m1:', u'i'),
  ('10', 'verb:imps', u'o'),
  ('11', 'subst:ger:sg:nom:n', u'ie'), # [bez przeczenia]
  ('11pg', 'subst:ger:pl:gen:n', u''),
  ('12', 'ppas:pl:nom:m1', u''),
]

def get_extra(lexical_class, forms, **extra):
  if lexical_class.symbol == 'subst':
    if 'tantum' in extra:
      tantum = extra['tantum']
    else:
      tantum = get_tantum(forms)
    tag = forms[0][1]
    gender = tag[-1][0]
    # powinienem częściej tak pisać zamiast for...else jeśli się da
    if all(form[1][-1] in ('m1', 'depr') for form in forms):
      gender = 'm1'
    if tantum == 'pl' or tag[1] == 'pltant':
      gender = 'p'
      tantum = 'pl'
    return {'gender': gender, 'tantum': tantum}
  elif lexical_class.symbol == 'adj':
    all_forms = [form[0] for form in forms]
    negated = [(form, tag) for (form, tag) in forms
               if form == 'nie' + all_forms[0]]
    negated += [(form, tag) for (form, tag) in forms
                if form.startswith('nie') and form[4:] in all_forms
                and form != 'nie' + all_forms[0]]
    return {'negated': negated}
  elif lexical_class.symbol == 'v':
    all_forms = [form[0] for form in forms]
    base_forms = []
    for label, tag_prefix, end in v_base_forms:
      for form, tag in forms:
        negated_form = (form.startswith(u'nie') and
                        form[len('nie'):] in all_forms)
        if (':'.join(tag).startswith(tag_prefix) and
            not (form.endswith(u'że') and tag[1] == 'impt')
            and not negated_form):
          if form.endswith(end): # ech...
            base_form = cut_end(form, end)
            if label == '10':
              bf10 = base_form
            base_forms.append((base_form, label))
    derived = set()
    labels = set(label for (form, label) in base_forms)
    if '10' in labels and bf10 + 'y' in all_forms:
      derived.add('ppas')
    for form, tag in forms:
      plain_tag = ':'.join(tag)
      if plain_tag.startswith('subst:ger') and '11' in labels:
        derived.add('ger')
      if plain_tag.startswith('pact') and '3' in labels:
        derived.add('pact')
    old_base_forms = base_forms
    if 'ppas' not in derived:
      base_forms = []
      for base_form, label in old_base_forms:
        if label != '12':
          base_forms.append((base_form, label))
    if DEBUG:
      for f, l in base_forms:
        print f, l
    return {'base_forms': base_forms, 'derived': derived}
  else:
    return {}


def relevant_subst(ending, gender, tantum):
  bfl = ending.base_form_label.entry
  tag = bfl.split(':')
  type = ending.pattern.type.entry
  return (not (gender in ('m1', 'p1') and bfl == 'pl:nom') and
          not (len(tag) >= 3 and gender[0] != 'p' and
               tag[2][0] != gender[0]) and
          not (tantum and tag[0] != tantum) and
          not (gender == 'p3' and bfl.startswith('pl:gen:') and (
                (type == 'n' and tag[2] == 'n') or
                (type == 'f' and tag[2] == 'm')
              )) and
          not (gender not in ('m1', 'p1') and bfl == 'pl:nom:mo'))

def relevant_adj(ending):
  tag = ending.base_form_label.entry
  return tag not in ('0', '3+')

def relevant_v(ending, base_forms, derived):
  tag = ending.base_form_label.entry
  if 'ppas' not in derived and tag == '12':
    return False
  if 'ger' not in derived and tag in ('11', '11pg'):
    return False
  return tag not in ("6'",)

def relevant(lexical_class, ending, **extra):
  if lexical_class.symbol == 'subst':
    return relevant_subst(ending, **extra)
  elif lexical_class.symbol == 'adj':
    return relevant_adj(ending)
  elif lexical_class.symbol == 'v':
    return relevant_v(ending, **extra)

from itertools import chain, combinations
def powerset(iterable):
  """powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"""
  s = list(iterable)
  return chain.from_iterable(
    combinations(s, r) for r in range(min(len(s)+1, 5)))

def find_minimal_sets(form_set, covered_forms, necessary_patterns,
                      included_patterns, ending_sets):
  unnecessary_patterns = list(set(included_patterns) - set(necessary_patterns))
  found = []
  for subset in powerset(unnecessary_patterns):
    for found_set in found:
      if set(tuple(necessary_patterns) + subset).issuperset(found_set):
        break
    else:
      new_covered = set(covered_forms)
      for pattern, root in subset:
        new_covered |= ending_sets[pattern]
      if form_set.issubset(new_covered):
        found.append(tuple(necessary_patterns) + subset)
  return found

sure_bfls_sg = tuple(
  BaseFormLabel.objects.filter(
    entry__in=['sg:dat', 'sg:gen', 'sg:inst']).values_list('pk', flat=True))
sure_bfls_pl = tuple(
  BaseFormLabel.objects.filter(
    entry__in=['pl:dat', 'pl:inst', 'pl:loc']).values_list('pk', flat=True))

def basic_form_endings(lexical_class, basic_form, form_set, **extra):
  if 'gender' in extra:
    key = (lexical_class, extra['gender'])
  else:
    key = lexical_class
  if lexical_class.symbol != 'subst':
    return basic_form_endings_dict[key].filter(string__in=suffixes(basic_form))
  else:
    # karkołomne, ale trochę przyśpiesza
    endings = basic_form_endings_dict[key]
    new_endings = Ending.objects.none()
    for suf in suffixes(basic_form):
      root = cut_end(basic_form, suf)
      n = len(root)
      ending_strings = tuple(
        form[n:] for form in form_set if form.startswith(root))
      endings_part = endings.filter(string=suf)
      pattern_pks = endings_part.values_list('pattern', flat=True)
      patterns = Pattern.objects.filter(pk__in=pattern_pks).extra(
        where=["(w_id = '0000' or not exists "
               "(select id from zakonczenia where w_id = wzory.id "
               "and zak not in %s and efobaz in %s) or not exists "
               "(select id from zakonczenia where w_id = wzory.id "
               "and zak not in %s and efobaz in %s))"],
        params=[ending_strings, sure_bfls_sg, ending_strings, sure_bfls_pl])
      new_endings = new_endings | endings_part.filter(pattern__in=patterns)
    return new_endings

memoized_good_endings = {}

def good_ending_set_subst(pattern, root, tantum, gender):
  if (pattern, tantum, gender) in memoized_good_endings:
    good_endings = memoized_good_endings[(pattern, tantum, gender)]
    return set(root + e for e in good_endings)
  endings = pattern.endings
  if tantum:
    endings = endings.filter(base_form_label__entry__startswith=tantum)
  if gender not in ('m1', 'p1'):
    endings = endings.exclude(base_form_label__entry='pl:nom:mo')
  if gender[0] != 'p':
    for g in list(set('mfn') - set(gender[0])):
      endings = endings.exclude(
        base_form_label__entry__startswith='pl:gen:' + g)
  if gender == 'p3':
    if pattern.type.entry == 'f':
      endings = endings.exclude(base_form_label__entry='pl:gen:m')
    if pattern.type.entry == 'n':
      endings = endings.exclude(base_form_label__entry='pl:gen:n')
  good_endings = list(endings.values_list('string', flat=True))
  memoized_good_endings[(pattern, tantum, gender)] = good_endings
  return set(root + e for e in good_endings)

def good_ending_set(lexical_class, pattern, root='', **extra):
  if lexical_class.symbol != 'subst':
    return pattern.ending_set(root)
  else:
    return good_ending_set_subst(pattern, root, **extra)

memoized_pattern_ics = {}

def bad_pattern_subst(pattern, gender, tantum):
  if (pattern, gender) in memoized_pattern_ics:
    return memoized_pattern_ics[(pattern, gender)]
  ics = genders2[gender]
  if not pattern.lexemeinflectionpattern_set.filter(
       inflection_characteristic__entry__in=ics).filter(lexeme__pk__in=sgjp):
    ret = True
  elif pattern.type.entry in 'mn' and gender == 'f':
    ret = True
  elif pattern.type.entry in 'fm' and gender == 'n':
    ret = True
  else:
    ret = False
  memoized_pattern_ics[(pattern, gender)] = ret
  return ret


def find_patterns(lexical_class, forms, **extra):
  basic_form = forms[0][0]
  patterns = Pattern.objects.filter(type__lexical_class=lexical_class)
  # znaleźć wszystkie zawarte i zawierające wzory
  form_set = get_form_set(forms)
  if lexical_class.symbol == 'v':
    form_set = set(form for form, label in extra['base_forms'])
    all_forms = set(form for form, tag in forms)
  ending_sets = {}
  included_patterns = set()
  including_patterns = set()
  matching_patterns = set()
  base_forms_changed = False
  for basic_ending in basic_form_endings(lexical_class, basic_form, form_set, **extra):
    pattern = basic_ending.pattern
    if lexical_class.symbol == 'subst' and bad_pattern_subst(pattern, **extra):
      #print 'odpadł:', pattern
      continue # olewamy komentarze że formy odrzucone przez charfle?
    root = basic_form[:len(basic_form) - len(basic_ending.string)]
    ending_sets[pattern] = good_ending_set(
      lexical_class, pattern, root, **extra)
    including = form_set.issubset(ending_sets[pattern])
    extra_base_forms = []
    bad_forms = set()
    for ending in pattern.endings.all():
      if relevant(lexical_class, ending, **extra):
        if root + ending.string not in form_set:
          bfl = ending.base_form_label.entry
          #print pattern.name, root, ending.string, bfl
          if lexical_class.symbol == 'v':
            for label, tag, end in v_base_forms:
              if label == bfl:
                if root + ending.string + end in all_forms:
                  for form, tag in forms:
                    if form == root + ending.string + end:
                      this_tag = tag
                      break
                  # ech... uwielbiam zgadywać błędy
                  if (bfl in ('4', '5', "6'", '10', '11', '11pg', '12') and
                      (this_tag == ('verb', 'irreg') or this_tag[1] == 'ger')):
                    extra_base_forms.append(
                      (root + ending.string, label))
                    break
            else: # jeśli to nie była niewykryta forma bazowa
              for label, tag, end in v_base_forms:
                if label == bfl:
                  bad_forms.add(root + ending.string + end)
                  break
          else: # inne części mowy nie mają istotnych sufiksów?
            bad_forms.add(root + ending.string)
    if not bad_forms:
      if extra_base_forms:
        extra['base_forms'] += extra_base_forms
        base_forms_changed = True
      included_patterns.add((pattern, root))
      if including:
        matching_patterns.add((pattern, root))
    elif including:
      including_patterns.add(((pattern, root), tuple(bad_forms)))

  if base_forms_changed:
    #print extra['base_forms']
    return find_patterns(lexical_class, forms, **extra)
  # nie wiem, czy to potrzebne, ale na wszelki wypadek
  included_patterns = list(included_patterns)
  including_patterns = list(including_patterns)
  matching_patterns = list(matching_patterns)
  if len(matching_patterns) > 0:
    if DEBUG:
      print u'dokładne wzory: %s' % join(matching_patterns)
    return 'match', matching_patterns, included_patterns, including_patterns
  # nic nie pasuje albo trzeba wybrać wiele wzorów
  if DEBUG and len(including_patterns) > 0:
    print u'zawierające: %s' % join(p for p, b_f in including_patterns)
  if DEBUG and len(included_patterns) > 0:
    print u'zawarte: %s' % join(included_patterns)
  return find_many_patterns(
    lexical_class, form_set, basic_form, included_patterns, ending_sets,
    **extra) + (included_patterns, including_patterns)

def find_many_patterns(lexical_class, form_set, basic_form, included_patterns,
                       ending_sets, **extra):
  necessary_patterns = set()
  missing_form = None
  for form in form_set:
    having = []
    for pattern, root in included_patterns:
      if form in ending_sets[pattern]:
        having.append((pattern, root))
    if len(having) == 1:
      necessary_patterns.add(having[0])
    if not having:
      missing_form = form
      break
  if missing_form:
    if DEBUG:
      print u"brak formy: %s" % missing_form
    return 'none', []
  covered_forms = set()
  for pattern, root in necessary_patterns:
    covered_forms |= ending_sets[pattern]
  if form_set.issubset(covered_forms):
    if DEBUG:
      print u"pokryte koniecznymi wzorami: %s" % join(necessary_patterns)
    return 'many', [list(necessary_patterns)]
  else:
    #for pattern, root in included_patterns:
    #  print pattern, ending_sets[pattern]
    minimal_sets = find_minimal_sets(
      form_set, covered_forms, necessary_patterns, included_patterns,
      ending_sets)
    return 'many', minimal_sets


def check_sgjp_v(entry, ic, patterns, base_forms, derived):
  lexemes = Lexeme.objects.distinct().filter(
    entry=entry, part_of_speech__symbol='v')
  lexemes = lexemes.filter(pk__in=sgjp)
  for lexeme in lexemes:
    sgjp_patterns = set()
    for lip in lexeme.lexemeinflectionpattern_set.all():
      sgjp_ic = lip.inflection_characteristic.entry
      # TODO co z q*?
      if sgjp_ic != ic and ic in ('dk', 'ndk'):
        break
      if sgjp_ic in ('dk', 'ndk') and ic not in ('dk', 'ndk'):
        break
      sgjp_patterns.add((lip.pattern, lip.root))
    else:
      if set(patterns) == sgjp_patterns:
        sgjp_derived = {}
        for pos in ('ger', 'pact', 'ppas'):
          derived_part = lexeme.refs_to.filter(
            type='ver' + pos, to_lexeme__deleted=False)
          if pos == 'ppas':
            derived_part = derived_part
          if derived_part:
            sgjp_derived[pos] = [der.to_lexeme for der in derived_part]
        if derived == set(sgjp_derived):
          return lexeme, sgjp_derived
  return False, None

def check_sgjp(lc_sym, entry, form_set, forms, **extra):
  if lc_sym == 'v':
    return False
  if lc_sym != 'adj':
    lexemes = Lexeme.objects.distinct().filter(
      entry=entry, part_of_speech__lexical_class__symbol=lc_sym)
  else:
    if forms[0][1][-1] == 'comp':
      parts_of_speech = ['adjcom']
    else:
      parts_of_speech = ['adj', 'appas']
    lexemes = Lexeme.objects.distinct().filter(
      entry=entry, part_of_speech__symbol__in=parts_of_speech)
  lexemes = lexemes.filter(pk__in=sgjp)
  matched_lexemes = []
  for lexeme in lexemes:
    if lc_sym == 'adj' and lexeme.refs_to.filter(
         type='nieadj', to_lexeme__deleted=False):
      continue
    if lc_sym == 'subst' and extra['tantum'] == 'sg':
      sgjp_forms = lexeme.all_forms(affixes=False, label_filter=r'sg:')
    elif lexeme.part_of_speech.symbol == 'appas':
      sgjp_forms = lexeme.all_forms(affixes=True)
    else:
      sgjp_forms = lexeme.all_forms(affixes=False)
    if sgjp_forms == form_set:
      matched_lexemes.append(lexeme)
      continue
    diff = sgjp_forms - form_set
    exceptions = []
    if lc_sym == 'subst':
      if lexeme.lexemeinflectionpattern_set.filter(
          inflection_characteristic__entry__in=('m1', 'p1')).exists():
        # depr
        exceptions = lexeme.all_forms(affixes=False, label_filter=r'^pl:nom$')
    elif lc_sym == 'adj':
      # -o
      exceptions = lexeme.all_forms(affixes=False, label_filter=r'^0$')
    if form_set.issubset(sgjp_forms) and diff.issubset(exceptions):
      matched_lexemes.append(lexeme)
  if len(matched_lexemes) > 1:
    if lc_sym == 'subst' and entry.endswith(u'ość'):
      matched_lexemes_subst = [
        l for l in matched_lexemes if l.part_of_speech.symbol == 'subst']
      if matched_lexemes_subst:
        matched_lexemes = matched_lexemes_subst
    if len(matched_lexemes) > 1:
      debug(entry, u'niejednoznaczność dopasowanych leksemów')
  if len(matched_lexemes) > 0:
    return matched_lexemes[0]
  return False

# dla przymiotników
def get_negation(lc_sym, lexeme):
  negations = [cf.to_lexeme for cf in CrossReference.objects.filter(
               from_lexeme=lexeme, to_lexeme__deleted=False, type='adjnie')]
  good_negation = None
  for n in negations:
    if n.entry == u'nie' + lexeme.entry:
      if all(lip.inflection_characteristic.entry == '0-'
             for lip in n.lexemeinflectionpattern_set.all()):
        good_negation = n
  return good_negation

def closest_lexeme_subst(entry, gender, patterns, included=None):
  lexemes = Lexeme.objects.filter(
    deleted=False, part_of_speech__lexical_class__symbol='subst')
  lexemes = lexemes.distinct()
  # ten sam rodzaj
  gender = gender[0] if gender != 'm1' else 'm1'
  if genders[gender]:
    lexemes = lexemes.filter(
      lexemeinflectionpattern__inflection_characteristic__entry__in=
      genders[gender])
  if not included:
    # posiada wzór zawierający się w pasujących
    lexemes = lexemes.filter(lexemeinflectionpattern__pattern__in=patterns)
  else:
    #print patterns, included
    new_lexemes = Lexeme.objects.none()
    # posiada wszystkie wzory z któregoś zestawu
    for pattern_set in patterns:
      part = lexemes
      for pattern, root in pattern_set:
        part = part.filter(lexemeinflectionpattern__pattern=pattern)
      new_lexemes = new_lexemes | part
    lexemes = new_lexemes.distinct()
  # nie posiada wzorów niezawierających się w pasujących, dobra wielkość
  uppercase = entry[0].isupper()
  good_lexemes = []
  for lexeme in lexemes:
    if lexeme.entry[0].isupper():
      for lip in lexeme.lexemeinflectionpattern_set.all():
        if not included:
          if lip.pattern not in patterns:
            break
        else:
          if lip.pattern not in included:
            break
      else:
        good_lexemes.append(lexeme)
  # najdłuższe wspólne zakończenie
  best = (-1, None)
  for lexeme in good_lexemes:
    common_suffix = 0
    for char1, char2 in zip(entry[::-1], lexeme.entry[::-1]):
      if char1 == char2:
        common_suffix += 1
      else:
        break
    if common_suffix > best[0]:
      best = (common_suffix, lexeme)
  return best[1]

def get_inflection_characteristic_subst(forms, patterns, gender, ic, tantum):
  if tantum == 'sg' and ic == 'm':
    ic = 'm3'
  elif ic in ('n', 'i'): # ech, to 'i' - nadal możliwe?
    ic = 'n2'
  elif ic == 'm':
    ic = 'm3'
    sg_acc = []
    for form in forms:
      if len(form[1]) >= 3 and form[1][1:3] == ('sg', 'acc'):
        sg_acc.append(form[0])
    if len(sg_acc) == 1:
      if sg_acc[0] != forms[0][0]:
        ic = 'm2'
  return ic

def get_inflection_characteristic_adj(forms, patterns, **extra):
  # charfl: 3+ jeśli jest adjp
  ic = '0-'
  for form in forms:
    if form[1][0] == 'adjp':
      ic = '3+'
  if patterns[0][0].endings.filter(base_form_label__entry='0') and ic != '3+':
    ic = ''
  return ic

def get_inflection_characteristic_v(forms, base_forms, **extra):
  # dk, ndk, ndk/dk
  # na podstawie istnienia -ąc (ndk), -szy (dk)
  ndk = dk = nonq = False
  for form in forms:
    if form[0].endswith(u'ąc') and not ':'.join(form[1]).startswith('verb:inf'):
      ndk = True
    elif form[0].endswith(u'wszy') or form[0].endswith(u'łszy'):
      dk = True
    elif ':'.join(form[1]).startswith('verb:praet:sg:pri'):
      nonq = True
  if not dk and not ndk: # mniej wiarygodne - czy mdlić może być dk?
    for form in forms:
      if form[1][1] == 'inf':
        if form[1][-1] == 'imperf':
          ndk = True
        elif form[1][-1] == 'perf':
          dk = True
        break
  if ndk and not dk:
    ic = 'ndk'
  elif dk and not ndk:
    ic = 'dk'
  elif ndk and dk:
    ic = 'ndk/dk'
  else:
    debug(forms[0][0], u'nie udało się ustalić aspektu')
    ic = None
  if ic and not nonq:
    ic = 'q' + ic
  return ic

def get_inflection_characteristic(lc_sym, forms, patterns, **extra):
  if lc_sym == 'subst':
    return get_inflection_characteristic_subst(forms, patterns, **extra)
  elif lc_sym == 'adj':
    return get_inflection_characteristic_adj(forms, patterns, **extra)
  elif lc_sym == 'v':
    return get_inflection_characteristic_v(forms, **extra)

def blacklist_filter(patterns):
  return [(pattern, root) for (pattern, root) in patterns
          if pattern.name not in blacklist]

def filter_patterns(filter, action_name, type, patterns, included, including,
                    lexical_class, form_set, entry, **extra):
  old_patterns = patterns
  old_included = included
  bad_patterns = False
  if type == 'many':
    if any(pattern_set != filter(pattern_set) for pattern_set in patterns):
      included = filter(included)
      ending_sets = {}
      for pattern, root in included:
        ending_sets[pattern] = good_ending_set(
          lexical_class, pattern, root, **extra)
      type, patterns = find_many_patterns(
        lexical_class, form_set, entry, included, ending_sets, **extra)
      if type != 'many':
        debug(entry, u'mnogie dopasowanie zepsute przez %s (%s)' %
                     (action_name, join_many(old_patterns)))
        type = 'many'
        patterns, included = old_patterns, old_included
        bad_patterns = True
  elif type == 'none':
    including_dict = dict(including)
    including = [(key, including_dict[key]) for key in filter(including_dict)]
  else: # type == 'match'
    patterns = filter(patterns)
    including_dict = dict(including)
    including = [(key, including_dict[key]) for key in filter(including_dict)]
    included = filter(included)
    if old_patterns and not patterns:
      ending_sets = {}
      for pattern, root in included:
        ending_sets[pattern] = good_ending_set(
          lexical_class, pattern, root, **extra)
      type, patterns = find_many_patterns(
        lexical_class, form_set, entry, included, ending_sets, **extra)
      if type == 'none':
        debug(entry, u'znikły wzory przez %s (%s)' %
                     (action_name, join(old_patterns)))
        type = 'match'
        patterns = old_patterns
        bad_patterns = True
  return type, patterns, included, including, bad_patterns

def create_derived(pos, base_forms, forms, patterns):
  tab = {'ger': ('11', u'ie'), 'pact': ('3', u'cy'), 'ppas': ('10', u'y')}
  entries = {}
  for pattern, root in patterns:
    bfl = tab[pos][0]
    ending = pattern.endings.get(base_form_label__entry=bfl)
    entry = root + ending.string + tab[pos][1]
    if entry not in entries:
      entries[entry] = []
    entries[entry].append(pattern.name)
  output = []
  for entry, patterns in entries.iteritems():
    if entry in (form[0] for form in forms):
      output.append((pos, entry, patterns))
  return output

def get_sgjp(lexeme):
  return {'source': 'sgjp', 'id': lexeme.pk, 'entry': lexeme.entry}

def create_lexeme(entry, homonym_number, part_of_speech, status, comment):
  return {
    'source': 'morfologik',
    'entry': entry,
    'homonym_number': homonym_number,
    'part_of_speech': part_of_speech,
    'status': status,
    'comment': comment,
  }

def create_lip(pattern, root, i, ic, part_of_speech):
  output = {
    'pattern': pattern if isinstance(pattern, basestring) else pattern.name,
    'ind': i,
    'ic': (ic, part_of_speech),
  }
  if root:
    output['root'] = {'type': 'string', 'root': root}
  else:
    output['root'] = {'type': 'compute'}
  return output

alternative_gender = {
  'p1': 'p3',
  'p3': 'p1',
  'm1': 'm2/m3',
  'm': 'm1',
}

alternative_gender2 = {
  'p1': 'p3',
  'p3': 'p1',
  'm1': 'm',
  'm': 'm1',
}

def lexeme_creation(lc_sym, entry, ic, forms, type, patterns, fitting,
                    bad_patterns, included, other_result, tantum=None,
                    gender=None, negated=None, base_forms=None, derived=None):
  homonym_number = 1
  status = 'desc' if type != 'none' else 'cand'
  comments = []
  copy_lips = False
  if lc_sym == 'subst':
    part_of_speech = 'subst' # co z osc i skrs?
    if ic in ('m2', 'm3'):
      sure = False
      if type != 'none' and len(fitting) == 1:
        for pattern, root in patterns:
          for e in patterns[0][0].endings.filter(base_form_label__entry='sg:gen'):
            if not e.string.endswith('u'):
              break
          else:
            continue
          break
        else: # wszystkie sg:gen kończą się na 'u'
          ic = 'm3'
          sure = True
        for pattern, root in patterns:
          if pattern.type.entry == 'f':
            ic = 'm2'
            sure = True
            break
      if not sure:
        status = 'cand'
    if tantum is None and ic == 'm1':
      for pattern, root in patterns:
        nmo_endings = pattern.endings.filter(base_form_label__entry='pl:nom')
        for e in nmo_endings:
          nmo_form = root + e.string
          if nmo_form not in (form[0] for form in forms):
            comments.append(u'Dodano formę depr')
            break
        else:
          continue
        break
    if ic == 'p1':
      for pattern, root in patterns:
        nmo_endings = pattern.endings.filter(base_form_label__entry='pl:nom')
        other_endings = pattern.endings.exclude(base_form_label__entry='pl:nom')
        other_strings = other_endings.values_list('string', flat=True)
        nmo_strings = [e.string for e in nmo_endings if e.string not in other_strings]
        nmo_forms = set(root + s for s in nmo_strings)
        if nmo_forms & set(form[0] for form in forms):
          comments.append(
            u'Usunięto formę depr: %s' % ', '.join(list(nmo_forms)))
          break
    if tantum == 'sg' and type != 'none':
      if type == 'match':
        search_patterns = [pattern for pattern, root in fitting]
        l = closest_lexeme_subst(entry, gender, search_patterns)
      else:
        included_patterns = [pattern for pattern, root in included]
        l = closest_lexeme_subst(entry, gender, fitting, included_patterns)
      if l:
        copy_lips = l.lexemeinflectionpattern_set.all()
        #print l
        comments.append(u'Automatycznie rozszerzone singulare tantum')
      else:
        if type == 'match':
          p = join(fitting)
        else:
          p = join_many(fitting)
        debug(entry, u'nie ma pasujących leksemów dla rozszerzenia sgtant '
                     u'dla wzorów %s' % p)
        comments.append(u'Nie udało się rozszerzyć singulare tantum')
        #status = 'cand'
      # dodać kwalifikator [po imporcie jednak]
  elif lc_sym == 'adj':
    if forms[0][1][-1] == 'comp':
      part_of_speech = 'adjcom'
      ic = ''
    else:
      part_of_speech = 'adj'
      if ic != '0-' and type != 'none':
        comments.append(u'Dodano formę -o')
  else: # lc_sym == 'v':
    part_of_speech = 'v'
    if type != 'none':
      if ic is None:
        comments.append(u'Nie udało się ustalić aspektu')
        ic = 'ndk'
        status = 'cand'
      elif ic.startswith('q') and derived != set():
        comments.append(u'Derywaty muszą mieć charfle bez q')
  if bad_patterns:
    comments.append(u'Wzory z czarnej listy!')
    status = 'cand'
  if len(fitting) > 1 or (type == 'none' and fitting):
    status = 'cand'
    if type == 'none':
      comments.append(u'Zawierające wzory:')
      for (pattern, root), bad_forms in fitting:
        comments.append('%s: %s' % (pattern.name, ', '.join(bad_forms)))
    elif type != 'many':
      comments.append(u'Pasujące wzory: %s' % join(fitting))
    else:
      comments.append(u'Pasujące zestawy wzorów: %s' % join_many(fitting))
  if other_result:
    status = 'cand'
    type2, patterns2, included2, including2 = other_result
    comments.append(u'Alternatywny rodzaj: %s' % alternative_gender[gender])
    if type2 == 'match':
      comments.append(u'Pasujące wzory: %s' % join(patterns2))
    elif type2 == 'many':
      comments.append(u'Pasujące zestawy wzorów: %s' % join_many(patterns2))
  # hm?
  if ic is None and type != 'none':
    comments.append(u'Dopasowane wzory: %s' % join(patterns))
  # zbieramy wzory do porównania [fuj, copypasta!]
  if len(fitting) > 1:
    if type == 'none':
      all_patterns = set(p for p, b_f in fitting)
    elif type != 'many':
      all_patterns = set(fitting)
    else:
      all_patterns = set()
      for pattern_set in fitting:
        all_patterns |= set(pattern_set)
    diff = compare_patterns(list(all_patterns))
    comments.append(u'Porównanie wzorów:')
    for p, diff_forms in diff.iteritems():
      comments.append(
        '%s: %s' % (p.name, ', '.join('%s: %s' % pair for pair in diff_forms)))
  if other_result and len(patterns2) > 1:
    if type2 != 'many':
      other_patterns = set(patterns2)
    else:
      other_patterns = set()
      for pattern_set in patterns2:
        other_patterns |= set(pattern_set)
    diff = compare_patterns(list(other_patterns))
    comments.append(u'Porównanie alternatywnych wzorów:')
    for p, diff_forms in diff.iteritems():
      comments.append(
        '%s: %s' % (p.name, ', '.join('%s: %s' % pair for pair in diff_forms)))
  comment = '\n'.join(comments)
  output = {
    'lexeme': create_lexeme(
      entry, homonym_number, part_of_speech, status, comment)
  }
  lips = []
  if ic is not None:
    if not copy_lips:
      for i, (pattern, root) in enumerate(patterns):
        lips.append(create_lip(pattern, root, i + 1, ic, part_of_speech))
    else:
      for lip in copy_lips:
        ic = lip.inflection_characteristic.entry
        lips.append(
          create_lip(lip.pattern, None, lip.index, ic, part_of_speech))
  output['lips'] = lips
  if lc_sym == 'adj' and negated:
    output['negated'] = True
  if lc_sym == 'v':
    derived_data = []
    for pos in derived:
      # wypadałoby informować, jeśli wyszło puste... (?)
      derived_data += create_derived(pos, base_forms, forms, patterns)
    output['derived'] = derived_data
  return output

def process_forms(lexical_class, forms, **extra):
  lc_sym = lexical_class.symbol
  other_result = None
  entry = forms[0][0]
  form_set = get_form_set(forms)
  check = check_sgjp(lc_sym, entry, form_set, forms, **extra)
  if check and not DEBUG:
    # dopisz leksem do słownika
    data = {'lexeme': get_sgjp(check)}
    if (lc_sym == 'adj' and extra['negated']):
      neg = get_negation(lc_sym, check)
      if neg:
        print_data({'lexeme': get_sgjp(neg)})
      else:
        data['negated'] = True
    print_data(data)
  else:
    if lexical_class.symbol == 'subst':
      ic = extra['gender']
    extra2 = dict(extra)
    # jeśli rzeczownik męski lub pltant, to puszczamy więcej razy
    if lexical_class.symbol == 'subst' and extra['gender'] in 'pm':
      if extra['gender'] == 'm':
        type2, patterns2, included2, including2 = find_patterns(
          lexical_class, forms, **extra)
        extra2['gender'] = 'm1'
        type1, patterns1, included1, including1 = find_patterns(
          lexical_class, forms, **extra2)
      elif extra['gender'] == 'p':
        extra2['gender'] = 'p1'
        type1, patterns1, included1, including1 = find_patterns(
          lexical_class, forms, **extra2)
        extra2['gender'] = 'p3'
        type2, patterns2, included2, including2 = find_patterns(
          lexical_class, forms, **extra2)
      if type1 != 'none' and type2 == 'none':
        type, patterns, included, including = type1, patterns1, included1, including1
        if extra['gender'] == 'm':
          ic = 'm1'
        else:
          ic = 'p1'
      elif type1 == 'none' and type2 != 'none':
        type, patterns, included, including = type2, patterns2, included2, including2
        if extra['gender'] == 'm':
          ic = 'm'
        else:
          ic = 'p3'
      elif type1 == type2 == 'none':
        type = 'none'
        patterns = []
        included = list(set(included1) | set(included2))
        including = list(set(including1) | set(including2))
        # chyba warto coś ustawić
        if extra['gender'] == 'm':
          ic = 'm'
        else:
          ic = 'p3'
      else: # z obu coś wyszło
        type, patterns, included, including = type1, patterns1, included1, including1
        if extra['gender'] == 'm':
          ic = 'm1'
        else:
          ic = 'p1'
        other_result = (type2, patterns2, included2, including2)
        if DEBUG:
          print u"dwie możliwości na rodzaj"
    else:
      type, patterns, included, including = find_patterns(
        lexical_class, forms, **extra)
    if type == 'none':
      if lexical_class.symbol == 'subst' and not extra['tantum']:
        extra['tantum'] = tantum_a_posteriori(
          form_set, [p for p, b_f in including])
        if extra['tantum']:
          if extra['tantum'] == 'pl':
            extra['gender'] = 'p'
            #if ic == 'm1':
            #  extra['gender'] = 'p1'
            #else:
            #  extra['gender'] = 'p3'
          return process_forms(lexical_class, forms, **extra)

    if lexical_class.symbol == 'subst':
      extra2['gender'] = ic
    type, patterns, included, including, bad_patterns = filter_patterns(
      blacklist_filter, u'czarną listę', type, patterns, included, including,
      lexical_class, form_set, entry, **extra2)
    if bad_patterns and other_result:
      type, patterns, included, including = other_result
      ic = extra2['gender'] = alternative_gender2[ic]
      type, patterns, included, including, bad_patterns = filter_patterns(
        blacklist_filter, u'czarną listę', type, patterns, included, including,
        lexical_class, form_set, entry, **extra2)
      other_result = None
    elif other_result:
      type2, patterns2, included2, including2 = other_result
      new_other_result = filter_patterns(
        blacklist_filter, u'czarną listę', type2, patterns2, included2,
        including2, lexical_class, form_set, entry, **extra2)
      if not new_other_result[4]:
        other_result = new_other_result[:4]
      else:
        other_result = None

    # wzory się już nie zmienią od tego miejsca
    if type == 'many':
      # albo patterns[0]...
      all_patterns = [p for pattern_set in patterns for p in pattern_set]
    else:
      all_patterns = patterns
    if type != 'none':
      # brzydko...
      if lexical_class.symbol == 'subst':
        extra['ic'] = ic
      ic = get_inflection_characteristic(lc_sym, forms, all_patterns, **extra)
      if lexical_class.symbol == 'subst':
        del extra['ic']
    else:
      ic = None
    if lc_sym == 'subst':
      # poprawka dla m2/m3
      if ic in ('m2', 'm3') and patterns and not bad_patterns:
        new_ic = ''
        for pattern, root in all_patterns:
          for ic2 in ('m2', 'm3'):
            # jeśli wszystkie użycia tego wzoru są przy ic2
            if not pattern.lexemeinflectionpattern_set.exclude(
                 inflection_characteristic__entry=ic2).filter(
                   lexeme__pk__in=sgjp).exists():
              if new_ic == '':
                new_ic = ic2
              elif new_ic != ic2:
                new_ic = None
        if new_ic:
          ic = new_ic

    if type == 'none':
      debug(entry, u'zawiera się w %s' % join(p for p, b_f in including))
      chosen = []
      fitting = including
      if lexical_class.symbol == 'adj' and including:
        print_forms(forms, 'rzeczownik#')
        return
    elif type == 'match':
      patterns.sort(key=lambda p: p[0].name)
      fitting = patterns
      chosen = patterns[:1]
    elif type == 'many':
      chosen = patterns[0]
      if DEBUG:
        print u'zestawy wielu wzorów: %s' % join_many(patterns)
      fitting = patterns
    if not DEBUG and lc_sym == 'v':
      check, sgjp_derived = check_sgjp_v(entry, ic, chosen, **extra)
      if check:
        print_data({'lexeme': get_sgjp(check)})
        for derived_lexemes in sgjp_derived.values():
          for derived_lexeme in derived_lexemes:
            print_data({'lexeme': get_sgjp(derived_lexeme)})
        return
    if not DEBUG:
      data = lexeme_creation(
        lc_sym, entry, ic, forms, type, chosen, fitting, bad_patterns, included,
        other_result, **extra2)
      print_data(data)

def get_pos_ndm(tag):
  if tag[0] == 'adv':
    return 'adv' if tag[-1] != 'comp' else 'advcom'
  elif tag[0] == 'xxx':
    return 'burk'
  else:
    return tag[0]

def process_ndm(forms):
  entry = forms[0][0]
  form_set = get_form_set(forms)
  pattern_name = 'ndm'
  ics = ['']
  tag = forms[0][1].split(':')
  pos = get_pos_ndm(tag)
  status = 'desc'
  comments = []
  if len(form_set) > 2:
    debug(entry, u'nieodmienne z ponad dwiema formami')
    comments.append(u'nieodmienne z ponad dwiema formami')
    status = 'cand'
    ics = []
  elif pos == 'prep':
    if len(tag) == 1:
      debug(entry, u'przyimek bez przypadku')
      comments.append(u'przyimek bez przypadku')
      status = 'cand'
      ics = []
    else:
      ics = tag[1].split('.')
    if len(form_set) == 2:
      if entry + u'e' == forms[1][0]:
        pattern_name = 'ndm e'
      else:
        debug(entry, u'nierozpoznana odmiana przyimka')
        comments.append(u'nierozpoznana odmiana przyimka')
        status = 'cand'
  elif len(form_set) == 2:
    if pos == 'adv' and forms[1][0] == u'nie' + entry:
      process_ndm(forms[:1])
      process_ndm(forms[1:])
      return
    else:
      debug(entry, u'nierozpoznane nieodmienne z dwiema formami')
      comments.append(u'nierozpoznane nieodmienne z dwiema formami')
      status = 'cand'
  # sprawdzić czy to co wyszło jest już w sgjp
  lexemes = Lexeme.objects.distinct().filter(entry=entry)
  lexemes = lexemes.filter(pk__in=sgjp)
  if pos == 'adv':
    lexemes = lexemes.filter(part_of_speech__symbol__in=('adv', 'advndm'))
  elif pos == 'conj':
    lexemes = lexemes.filter(part_of_speech__symbol__in=('conj', 'comp'))
  else:
    lexemes = lexemes.filter(part_of_speech__symbol=pos)
  in_sgjp = bool(lexemes)
  for lexeme in lexemes:
    sgjp_ics = set()
    for lip in lexeme.lexemeinflectionpattern_set.all():
      if lip.pattern.name != pattern_name:
        in_sgjp = False
        break
      sgjp_ics.add(lip.inflection_characteristic.entry)
    if set(ics) != sgjp_ics:
      in_sgjp = False
    if in_sgjp:
      break
  if in_sgjp:
    print_data({'lexeme': get_sgjp(lexeme)})
  else:
    homonym_number = 1
    data = {
      'lexeme': create_lexeme(
        entry, homonym_number, pos, status, '\n'.join(comments))
    }
    lips = []
    for i, ic in enumerate(ics):
      lips.append(create_lip(pattern_name, entry, i + 1, ic, pos))
    data['lips'] = lips
    print_data(data)

def print_data(data):
  print json.dumps(data)

def print_forms(forms, prefix):
  for form, tag in forms:
    try:
      end = tag.index('pos')
    except ValueError:
      end = tag.index('comp')
    new_tag = ':'.join(tag[:end]).replace('adj', 'subst')
    print>>sys.stderr, ('%s%s\t%s' % (prefix, form, new_tag)).encode('utf-8')
  print>>sys.stderr, prefix

def parse_file(lc_sym, path):
  with open(path) as file:
    lexical_class = LexicalClass.objects.get(symbol=lc_sym)
    forms = []
    for line in file:
      line = line.decode('utf-8').rstrip('\n')
      if line == '':
        if lc_sym == 'ndm':
          process_ndm(forms)
        else:
          forms = expand_forms(forms)
          extra = get_extra(lexical_class, forms)
          process_forms(lexical_class, forms, **extra)
        forms = []
      else:
        form, tag = line.split('\t')
        if DEBUG and forms == []:
          print form
        forms.append((form, tag))