SGJP / Kuznia

# -*- coding: utf-8 -*-

import sys
import json

from django.core.management.base import BaseCommand

from common.util import suffixes, cut_end, debug, GroupDict, uniopen
from dictionary.models import Pattern, Lexeme, InflectionCharacteristic, \
    Ending, LexicalClass, CrossReference, BaseFormLabel, Vocabulary
from dictionary.util import expand_tag, compare_patterns
from patterns.pattern_blacklist import blacklist


class Command(BaseCommand):
    args = '<symbol części mowy> <nazwa pliku wejściowego>'
    help = 'Creates scripts for importing data from Morfologik'

    def handle(self, lc_sym, input_file, **options):
        raise Exception("stale code")
        #parse_file(lc_sym, input_file)


DEBUG = False

#sgjp = Vocabulary.objects.get(id='SGJP').owned_lexemes_pk()
morf = Vocabulary.objects.get(id='Morfologik').owned_lexemes_pk()
sgjp = Lexeme.objects.filter(source='SGJP')

# może być nierozpoznane pltant
ending_genders = {
    'm1': ['m1', 'p1'],
    'm': ['m2', 'm3', 'p2', 'p3'],
    'f': ['f', 'p2', 'p3'],
    'n': ['n1', 'n2', 'p2', 'p3'],
    'p1': ['p1'],
    'p3': ['p2', 'p3'],
    'i': [], # wszystkie
}

genders = {
    'm1': ['m1'],
    'm': ['m2', 'm3'], #, 'p1', 'p2', 'p3'],
    'f': ['f'], #, 'p2', 'p3'],
    'n': ['n1', 'n2'], #, 'p2', 'p3'],
    'p1': ['p1'],
    'p3': ['p2', 'p3'],
    'i': [], # wszystkie
}

# piękne nazwy...
genders2 = {
    'm1': ['m1', 'p1'],
    'm': ['m2', 'm3', 'p2', 'p3'],
    'f': ['f', 'p2', 'p3'],
    'n': ['n1', 'n2', 'p2', 'p3'],
    'p1': ['p1', 'm1'],
    'p3': ['p2', 'p3', 'f', 'm2', 'm3', 'n1', 'n2'],
    'i': [], # wszystkie
}


def get_basic_endings(lexical_class, parts_of_speech, gender=None):
    if lexical_class.symbol == 'v':
        return Ending.objects.filter(
            base_form_label__symbol='5',
            pattern__type__lexical_class=lexical_class)
    ics = InflectionCharacteristic.objects.filter(
        part_of_speech__in=parts_of_speech)
    if gender and ending_genders[gender]:
        ics = ics.filter(symbol__in=ending_genders[gender])
    basic_form_labels = ics.values_list('basic_form_label',
        flat=True).distinct()
    return Ending.objects.filter(base_form_label__pk__in=basic_form_labels,
        pattern__type__lexical_class=lexical_class)


basic_form_endings_dict = {}
for lexical_class in LexicalClass.objects.all():
    parts_of_speech = lexical_class.partofspeech_set.all()
    if lexical_class.symbol == 'subst':
        for gender in ending_genders:
            basic_form_endings_dict[
                (lexical_class, gender)] = get_basic_endings(
                lexical_class, parts_of_speech, gender)
    else:
        basic_form_endings_dict[lexical_class] = get_basic_endings(
            lexical_class, parts_of_speech)


def join(patterns):
    return ', '.join(sorted(p[0].name for p in patterns))


def join_many(patterns):
    return ', '.join('[%s]' % join(set) for set in patterns)


def prepare_endings(forms):
    root = forms[0][0]
    for form in forms[1:]:
        new_root = ''
        for char1, char2 in zip(root, form[0]):
            if char1 == char2:
                new_root += char1
            else:
                break
        root = new_root
    endings = set()
    for form in forms:
        endings.add((form[0][len(root):], form[1]))
    return endings, root


def get_form_set(forms):
    form_set = set(form[0] for form in forms)
    return form_set - set('nie' + form for form in form_set)


def expand_forms(forms):
    expanded = []
    for form in forms:
        tags = expand_tag(form[1])
        expanded += [(form[0], tag) for tag in tags]
    return expanded


def get_tantum(forms):
    sg = set()
    pl = set()
    for form in forms:
        if form[1][1].startswith('pl'):
            pl.add(form[0])
        elif form[1][1].startswith('sg'): # ignorujemy irreg
            sg.add(form[0])
    if sg == pl:
        return
    # za dużo fałszywych pozytywów
    #elif pl.issubset(sg):
    #  return 'sg'
    elif sg.issubset(pl): # or len(sg) == 1: -- czasem psuje
        return 'pl'
    else:
        return


def tantum_a_posteriori(form_set, patterns):
    tantum = None
    for pattern, root in patterns:
        tantum_forms = {
            'sg': set(root + e for e in
                pattern.endings.filter(
                    base_form_label__symbol__startswith='sg')
                .values_list('string', flat=True)),
            'pl': set(root + e for e in
                pattern.endings.filter(
                    base_form_label__symbol__startswith='pl')
                .values_list('string', flat=True)),
        }
        for num in ('sg', 'pl'):
            if form_set.issubset(tantum_forms[num]):
                tantum = num
        if tantum:
            return tantum
    if not patterns:
        return 'sg'
    return None


v_base_forms = [
    ('1', 'verb:fin:sg:ter:', u''),
    ('2', 'verb:fin:sg:pri:', u''),
    ('3', 'verb:fin:pl:ter:', u''),
    ('4', 'verb:impt:sg:sec:', u''),
    ('5', 'verb:inf:', u''),
    ('6', 'verb:praet:sg:ter:m:', u''),
    ("6'", 'pant:perf', u'szy'),
    ('7', 'verb:praet:sg:pri:m:', u'em'),
    ('8', 'verb:praet:sg:pri:f:', u'am'),
    ('9', 'verb:praet:pl:ter:m1:', u'i'),
    ('10', 'verb:imps', u'o'),
    ('11', 'subst:ger:sg:nom:n', u'ie'), # [bez przeczenia]
    ('11pg', 'subst:ger:pl:gen:n', u''),
    ('12', 'ppas:pl:nom:m1', u''),
]


def get_extra(lexical_class, forms, **extra):
    if lexical_class.symbol == 'subst':
        if 'tantum' in extra:
            tantum = extra['tantum']
        else:
            tantum = get_tantum(forms)
        tag = forms[0][1]
        gender = tag[-1][0]
        # powinienem częściej tak pisać zamiast for...else jeśli się da
        if all(form[1][-1] in ('m1', 'depr') for form in forms):
            gender = 'm1'
        if tantum == 'pl' or tag[1] == 'pltant':
            gender = 'p'
            tantum = 'pl'
        return {'gender': gender, 'tantum': tantum}
    elif lexical_class.symbol == 'adj':
        all_forms = [form[0] for form in forms]
        negated = [(form, tag) for (form, tag) in forms
            if form == 'nie' + all_forms[0]]
        negated += [(form, tag) for (form, tag) in forms
            if form.startswith('nie') and form[4:] in all_forms
            and form != 'nie' + all_forms[0]]
        return {'negated': negated}
    elif lexical_class.symbol == 'v':
        all_forms = [form[0] for form in forms]
        base_forms = []
        for label, tag_prefix, end in v_base_forms:
            for form, tag in forms:
                negated_form = (form.startswith(u'nie') and
                                form[len('nie'):] in all_forms)
                if (':'.join(tag).startswith(tag_prefix) and
                    not (form.endswith(u'że') and tag[1] == 'impt')
                and not negated_form):
                    if form.endswith(end): # ech...
                        base_form = cut_end(form, end)
                        if label == '10':
                            bf10 = base_form
                        base_forms.append((base_form, label))
        derived = set()
        labels = set(label for (form, label) in base_forms)
        if '10' in labels and bf10 + 'y' in all_forms:
            derived.add('ppas')
        for form, tag in forms:
            plain_tag = ':'.join(tag)
            if plain_tag.startswith('subst:ger') and '11' in labels:
                derived.add('ger')
            if plain_tag.startswith('pact') and '3' in labels:
                derived.add('pact')
        old_base_forms = base_forms
        if 'ppas' not in derived:
            base_forms = []
            for base_form, label in old_base_forms:
                if label != '12':
                    base_forms.append((base_form, label))
        if DEBUG:
            for f, l in base_forms:
                print f, l
        return {'base_forms': base_forms, 'derived': derived}
    else:
        return {}


def relevant_subst(ending, gender, tantum):
    bfl = ending.base_form_label.symbol
    tag = bfl.split(':')
    pattern_type = ending.pattern.type.symbol
    return (not (gender in ('m1', 'p1') and bfl == 'pl:nom') and
            not (len(tag) >= 3 and gender[0] != 'p' and
                 tag[2][0] != gender[0]) and
            not (tantum and tag[0] != tantum) and
            not (gender == 'p3' and bfl.startswith('pl:gen:') and (
                (pattern_type == 'n' and tag[2] == 'n') or
                (pattern_type == 'f' and tag[2] == 'm')
            )) and
            not (gender not in ('m1', 'p1') and bfl == 'pl:nom:mo'))


def relevant_adj(ending):
    tag = ending.base_form_label.symbol
    return tag not in ('0', '3+')


def relevant_v(ending, base_forms, derived):
    tag = ending.base_form_label.symbol
    if 'ppas' not in derived and tag == '12':
        return False
    if 'ger' not in derived and tag in ('11', '11pg'):
        return False
    return tag not in ("6'",)


def relevant(lexical_class, ending, **extra):
    if lexical_class.symbol == 'subst':
        return relevant_subst(ending, **extra)
    elif lexical_class.symbol == 'adj':
        return relevant_adj(ending)
    elif lexical_class.symbol == 'v':
        return relevant_v(ending, **extra)


from itertools import chain, combinations


def powerset(iterable):
    """powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"""
    s = list(iterable)
    return chain.from_iterable(
        combinations(s, r) for r in range(min(len(s) + 1, 5)))


def find_minimal_sets(form_set, covered_forms, necessary_patterns,
                      included_patterns, ending_sets):
    unnecessary_patterns = list(
        set(included_patterns) - set(necessary_patterns))
    found = []
    for subset in powerset(unnecessary_patterns):
        for found_set in found:
            if set(tuple(necessary_patterns) + subset).issuperset(found_set):
                break
        else:
            new_covered = set(covered_forms)
            for pattern, root in subset:
                new_covered |= ending_sets[pattern]
            if form_set.issubset(new_covered):
                found.append(tuple(necessary_patterns) + subset)
    return found


sure_bfls_sg = tuple(
    BaseFormLabel.objects.filter(
        symbol__in=['sg:dat', 'sg:gen', 'sg:inst']).values_list('pk',
        flat=True))
sure_bfls_pl = tuple(
    BaseFormLabel.objects.filter(
        symbol__in=['pl:dat', 'pl:inst', 'pl:loc']).values_list('pk',
        flat=True))


def basic_form_endings(lexical_class, basic_form, form_set, **extra):
    if 'gender' in extra:
        key = (lexical_class, extra['gender'])
    else:
        key = lexical_class
    if lexical_class.symbol != 'subst':
        return basic_form_endings_dict[key].filter(
            string__in=suffixes(basic_form))
    else:
        # karkołomne, ale trochę przyśpiesza
        endings = basic_form_endings_dict[key]
        new_endings = Ending.objects.none()
        for suf in suffixes(basic_form):
            root = cut_end(basic_form, suf)
            n = len(root)
            ending_strings = tuple(
                form[n:] for form in form_set if form.startswith(root))
            endings_part = endings.filter(string=suf)
            pattern_pks = endings_part.values_list('pattern', flat=True)
            patterns = Pattern.objects.filter(pk__in=pattern_pks).extra(
                where=["(w_id = '0000' or not exists "
                       "(select id from zakonczenia where w_id = wzory.id "
                       "and zak not in %s and efobaz in %s) or not exists "
                       "(select id from zakonczenia where w_id = wzory.id "
                       "and zak not in %s and efobaz in %s))"],
                params=[ending_strings, sure_bfls_sg, ending_strings,
                    sure_bfls_pl])
            new_endings = new_endings | endings_part.filter(
                pattern__in=patterns)
        return new_endings


memoized_good_endings = {}


def good_ending_set_subst(pattern, root, tantum, gender):
    if (pattern, tantum, gender) in memoized_good_endings:
        good_endings = memoized_good_endings[(pattern, tantum, gender)]
        return set(root + e for e in good_endings)
    endings = pattern.endings
    if tantum:
        endings = endings.filter(base_form_label__symbol__startswith=tantum)
    if gender not in ('m1', 'p1'):
        endings = endings.exclude(base_form_label__symbol='pl:nom:mo')
    if gender[0] != 'p':
        for g in list(set('mfn') - set(gender[0])):
            endings = endings.exclude(
                base_form_label__symbol__startswith='pl:gen:' + g)
    if gender == 'p3':
        if pattern.type.symbol == 'f':
            endings = endings.exclude(base_form_label__symbol='pl:gen:m')
        if pattern.type.symbol == 'n':
            endings = endings.exclude(base_form_label__symbol='pl:gen:n')
    good_endings = list(endings.values_list('string', flat=True))
    memoized_good_endings[(pattern, tantum, gender)] = good_endings
    return set(root + e for e in good_endings)


def good_ending_set(lexical_class, pattern, root='', **extra):
    if lexical_class.symbol != 'subst':
        return pattern.ending_set(root)
    else:
        return good_ending_set_subst(pattern, root, **extra)


memoized_pattern_ics = {}


def bad_pattern_subst(pattern, gender, tantum):
    if (pattern, gender) in memoized_pattern_ics:
        return memoized_pattern_ics[(pattern, gender)]
    ics = genders2[gender]
    if not pattern.lexemeinflectionpattern_set.filter(
            inflection_characteristic__symbol__in=ics).filter(
            lexeme__pk__in=sgjp):
        ret = True
    elif pattern.type.symbol in 'mn' and gender == 'f':
        ret = True
    elif pattern.type.symbol in 'fm' and gender == 'n':
        ret = True
    else:
        ret = False
    memoized_pattern_ics[(pattern, gender)] = ret
    return ret


def find_patterns(lexical_class, forms, **extra):
    basic_form = forms[0][0]
    patterns = Pattern.objects.filter(type__lexical_class=lexical_class)
    # znaleźć wszystkie zawarte i zawierające wzory
    form_set = get_form_set(forms)
    if lexical_class.symbol == 'v':
        form_set = set(form for form, label in extra['base_forms'])
        all_forms = set(form for form, tag in forms)
    ending_sets = {}
    included_patterns = set()
    including_patterns = set()
    matching_patterns = set()
    base_forms_changed = False
    for basic_ending in basic_form_endings(lexical_class, basic_form, form_set,
            **extra):
        pattern = basic_ending.pattern
        if lexical_class.symbol == 'subst' and bad_pattern_subst(pattern,
                **extra):
            #print 'odpadł:', pattern
            continue # olewamy komentarze że formy odrzucone przez charfle?
        root = basic_form[:len(basic_form) - len(basic_ending.string)]
        ending_sets[pattern] = good_ending_set(
            lexical_class, pattern, root, **extra)
        including = form_set.issubset(ending_sets[pattern])
        extra_base_forms = []
        bad_forms = set()
        for ending in pattern.endings.all():
            if relevant(lexical_class, ending, **extra):
                if root + ending.string not in form_set:
                    bfl = ending.base_form_label.symbol
                    #print pattern.name, root, ending.string, bfl
                    if lexical_class.symbol == 'v':
                        for label, tag, end in v_base_forms:
                            if label == bfl:
                                if root + ending.string + end in all_forms:
                                    for form, tag in forms:
                                        if form == root + ending.string + end:
                                            this_tag = tag
                                            break
                                            # ech... uwielbiam zgadywać błędy
                                    if (bfl in (
                                        '4', '5', "6'", '10', '11', '11pg',
                                        '12') and
                                        (this_tag == ('verb', 'irreg') or
                                                this_tag[1] == 'ger')):
                                        extra_base_forms.append(
                                            (root + ending.string, label))
                                        break
                        else: # jeśli to nie była niewykryta forma bazowa
                            for label, tag, end in v_base_forms:
                                if label == bfl:
                                    bad_forms.add(root + ending.string + end)
                                    break
                    else: # inne części mowy nie mają istotnych sufiksów?
                        bad_forms.add(root + ending.string)
        if not bad_forms:
            if extra_base_forms:
                extra['base_forms'] += extra_base_forms
                base_forms_changed = True
            included_patterns.add((pattern, root))
            if including:
                matching_patterns.add((pattern, root))
        elif including:
            including_patterns.add(((pattern, root), tuple(bad_forms)))

    if base_forms_changed:
        #print extra['base_forms']
        return find_patterns(lexical_class, forms, **extra)
        # nie wiem, czy to potrzebne, ale na wszelki wypadek
    included_patterns = list(included_patterns)
    including_patterns = list(including_patterns)
    matching_patterns = list(matching_patterns)
    if len(matching_patterns) > 0:
        if DEBUG:
            print u'dokładne wzory: %s' % join(matching_patterns)
        return 'match', matching_patterns, included_patterns, including_patterns
        # nic nie pasuje albo trzeba wybrać wiele wzorów
    if DEBUG and len(including_patterns) > 0:
        print u'zawierające: %s' % join(p for p, b_f in including_patterns)
    if DEBUG and len(included_patterns) > 0:
        print u'zawarte: %s' % join(included_patterns)
    return find_many_patterns(
        lexical_class, form_set, basic_form, included_patterns, ending_sets,
        **extra) + (included_patterns, including_patterns)


def find_many_patterns(lexical_class, form_set, basic_form, included_patterns,
                       ending_sets, **extra):
    necessary_patterns = set()
    missing_form = None
    for form in form_set:
        having = []
        for pattern, root in included_patterns:
            if form in ending_sets[pattern]:
                having.append((pattern, root))
        if len(having) == 1:
            necessary_patterns.add(having[0])
        if not having:
            missing_form = form
            break
    if missing_form:
        if DEBUG:
            print u"brak formy: %s" % missing_form
        return 'none', []
    covered_forms = set()
    for pattern, root in necessary_patterns:
        covered_forms |= ending_sets[pattern]
    if form_set.issubset(covered_forms):
        if DEBUG:
            print u"pokryte koniecznymi wzorami: %s" % join(necessary_patterns)
        return 'many', [list(necessary_patterns)]
    else:
        #for pattern, root in included_patterns:
        #  print pattern, ending_sets[pattern]
        minimal_sets = find_minimal_sets(
            form_set, covered_forms, necessary_patterns, included_patterns,
            ending_sets)
        return 'many', minimal_sets


def check_sgjp_v(entry, ic, patterns, base_forms, derived):
    lexemes = Lexeme.objects.distinct().filter(
        entry=entry, part_of_speech__symbol='v')
    lexemes = lexemes.filter(pk__in=sgjp)
    for lexeme in lexemes:
        sgjp_patterns = set()
        for lip in lexeme.lexemeinflectionpattern_set.all():
            sgjp_ic = lip.inflection_characteristic.symbol
            # TODO co z q*?
            if sgjp_ic != ic and ic in ('dk', 'ndk'):
                break
            if sgjp_ic in ('dk', 'ndk') and ic not in ('dk', 'ndk'):
                break
            sgjp_patterns.add((lip.pattern, lip.root))
        else:
            if set(patterns) == sgjp_patterns:
                sgjp_derived = {}
                for pos in ('ger', 'pact', 'ppas'):
                    derived_part = lexeme.refs_to.filter(type='ver' + pos)
                    if pos == 'ppas':
                        derived_part = derived_part
                    if derived_part:
                        sgjp_derived[pos] = [der.to_lexeme for der in
                            derived_part]
                if derived == set(sgjp_derived):
                    return lexeme, sgjp_derived
    return False, None


def check_sgjp(lc_sym, entry, form_set, forms, **extra):
    if lc_sym == 'v':
        return False
    if lc_sym != 'adj':
        lexemes = Lexeme.objects.distinct().filter(
            entry=entry, part_of_speech__lexical_class__symbol=lc_sym)
    else:
        if forms[0][1][-1] == 'comp':
            parts_of_speech = ['adjcom']
        else:
            parts_of_speech = ['adj', 'appas']
        lexemes = Lexeme.objects.distinct().filter(
            entry=entry, part_of_speech__symbol__in=parts_of_speech)
    lexemes = lexemes.filter(pk__in=sgjp)
    matched_lexemes = []
    for lexeme in lexemes:
        if lc_sym == 'adj' and lexeme.refs_to.filter(type='nieadj'):
            continue
        if lc_sym == 'subst' and extra['tantum'] == 'sg':
            sgjp_forms = lexeme.all_forms(label_filter=r'sg:')
        elif lexeme.part_of_speech.symbol == 'appas':
            sgjp_forms = lexeme.all_forms()
        else:
            sgjp_forms = lexeme.all_forms()
        if sgjp_forms == form_set:
            matched_lexemes.append(lexeme)
            continue
        diff = sgjp_forms - form_set
        exceptions = []
        if lc_sym == 'subst':
            if lexeme.lexemeinflectionpattern_set.filter(
                    inflection_characteristic__symbol__in=(
                        'm1', 'p1')).exists():
                # depr
                exceptions = lexeme.all_forms(label_filter=r'^pl:nom$')
        elif lc_sym == 'adj':
            # -o
            exceptions = lexeme.all_forms(label_filter=r'^0$')
        if form_set.issubset(sgjp_forms) and diff.issubset(exceptions):
            matched_lexemes.append(lexeme)
    if len(matched_lexemes) > 1:
        if lc_sym == 'subst' and entry.endswith(u'ość'):
            matched_lexemes_subst = [
                l for l in matched_lexemes if
                l.part_of_speech.symbol == 'subst']
            if matched_lexemes_subst:
                matched_lexemes = matched_lexemes_subst
        if len(matched_lexemes) > 1:
            debug(entry, u'niejednoznaczność dopasowanych leksemów')
    if len(matched_lexemes) > 0:
        return matched_lexemes[0]
    return False

# dla przymiotników
def get_negation(lc_sym, lexeme):
    negations = [cf.to_lexeme for cf in CrossReference.objects.filter(
        from_lexeme=lexeme, type='adjnie')]
    good_negation = None
    for n in negations:
        if n.entry == u'nie' + lexeme.entry:
            if all(lip.inflection_characteristic.symbol == '0-'
                for lip in n.lexemeinflectionpattern_set.all()):
                good_negation = n
    return good_negation


def closest_lexeme_subst(entry, gender, patterns, included=None):
    lexemes = Lexeme.objects.filter(
        part_of_speech__lexical_class__symbol='subst')
    lexemes = lexemes.distinct()
    # ten sam rodzaj
    gender = gender[0] if gender != 'm1' else 'm1'
    if genders[gender]:
        lexemes = lexemes.filter(
            lexemeinflectionpattern__inflection_characteristic__symbol__in=
            genders[gender])
    if not included:
        # posiada wzór zawierający się w pasujących
        lexemes = lexemes.filter(lexemeinflectionpattern__pattern__in=patterns)
    else:
        #print patterns, included
        new_lexemes = Lexeme.objects.none()
        # posiada wszystkie wzory z któregoś zestawu
        for pattern_set in patterns:
            part = lexemes
            for pattern, root in pattern_set:
                part = part.filter(lexemeinflectionpattern__pattern=pattern)
            new_lexemes = new_lexemes | part
        lexemes = new_lexemes.distinct()
        # nie posiada wzorów niezawierających się w pasujących, dobra wielkość
    uppercase = entry[0].isupper()
    good_lexemes = []
    for lexeme in lexemes:
        if lexeme.entry[0].isupper():
            for lip in lexeme.lexemeinflectionpattern_set.all():
                if not included:
                    if lip.pattern not in patterns:
                        break
                else:
                    if lip.pattern not in included:
                        break
            else:
                good_lexemes.append(lexeme)
                # najdłuższe wspólne zakończenie
    best = (-1, None)
    for lexeme in good_lexemes:
        common_suffix = 0
        for char1, char2 in zip(entry[::-1], lexeme.entry[::-1]):
            if char1 == char2:
                common_suffix += 1
            else:
                break
        if common_suffix > best[0]:
            best = (common_suffix, lexeme)
    return best[1]


def get_inflection_characteristic_subst(forms, patterns, gender, ic, tantum):
    if tantum == 'sg' and ic == 'm':
        ic = 'm3'
    elif ic in ('n', 'i'): # ech, to 'i' - nadal możliwe?
        ic = 'n2'
    elif ic == 'm':
        ic = 'm3'
        sg_acc = []
        for form in forms:
            if len(form[1]) >= 3 and form[1][1:3] == ('sg', 'acc'):
                sg_acc.append(form[0])
        if len(sg_acc) == 1:
            if sg_acc[0] != forms[0][0]:
                ic = 'm2'
    return ic


def get_inflection_characteristic_adj(forms, patterns, **extra):
    # charfl: 3+ jeśli jest adjp
    ic = '0-'
    for form in forms:
        if form[1][0] == 'adjp':
            ic = '3+'
    if patterns[0][0].endings.filter(
            base_form_label__symbol='0') and ic != '3+':
        ic = ''
    return ic


def get_inflection_characteristic_v(forms, base_forms, **extra):
    # dk, ndk, ndk/dk
    # na podstawie istnienia -ąc (ndk), -szy (dk)
    ndk = dk = nonq = False
    for form in forms:
        if form[0].endswith(u'ąc') and not ':'.join(form[1]).startswith(
                'verb:inf'):
            ndk = True
        elif form[0].endswith(u'wszy') or form[0].endswith(u'łszy'):
            dk = True
        elif ':'.join(form[1]).startswith('verb:praet:sg:pri'):
            nonq = True
    if not dk and not ndk: # mniej wiarygodne - czy mdlić może być dk?
        for form in forms:
            if form[1][1] == 'inf':
                if form[1][-1] == 'imperf':
                    ndk = True
                elif form[1][-1] == 'perf':
                    dk = True
                break
    if ndk and not dk:
        ic = 'ndk'
    elif dk and not ndk:
        ic = 'dk'
    elif ndk and dk:
        ic = 'ndk/dk'
    else:
        debug(forms[0][0], u'nie udało się ustalić aspektu')
        ic = None
    if ic and not nonq:
        ic = 'q' + ic
    return ic


def get_inflection_characteristic(lc_sym, forms, patterns, **extra):
    if lc_sym == 'subst':
        return get_inflection_characteristic_subst(forms, patterns, **extra)
    elif lc_sym == 'adj':
        return get_inflection_characteristic_adj(forms, patterns, **extra)
    elif lc_sym == 'v':
        return get_inflection_characteristic_v(forms, **extra)


def blacklist_filter(patterns):
    return [(pattern, root) for (pattern, root) in patterns
        if pattern.name not in blacklist]


def filter_patterns(filter, action_name, type, patterns, included, including,
                    lexical_class, form_set, entry, **extra):
    old_patterns = patterns
    old_included = included
    bad_patterns = False
    if type == 'many':
        if any(pattern_set != filter(pattern_set) for pattern_set in patterns):
            included = filter(included)
            ending_sets = {}
            for pattern, root in included:
                ending_sets[pattern] = good_ending_set(
                    lexical_class, pattern, root, **extra)
            type, patterns = find_many_patterns(
                lexical_class, form_set, entry, included, ending_sets, **extra)
            if type != 'many':
                debug(entry, u'mnogie dopasowanie zepsute przez %s (%s)' %
                             (action_name, join_many(old_patterns)))
                type = 'many'
                patterns, included = old_patterns, old_included
                bad_patterns = True
    elif type == 'none':
        including_dict = dict(including)
        including = [(key, including_dict[key]) for key in
            filter(including_dict)]
    else: # type == 'match'
        patterns = filter(patterns)
        including_dict = dict(including)
        including = [(key, including_dict[key]) for key in
            filter(including_dict)]
        included = filter(included)
        if old_patterns and not patterns:
            ending_sets = {}
            for pattern, root in included:
                ending_sets[pattern] = good_ending_set(
                    lexical_class, pattern, root, **extra)
            type, patterns = find_many_patterns(
                lexical_class, form_set, entry, included, ending_sets, **extra)
            if type == 'none':
                debug(entry, u'znikły wzory przez %s (%s)' %
                             (action_name, join(old_patterns)))
                type = 'match'
                patterns = old_patterns
                bad_patterns = True
    return type, patterns, included, including, bad_patterns


def create_derived(pos, base_forms, forms, patterns):
    tab = {'ger': ('11', u'ie'), 'pact': ('3', u'cy'), 'ppas': ('10', u'y')}
    entries = GroupDict()
    for pattern, root in patterns:
        bfl = tab[pos][0]
        ending = pattern.endings.get(base_form_label__symbol=bfl)
        entry = root + ending.string + tab[pos][1]
        entries.add(entry, pattern.name)
    output = []
    for entry, patterns in entries.iteritems():
        if entry in (form[0] for form in forms):
            output.append((pos, entry, patterns))
    return output


def get_sgjp(lexeme):
    return {'source': 'sgjp', 'id': lexeme.pk, 'entry': lexeme.entry}


def create_lexeme(entry, part_of_speech, status, comment):
    return {
        'source': 'morfologik',
        'entry': entry,
        'part_of_speech': part_of_speech,
        'status': status,
        'comment': comment,
    }


def create_lip(pattern, root, i, ic, part_of_speech):
    output = {
        'pattern': pattern if isinstance(pattern, basestring) else pattern.name,
        'ind': i,
        'ic': (ic, part_of_speech),
    }
    if root:
        output['root'] = {'type': 'string', 'root': root}
    else:
        output['root'] = {'type': 'compute'}
    return output


alternative_gender = {
    'p1': 'p3',
    'p3': 'p1',
    'm1': 'm2/m3',
    'm': 'm1',
}

alternative_gender2 = {
    'p1': 'p3',
    'p3': 'p1',
    'm1': 'm',
    'm': 'm1',
}


def lexeme_creation(lc_sym, entry, ic, forms, type, patterns, fitting,
                    bad_patterns, included, other_result, tantum=None,
                    gender=None, negated=None, base_forms=None, derived=None):
    status = 'desc' if type != 'none' else 'cand'
    comments = []
    copy_lips = False
    if lc_sym == 'subst':
        part_of_speech = 'subst' # co z osc i skrs?
        if ic in ('m2', 'm3'):
            sure = False
            if type != 'none' and len(fitting) == 1:
                for pattern, root in patterns:
                    for e in patterns[0][0].endings.filter(
                            base_form_label__symbol='sg:gen'):
                        if not e.string.endswith('u'):
                            break
                    else:
                        continue
                    break
                else: # wszystkie sg:gen kończą się na 'u'
                    ic = 'm3'
                    sure = True
                for pattern, root in patterns:
                    if pattern.type.symbol == 'f':
                        ic = 'm2'
                        sure = True
                        break
            if not sure:
                status = 'cand'
        if tantum is None and ic == 'm1':
            for pattern, root in patterns:
                nmo_endings = pattern.endings.filter(
                    base_form_label__symbol='pl:nom')
                for e in nmo_endings:
                    nmo_form = root + e.string
                    if nmo_form not in (form[0] for form in forms):
                        comments.append(u'Dodano formę depr')
                        break
                else:
                    continue
                break
        if ic == 'p1':
            for pattern, root in patterns:
                nmo_endings = pattern.endings.filter(
                    base_form_label__symbol='pl:nom')
                other_endings = pattern.endings.exclude(
                    base_form_label__symbol='pl:nom')
                other_strings = other_endings.values_list('string', flat=True)
                nmo_strings = [e.string for e in nmo_endings if
                    e.string not in other_strings]
                nmo_forms = set(root + s for s in nmo_strings)
                if nmo_forms & set(form[0] for form in forms):
                    comments.append(
                        u'Usunięto formę depr: %s' % ', '.join(list(nmo_forms)))
                    break
        if tantum == 'sg' and type != 'none':
            if type == 'match':
                search_patterns = [pattern for pattern, root in fitting]
                l = closest_lexeme_subst(entry, gender, search_patterns)
            else:
                included_patterns = [pattern for pattern, root in included]
                l = closest_lexeme_subst(entry, gender, fitting,
                    included_patterns)
            if l:
                copy_lips = l.lexemeinflectionpattern_set.all()
                #print l
                comments.append(u'Automatycznie rozszerzone singulare tantum')
            else:
                if type == 'match':
                    p = join(fitting)
                else:
                    p = join_many(fitting)
                debug(entry,
                    u'nie ma pasujących leksemów dla rozszerzenia sgtant '
                    u'dla wzorów %s' % p)
                comments.append(u'Nie udało się rozszerzyć singulare tantum')
                #status = 'cand'
                # dodać kwalifikator [po imporcie jednak]
    elif lc_sym == 'adj':
        if forms[0][1][-1] == 'comp':
            part_of_speech = 'adjcom'
            ic = ''
        else:
            part_of_speech = 'adj'
            if ic != '0-' and type != 'none':
                comments.append(u'Dodano formę -o')
    else: # lc_sym == 'v':
        part_of_speech = 'v'
        if type != 'none':
            if ic is None:
                comments.append(u'Nie udało się ustalić aspektu')
                ic = 'ndk'
                status = 'cand'
            elif ic.startswith('q') and derived != set():
                comments.append(u'Derywaty muszą mieć charfle bez q')
    if bad_patterns:
        comments.append(u'Wzory z czarnej listy!')
        status = 'cand'
    if len(fitting) > 1 or (type == 'none' and fitting):
        status = 'cand'
        if type == 'none':
            comments.append(u'Zawierające wzory:')
            for (pattern, root), bad_forms in fitting:
                comments.append('%s: %s' % (pattern.name, ', '.join(bad_forms)))
        elif type != 'many':
            comments.append(u'Pasujące wzory: %s' % join(fitting))
        else:
            comments.append(u'Pasujące zestawy wzorów: %s' % join_many(fitting))
    if other_result:
        status = 'cand'
        type2, patterns2, included2, including2 = other_result
        comments.append(u'Alternatywny rodzaj: %s' % alternative_gender[gender])
        if type2 == 'match':
            comments.append(u'Pasujące wzory: %s' % join(patterns2))
        elif type2 == 'many':
            comments.append(
                u'Pasujące zestawy wzorów: %s' % join_many(patterns2))
            # hm?
    if ic is None and type != 'none':
        comments.append(u'Dopasowane wzory: %s' % join(patterns))
        # zbieramy wzory do porównania [fuj, copypasta!]
    if len(fitting) > 1:
        if type == 'none':
            all_patterns = set(p for p, b_f in fitting)
        elif type != 'many':
            all_patterns = set(fitting)
        else:
            all_patterns = set()
            for pattern_set in fitting:
                all_patterns |= set(pattern_set)
        diff = compare_patterns(list(all_patterns))
        comments.append(u'Porównanie wzorów:')
        for p, diff_forms in diff.iteritems():
            comments.append(
                '%s: %s' % (
                    p.name, ', '.join('%s: %s' % pair for pair in diff_forms)))
    if other_result and len(patterns2) > 1:
        if type2 != 'many':
            other_patterns = set(patterns2)
        else:
            other_patterns = set()
            for pattern_set in patterns2:
                other_patterns |= set(pattern_set)
        diff = compare_patterns(list(other_patterns))
        comments.append(u'Porównanie alternatywnych wzorów:')
        for p, diff_forms in diff.iteritems():
            comments.append(
                '%s: %s' % (
                    p.name, ', '.join('%s: %s' % pair for pair in diff_forms)))
    comment = '\n'.join(comments)
    output = {
        'lexeme': create_lexeme(entry, part_of_speech, status, comment)
    }
    lips = []
    if ic is not None:
        if not copy_lips:
            for i, (pattern, root) in enumerate(patterns):
                lips.append(
                    create_lip(pattern, root, i + 1, ic, part_of_speech))
        else:
            for lip in copy_lips:
                ic = lip.inflection_characteristic.symbol
                lips.append(
                    create_lip(lip.pattern, None, lip.index, ic,
                        part_of_speech))
    output['lips'] = lips
    if lc_sym == 'adj' and negated:
        output['negated'] = True
    if lc_sym == 'v':
        derived_data = []
        for pos in derived:
            # wypadałoby informować, jeśli wyszło puste... (?)
            derived_data += create_derived(pos, base_forms, forms, patterns)
        output['derived'] = derived_data
    return output


def process_forms(lexical_class, forms, **extra):
    lc_sym = lexical_class.symbol
    other_result = None
    entry = forms[0][0]
    form_set = get_form_set(forms)
    check = check_sgjp(lc_sym, entry, form_set, forms, **extra)
    if check and not DEBUG:
        # dopisz leksem do słownika
        data = {'lexeme': get_sgjp(check)}
        if (lc_sym == 'adj' and extra['negated']):
            neg = get_negation(lc_sym, check)
            if neg:
                print_data({'lexeme': get_sgjp(neg)})
            else:
                data['negated'] = True
        print_data(data)
    else:
        if lexical_class.symbol == 'subst':
            ic = extra['gender']
        extra2 = dict(extra)
        # jeśli rzeczownik męski lub pltant, to puszczamy więcej razy
        if lexical_class.symbol == 'subst' and extra['gender'] in 'pm':
            if extra['gender'] == 'm':
                type2, patterns2, included2, including2 = find_patterns(
                    lexical_class, forms, **extra)
                extra2['gender'] = 'm1'
                type1, patterns1, included1, including1 = find_patterns(
                    lexical_class, forms, **extra2)
            elif extra['gender'] == 'p':
                extra2['gender'] = 'p1'
                type1, patterns1, included1, including1 = find_patterns(
                    lexical_class, forms, **extra2)
                extra2['gender'] = 'p3'
                type2, patterns2, included2, including2 = find_patterns(
                    lexical_class, forms, **extra2)
            if type1 != 'none' and type2 == 'none':
                type, patterns, included, including = type1, patterns1, included1, including1
                if extra['gender'] == 'm':
                    ic = 'm1'
                else:
                    ic = 'p1'
            elif type1 == 'none' and type2 != 'none':
                type, patterns, included, including = type2, patterns2, included2, including2
                if extra['gender'] == 'm':
                    ic = 'm'
                else:
                    ic = 'p3'
            elif type1 == type2 == 'none':
                type = 'none'
                patterns = []
                included = list(set(included1) | set(included2))
                including = list(set(including1) | set(including2))
                # chyba warto coś ustawić
                if extra['gender'] == 'm':
                    ic = 'm'
                else:
                    ic = 'p3'
            else: # z obu coś wyszło
                type, patterns, included, including = type1, patterns1, included1, including1
                if extra['gender'] == 'm':
                    ic = 'm1'
                else:
                    ic = 'p1'
                other_result = (type2, patterns2, included2, including2)
                if DEBUG:
                    print u"dwie możliwości na rodzaj"
        else:
            type, patterns, included, including = find_patterns(
                lexical_class, forms, **extra)
        if type == 'none':
            if lexical_class.symbol == 'subst' and not extra['tantum']:
                extra['tantum'] = tantum_a_posteriori(
                    form_set, [p for p, b_f in including])
                if extra['tantum']:
                    if extra['tantum'] == 'pl':
                        extra['gender'] = 'p'
                        #if ic == 'm1':
                        #  extra['gender'] = 'p1'
                        #else:
                        #  extra['gender'] = 'p3'
                    return process_forms(lexical_class, forms, **extra)

        if lexical_class.symbol == 'subst':
            extra2['gender'] = ic
        type, patterns, included, including, bad_patterns = filter_patterns(
            blacklist_filter, u'czarną listę', type, patterns, included,
            including,
            lexical_class, form_set, entry, **extra2)
        if bad_patterns and other_result:
            type, patterns, included, including = other_result
            ic = extra2['gender'] = alternative_gender2[ic]
            type, patterns, included, including, bad_patterns = filter_patterns(
                blacklist_filter, u'czarną listę', type, patterns, included,
                including,
                lexical_class, form_set, entry, **extra2)
            other_result = None
        elif other_result:
            type2, patterns2, included2, including2 = other_result
            new_other_result = filter_patterns(
                blacklist_filter, u'czarną listę', type2, patterns2, included2,
                including2, lexical_class, form_set, entry, **extra2)
            if not new_other_result[4]:
                other_result = new_other_result[:4]
            else:
                other_result = None

        # wzory się już nie zmienią od tego miejsca
        if type == 'many':
            # albo patterns[0]...
            all_patterns = [p for pattern_set in patterns for p in pattern_set]
        else:
            all_patterns = patterns
        if type != 'none':
            # brzydko...
            if lexical_class.symbol == 'subst':
                extra['ic'] = ic
            ic = get_inflection_characteristic(lc_sym, forms, all_patterns,
                **extra)
            if lexical_class.symbol == 'subst':
                del extra['ic']
        else:
            ic = None
        if lc_sym == 'subst':
            # poprawka dla m2/m3
            if ic in ('m2', 'm3') and patterns and not bad_patterns:
                new_ic = ''
                for pattern, root in all_patterns:
                    for ic2 in ('m2', 'm3'):
                        # jeśli wszystkie użycia tego wzoru są przy ic2
                        if not pattern.lexemeinflectionpattern_set.exclude(
                                inflection_characteristic__symbol=ic2).filter(
                                lexeme__pk__in=sgjp).exists():
                            if new_ic == '':
                                new_ic = ic2
                            elif new_ic != ic2:
                                new_ic = None
                if new_ic:
                    ic = new_ic

        if type == 'none':
            debug(entry, u'zawiera się w %s' % join(p for p, b_f in including))
            chosen = []
            fitting = including
            if lexical_class.symbol == 'adj' and including:
                print_forms(forms, 'rzeczownik#')
                return
        elif type == 'match':
            patterns.sort(key=lambda p: p[0].name)
            fitting = patterns
            chosen = patterns[:1]
        elif type == 'many':
            chosen = patterns[0]
            if DEBUG:
                print u'zestawy wielu wzorów: %s' % join_many(patterns)
            fitting = patterns
        if not DEBUG and lc_sym == 'v':
            check, sgjp_derived = check_sgjp_v(entry, ic, chosen, **extra)
            if check:
                print_data({'lexeme': get_sgjp(check)})
                for derived_lexemes in sgjp_derived.values():
                    for derived_lexeme in derived_lexemes:
                        print_data({'lexeme': get_sgjp(derived_lexeme)})
                return
        if not DEBUG:
            data = lexeme_creation(
                lc_sym, entry, ic, forms, type, chosen, fitting, bad_patterns,
                included,
                other_result, **extra2)
            print_data(data)


def get_pos_ndm(tag):
    if tag[0] == 'adv':
        return 'adv' if tag[-1] != 'comp' else 'advcom'
    elif tag[0] == 'xxx':
        return 'burk'
    else:
        return tag[0]


def process_ndm(forms):
    entry = forms[0][0]
    form_set = get_form_set(forms)
    pattern_name = 'ndm'
    ics = ['']
    tag = forms[0][1].split(':')
    pos = get_pos_ndm(tag)
    status = 'desc'
    comments = []
    if len(form_set) > 2:
        debug(entry, u'nieodmienne z ponad dwiema formami')
        comments.append(u'nieodmienne z ponad dwiema formami')
        status = 'cand'
        ics = []
    elif pos == 'prep':
        if len(tag) == 1:
            debug(entry, u'przyimek bez przypadku')
            comments.append(u'przyimek bez przypadku')
            status = 'cand'
            ics = []
        else:
            ics = tag[1].split('.')
        if len(form_set) == 2:
            if entry + u'e' == forms[1][0]:
                pattern_name = 'ndm e'
            else:
                debug(entry, u'nierozpoznana odmiana przyimka')
                comments.append(u'nierozpoznana odmiana przyimka')
                status = 'cand'
    elif len(form_set) == 2:
        if pos == 'adv' and forms[1][0] == u'nie' + entry:
            process_ndm(forms[:1])
            process_ndm(forms[1:])
            return
        else:
            debug(entry, u'nierozpoznane nieodmienne z dwiema formami')
            comments.append(u'nierozpoznane nieodmienne z dwiema formami')
            status = 'cand'
            # sprawdzić czy to co wyszło jest już w sgjp
    lexemes = Lexeme.objects.distinct().filter(entry=entry)
    lexemes = lexemes.filter(pk__in=sgjp)
    if pos == 'adv':
        lexemes = lexemes.filter(part_of_speech__symbol__in=('adv', 'advndm'))
    elif pos == 'conj':
        lexemes = lexemes.filter(part_of_speech__symbol__in=('conj', 'comp'))
    else:
        lexemes = lexemes.filter(part_of_speech__symbol=pos)
    in_sgjp = bool(lexemes)
    for lexeme in lexemes:
        sgjp_ics = set()
        for lip in lexeme.lexemeinflectionpattern_set.all():
            if lip.pattern.name != pattern_name:
                in_sgjp = False
                break
            sgjp_ics.add(lip.inflection_characteristic.symbol)
        if set(ics) != sgjp_ics:
            in_sgjp = False
        if in_sgjp:
            break
    if in_sgjp:
        print_data({'lexeme': get_sgjp(lexeme)})
    else:
        data = {
            'lexeme': create_lexeme(
                entry, pos, status, '\n'.join(comments))
        }
        lips = []
        for i, ic in enumerate(ics):
            lips.append(create_lip(pattern_name, entry, i + 1, ic, pos))
        data['lips'] = lips
        print_data(data)


def print_data(data):
    print json.dumps(data)


def print_forms(forms, prefix):
    for form, tag in forms:
        try:
            end = tag.index('pos')
        except ValueError:
            end = tag.index('comp')
        new_tag = ':'.join(tag[:end]).replace('adj', 'subst')
        print>> sys.stderr, ('%s%s\t%s' % (prefix, form, new_tag)).encode(
            'utf-8')
    print>> sys.stderr, prefix


def parse_file(lc_sym, path):
    lexical_class = LexicalClass.objects.get(symbol=lc_sym)
    forms = []
    for line in uniopen(path):
        line = line.decode('utf-8').rstrip('\n')
        if line == '':
            if lc_sym == 'ndm':
                process_ndm(forms)
            else:
                forms = expand_forms(forms)
                extra = get_extra(lexical_class, forms)
                process_forms(lexical_class, forms, **extra)
            forms = []
        else:
            form, tag = line.split('\t')
            if DEBUG and forms == []:
                print form
            forms.append((form, tag))