cothec / Corneferencer

import math
import numpy
import random
import re

import conf
from corneferencer.resolvers import constants


# mention features
def head_vec(mention):
    head_base = mention.head_orth
    if mention.head is not None:
        head_base = mention.head['base']
    return list(get_wv(conf.W2V_MODEL, head_base))


def first_word_vec(mention):
    return list(get_wv(conf.W2V_MODEL, mention.words[0]['base']))


def last_word_vec(mention):
    return list(get_wv(conf.W2V_MODEL, mention.words[-1]['base']))


def first_after_vec(mention):
    if len(mention.follow_context) > 0:
        vec = list(get_wv(conf.W2V_MODEL, mention.follow_context[0]['base']))
    else:
        vec = [0.0] * conf.W2V_SIZE
    return vec


def second_after_vec(mention):
    if len(mention.follow_context) > 1:
        vec = list(get_wv(conf.W2V_MODEL, mention.follow_context[1]['base']))
    else:
        vec = [0.0] * conf.W2V_SIZE
    return vec


def first_before_vec(mention):
    if len(mention.prec_context) > 0:
        vec = list(get_wv(conf.W2V_MODEL, mention.prec_context[-1]['base']))
    else:
        vec = [0.0] * conf.W2V_SIZE
    return vec


def second_before_vec(mention):
    if len(mention.prec_context) > 1:
        vec = list(get_wv(conf.W2V_MODEL, mention.prec_context[-2]['base']))
    else:
        vec = [0.0] * conf.W2V_SIZE
    return vec


def preceding_context_vec(mention):
    return list(get_context_vec(mention.prec_context, conf.W2V_MODEL))


def following_context_vec(mention):
    return list(get_context_vec(mention.follow_context, conf.W2V_MODEL))


def mention_vec(mention):
    return list(get_context_vec(mention.words, conf.W2V_MODEL))


def sentence_vec(mention):
    return list(get_context_vec(mention.sentence, conf.W2V_MODEL))


def mention_type(mention):
    type_vec = [0.0] * 4
    if mention.head is None:
        type_vec[3] = 1.0
    elif mention.head['ctag'] in constants.NOUN_TAGS:
        type_vec[0] = 1.0
    elif mention.head['ctag'] in constants.PPRON_TAGS:
        type_vec[1] = 1.0
    elif mention.head['ctag'] in constants.ZERO_TAGS:
        type_vec[2] = 1.0
    else:
        type_vec[3] = 1.0
    return type_vec


def is_first_second_person(mention):
    if mention.head is None:
        return 0.0
    if mention.head['person'] in constants.FIRST_SECOND_PERSON:
        return 1.0
    return 0.0


def is_demonstrative(mention):
    if mention.words[0]['base'].lower() in constants.INDICATIVE_PRONS_BASES:
        return 1.0
    return 0.0


def is_demonstrative_nominal(mention):
    if mention.head is None:
        return 0.0
    if is_demonstrative(mention) and mention.head['ctag'] in constants.NOUN_TAGS:
        return 1.0
    return 0.0


def is_demonstrative_pronoun(mention):
    if mention.head is None:
        return 0.0
    if (is_demonstrative(mention) and
            (mention.head['ctag'] in constants.PPRON_TAGS or mention.head['ctag'] in constants.ZERO_TAGS)):
        return 1.0
    return 0.0


def is_refl_pronoun(mention):
    if mention.head is None:
        return 0.0
    if mention.head['ctag'] in constants.SIEBIE_TAGS:
        return 1.0
    return 0.0


def is_first_in_sentence(mention):
    if mention.first_in_sentence:
        return 1.0
    return 0.0


def is_zero_or_pronoun(mention):
    if mention.head is None:
        return 0.0
    if mention.head['ctag'] in constants.PPRON_TAGS or mention.head['ctag'] in constants.ZERO_TAGS:
        return 1.0
    return 0.0


def head_contains_digit(mention):
    _digits = re.compile('\d')
    if _digits.search(mention.head_orth):
        return 1.0
    return 0.0


def mention_contains_digit(mention):
    _digits = re.compile('\d')
    if _digits.search(mention.text):
        return 1.0
    return 0.0


def contains_letter(mention):
    if any(c.isalpha() for c in mention.text):
        return 1.0
    return 0.0


def post_modified(mention):
    if mention.head_orth != mention.words[-1]['orth']:
        return 1.0
    return 0.0


# pair features
def distances_vec(ante, ana):
    vec = []

    mnts_intersect = pair_intersect(ante, ana)

    words_dist = [0.0] * 11
    words_bucket = 0
    if mnts_intersect != 1.0:
        words_bucket = get_distance_bucket(ana.start_in_words - ante.end_in_words)
    words_dist[words_bucket] = 1.0
    vec.extend(words_dist)

    mentions_dist = [0.0] * 11
    mentions_bucket = 0
    if mnts_intersect != 1.0:
        mentions_bucket = get_distance_bucket(ana.position_in_mentions - ante.position_in_mentions)
    if words_bucket == 10:
        mentions_bucket = 10
    mentions_dist[mentions_bucket] = 1.0
    vec.extend(mentions_dist)

    vec.append(mnts_intersect)

    return vec


def pair_intersect(ante, ana):
    for ante_word in ante.words:
        for ana_word in ana.words:
            if ana_word['id'] == ante_word['id']:
                return 1.0
    return 0.0


def head_match(ante, ana):
    if ante.head_orth.lower() == ana.head_orth.lower():
        return 1.0
    return 0.0


def exact_match(ante, ana):
    if ante.text.lower() == ana.text.lower():
        return 1.0
    return 0.0


def base_match(ante, ana):
    if ante.lemmatized_text.lower() == ana.lemmatized_text.lower():
        return 1.0
    return 0.0


def ante_contains_rarest_from_ana(ante, ana):
    ana_rarest = ana.rarest
    for word in ante.words:
        if word['base'] == ana_rarest['base']:
            return 1.0
    return 0.0


def agreement(ante, ana, tag_name):
    agr_vec = [0.0] * 3
    if (ante.head is None or ana.head is None or
            ante.head[tag_name] == 'unk' or ana.head[tag_name] == 'unk'):
        agr_vec[2] = 1.0
    elif ante.head[tag_name] == ana.head[tag_name]:
        agr_vec[0] = 1.0
    else:
        agr_vec[1] = 1.0
    return agr_vec


def is_acronym(ante, ana):
    if ana.text.upper() == ana.text:
        return check_one_way_acronym(ana.text, ante.text)
    if ante.text.upper() == ante.text:
        return check_one_way_acronym(ante.text, ana.text)
    return 0.0


def same_sentence(ante, ana):
    if ante.sentence_id == ana.sentence_id:
        return 1.0
    return 0.0


def neighbouring_sentence(ante, ana):
    if ana.sentence_id - ante.sentence_id == 1:
        return 1.0
    return 0.0


def cousin_sentence(ante, ana):
    if ana.sentence_id - ante.sentence_id == 2:
        return 1.0
    return 0.0


def distant_sentence(ante, ana):
    if ana.sentence_id - ante.sentence_id > 2:
        return 1.0
    return 0.0


def same_paragraph(ante, ana):
    if ante.paragraph_id == ana.paragraph_id:
        return 1.0
    return 0.0


def flat_gender_agreement(ante, ana):
    agr_vec = [0.0] * 3
    if (ante.head is None or ana.head is None or
            ante.head['gender'] == 'unk' or ana.head['gender'] == 'unk'):
        agr_vec[2] = 1.0
    elif (ante.head['gender'] == ana.head['gender'] or
            (ante.head['gender'] in constants.MASCULINE_TAGS and ana.head['gender'] in constants.MASCULINE_TAGS)):
        agr_vec[0] = 1.0
    else:
        agr_vec[1] = 1.0
    return agr_vec


def left_match(ante, ana):
    if (ante.text.lower().startswith(ana.text.lower()) or
            ana.text.lower().startswith(ante.text.lower())):
        return 1.0
    return 0.0


def right_match(ante, ana):
    if (ante.text.lower().endswith(ana.text.lower()) or
            ana.text.lower().endswith(ante.text.lower())):
        return 1.0
    return 0.0


def abbrev2(ante, ana):
    ante_abbrev = get_abbrev(ante)
    ana_abbrev = get_abbrev(ana)
    if ante.head_orth == ana_abbrev or ana.head_orth == ante_abbrev:
        return 1.0
    return 0.0


def string_kernel(ante, ana):
    s1 = ante.text
    s2 = ana.text
    return sk(s1, s2) / (math.sqrt(sk(s1, s1) * sk(s2, s2)))


def head_string_kernel(ante, ana):
    s1 = ante.head_orth
    s2 = ana.head_orth
    return sk(s1, s2) / (math.sqrt(sk(s1, s1) * sk(s2, s2)))


def wordnet_synonyms(ante, ana):
    ante_synonyms = set()
    if ante.head is None or ana.head is None:
        return 0.0

    if ante.head['base'] in conf.LEMMA2SYNONYMS:
        ante_synonyms = conf.LEMMA2SYNONYMS[ante.head['base']]

    ana_synonyms = set()
    if ana.head['base'] in conf.LEMMA2SYNONYMS:
        ana_synonyms = conf.LEMMA2SYNONYMS[ana.head['base']]

    if ana.head['base'] in ante_synonyms or ante.head['base'] in ana_synonyms:
        return 1.0
    return 0.0


def wordnet_ana_is_hypernym(ante, ana):
    if ante.head is None or ana.head is None:
        return 0.0

    ante_hypernyms = set()
    if ante.head['base'] in conf.LEMMA2HYPERNYMS:
        ante_hypernyms = conf.LEMMA2HYPERNYMS[ante.head['base']]

    ana_hypernyms = set()
    if ana.head['base'] in conf.LEMMA2HYPERNYMS:
        ana_hypernyms = conf.LEMMA2HYPERNYMS[ana.head['base']]

    if not ante_hypernyms or not ana_hypernyms:
        return 0.0

    if ana.head['base'] in ante_hypernyms:
        return 1.0
    return 0.0


def wordnet_ante_is_hypernym(ante, ana):
    if ante.head is None or ana.head is None:
        return 0.0

    ana_hypernyms = set()
    if ana.head['base'] in conf.LEMMA2HYPERNYMS:
        ana_hypernyms = conf.LEMMA2HYPERNYMS[ana.head['base']]

    ante_hypernyms = set()
    if ante.head['base'] in conf.LEMMA2HYPERNYMS:
        ante_hypernyms = conf.LEMMA2HYPERNYMS[ante.head['base']]

    if not ante_hypernyms or not ana_hypernyms:
        return 0.0

    if ante.head['base'] in ana_hypernyms:
        return 1.0
    return 0.0


def wikipedia_link(ante, ana):
    ante_base = ante.lemmatized_text.lower()
    ana_base = ana.lemmatized_text.lower()
    if ante_base == ana_base:
        return 1.0

    ante_links = set()
    if ante_base in conf.TITLE2LINKS:
        ante_links = conf.TITLE2LINKS[ante_base]

    ana_links = set()
    if ana_base in conf.TITLE2LINKS:
        ana_links = conf.TITLE2LINKS[ana_base]

    if ana_base in ante_links or ante_base in ana_links:
        return 1.0

    return 0.0


def wikipedia_mutual_link(ante, ana):
    ante_base = ante.lemmatized_text.lower()
    ana_base = ana.lemmatized_text.lower()
    if ante_base == ana_base:
        return 1.0

    ante_links = set()
    if ante_base in conf.TITLE2LINKS:
        ante_links = conf.TITLE2LINKS[ante_base]

    ana_links = set()
    if ana_base in conf.TITLE2LINKS:
        ana_links = conf.TITLE2LINKS[ana_base]

    if ana_base in ante_links and ante_base in ana_links:
        return 1.0

    return 0.0


def wikipedia_redirect(ante, ana):
    ante_base = ante.lemmatized_text.lower()
    ana_base = ana.lemmatized_text.lower()
    if ante_base == ana_base:
        return 1.0

    if ante_base in conf.TITLE2REDIRECT and conf.TITLE2REDIRECT[ante_base] == ana_base:
        return 1.0

    if ana_base in conf.TITLE2REDIRECT and conf.TITLE2REDIRECT[ana_base] == ante_base:
        return 1.0

    return 0.0


def samesent_anapron_antefirstinpar(ante, ana):
    if same_sentence(ante, ana) and is_zero_or_pronoun(ana) and ante.first_in_paragraph:
        return 1.0
    return 0.0


def samesent_antefirstinpar_personnumbermatch(ante, ana):
    if (same_sentence(ante, ana) and ante.first_in_paragraph
            and agreement(ante, ana, 'number')[0] and agreement(ante, ana, 'person')[0]):
        return 1.0
    return 0.0


def adjsent_anapron_adjmen_personnumbermatch(ante, ana):
    if (neighbouring_sentence(ante, ana) and is_zero_or_pronoun(ana)
            and ana.position_in_mentions - ante.position_in_mentions == 1
            and agreement(ante, ana, 'number')[0] and agreement(ante, ana, 'person')[0]):
        return 1.0
    return 0.0


def adjsent_anapron_adjmen(ante, ana):
    if (neighbouring_sentence(ante, ana) and is_zero_or_pronoun(ana)
            and ana.position_in_mentions - ante.position_in_mentions == 1):
        return 1.0
    return 0.0


# supporting functions
def get_wv(model, lemma, use_random_vec=True):
    vec = None
    if use_random_vec:
        vec = random_vec()
    try:
        vec = model.wv[lemma]
    except KeyError:
        pass
    except TypeError:
        pass
    return vec


def random_vec():
    return numpy.asarray([random.uniform(-0.25, 0.25) for i in range(0, conf.W2V_SIZE)], dtype=numpy.float32)


def get_context_vec(words, model):
    vec = numpy.zeros(conf.W2V_SIZE, dtype=numpy.float32)
    unknown_count = 0
    if len(words) != 0:
        for word in words:
            word_vec = get_wv(model, word['base'], conf.RANDOM_WORD_VECTORS)
            if word_vec is None:
                unknown_count += 1
            else:
                vec += word_vec
        significant_words = len(words) - unknown_count
        if significant_words != 0:
            vec = vec / float(significant_words)
        else:
            vec = random_vec()
    return vec


def get_distance_bucket(distance):
    if 0 <= distance <= 4:
        return distance
    elif 5 <= distance <= 7:
        return 5
    elif 8 <= distance <= 15:
        return 6
    elif 16 <= distance <= 31:
        return 7
    elif 32 <= distance <= 63:
        return 8
    elif distance >= 64:
        return 9
    return 10


def check_one_way_acronym(acronym, expression):
    initials = u''
    for expr1 in expression.split('-'):
        for expr2 in expr1.split():
            expr2 = expr2.strip()
            if expr2:
                initials += expr2[0].upper()
    if acronym == initials:
        return 1.0
    return 0.0


def get_abbrev(mention):
    abbrev = u''
    for word in mention.words:
        if word['orth'][0].isupper():
            abbrev += word['orth'][0]
    return abbrev


def sk(s1, s2):
    lam = 0.4

    p = len(s1)
    if len(s2) < len(s1):
        p = len(s2)

    h, w = len(s1)+1, len(s2)+1
    dps = [[0.0] * w for i in range(h)]
    dp = [[0.0] * w for i in range(h)]

    kernel_mat = [0.0] * (len(s1) + 1)

    for i in range(len(s1)+1):
        if i == 0:
            continue
        for j in range(len(s2)+1):
            if j == 0:
                continue
            if s1[i-1] == s2[j-1]:
                dps[i][j] = lam * lam
                kernel_mat[0] += dps[i][j]
            else:
                dps[i][j] = 0.0

    for m in range(p):
        if m == 0:
            continue

        kernel_mat[m] = 0.0
        for j in range(len(s2)+1):
            dp[m-1][j] = 0.0

        for i in range(len(s1)+1):
            dp[i][m-1] = 0.0

        for i in range(len(s1)+1):
            if i < m:
                continue
            for j in range(len(s2)+1):
                if j < m:
                    continue
                dp[i][j] = dps[i][j] + lam * dp[i - 1][j] + lam * dp[i][j - 1] - lam * lam * dp[i - 1][j - 1]

                if s1[i-1] == s2[j-1]:
                    dps[i][j] = lam * lam * dp[i - 1][j - 1]
                    kernel_mat[m] += dps[i][j]

    k = 0.0
    for i in range(p):
        k += kernel_mat[i]
    return k