Added new features to feature vector.

Bartłomiej Nitoń
1 parent c2871e0d
Showing 1 changed file with 179 additions and 19 deletions
preparator.py
@@ -14,10 +14,11 @@ from gensim.models.word2vec import Word2Vec
 TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data', 'test-prepared'))
 TRAIN_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data', 'train-prepared'))
+FREQ_300M_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data', 'freq', 'base.lst'))
 ANNO_PATH = TEST_PATH
 OUT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data',
-                                        'test.csv'))
+                                        'test-20170627.csv'))
 EACH_TEXT_SEPARATELLY = False
 CONTEXT = 5
@@ -25,7 +26,12 @@ W2V_SIZE = 50
 MODEL = os.path.abspath(os.path.join(os.path.dirname(__file__), 'models',
                                      '%d' % W2V_SIZE,
                                      'w2v_allwiki_nkjpfull_%d.model' % W2V_SIZE))
+
+NOUN_TAGS = ['subst', 'ger', 'depr']
+PPRON_TAGS = ['ppron12', 'ppron3']
+ZERO_TAGS = ['fin', 'praet', 'bedzie', 'impt', 'winien', 'aglt']
 POSSIBLE_HEADS = [u'§', u'%', u'*', u'"', u'„', u'&', u'-']
+
 NEG_PROPORTION = 1
 RANDOM_VECTORS = True
@@ -38,8 +44,9 @@ UNKNONW_WORDS = 0
 def main():
     model = Word2Vec.load(MODEL)
+    freq_list = load_freq_list(FREQ_300M_PATH)
     try:
-        create_data_vectors(model)
+        create_data_vectors(model, freq_list)
     finally:
         print 'Unknown words: ', UNKNONW_WORDS
         print 'All words: ', ALL_WORDS
@@ -47,7 +54,20 @@ def main():
         print 'Negatives: ', NEG_COUNT
-def create_data_vectors(model):
+def load_freq_list(freq_path):
+    freq_list = {}
+    with codecs.open(freq_path, 'r', 'utf-8') as freq_file:
+        lines = freq_file.readlines()
+        for line in lines:
+            line_parts = line.split()
+            freq = int(line_parts[0])
+            base = line_parts[1]
+            if base not in freq_list:
+                freq_list[base] = freq
+    return freq_list
+
+
+def create_data_vectors(model, freq_list):
     features_file = None
     if not EACH_TEXT_SEPARATELLY:
         features_file = codecs.open(OUT_PATH, 'wt', 'utf-8')
@@ -72,7 +92,7 @@ def create_data_vectors(model):
                 print len(negatives)
             words_path = os.path.join(ANNO_PATH, '%s_words.xml' % textname)
-            mentions_dict = markables_level_2_dict(mentions_path, words_path)
+            mentions_dict = markables_level_2_dict(mentions_path, words_path, freq_list)
             if EACH_TEXT_SEPARATELLY:
                 text_features_path = os.path.join(OUT_PATH, '%s.csv' % textname)
@@ -185,8 +205,8 @@ def get_mention_features(mention_span, mentions_dict, model):
     mention = get_mention_by_attr(mentions_dict, 'span', mention_span)
     if DEBUG:
-        features.append(mention['head_base'])
-    head_vec = get_wv(model, mention['head_base'])
+        features.append(mention['head']['base'])
+    head_vec = get_wv(model, mention['head']['base'])
     features.extend(list(head_vec))
     if DEBUG:
@@ -257,9 +277,25 @@ def get_mention_features(mention_span, mentions_dict, model):
     sentence_vec = get_context_vec(mention['sentence'], model)
     features.extend(list(sentence_vec))
+    # cechy uzupelniajace
+    features.extend(mention_type(mention))
+
     return features
+def mention_type(mention):
+    type_vec = [0] * 4
+    if mention['head']['ctag'] in NOUN_TAGS:
+        type_vec[0] = 1
+    elif mention['head']['ctag'] in PPRON_TAGS:
+        type_vec[1] = 1
+    elif mention['head']['ctag'] in ZERO_TAGS:
+        type_vec[2] = 1
+    else:
+        type_vec[3] = 1
+    return type_vec
+
+
 def get_wv(model, lemma, random=True):
     global ALL_WORDS
     global UNKNONW_WORDS
@@ -330,10 +366,14 @@ def get_pair_features(pair, mentions_dict):
     features.append(exact_match(ante, ana))
     features.append(base_match(ante, ana))
-    if len(mentions_dict) > 100:
-        features.append(1)
-    else:
-        features.append(0)
+    # cechy uzupelniajace
+    features.append(ante_contains_rarest_from_ana(ante, ana))
+    features.extend(agreement(ante, ana, 'gender'))
+    features.extend(agreement(ante, ana, 'number'))
+    features.extend(agreement(ante, ana, 'person'))
+    features.append(is_acronym(ante, ana))
+    features.append(same_sentence(ante, ana))
+    features.append(same_paragraph(ante, ana))
     return features
@@ -382,7 +422,58 @@ def base_match(ante, ana):
     return 0
-def markables_level_2_dict(markables_path, words_path, namespace='www.eml.org/NameSpaces/mention'):
+def ante_contains_rarest_from_ana(ante, ana):
+    ana_rarest = ana['rarest']
+    for word in ante['words']:
+        if word['base'] == ana_rarest['base']:
+            return 1
+    return 0
+
+
+def agreement(ante, ana, tag_name):
+    agr_vec = [0] * 3
+    if ante['head'][tag_name] == 'unk' or ana['head'][tag_name] == 'unk':
+        agr_vec[2] = 1
+    elif ante['head'][tag_name] == ana['head'][tag_name]:
+        agr_vec[0] = 1
+    else:
+        agr_vec[1] = 1
+    return agr_vec
+
+
+def is_acronym(ante, ana):
+    if ana['text'].upper() == ana['text']:
+        return check_one_way_acronym(ana['text'], ante['text'])
+    if ante['text'].upper() == ante['text']:
+        return check_one_way_acronym(ante['text'], ana['text']);
+    return 0;
+
+
+def check_one_way_acronym(acronym, expression):
+    initials = u''
+    for expr1 in expression.split('-'):
+        for expr2 in expr1.split():
+            expr2 = expr2.strip()
+            if expr2:
+                initials += unicode(expr2[0]).upper()
+    if acronym == initials:
+        return 1;
+    return 0;
+
+
+def same_sentence(ante, ana):
+    if ante['sentence_id'] == ana['sentence_id']:
+        return 1
+    return 0
+
+
+def same_paragraph(ante, ana):
+    if ante['paragraph_id'] == ana['paragraph_id']:
+        return 1
+    return 0
+
+
+def markables_level_2_dict(markables_path, words_path, freq_list, namespace='www.eml.org/NameSpaces/mention'):
     markables_dicts = []
     markables_tree = etree.parse(markables_path)
     markables = markables_tree.xpath("//ns:markable", namespaces={'ns': namespace})
@@ -401,9 +492,9 @@ def markables_level_2_dict(markables_path, words_path, namespace=&#39;www.eml.org/Na
             if head_orth not in POSSIBLE_HEADS:
                 mention_words = span_to_words(span, words)
-                prec_context, follow_context, sentence, mnt_start_position, mnt_end_position = get_context(mention_words, words)
+                prec_context, follow_context, sentence, mnt_start_position, mnt_end_position, paragraph_id, sentence_id = get_context(mention_words, words)
-                head_base = get_head_base(head_orth, mention_words)
+                head = get_head(head_orth, mention_words)
                 markables_dicts.append({'id': markable.attrib['id'],
                                         'set': markable.attrib['mention_group'],
                                         'text': span_to_text(span, words, 'orth'),
@@ -411,7 +502,7 @@ def markables_level_2_dict(markables_path, words_path, namespace=&#39;www.eml.org/Na
                                         'words': mention_words,
                                         'span': span,
                                         'head_orth': head_orth,
-                                        'head_base': head_base,
+                                        'head': head,
                                         'dominant': dominant,
                                         'node': markable,
                                         'prec_context': prec_context,
@@ -419,7 +510,10 @@ def markables_level_2_dict(markables_path, words_path, namespace=&#39;www.eml.org/Na
                                         'sentence': sentence,
                                         'position_in_mentions': idx,
                                         'start_in_words': mnt_start_position,
-                                        'end_in_words': mnt_end_position})
+                                        'end_in_words': mnt_end_position,
+                                        'rarest': get_rarest_word(mention_words, freq_list),
+                                        'paragraph_id': paragraph_id,
+                                        'sentence_id': sentence_id})
         else:
             print 'Zduplikowana wzmianka: %s' % span
@@ -427,6 +521,8 @@ def markables_level_2_dict(markables_path, words_path, namespace=&#39;www.eml.org/Na
 def get_context(mention_words, words):
+    paragraph_id = 0
+    sentence_id = 0
     prec_context = []
     follow_context = []
     sentence = []
@@ -442,7 +538,11 @@ def get_context(mention_words, words):
             sentence = get_sentence(idx, words)
             mnt_end_position = get_mention_end(last_word, words)
             break
-    return prec_context, follow_context, sentence, mnt_start_position, mnt_end_position
+        if word['lastinsent']:
+            sentence_id += 1
+        if word['lastinpar']:
+            paragraph_id += 1
+    return prec_context, follow_context, sentence, mnt_start_position, mnt_end_position, paragraph_id, sentence_id
 def get_prec_context(mention_start, words):
@@ -514,10 +614,10 @@ def get_sentence_end(words, word_idx):
     return len(words) - 1
-def get_head_base(head_orth, words):
+def get_head(head_orth, words):
     for word in words:
         if word['orth'].lower() == head_orth.lower() or word['orth'] == head_orth:
-            return word['base']
+            return word
     return None
@@ -531,15 +631,61 @@ def get_words(filepath):
         lastinsent = False
         if 'lastinsent' in word.attrib and word.attrib['lastinsent'] == 'true':
             lastinsent = True
+        lastinpar = False
+        if 'lastinpar' in word.attrib and word.attrib['lastinpar'] == 'true':
+            lastinpar = True
         words.append({'id': word.attrib['id'],
                       'orth': word.text,
                       'base': word.attrib['base'],
                       'hasnps': hasnps,
                       'lastinsent': lastinsent,
-                      'ctag': word.attrib['ctag']})
+                      'lastinpar': lastinpar,
+                      'ctag': word.attrib['ctag'],
+                      'msd': word.attrib['msd'],
+                      'gender': get_gender(word.attrib['msd']),
+                      'person': get_person(word.attrib['msd']),
+                      'number': get_number(word.attrib['msd'])})
     return words
+def get_gender(msd):
+    tags = msd.split(':')
+    if 'm1' in tags:
+        return 'm1'
+    elif 'm2' in tags:
+        return 'm2'
+    elif 'm3' in tags:
+        return 'm3'
+    elif 'f' in tags:
+        return 'f'
+    elif 'n' in tags:
+        return 'n'
+    else:
+        return 'unk'
+
+
+def get_person(msd):
+    tags = msd.split(':')
+    if 'pri' in tags:
+        return 'pri'
+    elif 'sec' in tags:
+        return 'sec'
+    elif 'ter' in tags:
+        return 'ter'
+    else:
+        return 'unk'
+
+
+def get_number(msd):
+    tags = msd.split(':')
+    if 'sg' in tags:
+        return 'sg'
+    elif 'pl' in tags:
+        return 'pl'
+    else:
+        return 'unk'
+
+
 def get_mention_by_attr(mentions, attr_name, value):
     for mention in mentions:
         if mention[attr_name] == value:
@@ -652,5 +798,19 @@ def word_to_ignore(word):
     return False
+def get_rarest_word(words, freq_list):
+    min_freq = 0
+    rarest_word = words[0]
+    for i, word in enumerate(words):
+        word_freq = 0
+        if word['base'] in freq_list:
+            word_freq = freq_list[word['base']]
+
+        if i == 0 or word_freq < min_freq:
+            min_freq = word_freq
+            rarest_word = word
+    return rarest_word
+
+
 if __name__ == '__main__':
     main()