importing examples from XML

Tomasz Bartosiak
1 parent ca3f9dba
Showing 13 changed files with 99 additions and 24 deletions
connections/models.py
examples/models.py
importer/Argument.py
importer/Entry.py
importer/Frame.py
importer/Meanings.py
importer/Position.py
importer/Realizations.py
importer/Semantics.py
importer/WalentyPreprocessXML.py
importer/WalentyXML.py
reset_db.sh
syntax/management/commands/import_tei.py
@@ -50,6 +50,7 @@ class Status(models.Model):
 class ExampleConnection(models.Model):
     example = models.ForeignKey(Example, related_name='example_connections', on_delete=models.PROTECT)
     lexical_unit = models.ForeignKey(LexicalUnit, related_name='example_connections', null=True, on_delete=models.PROTECT)
+    arguments = models.ManyToManyField(Argument, related_name='example_connections')
     schema_connections = models.ManyToManyField('SchemaHook', related_name='example_connections')
 from django.db import models
-
 class Example(models.Model):
     entry = models.ForeignKey('connections.Entry', related_name='examples', on_delete=models.PROTECT)
     sentence = models.TextField()
     opinion = models.ForeignKey('ExampleOpinion', related_name='examples', on_delete=models.PROTECT)
     source = models.ForeignKey('ExampleSource', related_name='examples', on_delete=models.PROTECT)
-
+    note = models.TextField(null=True)
+    
     def __str__(self):
         return self.sentence
@@ -31,8 +31,8 @@ class Relation:
         pref.save()
         argument.relations.add(pref)
-    def __unicode__(self):
-        return self._type + '->' + unicode(self._to._semantic_role)
+    def __str__(self):
+        return self._type + '->' + str(self._to._semantic_role)
 class SelectionalPreference:
@@ -84,8 +84,8 @@ class SelectionalPreference:
             print(type, values)
             raise UnknownError()
-    def __unicode__(self):
-        return unicode(self._value)
+    def __str__(self):
+        return str(self._value)
 class SemanticRole:
@@ -106,7 +106,7 @@ class SemanticRole:
         argument_role = semantics.models.ArgumentRole.objects.get(role=role, attribute=attribute)
         return argument_role
-    def __unicode__(self):
+    def __str__(self):
         if self._attribute is None:
             return self._value.lower()
         else:
@@ -121,6 +121,7 @@ class Argument:
         self._selectional_preferences = selectional_preferences
         self._references = references
         self._id = id
+        self._db_id = None
     @classmethod
     def fromTree(cls, tree):
@@ -158,12 +159,13 @@ class Argument:
                                              frame=frame,
                                              preferences_count=len(self._selectional_preferences))
         argument.save()
+        self._db_id = argument.id
         return argument
     def store_preferences(self, frame, argument):
         for preference in self._selectional_preferences:
             preference.store(frame, argument)
-    def __unicode__(self):
-        return unicode(self._semantic_role) + '[' + ','.join([unicode(pref) for pref in self._selectional_preferences]) + ']'
+    def __str__(self):
+        return str(self._semantic_role) + '[' + ','.join([str(pref) for pref in self._selectional_preferences]) + ']'
@@ -4,23 +4,24 @@
 from importer.Syntax import Syntax
 from importer.Meanings import Meanings
 from importer.Semantics import Semantics
+from importer.Examples import Examples
 from connections.models import POS, Status
 import connections.models
 class Entry:
-    def __init__(self, entry_tree, entry_meanings, meanings, frames):
+    def __init__(self, entry_tree, entry_meanings, meanings, frames, examples_in_data, examples_out_file, misconnected_out_file):
         self._base = entry_tree._children[0]._children[0]._content
         self._pos = entry_tree._children[0]._children[1]._content
+        print("processing: " + self._base)         
         self._status = entry_tree._children[1]._children[0]._children[0]._content
-        print(self._base)
         self._syntax = Syntax.fromTree(entry_tree._children[2])
+        self._meanings = None
+        self._semantics = None
         if len(entry_tree._children) >= 7:
             self._meanings = Meanings.fromTree(entry_tree._children[5])
             self._semantics = Semantics.fromTree(self._base, self._pos, entry_tree._children[4], frames, self._meanings, self._syntax, entry_tree._children[6])
-        else:
-            self._meanings = None
-            self._semantics = None
+        self._examples = Examples.fromTree(entry_tree._children[3], self._syntax, self._semantics, self._base, self._meanings, examples_in_data, examples_out_file, misconnected_out_file)
     def store(self, all_meanings, stored_positions):
         pos = POS.objects.get(tag=self._pos)
@@ -31,6 +32,7 @@ class Entry:
         self._syntax.store(entry, stored_positions)
         if self._semantics is not None:
             self._semantics.store(entry, all_meanings)
+        self._examples.store(entry, all_meanings)
     def __str__(self):
         return self._pos + '(' + self._base + ',' + str(self._syntax) + ').'
@@ -69,7 +69,8 @@ class Frame:
     def getSignature(self):
-        return self._base + '\t[' + ','.join([unicode(meaning) for meaning in self._meanings]) + ']'
+        return self._base + '\t[' + ','.join([str(meaning) for meaning in self._meanings]) + ']'
+
+    def __str__(self):
+        return self.getSignature() + '\t[' + ','.join([str(argument) for argument in self._arguments.values()]) + ']'
-    def __unicode__(self):
-        return self.getSignature() + '\t[' + ','.join([unicode(argument) for argument in self._arguments.values()]) + ']'
@@ -62,7 +62,15 @@ class Meaning:
             unit.entry = entry
             unit.save()
         frame.lexical_units.add(unit)
-    
+
+    def get(self):
+        units = LexicalUnit.objects.filter(base=self._name, sense=self._variant)
+        if len(units) == 0:
+            print(self)
+            raise UnknownError()
+        else:
+            return units[0]
+        
     def __unicode__(self):
         return '\'' + self._name + '\'-' + self._variant
@@ -86,7 +94,7 @@ class Meanings:
         return self._meanings
     def locate(self, meaning_id):
-        if meanings_id in self._meanings:
+        if meaning_id in self._meanings:
             return self._meanings[meaning_id]
         else:
             return None
@@ -112,7 +112,8 @@ class Position:
                                  pred_control=position.pred_control,
                                  phrase_str=phrase_text).count()
                 if desc_count == 0:
-                    desc_text = phrase_description2(phrase, self, negativity)
+                    #desc_text = phrase_description2(phrase, self, negativity)
+                    desc_text = "tu pójdzie opis"
                     desc = NaturalLanguageDescription(
                                      negativity=negativity,
                                      function=position.function,
@@ -162,3 +163,18 @@ class Position:
                 return pre + 'obj([' + ','.join(temp) + '])' + post
             elif self._function._value == 'head':
                 return pre + 'head([' + ','.join(temp) + '])' + post
+
+    def subposition(self, phrases=None):
+        c = ''
+        if self._control is not None:
+            c = ','.join([control._function for control in self._control])
+        f = ''
+        if self._function is not None: 
+            f = self._function._value
+        if f != '' and c != '':
+            f += ','
+        if phrases is None:
+            return f + c + '{' + ','.join([str(phrase) for phrase in self._phrases]) + '}'
+        else:
+            temp = [str(phrase) for phrase in self._phrases if phrase in phrases]
+            return f + c + '{' + ','.join(temp) + '}'
@@ -35,7 +35,12 @@ class ArgumentRealization:
             hook.save()
             connection.schema_connections.add(hook)
-        
+    def matches(self, phrases):
+        for phrase in phrases:
+            if phrase not in self._phrases:
+                return False
+        return True
+            
 class FrameRealization:
     def __init__(self, frame, schema, argument_realizations):
@@ -67,5 +72,12 @@ class FrameRealization:
         subentry = self._schema.getSubentry(entry)
         for ar in self._argument_realizations:
             ar.store(subentry, frame, schema, alternation)
+
+    def findMatchingArgument(self, phrases):
+        for ar in self._argument_realizations:
+            if ar.matches(phrases):
+                return ar._argument
+        return None
+        
@@ -39,6 +39,16 @@ class Semantics:
     def getPhraseIds(self):
         return self._phrases
+    def findFrame(self, meaning):
+        for frame in self._frames:
+            if meaning in frame._meanings:
+                realizations = []
+                for realization in self._realizations:
+                    if realization._frame._id == frame._id:
+                        realizations.append(realization)
+                return frame, realizations
+        return None
+    
     # def preferencesToUnicode(self):
     #     return '\n'.join([unicode(frame) for frame in self._frames])
@@ -80,6 +80,7 @@ class WalentyPreprocessTeiHandler(handler.ContentHandler):
         for entry_data, meaning in self.meanings.values():
             name, pos = entry_data
             meaning.save(pos)
+        print("Stored")
     def extend(self, base, pos, meanings, frames):
         self.entry_meanings[(base, pos)] = [id for id in meanings._meanings]
@@ -4,6 +4,9 @@
 from xml.sax import handler
 from importer.Entry import Entry
+examples_out_file = 'examples_ambig.txt'
+misconnected_examples_out_file = 'examples_to_reattach.txt'
+
 class XMLNode:
@@ -37,6 +40,9 @@ class WalentyTeiHandler(handler.ContentHandler):
         self._meanings = meanings
         self._frames = frames
         self._stored_positions = {}
+        self._examples_in = None # @TODO: read disambiguated file
+        self._examples_out = open(examples_out_file, "w")
+        self._misconnected_out = open(misconnected_examples_out_file, "w")
     def startElement(self, name, attrs):
         if name == 'date':
@@ -61,7 +67,7 @@ class WalentyTeiHandler(handler.ContentHandler):
             if name == 'entry':
                 if self._current is not None:
                     raise TEIStructureError()
-                entry = Entry(self._subtree, self._entry_meanings, self._meanings, self._frames)
+                entry = Entry(self._subtree, self._entry_meanings, self._meanings, self._frames, self._examples_in, self._examples_out, self._misconnected_out)
                 entry.store(self._meanings, self._stored_positions)
             self._content = ''
         else:
@@ -78,5 +84,6 @@ class WalentyTeiHandler(handler.ContentHandler):
         self._content += content
     def endDocument(self):
-        pass
+        self._examples_out.close()
+        self._misconnected_out.close()
@@ -4,3 +4,5 @@ dropdb shellvalier
 createdb shellvalier -E UTF8 -T template0 -l pl_PL.utf8
 python3 manage.py migrate
 python3 manage.py import_plWordnet
+python3 manage.py import_tei
+
@@ -9,6 +9,7 @@ from importer.WalentyXML import WalentyTeiHandler
 from importer.WalentyPreprocessXML import WalentyPreprocessTeiHandler
 from shellvalier.settings import BASE_DIR
 from connections.models import POS, Status
+from examples.models import ExampleOpinion, ExampleSource
 from syntax.models import SchemaOpinion, Aspect, InherentSie, Negativity, Predicativity, SyntacticFunction, Control, PredicativeControl
 from semantics.models import FrameOpinion, ArgumentRole, SemanticRole, RoleAttribute, PredefinedSelectionalPreference, SelectionalPreferenceRelation
@@ -54,6 +55,8 @@ def import_constants():
     import_semantic_roles()
     import_predefined_preferences()
     import_preference_relations()
+    import_examples_sources()
+    import_examples_opinions()
     pass
 def import_poses():
@@ -149,5 +152,15 @@ def import_preference_relations():
         relat = SelectionalPreferenceRelation(plwn_id=id, key=name)
         relat.save()
-    
+def import_examples_sources():
+    sources = [(0, u'NKJP0.5M'), (1, u'NKJP1.2M'), (2, u'NKJP30M'), (3, u'NKJP250M'), (4, u'NKJP300M'), (5, u'NKJP500M'), (6, u'NKJP1800M'), (7, u'linguistic_literature'), (8, u'other_literature'), (9, u'own')]
+    for pri, name in sources:
+        es = ExampleSource(key=name, priority=pri)
+        es.save()
+
+def import_examples_opinions():
+    opinions = [(0, 'zły'), (1, 'wątpliwy'), (2, 'dobry')]
+    for pri, name in opinions:
+        eo = ExampleOpinion(key=name, priority=pri)
+        eo.save()