limanling · dependabot · Jun 2, 2021 · Oct 19, 2021 · Oct 19, 2021
diff --git a/src/dataflow/numpy/.DS_Store → .DS_Store b/src/dataflow/numpy/.DS_Store → .DS_Store
diff --git a/.idea/dictionaries/lynn.xml b/.idea/dictionaries/lynn.xml
diff --git a/.idea/m2e2-multimedia-event-extraction.iml b/.idea/m2e2-multimedia-event-extraction.iml
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/.idea/modules.xml b/.idea/modules.xml
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
diff --git a/README.md b/README.md
@@ -29,10 +29,10 @@ pip install -r requirements.txt
 We download situation recognition data from [imSitu](http://imsitu.org/). Please find the preprocessed data in [PreprcessedSR](https://drive.google.com/drive/folders/1h0qwYWeGEoCx8m-zwH-XcoPSyffmrC-c?usp=sharing).
 
 ### ACE (Text Event Extraction Data)
-We preprcoessed ACE following [JMEE](https://github.com/lx865712528/EMNLP2018-JMEE/tree/master). The preprocessing script is in `dataflow/preprocess_ace_JMEE.py`, and the sample data format is in [sample.json](https://github.com/lx865712528/EMNLP2018-JMEE/blob/master/ace-05-splits/sample.json). Due to license reason, the ACE 2005 dataset is only accessible to those with LDC2006T06 license, please drop me an email showing your possession of the license for the processed data.
+We preprcoessed ACE following [JMEE](https://github.com/lx865712528/EMNLP2018-JMEE/tree/master). The sample data format is in [sample.json](https://github.com/lx865712528/EMNLP2018-JMEE/blob/master/ace-05-splits/sample.json). Due to license reason, the ACE 2005 dataset is only accessible to those with LDC2006T06 license, please drop me an email `manling2@illinois.edu` showing your possession of the license for the processed data.
 
 ### Voice of America Image-Caption Pairs
-We crawled VOA image-captions to train the common space, the [image-caption pairs](https://uofi.box.com/s/xtn9p6m8z5qtjbbi5tqrl45tn6apew4x) and images can be downloaded using the URLs (We share image URLs instead of downloaded images due to license issue). We preprocess the data including object detection, and parse text sentences. The preprocessed data is in [PreprocessedVOA](https://drive.google.com/drive/folders/1I9vMGIhWZpKqxQYip91eLoDRnrkqRxnt?usp=sharing).
+We crawled VOA image-captions to train the common space, the [image-caption pairs](https://uofi.box.com/s/xtn9p6m8z5qtjbbi5tqrl45tn6apew4x) and images can be downloaded using the URLs (We share image URLs instead of downloaded images due to license issue) using script in [dataset_image_download.py](https://github.com/limanling/m2e2/blob/master/src/dataflow/numpy/dataset_image_download.py). We preprocess the data including object detection, and parse text sentences. The preprocessed data is in [PreprocessedVOA](https://drive.google.com/drive/folders/1I9vMGIhWZpKqxQYip91eLoDRnrkqRxnt?usp=sharing).
 
 ### M2E2 (Multimedia Event Extraction Benchmark)
 

diff --git a/data/ace/ace_sr_mapping.txt b/data/ace/ace_sr_mapping.txt
diff --git a/data/object/class-descriptions-boxable.csv b/data/object/class-descriptions-boxable.csv
diff --git a/requirements.txt b/requirements.txt
@@ -3,7 +3,7 @@ ujson==1.35
 seqeval==0.0.12
 six==1.12.0
 jieba==0.39
-nltk==3.4.5
+nltk==3.6.5
 scipy==1.3.0
 torchvision==0.3.0
 Pillow==8.1.1

diff --git a/scripts/.gitkeep b/scripts/.gitkeep
diff --git a/scripts/eval/.gitkeep b/scripts/eval/.gitkeep
diff --git a/scripts/train/.gitkeep b/scripts/train/.gitkeep
diff --git a/src/.DS_Store b/src/.DS_Store
diff --git a/src/.gitkeep b/src/.gitkeep
diff --git a/src/__init__.py b/src/__init__.py
diff --git a/src/dataflow/.DS_Store b/src/dataflow/.DS_Store
diff --git a/src/dataflow/.gitkeep b/src/dataflow/.gitkeep
diff --git a/src/dataflow/__init__.py b/src/dataflow/__init__.py
diff --git a/src/dataflow/numpy/__init__.py b/src/dataflow/numpy/__init__.py
diff --git a/src/dataflow/numpy/anno_mapping.py b/src/dataflow/numpy/anno_mapping.py
@@ -0,0 +1,103 @@
+from collections import defaultdict
+
+def event_type_norm(type_str):
+    return type_str.replace('.', '||').replace(':', '||').replace('-', '|').upper()
+
+
+def role_name_norm(type_str):
+    return type_str.upper()
+
+entity_type_mapping_brat = {
+    'PER': 'PER',
+    'ORG': 'ORG',
+    'GPE': 'GPE',
+    'LOC': 'LOC',
+    'FAC': 'FAC',
+    'VEH': 'VEH',
+    'WEA': 'WEA',
+    # 'TIM': 'TIME',
+    # 'NUM': 'VALUE',
+    # 'TIT',
+    'MON': 'VALUE',
+    # 'URL',
+    # 'RES',
+    # 'BALLOT'
+}
+
+event_type_mapping_brat2ace = {
+    'Die': 'Life:Die',
+    # 'Injure': 'Life:Injure',
+    'TransferMoney': 'Transaction:Transfer-Money',
+    'Attack': 'Conflict:Attack',
+    'Demonstrate': 'Conflict:Demonstrate',
+    'Correspondence': 'Contact:Phone-Write',
+    'Meet': 'Contact:Meet',
+    'ArrestJail': 'Justice:Arrest-Jail',
+    # 'ReleaseParole': 'Justice:Release-Parole',
+    'TransportPerson': 'Movement:Transport',
+    'TransportArtifact': 'Movement:Transport',
+}
+
+event_type_mapping_ace2brat = {event_type_norm(v): k for k, v in event_type_mapping_brat2ace.items()}
+# print(event_type_mapping_ace2brat)
+
+event_type_mapping_aida2brat = {
+    'Life.Die': 'Die',
+    'Life.Injure': 'Injure',
+    'Transaction.TransferMoney': 'TransferMoney',
+    'Conflict.Attack': 'Attack',
+    'Conflict.Demonstrate': 'Demonstrate',
+    'Contact.Correspondence': 'Correspondence',
+    'Contact.Meet': 'Meet',
+    'Justice.ArrestJail': 'ArrestJail',
+    'Justice.ReleaseParole': 'ReleaseParole',
+    'Movement.TransportPerson': 'TransportPerson',
+    'Movement.TransportArtifact': 'TransportArtifact',
+}
+
+event_role_mapping_brat2ace = { # delete time-related ones when testing
+    'Die': {'Victim': 'Victim', 'Agent': 'Agent', 'Instrument': 'Instrument', 'Place': 'Place'},  #, 'Time': 'Time'},
+    # 'Injure': {'Victim': 'Victim', 'Agent': 'Agent', 'Instrument': 'Instrument', 'Place': 'Place'},  #, 'Time': 'Time'},
+    'TransferMoney': {'Giver': 'Giver', 'Recipient': 'Recipient', 'Beneficiary': 'Beneficiary', 'Money': 'Money', 'Place': 'Place'},  #, 'Time': 'Time'},
+    'Attack': {'Attacker': 'Attacker', 'Instrument': 'Instrument', 'Place': 'Place', 'Target': 'Target'},  #, 'Time': 'Time'},
+    'Demonstrate': {'Demonstrator': 'Entity', 'Place': 'Place'},  #, 'Time': 'Time'},
+    'Correspondence': {'Participant': 'Entity', 'Place': 'Place'},  #, 'Time': 'Time'},
+    'Meet': {'Participant': 'Entity', 'Place': 'Place'},  #, 'Time': 'Time'},
+    'ArrestJail': {'Agent': 'Agent', 'Person': 'Person', 'Place': 'Place'},  #, 'Time': 'Time'},
+    # 'ReleaseParole': {'Agent': 'Entity', 'Person': 'Person', 'Place': 'Place'},  #, 'Time': 'Time'},
+    'TransportPerson': {'Agent': 'Agent', 'Person': 'Artifact', 'Instrument': 'Vehicle', 'Destination': 'Destination', 'Origin': 'Origin'},  #, 'Time': 'Time'},
+    'TransportArtifact': {'Agent': 'Agent', 'Artifact': 'Artifact', 'Instrument': 'Vehicle', 'Destination': 'Destination', 'Origin': 'Origin'},  #, 'Time': 'Time'},
+}
+
+# event_role_mapping_ace2brat = {role_name_norm(v): k for t in event_role_mapping_brat2ace.items() for k, v in event_role_mapping_brat2ace[t]}
+event_role_mapping_ace2brat = defaultdict(lambda : defaultdict())
+for t in event_role_mapping_brat2ace:
+    for k, v in event_role_mapping_brat2ace[t].items():
+        event_type_ace = event_type_norm(event_type_mapping_brat2ace[t])
+        event_role_mapping_ace2brat[event_type_ace][role_name_norm(v)] = k
+# print(event_role_mapping_ace2brat)
+
+event_type_mapping_image2ace = {
+    'Movement.TransportPerson': 'Movement:Transport',
+    'Movement.TransportArtifact': 'Movement:Transport',
+    'Life.Die': 'Life:Die',
+    'Conflict.Attack': 'Conflict:Attack',
+    'Conflict.Demonstrate': 'Conflict:Demonstrate',
+    'Contact.Phone-Write': 'Contact:Phone-Write',
+    'Contact.Meet': 'Contact:Meet',
+    'Transaction.TransferMoney': 'Transaction:Transfer-Money',
+    'Justice.ArrestJail': 'Justice:Arrest-Jail',
+}
+
+event_role_mapping_image2ace = {
+    'Life.Die': {'victim': 'Victim', 'agent': 'Agent', 'instrument': 'Instrument', 'place': 'Place'},  #, 'Time': 'Time'},
+    'Transaction.TransferMoney': {'instrument': 'Instrument', 'giver': 'Giver', 'recipient': 'Recipient', 'beneficiary': 'Beneficiary', 'money': 'Money', 'place': 'Place'},  #, 'Time': 'Time'},
+    'Conflict.Attack': {'attacker': 'Attacker', 'instrument': 'Instrument', 'place': 'Place', 'target': 'Target', 'victim': 'Target'},  #, 'Time': 'Time'},
+    'Conflict.Demonstrate': {'police':'Police', 'instrument': 'Instrument', 'demonstrator': 'Entity', 'place': 'Place', 'participant': 'Entity'},  #, 'Time': 'Time'},
+    'Contact.Phone-Write': {'instrument':'Instrument', 'participant': 'Entity', 'place': 'Place'},  #, 'Time': 'Time'},
+    'Contact.Meet': {'participant': 'Entity', 'place': 'Place'},  #, 'Time': 'Time'},
+    'Justice.ArrestJail': {'instrument': 'Instrument', 'agent': 'Agent', 'person': 'Person', 'place': 'Place'},  #, 'Time': 'Time'},
+    'Movement.TransportPerson': {'agent': 'Agent', 'person': 'Artifact', 'instrument': 'Instrument', 'destination': 'Destination', 'origin': 'Origin'},  #, 'Time': 'Time'},
+    'Movement.TransportArtifact': {'person': 'Artifact', 'agent': 'Agent', 'artifact': 'Artifact', 'instrument': 'Instrument', 'destination': 'Destination', 'origin': 'Origin'},  #, 'Time': 'Time'},
+}
+# 'TransportArtifact' has 'person' in image annotation
diff --git a/src/dataflow/numpy/data_loader_ace_role.py b/src/dataflow/numpy/data_loader_ace_role.py
diff --git a/src/dataflow/numpy/data_loader_grounding.py b/src/dataflow/numpy/data_loader_grounding.py
diff --git a/src/dataflow/numpy/data_loader_situation.py b/src/dataflow/numpy/data_loader_situation.py
diff --git a/src/dataflow/numpy/dataset_image_download.py b/src/dataflow/numpy/dataset_image_download.py
@@ -13,9 +13,10 @@ def download_image_list(meta_json, dir_save):
     for doc_id in metadata:
         for img_id in metadata[doc_id]:
             url_image = metadata[doc_id][img_id]['url']
-            image_path_save = os.path.join(dir_save, '%s_%s.jpg' % (doc_id, img_id))
-            if not url_image.endswith('.jpg'):
-                print(image_path_save, url_image)
+            suffix_image = metadata[doc_id][img_id]['url'].split('.')[-1]
+            image_path_save = os.path.join(dir_save, '%s_%s.%s' % (doc_id, img_id, suffix_image))
+            # if not url_image.endswith('.jpg'):
+            #     print(image_path_save, url_image)
             download_image(url_image, image_path_save)
 
 

diff --git a/src/dataflow/numpy/prepare_vocab.py b/src/dataflow/numpy/prepare_vocab.py
diff --git a/src/dataflow/torch/Corpus.py b/src/dataflow/torch/Corpus.py
diff --git a/src/dataflow/torch/Data.py b/src/dataflow/torch/Data.py
diff --git a/src/dataflow/torch/Sentence.py b/src/dataflow/torch/Sentence.py
diff --git a/src/dataflow/torch/__init__.py b/src/dataflow/torch/__init__.py
diff --git a/src/engine/.gitkeep b/src/engine/.gitkeep
diff --git a/src/engine/EErunner.py b/src/engine/EErunner.py
diff --git a/src/engine/EEtraining.py b/src/engine/EEtraining.py
diff --git a/src/engine/Groundingrunner.py b/src/engine/Groundingrunner.py
diff --git a/src/engine/Groundingtraining.py b/src/engine/Groundingtraining.py
diff --git a/src/engine/JOINTRunner.py b/src/engine/JOINTRunner.py
diff --git a/src/engine/JOINTTraining_sample.py b/src/engine/JOINTTraining_sample.py
diff --git a/src/engine/SRrunner.py b/src/engine/SRrunner.py
diff --git a/src/engine/SRtraining.py b/src/engine/SRtraining.py
diff --git a/src/engine/TestRunnerEE.py b/src/engine/TestRunnerEE.py
diff --git a/src/engine/TestRunnerEE_m2e2.py b/src/engine/TestRunnerEE_m2e2.py
diff --git a/src/engine/TestRunnerJOINT.py b/src/engine/TestRunnerJOINT.py
diff --git a/src/engine/TestRunnerSR.py b/src/engine/TestRunnerSR.py
diff --git a/src/engine/TestRunnerSR_m2e2.py b/src/engine/TestRunnerSR_m2e2.py
diff --git a/src/engine/__init__.py b/src/engine/__init__.py
diff --git a/src/eval/.gitkeep b/src/eval/.gitkeep
diff --git a/src/eval/EEtesting.py b/src/eval/EEtesting.py
diff --git a/src/eval/EEvisualizing.py b/src/eval/EEvisualizing.py
diff --git a/src/eval/Groundingtesting.py b/src/eval/Groundingtesting.py
diff --git a/src/eval/JOINTtesting.py b/src/eval/JOINTtesting.py
diff --git a/src/eval/SRtesting.py b/src/eval/SRtesting.py
diff --git a/src/eval/__init__.py b/src/eval/__init__.py
diff --git a/src/models/.gitkeep b/src/models/.gitkeep
diff --git a/src/models/__init__.py b/src/models/__init__.py
diff --git a/src/models/ace_classifier.py b/src/models/ace_classifier.py
diff --git a/src/models/ee.py b/src/models/ee.py
diff --git a/src/models/grounding.py b/src/models/grounding.py
diff --git a/src/models/modules/.gitkeep b/src/models/modules/.gitkeep
diff --git a/src/models/modules/DynamicLSTM.py b/src/models/modules/DynamicLSTM.py
diff --git a/src/models/modules/EmbeddingLayer.py b/src/models/modules/EmbeddingLayer.py
diff --git a/src/models/modules/EmbeddingLayerImage.py b/src/models/modules/EmbeddingLayerImage.py
diff --git a/src/models/modules/FMapLayerImage.py b/src/models/modules/FMapLayerImage.py
diff --git a/src/models/modules/GCN.py b/src/models/modules/GCN.py
diff --git a/src/models/modules/HighWay.py b/src/models/modules/HighWay.py
diff --git a/src/models/modules/SelfAttention.py b/src/models/modules/SelfAttention.py
diff --git a/src/models/modules/__init__.py b/src/models/modules/__init__.py
diff --git a/src/models/modules/model.py b/src/models/modules/model.py
diff --git a/src/models/sr.py b/src/models/sr.py
diff --git a/src/models/sr_object.py b/src/models/sr_object.py
diff --git a/src/util/.gitkeep b/src/util/.gitkeep
diff --git a/src/util/__init__.py b/src/util/__init__.py
diff --git a/src/util/constant.py b/src/util/constant.py
diff --git a/src/util/consts.py b/src/util/consts.py
diff --git a/src/util/helper.py b/src/util/helper.py
diff --git a/src/util/util_img.py b/src/util/util_img.py
diff --git a/src/util/util_model.py b/src/util/util_model.py
diff --git a/src/util/vocab.py b/src/util/vocab.py
diff --git a/training-testing.jpg b/training-testing.jpg