diff --git a/src/dataflow/numpy/.DS_Store b/.DS_Store similarity index 89% rename from src/dataflow/numpy/.DS_Store rename to .DS_Store index 5008ddf..0e73bdd 100644 Binary files a/src/dataflow/numpy/.DS_Store and b/.DS_Store differ diff --git a/.idea/dictionaries/lynn.xml b/.idea/dictionaries/lynn.xml old mode 100644 new mode 100755 diff --git a/.idea/m2e2-multimedia-event-extraction.iml b/.idea/m2e2-multimedia-event-extraction.iml old mode 100644 new mode 100755 diff --git a/.idea/misc.xml b/.idea/misc.xml old mode 100644 new mode 100755 diff --git a/.idea/modules.xml b/.idea/modules.xml old mode 100644 new mode 100755 diff --git a/.idea/vcs.xml b/.idea/vcs.xml old mode 100644 new mode 100755 diff --git a/.idea/workspace.xml b/.idea/workspace.xml old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 index fbef72f..62fbbdf --- a/README.md +++ b/README.md @@ -29,10 +29,10 @@ pip install -r requirements.txt We download situation recognition data from [imSitu](http://imsitu.org/). Please find the preprocessed data in [PreprcessedSR](https://drive.google.com/drive/folders/1h0qwYWeGEoCx8m-zwH-XcoPSyffmrC-c?usp=sharing). ### ACE (Text Event Extraction Data) -We preprcoessed ACE following [JMEE](https://github.com/lx865712528/EMNLP2018-JMEE/tree/master). The preprocessing script is in `dataflow/preprocess_ace_JMEE.py`, and the sample data format is in [sample.json](https://github.com/lx865712528/EMNLP2018-JMEE/blob/master/ace-05-splits/sample.json). Due to license reason, the ACE 2005 dataset is only accessible to those with LDC2006T06 license, please drop me an email showing your possession of the license for the processed data. +We preprcoessed ACE following [JMEE](https://github.com/lx865712528/EMNLP2018-JMEE/tree/master). The sample data format is in [sample.json](https://github.com/lx865712528/EMNLP2018-JMEE/blob/master/ace-05-splits/sample.json). Due to license reason, the ACE 2005 dataset is only accessible to those with LDC2006T06 license, please drop me an email `manling2@illinois.edu` showing your possession of the license for the processed data. ### Voice of America Image-Caption Pairs -We crawled VOA image-captions to train the common space, the [image-caption pairs](https://uofi.box.com/s/xtn9p6m8z5qtjbbi5tqrl45tn6apew4x) and images can be downloaded using the URLs (We share image URLs instead of downloaded images due to license issue). We preprocess the data including object detection, and parse text sentences. The preprocessed data is in [PreprocessedVOA](https://drive.google.com/drive/folders/1I9vMGIhWZpKqxQYip91eLoDRnrkqRxnt?usp=sharing). +We crawled VOA image-captions to train the common space, the [image-caption pairs](https://uofi.box.com/s/xtn9p6m8z5qtjbbi5tqrl45tn6apew4x) and images can be downloaded using the URLs (We share image URLs instead of downloaded images due to license issue) using script in [dataset_image_download.py](https://github.com/limanling/m2e2/blob/master/src/dataflow/numpy/dataset_image_download.py). We preprocess the data including object detection, and parse text sentences. The preprocessed data is in [PreprocessedVOA](https://drive.google.com/drive/folders/1I9vMGIhWZpKqxQYip91eLoDRnrkqRxnt?usp=sharing). ### M2E2 (Multimedia Event Extraction Benchmark) diff --git a/data/ace/ace_sr_mapping.txt b/data/ace/ace_sr_mapping.txt old mode 100644 new mode 100755 diff --git a/data/object/class-descriptions-boxable.csv b/data/object/class-descriptions-boxable.csv old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt index 0c934f7..a01e6de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ ujson==1.35 seqeval==0.0.12 six==1.12.0 jieba==0.39 -nltk==3.4.5 +nltk==3.6.5 scipy==1.3.0 torchvision==0.3.0 Pillow==8.1.1 diff --git a/scripts/.gitkeep b/scripts/.gitkeep old mode 100644 new mode 100755 diff --git a/scripts/eval/.gitkeep b/scripts/eval/.gitkeep old mode 100644 new mode 100755 diff --git a/scripts/train/.gitkeep b/scripts/train/.gitkeep old mode 100644 new mode 100755 diff --git a/src/.DS_Store b/src/.DS_Store new file mode 100644 index 0000000..b236685 Binary files /dev/null and b/src/.DS_Store differ diff --git a/src/.gitkeep b/src/.gitkeep old mode 100644 new mode 100755 diff --git a/src/__init__.py b/src/__init__.py old mode 100644 new mode 100755 diff --git a/src/dataflow/.DS_Store b/src/dataflow/.DS_Store new file mode 100644 index 0000000..1fce158 Binary files /dev/null and b/src/dataflow/.DS_Store differ diff --git a/src/dataflow/.gitkeep b/src/dataflow/.gitkeep old mode 100644 new mode 100755 diff --git a/src/dataflow/__init__.py b/src/dataflow/__init__.py old mode 100644 new mode 100755 diff --git a/src/dataflow/numpy/__init__.py b/src/dataflow/numpy/__init__.py old mode 100644 new mode 100755 diff --git a/src/dataflow/numpy/anno_mapping.py b/src/dataflow/numpy/anno_mapping.py new file mode 100644 index 0000000..f7b5bed --- /dev/null +++ b/src/dataflow/numpy/anno_mapping.py @@ -0,0 +1,103 @@ +from collections import defaultdict + +def event_type_norm(type_str): + return type_str.replace('.', '||').replace(':', '||').replace('-', '|').upper() + + +def role_name_norm(type_str): + return type_str.upper() + +entity_type_mapping_brat = { + 'PER': 'PER', + 'ORG': 'ORG', + 'GPE': 'GPE', + 'LOC': 'LOC', + 'FAC': 'FAC', + 'VEH': 'VEH', + 'WEA': 'WEA', + # 'TIM': 'TIME', + # 'NUM': 'VALUE', + # 'TIT', + 'MON': 'VALUE', + # 'URL', + # 'RES', + # 'BALLOT' +} + +event_type_mapping_brat2ace = { + 'Die': 'Life:Die', + # 'Injure': 'Life:Injure', + 'TransferMoney': 'Transaction:Transfer-Money', + 'Attack': 'Conflict:Attack', + 'Demonstrate': 'Conflict:Demonstrate', + 'Correspondence': 'Contact:Phone-Write', + 'Meet': 'Contact:Meet', + 'ArrestJail': 'Justice:Arrest-Jail', + # 'ReleaseParole': 'Justice:Release-Parole', + 'TransportPerson': 'Movement:Transport', + 'TransportArtifact': 'Movement:Transport', +} + +event_type_mapping_ace2brat = {event_type_norm(v): k for k, v in event_type_mapping_brat2ace.items()} +# print(event_type_mapping_ace2brat) + +event_type_mapping_aida2brat = { + 'Life.Die': 'Die', + 'Life.Injure': 'Injure', + 'Transaction.TransferMoney': 'TransferMoney', + 'Conflict.Attack': 'Attack', + 'Conflict.Demonstrate': 'Demonstrate', + 'Contact.Correspondence': 'Correspondence', + 'Contact.Meet': 'Meet', + 'Justice.ArrestJail': 'ArrestJail', + 'Justice.ReleaseParole': 'ReleaseParole', + 'Movement.TransportPerson': 'TransportPerson', + 'Movement.TransportArtifact': 'TransportArtifact', +} + +event_role_mapping_brat2ace = { # delete time-related ones when testing + 'Die': {'Victim': 'Victim', 'Agent': 'Agent', 'Instrument': 'Instrument', 'Place': 'Place'}, #, 'Time': 'Time'}, + # 'Injure': {'Victim': 'Victim', 'Agent': 'Agent', 'Instrument': 'Instrument', 'Place': 'Place'}, #, 'Time': 'Time'}, + 'TransferMoney': {'Giver': 'Giver', 'Recipient': 'Recipient', 'Beneficiary': 'Beneficiary', 'Money': 'Money', 'Place': 'Place'}, #, 'Time': 'Time'}, + 'Attack': {'Attacker': 'Attacker', 'Instrument': 'Instrument', 'Place': 'Place', 'Target': 'Target'}, #, 'Time': 'Time'}, + 'Demonstrate': {'Demonstrator': 'Entity', 'Place': 'Place'}, #, 'Time': 'Time'}, + 'Correspondence': {'Participant': 'Entity', 'Place': 'Place'}, #, 'Time': 'Time'}, + 'Meet': {'Participant': 'Entity', 'Place': 'Place'}, #, 'Time': 'Time'}, + 'ArrestJail': {'Agent': 'Agent', 'Person': 'Person', 'Place': 'Place'}, #, 'Time': 'Time'}, + # 'ReleaseParole': {'Agent': 'Entity', 'Person': 'Person', 'Place': 'Place'}, #, 'Time': 'Time'}, + 'TransportPerson': {'Agent': 'Agent', 'Person': 'Artifact', 'Instrument': 'Vehicle', 'Destination': 'Destination', 'Origin': 'Origin'}, #, 'Time': 'Time'}, + 'TransportArtifact': {'Agent': 'Agent', 'Artifact': 'Artifact', 'Instrument': 'Vehicle', 'Destination': 'Destination', 'Origin': 'Origin'}, #, 'Time': 'Time'}, +} + +# event_role_mapping_ace2brat = {role_name_norm(v): k for t in event_role_mapping_brat2ace.items() for k, v in event_role_mapping_brat2ace[t]} +event_role_mapping_ace2brat = defaultdict(lambda : defaultdict()) +for t in event_role_mapping_brat2ace: + for k, v in event_role_mapping_brat2ace[t].items(): + event_type_ace = event_type_norm(event_type_mapping_brat2ace[t]) + event_role_mapping_ace2brat[event_type_ace][role_name_norm(v)] = k +# print(event_role_mapping_ace2brat) + +event_type_mapping_image2ace = { + 'Movement.TransportPerson': 'Movement:Transport', + 'Movement.TransportArtifact': 'Movement:Transport', + 'Life.Die': 'Life:Die', + 'Conflict.Attack': 'Conflict:Attack', + 'Conflict.Demonstrate': 'Conflict:Demonstrate', + 'Contact.Phone-Write': 'Contact:Phone-Write', + 'Contact.Meet': 'Contact:Meet', + 'Transaction.TransferMoney': 'Transaction:Transfer-Money', + 'Justice.ArrestJail': 'Justice:Arrest-Jail', +} + +event_role_mapping_image2ace = { + 'Life.Die': {'victim': 'Victim', 'agent': 'Agent', 'instrument': 'Instrument', 'place': 'Place'}, #, 'Time': 'Time'}, + 'Transaction.TransferMoney': {'instrument': 'Instrument', 'giver': 'Giver', 'recipient': 'Recipient', 'beneficiary': 'Beneficiary', 'money': 'Money', 'place': 'Place'}, #, 'Time': 'Time'}, + 'Conflict.Attack': {'attacker': 'Attacker', 'instrument': 'Instrument', 'place': 'Place', 'target': 'Target', 'victim': 'Target'}, #, 'Time': 'Time'}, + 'Conflict.Demonstrate': {'police':'Police', 'instrument': 'Instrument', 'demonstrator': 'Entity', 'place': 'Place', 'participant': 'Entity'}, #, 'Time': 'Time'}, + 'Contact.Phone-Write': {'instrument':'Instrument', 'participant': 'Entity', 'place': 'Place'}, #, 'Time': 'Time'}, + 'Contact.Meet': {'participant': 'Entity', 'place': 'Place'}, #, 'Time': 'Time'}, + 'Justice.ArrestJail': {'instrument': 'Instrument', 'agent': 'Agent', 'person': 'Person', 'place': 'Place'}, #, 'Time': 'Time'}, + 'Movement.TransportPerson': {'agent': 'Agent', 'person': 'Artifact', 'instrument': 'Instrument', 'destination': 'Destination', 'origin': 'Origin'}, #, 'Time': 'Time'}, + 'Movement.TransportArtifact': {'person': 'Artifact', 'agent': 'Agent', 'artifact': 'Artifact', 'instrument': 'Instrument', 'destination': 'Destination', 'origin': 'Origin'}, #, 'Time': 'Time'}, +} +# 'TransportArtifact' has 'person' in image annotation \ No newline at end of file diff --git a/src/dataflow/numpy/data_loader_ace_role.py b/src/dataflow/numpy/data_loader_ace_role.py old mode 100644 new mode 100755 diff --git a/src/dataflow/numpy/data_loader_grounding.py b/src/dataflow/numpy/data_loader_grounding.py old mode 100644 new mode 100755 diff --git a/src/dataflow/numpy/data_loader_situation.py b/src/dataflow/numpy/data_loader_situation.py old mode 100644 new mode 100755 diff --git a/src/dataflow/numpy/dataset_image_download.py b/src/dataflow/numpy/dataset_image_download.py old mode 100644 new mode 100755 index 04128ce..27e7cd2 --- a/src/dataflow/numpy/dataset_image_download.py +++ b/src/dataflow/numpy/dataset_image_download.py @@ -13,9 +13,10 @@ def download_image_list(meta_json, dir_save): for doc_id in metadata: for img_id in metadata[doc_id]: url_image = metadata[doc_id][img_id]['url'] - image_path_save = os.path.join(dir_save, '%s_%s.jpg' % (doc_id, img_id)) - if not url_image.endswith('.jpg'): - print(image_path_save, url_image) + suffix_image = metadata[doc_id][img_id]['url'].split('.')[-1] + image_path_save = os.path.join(dir_save, '%s_%s.%s' % (doc_id, img_id, suffix_image)) + # if not url_image.endswith('.jpg'): + # print(image_path_save, url_image) download_image(url_image, image_path_save) diff --git a/src/dataflow/numpy/prepare_vocab.py b/src/dataflow/numpy/prepare_vocab.py old mode 100644 new mode 100755 diff --git a/src/dataflow/torch/Corpus.py b/src/dataflow/torch/Corpus.py old mode 100644 new mode 100755 diff --git a/src/dataflow/torch/Data.py b/src/dataflow/torch/Data.py old mode 100644 new mode 100755 diff --git a/src/dataflow/torch/Sentence.py b/src/dataflow/torch/Sentence.py old mode 100644 new mode 100755 diff --git a/src/dataflow/torch/__init__.py b/src/dataflow/torch/__init__.py old mode 100644 new mode 100755 diff --git a/src/engine/.gitkeep b/src/engine/.gitkeep old mode 100644 new mode 100755 diff --git a/src/engine/EErunner.py b/src/engine/EErunner.py old mode 100644 new mode 100755 diff --git a/src/engine/EEtraining.py b/src/engine/EEtraining.py old mode 100644 new mode 100755 diff --git a/src/engine/Groundingrunner.py b/src/engine/Groundingrunner.py old mode 100644 new mode 100755 diff --git a/src/engine/Groundingtraining.py b/src/engine/Groundingtraining.py old mode 100644 new mode 100755 diff --git a/src/engine/JOINTRunner.py b/src/engine/JOINTRunner.py old mode 100644 new mode 100755 diff --git a/src/engine/JOINTTraining_sample.py b/src/engine/JOINTTraining_sample.py old mode 100644 new mode 100755 diff --git a/src/engine/SRrunner.py b/src/engine/SRrunner.py old mode 100644 new mode 100755 diff --git a/src/engine/SRtraining.py b/src/engine/SRtraining.py old mode 100644 new mode 100755 diff --git a/src/engine/TestRunnerEE.py b/src/engine/TestRunnerEE.py old mode 100644 new mode 100755 diff --git a/src/engine/TestRunnerEE_m2e2.py b/src/engine/TestRunnerEE_m2e2.py old mode 100644 new mode 100755 diff --git a/src/engine/TestRunnerJOINT.py b/src/engine/TestRunnerJOINT.py old mode 100644 new mode 100755 diff --git a/src/engine/TestRunnerSR.py b/src/engine/TestRunnerSR.py old mode 100644 new mode 100755 diff --git a/src/engine/TestRunnerSR_m2e2.py b/src/engine/TestRunnerSR_m2e2.py old mode 100644 new mode 100755 diff --git a/src/engine/__init__.py b/src/engine/__init__.py old mode 100644 new mode 100755 diff --git a/src/eval/.gitkeep b/src/eval/.gitkeep old mode 100644 new mode 100755 diff --git a/src/eval/EEtesting.py b/src/eval/EEtesting.py old mode 100644 new mode 100755 diff --git a/src/eval/EEvisualizing.py b/src/eval/EEvisualizing.py old mode 100644 new mode 100755 diff --git a/src/eval/Groundingtesting.py b/src/eval/Groundingtesting.py old mode 100644 new mode 100755 diff --git a/src/eval/JOINTtesting.py b/src/eval/JOINTtesting.py old mode 100644 new mode 100755 diff --git a/src/eval/SRtesting.py b/src/eval/SRtesting.py old mode 100644 new mode 100755 diff --git a/src/eval/__init__.py b/src/eval/__init__.py old mode 100644 new mode 100755 diff --git a/src/models/.gitkeep b/src/models/.gitkeep old mode 100644 new mode 100755 diff --git a/src/models/__init__.py b/src/models/__init__.py old mode 100644 new mode 100755 diff --git a/src/models/ace_classifier.py b/src/models/ace_classifier.py old mode 100644 new mode 100755 diff --git a/src/models/ee.py b/src/models/ee.py old mode 100644 new mode 100755 diff --git a/src/models/grounding.py b/src/models/grounding.py old mode 100644 new mode 100755 diff --git a/src/models/modules/.gitkeep b/src/models/modules/.gitkeep old mode 100644 new mode 100755 diff --git a/src/models/modules/DynamicLSTM.py b/src/models/modules/DynamicLSTM.py old mode 100644 new mode 100755 diff --git a/src/models/modules/EmbeddingLayer.py b/src/models/modules/EmbeddingLayer.py old mode 100644 new mode 100755 diff --git a/src/models/modules/EmbeddingLayerImage.py b/src/models/modules/EmbeddingLayerImage.py old mode 100644 new mode 100755 diff --git a/src/models/modules/FMapLayerImage.py b/src/models/modules/FMapLayerImage.py old mode 100644 new mode 100755 diff --git a/src/models/modules/GCN.py b/src/models/modules/GCN.py old mode 100644 new mode 100755 diff --git a/src/models/modules/HighWay.py b/src/models/modules/HighWay.py old mode 100644 new mode 100755 diff --git a/src/models/modules/SelfAttention.py b/src/models/modules/SelfAttention.py old mode 100644 new mode 100755 diff --git a/src/models/modules/__init__.py b/src/models/modules/__init__.py old mode 100644 new mode 100755 diff --git a/src/models/modules/model.py b/src/models/modules/model.py old mode 100644 new mode 100755 diff --git a/src/models/sr.py b/src/models/sr.py old mode 100644 new mode 100755 diff --git a/src/models/sr_object.py b/src/models/sr_object.py old mode 100644 new mode 100755 diff --git a/src/util/.gitkeep b/src/util/.gitkeep old mode 100644 new mode 100755 diff --git a/src/util/__init__.py b/src/util/__init__.py old mode 100644 new mode 100755 diff --git a/src/util/constant.py b/src/util/constant.py old mode 100644 new mode 100755 diff --git a/src/util/consts.py b/src/util/consts.py old mode 100644 new mode 100755 diff --git a/src/util/helper.py b/src/util/helper.py old mode 100644 new mode 100755 diff --git a/src/util/util_img.py b/src/util/util_img.py old mode 100644 new mode 100755 diff --git a/src/util/util_model.py b/src/util/util_model.py old mode 100644 new mode 100755 diff --git a/src/util/vocab.py b/src/util/vocab.py old mode 100644 new mode 100755 diff --git a/training-testing.jpg b/training-testing.jpg old mode 100644 new mode 100755