From 9f6e6e7d218159f0392cc50da8e892bce0af1e08 Mon Sep 17 00:00:00 2001 From: limanling Date: Tue, 1 Jun 2021 23:01:57 -0500 Subject: [PATCH 1/2] add image downloading --- README.md | 4 ++-- src/dataflow/numpy/dataset_image_download.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index fbef72f..62fbbdf 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,10 @@ pip install -r requirements.txt We download situation recognition data from [imSitu](http://imsitu.org/). Please find the preprocessed data in [PreprcessedSR](https://drive.google.com/drive/folders/1h0qwYWeGEoCx8m-zwH-XcoPSyffmrC-c?usp=sharing). ### ACE (Text Event Extraction Data) -We preprcoessed ACE following [JMEE](https://github.com/lx865712528/EMNLP2018-JMEE/tree/master). The preprocessing script is in `dataflow/preprocess_ace_JMEE.py`, and the sample data format is in [sample.json](https://github.com/lx865712528/EMNLP2018-JMEE/blob/master/ace-05-splits/sample.json). Due to license reason, the ACE 2005 dataset is only accessible to those with LDC2006T06 license, please drop me an email showing your possession of the license for the processed data. +We preprcoessed ACE following [JMEE](https://github.com/lx865712528/EMNLP2018-JMEE/tree/master). The sample data format is in [sample.json](https://github.com/lx865712528/EMNLP2018-JMEE/blob/master/ace-05-splits/sample.json). Due to license reason, the ACE 2005 dataset is only accessible to those with LDC2006T06 license, please drop me an email `manling2@illinois.edu` showing your possession of the license for the processed data. ### Voice of America Image-Caption Pairs -We crawled VOA image-captions to train the common space, the [image-caption pairs](https://uofi.box.com/s/xtn9p6m8z5qtjbbi5tqrl45tn6apew4x) and images can be downloaded using the URLs (We share image URLs instead of downloaded images due to license issue). We preprocess the data including object detection, and parse text sentences. The preprocessed data is in [PreprocessedVOA](https://drive.google.com/drive/folders/1I9vMGIhWZpKqxQYip91eLoDRnrkqRxnt?usp=sharing). +We crawled VOA image-captions to train the common space, the [image-caption pairs](https://uofi.box.com/s/xtn9p6m8z5qtjbbi5tqrl45tn6apew4x) and images can be downloaded using the URLs (We share image URLs instead of downloaded images due to license issue) using script in [dataset_image_download.py](https://github.com/limanling/m2e2/blob/master/src/dataflow/numpy/dataset_image_download.py). We preprocess the data including object detection, and parse text sentences. The preprocessed data is in [PreprocessedVOA](https://drive.google.com/drive/folders/1I9vMGIhWZpKqxQYip91eLoDRnrkqRxnt?usp=sharing). ### M2E2 (Multimedia Event Extraction Benchmark) diff --git a/src/dataflow/numpy/dataset_image_download.py b/src/dataflow/numpy/dataset_image_download.py index 04128ce..27e7cd2 100644 --- a/src/dataflow/numpy/dataset_image_download.py +++ b/src/dataflow/numpy/dataset_image_download.py @@ -13,9 +13,10 @@ def download_image_list(meta_json, dir_save): for doc_id in metadata: for img_id in metadata[doc_id]: url_image = metadata[doc_id][img_id]['url'] - image_path_save = os.path.join(dir_save, '%s_%s.jpg' % (doc_id, img_id)) - if not url_image.endswith('.jpg'): - print(image_path_save, url_image) + suffix_image = metadata[doc_id][img_id]['url'].split('.')[-1] + image_path_save = os.path.join(dir_save, '%s_%s.%s' % (doc_id, img_id, suffix_image)) + # if not url_image.endswith('.jpg'): + # print(image_path_save, url_image) download_image(url_image, image_path_save) From b3ebd0049f187a12028a16f4d4381f7657c62a8c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Sep 2021 02:16:41 +0000 Subject: [PATCH 2/2] Bump pillow from 8.1.1 to 8.3.2 Bumps [pillow](https://github.com/python-pillow/Pillow) from 8.1.1 to 8.3.2. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/master/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/8.1.1...8.3.2) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0c934f7..eda234c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ jieba==0.39 nltk==3.4.5 scipy==1.3.0 torchvision==0.3.0 -Pillow==8.1.1 +Pillow==8.3.2 common==0.1.2 constants==0.6.0 graphstate==1.0.6