Skip to content

xopen: open gzipped files transparently #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kristjaneerik wants to merge 4 commits into
base: master
Choose a base branch
from
Open

xopen: open gzipped files transparently #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f7c5552
Merge pull request #1 from counsyl/master
kristjaneerik Apr 19, 2018
f95c130
Sem-Ver: feature - xopen: transparently handle gzipped-files
kristjaneerik Apr 20, 2018
9c4dbd8
add file with some unicode
kristjaneerik Apr 20, 2018
08e3b3e
add emoji
kristjaneerik Apr 20, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions stor/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ def wrapper(path, *args, **kwargs):

# extra compat!
open = _delegate_to_path('open')
xopen = _delegate_to_path('xopen')
abspath = _delegate_to_path('abspath')
normcase = _delegate_to_path('normcase')
normpath = _delegate_to_path('normpath')
Expand Down
17 changes: 17 additions & 0 deletions stor/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import shutil
import sys
import warnings
import gzip

from six.moves import builtins
from six import text_type
Expand Down Expand Up @@ -272,6 +273,22 @@ def open(self, *args, **kwargs):

raise NotImplementedError

def xopen(self, mode='r', *args, **kwargs):
"""Open a file-like object, transparently handling gzip compression.

See: :func:`open`
"""
if self.endswith('.gz'):
if mode == 'r':
mode = 'rb'
if mode == 'w':

@jtratner jtratner Jun 28, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think these codepaths aren't actually tested (that's what is causing CI to fail).

Additionally, you need to pass the original mode to the gzip.GzipFile object, so perhaps you want something like:

if mode in ('r', 'rb'): fp_mode='rb'
if mode in ('w', 'wb'): fp_mode = 'wb'
fp = self.open(fp_mode, *args, **kwargs)
gzfp = gzip.GzipFile(mode=mode, fileobj=fp)

Additionally, GzipFile's docs say that it doesn't automatically close the underlying file object - which means that if you do something like this:

with stor.xopen('s3://unauthed-bucket/myfile.csv.gz') as fp:
    fp.write('somedata')

the exception will not bubble up to the user (I believe)

mode = 'wb'
fp = self.open(mode, *args, **kwargs)
gzfp = gzip.GzipFile(fileobj=fp)
return gzfp
else:
return self.open(mode, *args, **kwargs)

def list(self, *args, **kwargs):
"""List all contents using the path as a prefix.

Expand Down
2 changes: 2 additions & 0 deletions stor/tests/file_data/utf8_file_with_unicode.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
a ≥ b
c 😄
33 changes: 33 additions & 0 deletions stor/tests/shared_obs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,39 @@ def test_works_with_gzip(self, mock_read_object):
gzip_fp.seek(3)
assert_same_data(fp, gzip_fp)

@mock_read_object
def test_xopen_gzip(self, mock_read_object):
gzip_path = stor.join(stor.dirname(__file__), 'file_data', 's_3_2126.bcl.gz')

correct_binary_data = stor.open(gzip_path, 'rb').read()
mock_read_object.return_value = correct_binary_data

with stor.xopen(stor.join(self.drive, 'A/C/s_3_2126.bcl.gz'), 'r') as fp:
with gzip.open(gzip_path) as gzip_fp:
assert_same_data(fp, gzip_fp)

with stor.xopen(stor.join(self.drive, 'A/C/s_3_2126.bcl.gz'), 'r') as fp:
with gzip.open(gzip_path) as gzip_fp:
# after seeking, the rest of the data should still be the same
fp.seek(3)
gzip_fp.seek(3)
assert_same_data(fp, gzip_fp)

@mock_read_object
def test_xopen_regular(self, mock_read_object):
fpath = stor.join(stor.dirname(__file__), 'file_data', 'utf8_file_with_unicode.txt')
correct_binary_data = stor.open(fpath, 'rb').read()
mock_read_object.return_value = correct_binary_data

for mode in 'r', 'rb':
with stor.xopen(fpath, mode) as xfp:
with stor.open(fpath, mode) as fp:
assert_same_data(xfp, fp)

with stor.xopen(stor.join(self.drive, 'A/C/utf8_file_with_unicode.txt'), 'rb') as xfp:
with stor.open(fpath, 'rb') as fp:
assert_same_data(xfp, fp)

def test_makedirs_p_does_nothing(self):
# dumb test... but why not?
self.normal_path.makedirs_p()
Expand Down