From 3e57efdef3a8aac0f7fec205ba566896b690c31d Mon Sep 17 00:00:00 2001 From: Anuj Kumar Date: Fri, 30 Nov 2018 12:59:58 -0800 Subject: [PATCH 01/10] separating utils and posix tests --- stor/tests/test_posix.py | 117 --------------------------- stor_dx/tests/test_utils_dx.py | 0 stor_s3/tests/test_utils_s3.py | 2 +- stor_swift/tests/test_utils_swift.py | 2 +- 4 files changed, 2 insertions(+), 119 deletions(-) delete mode 100644 stor_dx/tests/test_utils_dx.py diff --git a/stor/tests/test_posix.py b/stor/tests/test_posix.py index 5a511a4d..d62c6cb8 100644 --- a/stor/tests/test_posix.py +++ b/stor/tests/test_posix.py @@ -1,4 +1,3 @@ -import mock import os import tempfile import unittest @@ -7,9 +6,6 @@ from stor import NamedTemporaryDirectory from stor import Path from stor import posix -from stor_s3 import s3 -from stor import settings -from stor_swift import swift from stor import windows @@ -27,20 +23,6 @@ def test_w_windows_path(self): with self.assertRaisesRegexp(TypeError, 'unsupported operand'): posix.PosixPath('my/path') / windows.WindowsPath(r'other\path') - def test_w_swift_component(self): - p = posix.PosixPath('my/path') / swift.SwiftPath('swift://t/c/name').name - self.assertEquals(p, posix.PosixPath('my/path/name')) - self.assertEquals(stor.join('my/path', - swift.SwiftPath('swift://t/c/name').name), - p) - - def test_w_s3_component(self): - p = posix.PosixPath('my/path') / s3.S3Path('s3://b/name').name - self.assertEquals(p, posix.PosixPath('my/path/name')) - self.assertEquals(stor.join('my/path', - s3.S3Path('s3://b/name').name), - p) - class TestAdd(unittest.TestCase): def test_success(self): @@ -51,14 +33,6 @@ def test_w_windows_path(self): with self.assertRaisesRegexp(TypeError, 'unsupported operand'): posix.PosixPath('my/path') + windows.WindowsPath(r'other\path') - def test_w_swift_component(self): - p = posix.PosixPath('my/path') + swift.SwiftPath('swift://t/c/name').name - self.assertEquals(p, posix.PosixPath('my/pathname')) - - def test_w_s3_component(self): - p = posix.PosixPath('my/path') + s3.S3Path('s3://b/name').name - self.assertEquals(p, posix.PosixPath('my/pathname')) - def test_invalid_radd(self): with self.assertRaisesRegexp(TypeError, 'unsupported operand'): 1 + posix.PosixPath('my/path') @@ -91,65 +65,6 @@ def test_posix_file_destination(self): self.assertTrue(dest.exists()) self.assertEquals(dest.open().read(), '1') - def test_ambigious_swift_resource_destination(self): - with stor.NamedTemporaryDirectory() as tmp_d: - source = tmp_d / '1' - with open(source, 'w') as tmp_file: - tmp_file.write('1') - - dest = 'swift://tenant/container/ambiguous-resource' - with self.assertRaisesRegexp(ValueError, 'Swift destination'): - stor.copy(source, dest) - - def test_ambigious_swift_container_destination(self): - with stor.NamedTemporaryDirectory() as tmp_d: - source = tmp_d / '1' - with open(source, 'w') as tmp_file: - tmp_file.write('1') - - dest = 'swift://tenant/ambiguous-container' - with self.assertRaisesRegexp(ValueError, 'Swift destination'): - stor.copy(source, dest) - - def test_tenant_swift_destination(self): - with stor.NamedTemporaryDirectory() as tmp_d: - source = tmp_d / 'source' - os.mkdir(source) - with open(source / '1.txt', 'w') as tmp_file: - tmp_file.write('1') - dest = 'swift://tenant/' - with self.assertRaisesRegexp(ValueError, 'copy to tenant'): - stor.copy(source / '1.txt', dest) - - @mock.patch.object(swift.SwiftPath, 'upload', autospec=True) - def test_swift_destination(self, mock_upload): - dest = Path('swift://tenant/container/file.txt') - with tempfile.NamedTemporaryFile() as tmp_f: - Path(tmp_f.name).copy(dest) - upload_args = mock_upload.call_args_list[0][0] - self.assertEquals(upload_args[0], dest.parent) - self.assertEquals(upload_args[1][0].source, tmp_f.name) - self.assertEquals(upload_args[1][0].object_name, 'file.txt') - - @mock.patch.object(s3.S3Path, 'upload', autospec=True) - def test_s3_destination(self, mock_upload): - dest = Path('s3://bucket/key/file.txt') - with tempfile.NamedTemporaryFile() as tmp_f: - Path(tmp_f.name).copy(dest) - upload_args = mock_upload.call_args_list[0][0] - self.assertEquals(upload_args[0], dest.parent) - self.assertEquals(upload_args[1][0].source, tmp_f.name) - self.assertEquals(upload_args[1][0].object_name, 'key/file.txt') - - def test_ambigious_s3_destination(self): - with stor.NamedTemporaryDirectory() as tmp_d: - source = tmp_d / '1' - with open(source, 'w') as tmp_file: - tmp_file.write('1') - dest = 's3://tenant/ambiguous-container' - with self.assertRaisesRegexp(ValueError, 'S3 destination'): - stor.copy(source, dest) - class TestCopytree(unittest.TestCase): def test_posix_destination(self): @@ -190,38 +105,6 @@ def test_posix_destination_w_error(self): with self.assertRaises(OSError): stor.copytree(invalid_source, dest) - @mock.patch.object(swift.SwiftPath, 'upload', autospec=True) - def test_swift_destination(self, mock_upload): - source = '.' - dest = Path('swift://tenant/container') - options = { - 'swift:upload': { - 'object_threads': 30, - 'segment_threads': 40 - } - } - - with settings.use(options): - stor.copytree(source, dest) - mock_upload.assert_called_once_with( - dest, - ['.'], - condition=None, - use_manifest=False, - headers=None) - - @mock.patch.object(s3.S3Path, 'upload', autospec=True) - def test_s3_destination(self, mock_upload): - source = '.' - dest = Path('s3://tenant/container') - stor.copytree(source, dest) - mock_upload.assert_called_once_with( - dest, - ['.'], - condition=None, - use_manifest=False, - headers=None) - class TestOpen(unittest.TestCase): def test_functional_open(self): diff --git a/stor_dx/tests/test_utils_dx.py b/stor_dx/tests/test_utils_dx.py deleted file mode 100644 index e69de29b..00000000 diff --git a/stor_s3/tests/test_utils_s3.py b/stor_s3/tests/test_utils_s3.py index cf10bcbb..30877418 100644 --- a/stor_s3/tests/test_utils_s3.py +++ b/stor_s3/tests/test_utils_s3.py @@ -49,4 +49,4 @@ def test_success(self): def test_path_no_perms(self): self.mock_copy.side_effect = stor.exceptions.FailedUploadError('foo') self.assertFalse(stor.utils.is_writeable('s3://stor-test/foo/bar')) - self.assertFalse(self.mock_remove.called) \ No newline at end of file + self.assertFalse(self.mock_remove.called) diff --git a/stor_swift/tests/test_utils_swift.py b/stor_swift/tests/test_utils_swift.py index 01900274..9110c995 100644 --- a/stor_swift/tests/test_utils_swift.py +++ b/stor_swift/tests/test_utils_swift.py @@ -97,4 +97,4 @@ def test_container_created_in_another_client(self): # is_writeable is called. self.mock_exists.side_effect = [False, True] self.mock_remove_container.side_effect = stor_swift.swift.ConflictError('foo') - self.assertTrue(stor_utils.is_writeable('swift://AUTH_stor_test/container/')) \ No newline at end of file + self.assertTrue(stor_utils.is_writeable('swift://AUTH_stor_test/container/')) From d029e03928aeab4a070d62b83463bb6e333bbcbb Mon Sep 17 00:00:00 2001 From: Anuj Kumar Date: Fri, 30 Nov 2018 13:02:08 -0800 Subject: [PATCH 02/10] forgot to add new posix files --- stor_s3/tests/test_posix_s3.py | 59 ++++++++++++++++++ stor_swift/tests/test_posix_swift.py | 90 ++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 stor_s3/tests/test_posix_s3.py create mode 100644 stor_swift/tests/test_posix_swift.py diff --git a/stor_s3/tests/test_posix_s3.py b/stor_s3/tests/test_posix_s3.py new file mode 100644 index 00000000..a431ac10 --- /dev/null +++ b/stor_s3/tests/test_posix_s3.py @@ -0,0 +1,59 @@ +import mock +import tempfile +import unittest + +import stor +from stor import Path +from stor import posix +from stor_s3 import s3 + + +class TestDiv(unittest.TestCase): + def test_w_s3_component(self): + p = posix.PosixPath('my/path') / s3.S3Path('s3://b/name').name + self.assertEquals(p, posix.PosixPath('my/path/name')) + self.assertEquals(stor.join('my/path', + s3.S3Path('s3://b/name').name), + p) + + +class TestAdd(unittest.TestCase): + def test_w_s3_component(self): + p = posix.PosixPath('my/path') + s3.S3Path('s3://b/name').name + self.assertEquals(p, posix.PosixPath('my/pathname')) + + +class TestCopy(unittest.TestCase): + @mock.patch.object(s3.S3Path, 'upload', autospec=True) + def test_s3_destination(self, mock_upload): + dest = Path('s3://bucket/key/file.txt') + with tempfile.NamedTemporaryFile() as tmp_f: + Path(tmp_f.name).copy(dest) + upload_args = mock_upload.call_args_list[0][0] + self.assertEquals(upload_args[0], dest.parent) + self.assertEquals(upload_args[1][0].source, tmp_f.name) + self.assertEquals(upload_args[1][0].object_name, 'key/file.txt') + + def test_ambigious_s3_destination(self): + with stor.NamedTemporaryDirectory() as tmp_d: + source = tmp_d / '1' + with open(source, 'w') as tmp_file: + tmp_file.write('1') + + dest = 's3://tenant/ambiguous-container' + with self.assertRaisesRegexp(ValueError, 'S3 destination'): + stor.copy(source, dest) + + +class TestCopytree(unittest.TestCase): + @mock.patch.object(s3.S3Path, 'upload', autospec=True) + def test_s3_destination(self, mock_upload): + source = '.' + dest = Path('s3://tenant/container') + stor.copytree(source, dest) + mock_upload.assert_called_once_with( + dest, + ['.'], + condition=None, + use_manifest=False, + headers=None) diff --git a/stor_swift/tests/test_posix_swift.py b/stor_swift/tests/test_posix_swift.py new file mode 100644 index 00000000..05fd2cef --- /dev/null +++ b/stor_swift/tests/test_posix_swift.py @@ -0,0 +1,90 @@ +import os +import mock +import tempfile +import unittest + +import stor +from stor import Path +from stor import posix +from stor import settings +from stor_swift import swift + + +class TestDiv(unittest.TestCase): + def test_w_swift_component(self): + p = posix.PosixPath('my/path') / swift.SwiftPath('swift://t/c/name').name + self.assertEquals(p, posix.PosixPath('my/path/name')) + self.assertEquals(stor.join('my/path', + swift.SwiftPath('swift://t/c/name').name), + p) + + +class TestAdd(unittest.TestCase): + def test_w_swift_component(self): + p = posix.PosixPath('my/path') + swift.SwiftPath('swift://t/c/name').name + self.assertEquals(p, posix.PosixPath('my/pathname')) + + +class TestCopy(unittest.TestCase): + @mock.patch.object(swift.SwiftPath, 'upload', autospec=True) + def test_swift_destination(self, mock_upload): + dest = Path('swift://tenant/container/file.txt') + with tempfile.NamedTemporaryFile() as tmp_f: + Path(tmp_f.name).copy(dest) + upload_args = mock_upload.call_args_list[0][0] + self.assertEquals(upload_args[0], dest.parent) + self.assertEquals(upload_args[1][0].source, tmp_f.name) + self.assertEquals(upload_args[1][0].object_name, 'file.txt') + + def test_ambigious_swift_resource_destination(self): + with stor.NamedTemporaryDirectory() as tmp_d: + source = tmp_d / '1' + with open(source, 'w') as tmp_file: + tmp_file.write('1') + + dest = 'swift://tenant/container/ambiguous-resource' + with self.assertRaisesRegexp(ValueError, 'Swift destination'): + stor.copy(source, dest) + + def test_ambigious_swift_container_destination(self): + with stor.NamedTemporaryDirectory() as tmp_d: + source = tmp_d / '1' + with open(source, 'w') as tmp_file: + tmp_file.write('1') + + dest = 'swift://tenant/ambiguous-container' + with self.assertRaisesRegexp(ValueError, 'Swift destination'): + stor.copy(source, dest) + + def test_tenant_swift_destination(self): + with stor.NamedTemporaryDirectory() as tmp_d: + source = tmp_d / 'source' + os.mkdir(source) + with open(source / '1.txt', 'w') as tmp_file: + tmp_file.write('1') + + dest = 'swift://tenant/' + with self.assertRaisesRegexp(ValueError, 'copy to tenant'): + stor.copy(source / '1.txt', dest) + + +class TestCopytree(unittest.TestCase): + @mock.patch.object(swift.SwiftPath, 'upload', autospec=True) + def test_swift_destination(self, mock_upload): + source = '.' + dest = Path('swift://tenant/container') + options = { + 'swift:upload': { + 'object_threads': 30, + 'segment_threads': 40 + } + } + + with settings.use(options): + stor.copytree(source, dest) + mock_upload.assert_called_once_with( + dest, + ['.'], + condition=None, + use_manifest=False, + headers=None) From acb03e301c62c8e960ee25c1e4c3b3ee672dfd91 Mon Sep 17 00:00:00 2001 From: Anuj Kumar Date: Fri, 30 Nov 2018 13:25:26 -0800 Subject: [PATCH 03/10] splitting windows tests --- stor/tests/test_windows.py | 18 ------------------ stor_s3/tests/test_windows_s3.py | 15 +++++++++++++++ stor_swift/tests/test_windows_swift.py | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 18 deletions(-) create mode 100644 stor_s3/tests/test_windows_s3.py create mode 100644 stor_swift/tests/test_windows_swift.py diff --git a/stor/tests/test_windows.py b/stor/tests/test_windows.py index d0555725..b58e8368 100644 --- a/stor/tests/test_windows.py +++ b/stor/tests/test_windows.py @@ -1,6 +1,4 @@ from stor import posix -from stor_s3 import s3 -from stor_swift import swift from stor import windows import unittest @@ -14,14 +12,6 @@ def test_w_posix_path(self): with self.assertRaisesRegexp(TypeError, 'unsupported operand'): windows.WindowsPath(r'my\path') / posix.PosixPath('other/path') - def test_w_swift_component(self): - with self.assertRaisesRegexp(TypeError, 'unsupported operand'): - windows.WindowsPath(r'my\path') / swift.SwiftPath('swift://t/c/name').name - - def test_w_s3_component(self): - with self.assertRaisesRegexp(TypeError, 'unsupported operand'): - windows.WindowsPath(r'my\path') / s3.S3Path('s3://b/name').name - class TestAdd(unittest.TestCase): def test_success(self): @@ -31,11 +21,3 @@ def test_success(self): def test_w_posix_path(self): with self.assertRaisesRegexp(TypeError, 'unsupported operand'): windows.WindowsPath(r'my\path') + posix.PosixPath('other/path') - - def test_w_swift_component(self): - with self.assertRaisesRegexp(TypeError, 'unsupported operand'): - windows.WindowsPath(r'my\path') + swift.SwiftPath('swift://t/c/name').name - - def test_w_s3_component(self): - with self.assertRaisesRegexp(TypeError, 'unsupported operand'): - windows.WindowsPath(r'my\path') + s3.S3Path('s3://b/name').name diff --git a/stor_s3/tests/test_windows_s3.py b/stor_s3/tests/test_windows_s3.py new file mode 100644 index 00000000..75cccd6e --- /dev/null +++ b/stor_s3/tests/test_windows_s3.py @@ -0,0 +1,15 @@ +from stor import windows +from stor_s3 import s3 +import unittest + + +class TestDiv(unittest.TestCase): + def test_w_s3_component(self): + with self.assertRaisesRegexp(TypeError, 'unsupported operand'): + windows.WindowsPath(r'my\path') / s3.S3Path('s3://b/name').name + + +class TestAdd(unittest.TestCase): + def test_w_s3_component(self): + with self.assertRaisesRegexp(TypeError, 'unsupported operand'): + windows.WindowsPath(r'my\path') + s3.S3Path('s3://b/name').name diff --git a/stor_swift/tests/test_windows_swift.py b/stor_swift/tests/test_windows_swift.py new file mode 100644 index 00000000..9a938050 --- /dev/null +++ b/stor_swift/tests/test_windows_swift.py @@ -0,0 +1,15 @@ +from stor import windows +from stor_swift import swift +import unittest + + +class TestDiv(unittest.TestCase): + def test_w_swift_component(self): + with self.assertRaisesRegexp(TypeError, 'unsupported operand'): + windows.WindowsPath(r'my\path') / swift.SwiftPath('swift://t/c/name').name + + +class TestAdd(unittest.TestCase): + def test_w_swift_component(self): + with self.assertRaisesRegexp(TypeError, 'unsupported operand'): + windows.WindowsPath(r'my\path') + swift.SwiftPath('swift://t/c/name').name From b4b5eea14292f5d32673f6042ca1549312544721 Mon Sep 17 00:00:00 2001 From: Anuj Kumar Date: Sat, 1 Dec 2018 13:40:17 -0800 Subject: [PATCH 04/10] refining tests --- stor_dx/tests/test_dx.py | 2 +- stor_swift/test.py | 2 +- stor_swift/tests/test_cli_swift.py | 10 ++++------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/stor_dx/tests/test_dx.py b/stor_dx/tests/test_dx.py index da1907e1..895c1814 100644 --- a/stor_dx/tests/test_dx.py +++ b/stor_dx/tests/test_dx.py @@ -10,8 +10,8 @@ from stor import exceptions from stor import Path -from stor_dx import utils from stor.tests.shared_obs import SharedOBSFileCases +from stor_dx import utils from stor_dx.dx import DXPath from stor_dx.test import DXTestCase import stor diff --git a/stor_swift/test.py b/stor_swift/test.py index b35c291e..51a5886d 100644 --- a/stor_swift/test.py +++ b/stor_swift/test.py @@ -106,4 +106,4 @@ def setUp(self): 'password': '__dummy__', 'auth_url': '__dummy__' } - }) \ No newline at end of file + }) diff --git a/stor_swift/tests/test_cli_swift.py b/stor_swift/tests/test_cli_swift.py index be5be25c..ac595683 100644 --- a/stor_swift/tests/test_cli_swift.py +++ b/stor_swift/tests/test_cli_swift.py @@ -33,16 +33,14 @@ def assertOutputMatches(self, exit_status=None, stdout='', stderr=''): assert False, 'SystemExit not raised' else: yield - actual_stdout = sys.stdout.getvalue() - actual_stderr = sys.stderr.getvalue() if not stdout: - self.assertEqual(actual_stdout, '', 'stdout') + self.assertEqual(sys.stdout.getvalue(), '', 'stdout') else: - self.assertRegexpMatches(actual_stdout, stdout, 'stdout') + self.assertRegexpMatches(sys.stdout.getvalue(), stdout, 'stdout') if not stderr: - self.assertEqual(actual_stderr, '', 'stderr') + self.assertEqual(sys.stderr.getvalue(), '', 'stderr') else: - self.assertRegexpMatches(actual_stderr, stderr, 'stderr') + self.assertRegexpMatches(sys.stderr.getvalue(), stderr, 'stderr') def parse_args(self, args): with mock.patch.object(sys, 'argv', args.split()): From 006eb34fd0a0263985f757d9c9ae78d0c9f85607 Mon Sep 17 00:00:00 2001 From: Anuj Kumar Date: Sat, 1 Dec 2018 15:01:58 -0800 Subject: [PATCH 05/10] preparing the packages to (finally) be separated --- docs/testing.rst | 15 ++++++++++++-- stor/__init__.py | 2 -- stor/base.py | 11 +++++----- stor/cli.py | 36 ++++++++++++++++++++------------ stor/test.py | 0 stor/utils.py | 50 ++++----------------------------------------- stor_dx/dx.py | 2 +- stor_swift/swift.py | 5 +++-- 8 files changed, 50 insertions(+), 71 deletions(-) delete mode 100644 stor/test.py diff --git a/docs/testing.rst b/docs/testing.rst index 5abf9c91..be1f4797 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -2,12 +2,23 @@ Testing ======= -Stor provides a test case to use when testing against Swift storage. It also +Stor provides a test case to use when testing against OBS storage. It also provides a mixin class that can be used when creating other base test classes -.. automodule:: stor.test .. autoclass:: stor_swift.test.SwiftTestMixin :members: .. autoclass:: stor_swift.test.SwiftTestCase :members: + +.. autoclass:: stor_s3.test.S3TestMixin + :members: + +.. autoclass:: stor_s3.test.S3TestCase + :members: + +.. autoclass:: stor_dx.test.DXTestMixin + :members: + +.. autoclass:: stor_dx.test.DXTestCase + :members: diff --git a/stor/__init__.py b/stor/__init__.py index 71d9edd6..6054840e 100644 --- a/stor/__init__.py +++ b/stor/__init__.py @@ -23,7 +23,6 @@ import pkg_resources from stor.utils import is_filesystem_path -from stor.utils import is_swift_path from stor.utils import is_obs_path from stor.utils import NamedTemporaryDirectory from stor.base import Path @@ -118,7 +117,6 @@ def glob(pth, pattern): # pragma: no cover 'rmtree', 'walkfiles', 'is_filesystem_path', - 'is_swift_path', 'is_obs_path', 'NamedTemporaryDirectory', ] diff --git a/stor/base.py b/stor/base.py index 19ca805d..b9e394d3 100644 --- a/stor/base.py +++ b/stor/base.py @@ -4,7 +4,6 @@ import os import ntpath import posixpath -import shlex import shutil import sys import warnings @@ -42,17 +41,19 @@ class Path(text_type): """ def __new__(cls, path): + from stor_swift import utils as swift_utils + from stor_s3 import utils as s3_utils + from stor_dx import utils as dx_utils if cls is Path: if not hasattr(path, 'startswith'): raise TypeError('must be a string like') - if utils.is_dx_path(path): - from stor_dx import utils as dx_utils + if dx_utils.is_dx_path(path): cls = dx_utils.find_dx_class(path) - elif utils.is_swift_path(path): + elif swift_utils.is_swift_path(path): from stor_swift.swift import SwiftPath cls = SwiftPath - elif utils.is_s3_path(path): + elif s3_utils.is_s3_path(path): from stor_s3.s3 import S3Path cls = S3Path diff --git a/stor/cli.py b/stor/cli.py index 3f6c9ff0..39e1b623 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -277,10 +277,13 @@ def get_path(pth, mode=None): def _wrapped_list(path, **kwargs): """Use iterative walkfiles for DX paths, rather than trying to generate full list first""" - if utils.is_dx_path(path): - func = stor.walkfiles - else: - func = stor.list + func = stor.list + try: + from stor_dx import utils as dx_utils + if dx_utils.is_dx_path(path): + func = stor.walkfiles + except ImportError: # pragma: no cover + pass return func(path, **kwargs) @@ -292,15 +295,22 @@ def _to_url(path): def _convert_swiftstack(path, bucket=None): path = stor.Path(path) - if utils.is_swift_path(path): - if not bucket: - # TODO (jtratner): print help here - raise ValueError('--bucket is required for swift paths') - return swiftstack.swift_to_s3(path, bucket=bucket) - elif utils.is_s3_path(path): - return swiftstack.s3_to_swift(path) - else: - raise ValueError("invalid path for conversion: '%s'" % path) + try: + from stor_swift import utils as swift_utils + if swift_utils.is_swift_path(path): + if not bucket: + # TODO (jtratner): print help here + raise ValueError('--bucket is required for swift paths') + return swiftstack.swift_to_s3(path, bucket=bucket) + except ImportError: # pragma: no cover + pass + try: + from stor_s3 import utils as s3_utils + if s3_utils.is_s3_path(path): + return swiftstack.s3_to_swift(path) + except ImportError: # pragma: no cover + pass + raise ValueError("invalid path for conversion: '%s'" % path) def create_parser(): diff --git a/stor/test.py b/stor/test.py deleted file mode 100644 index e69de29b..00000000 diff --git a/stor/utils.py b/stor/utils.py index 32e3f147..2c55b4a9 100644 --- a/stor/utils.py +++ b/stor/utils.py @@ -173,21 +173,6 @@ def validate_manifest_list(expected_objs, list_results): return set(expected_objs).issubset(listed_objs) -def is_swift_path(p): - """Determines if the path is a Swift path. - - All Swift paths start with swift:// - - Args: - p (str): The path string - - Returns: - bool: True if p is a Swift path, False otherwise. - """ - from stor_swift.swift import SwiftPath - return p.startswith(SwiftPath.drive) - - def is_filesystem_path(p): """Determines if the path is posix or windows filesystem. @@ -197,24 +182,12 @@ def is_filesystem_path(p): Returns: bool: True if p is a Windows path, False otherwise. """ + from stor_swift.utils import is_swift_path + from stor_s3.utils import is_s3_path + from stor_dx.utils import is_dx_path return not (is_swift_path(p) or is_s3_path(p) or is_dx_path(p)) -def is_s3_path(p): - """Determines if the path is a S3 path. - - All S3 paths start with ``s3://`` - - Args: - p (str): The path string - - Returns - bool: True if p is a S3 path, False otherwise. - """ - from stor_s3.s3 import S3Path - return p.startswith(S3Path.drive) - - def is_obs_path(p): """Determines if the path is an OBS path (an S3 or Swift path). @@ -224,22 +197,7 @@ def is_obs_path(p): Returns bool: True if p is an OBS path, False otherwise. """ - return is_s3_path(p) or is_swift_path(p) or is_dx_path(p) - - -def is_dx_path(p): - """Determines if the path is a DX path. - - All DX paths start with ``dx://`` - - Args: - p (str): The path string - - Returns - bool: True if p is a DX path, False otherwise. - """ - from stor_dx.dx import DXPath - return p.startswith(DXPath.drive) + return not is_filesystem_path(p) def is_writeable(path, swift_retry_options=None): diff --git a/stor_dx/dx.py b/stor_dx/dx.py index ab18cc3b..06b5a796 100644 --- a/stor_dx/dx.py +++ b/stor_dx/dx.py @@ -1302,4 +1302,4 @@ def _wait_on_close(self): # ignore timeout because we want client code to deal with it except dxpy.DXError as e: # pragma: no cover if 'timeout' not in str(e): - raise \ No newline at end of file + raise diff --git a/stor_swift/swift.py b/stor_swift/swift.py index afc129e0..0bcee451 100644 --- a/stor_swift/swift.py +++ b/stor_swift/swift.py @@ -50,7 +50,6 @@ from swiftclient.utils import generate_temp_url from stor import exceptions as stor_exceptions -from stor import is_swift_path from stor import settings from stor import utils from stor.base import Path @@ -58,6 +57,7 @@ from stor.obs import OBSUploadObject from stor.posix import PosixPath from stor.third_party.backoff import with_backoff +from stor_swift.utils import is_swift_path logger = logging.getLogger(__name__) @@ -1052,7 +1052,8 @@ def upload(self, to_upload, condition=None, use_manifest=False, - headers=None): + headers=None, + **kwargs): """Uploads a list of files and directories to swift. This method retries ``num_retries`` times if swift is unavailable or if From 2f89a4a847dc87e7b35e08149c0fbe7d674ead53 Mon Sep 17 00:00:00 2001 From: Anuj Kumar Date: Mon, 3 Dec 2018 15:28:30 -0800 Subject: [PATCH 06/10] making is_writeable ready for package split --- stor/utils.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/stor/utils.py b/stor/utils.py index 2c55b4a9..5674b40c 100644 --- a/stor/utils.py +++ b/stor/utils.py @@ -235,10 +235,9 @@ def is_writeable(path, swift_retry_options=None): from stor import join from stor import Path from stor import remove - from stor_swift.swift import ConflictError - from stor_swift.swift import SwiftPath - from stor_swift.swift import UnauthorizedError - from stor_swift.swift import UnavailableError + from stor.exceptions import ConflictError + from stor.exceptions import UnauthorizedError + from stor.exceptions import UnavailableError import stor path = with_trailing_slash(Path(path)) @@ -248,18 +247,24 @@ def is_writeable(path, swift_retry_options=None): container_path = None container_existed = None - if is_swift_path(path): - # We want this function to behave as a no-op with regards to the underlying - # container structure. Therefore we need to remove any containers created by this - # function that were not present when it was called. The `container_existed` - # defined below will store whether the container that we're checking existed when - # calling this function, so that we know if it should be removed at the end. - container_path = Path('{}{}/{}/'.format( - SwiftPath.drive, - path.tenant, - path.container - )) - container_existed = container_path.exists() + try: + from stor_swift.swift import SwiftPath + from stor_swift.utils import is_swift_path + except ImportError as e: # pragma: no cover + logger.warn('Cannot import Swift module: {}'.format(e)) + else: + if is_swift_path(path): + # We want this function to behave as a no-op with regards to the underlying + # container structure. Therefore we need to remove any containers created by this + # function that were not present when it was called. The `container_existed` + # defined below will store whether the container that we're checking existed when + # calling this function, so that we know if it should be removed at the end. + container_path = Path('{}{}/{}/'.format( + SwiftPath.drive, + path.tenant, + path.container + )) + container_existed = container_path.exists() with tempfile.NamedTemporaryFile() as tmpfile: try: From 911595a071e702fe459964e6d9efb9f1b2fdd06d Mon Sep 17 00:00:00 2001 From: Anuj Kumar Date: Tue, 4 Dec 2018 00:35:50 -0800 Subject: [PATCH 07/10] addressed review comments --- stor/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stor/utils.py b/stor/utils.py index 5674b40c..b8093ed0 100644 --- a/stor/utils.py +++ b/stor/utils.py @@ -250,9 +250,7 @@ def is_writeable(path, swift_retry_options=None): try: from stor_swift.swift import SwiftPath from stor_swift.utils import is_swift_path - except ImportError as e: # pragma: no cover - logger.warn('Cannot import Swift module: {}'.format(e)) - else: + if is_swift_path(path): # We want this function to behave as a no-op with regards to the underlying # container structure. Therefore we need to remove any containers created by this @@ -265,6 +263,8 @@ def is_writeable(path, swift_retry_options=None): path.container )) container_existed = container_path.exists() + except ImportError as e: # pragma: no cover + pass with tempfile.NamedTemporaryFile() as tmpfile: try: From af95eb9e49b1e14d88c4e28653d550eaf69cd2d9 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Tue, 4 Dec 2018 08:13:37 -0800 Subject: [PATCH 08/10] Update docs/testing.rst Co-Authored-By: anujkumar93 --- docs/testing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing.rst b/docs/testing.rst index be1f4797..fab858a6 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -2,7 +2,7 @@ Testing ======= -Stor provides a test case to use when testing against OBS storage. It also +Stor provides test case classes to use when testing against OBS storage. It also provides a mixin class that can be used when creating other base test classes .. autoclass:: stor_swift.test.SwiftTestMixin From b65970cb72241e48b5d210feafe556d74a01f032 Mon Sep 17 00:00:00 2001 From: Anuj Kumar Date: Tue, 4 Dec 2018 08:14:46 -0800 Subject: [PATCH 09/10] lint fix --- stor/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stor/utils.py b/stor/utils.py index b8093ed0..be7d1cbb 100644 --- a/stor/utils.py +++ b/stor/utils.py @@ -263,7 +263,7 @@ def is_writeable(path, swift_retry_options=None): path.container )) container_existed = container_path.exists() - except ImportError as e: # pragma: no cover + except ImportError: # pragma: no cover pass with tempfile.NamedTemporaryFile() as tmpfile: From a9ac93996f7306681155054bb0f6e775e639fd73 Mon Sep 17 00:00:00 2001 From: Anuj Kumar Date: Tue, 4 Dec 2018 08:26:21 -0800 Subject: [PATCH 10/10] addressed needing both swift and s3 for convert-swiftstack --- stor/cli.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 39e1b623..672570b1 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -294,22 +294,20 @@ def _to_url(path): def _convert_swiftstack(path, bucket=None): - path = stor.Path(path) try: from stor_swift import utils as swift_utils - if swift_utils.is_swift_path(path): - if not bucket: - # TODO (jtratner): print help here - raise ValueError('--bucket is required for swift paths') - return swiftstack.swift_to_s3(path, bucket=bucket) - except ImportError: # pragma: no cover - pass - try: from stor_s3 import utils as s3_utils - if s3_utils.is_s3_path(path): - return swiftstack.s3_to_swift(path) except ImportError: # pragma: no cover - pass + raise ImportError( + 'stor-swift and stor-s3 must be installed to use convert-swiftstack command') + path = stor.Path(path) + if swift_utils.is_swift_path(path): + if not bucket: + # TODO (jtratner): print help here + raise ValueError('--bucket is required for swift paths') + return swiftstack.swift_to_s3(path, bucket=bucket) + elif s3_utils.is_s3_path(path): + return swiftstack.s3_to_swift(path) raise ValueError("invalid path for conversion: '%s'" % path)