Preparing the modules to be split

docs/testing.rst

@@ -2,12 +2,23 @@
 
 Testing
 =======
-Stor provides a test case to use when testing against Swift storage. It also
+Stor provides test case classes to use when testing against OBS storage. It also
 provides a mixin class that can be used when creating other base test classes
 
-.. automodule:: stor.test
 .. autoclass:: stor_swift.test.SwiftTestMixin
     :members:
 
 .. autoclass:: stor_swift.test.SwiftTestCase
     :members:
+
+.. autoclass:: stor_s3.test.S3TestMixin
+    :members:
+
+.. autoclass:: stor_s3.test.S3TestCase
+    :members:
+
+.. autoclass:: stor_dx.test.DXTestMixin
+    :members:
+
+.. autoclass:: stor_dx.test.DXTestCase
+    :members:

stor/__init__.py

@@ -23,7 +23,6 @@
 import pkg_resources
 
 from stor.utils import is_filesystem_path
-from stor.utils import is_swift_path
 from stor.utils import is_obs_path
 from stor.utils import NamedTemporaryDirectory
 from stor.base import Path
@@ -118,7 +117,6 @@ def glob(pth, pattern):  # pragma: no cover
     'rmtree',
     'walkfiles',
     'is_filesystem_path',
-    'is_swift_path',
     'is_obs_path',
     'NamedTemporaryDirectory',
 ]

stor/base.py

@@ -4,7 +4,6 @@
 import os
 import ntpath
 import posixpath
-import shlex
 import shutil
 import sys
 import warnings
@@ -42,17 +41,19 @@ class Path(text_type):
     """
 
     def __new__(cls, path):
+        from stor_swift import utils as swift_utils
+        from stor_s3 import utils as s3_utils
+        from stor_dx import utils as dx_utils
         if cls is Path:
             if not hasattr(path, 'startswith'):
                 raise TypeError('must be a string like')
-            if utils.is_dx_path(path):
-                from stor_dx import utils as dx_utils
+            if dx_utils.is_dx_path(path):
                 cls = dx_utils.find_dx_class(path)
-            elif utils.is_swift_path(path):
+            elif swift_utils.is_swift_path(path):
                 from stor_swift.swift import SwiftPath
 
                 cls = SwiftPath
-            elif utils.is_s3_path(path):
+            elif s3_utils.is_s3_path(path):
                 from stor_s3.s3 import S3Path
 
                 cls = S3Path

stor/cli.py

@@ -277,10 +277,13 @@ def get_path(pth, mode=None):
 
 def _wrapped_list(path, **kwargs):
     """Use iterative walkfiles for DX paths, rather than trying to generate full list first"""
-    if utils.is_dx_path(path):
-        func = stor.walkfiles
-    else:
-        func = stor.list
+    func = stor.list
+    try:
+        from stor_dx import utils as dx_utils
+        if dx_utils.is_dx_path(path):
+            func = stor.walkfiles
+    except ImportError:  # pragma: no cover
+        pass
     return func(path, **kwargs)
 
 
@@ -291,16 +294,21 @@ def _to_url(path):
 
 
 def _convert_swiftstack(path, bucket=None):
+    try:
+        from stor_swift import utils as swift_utils
+        from stor_s3 import utils as s3_utils
+    except ImportError:  # pragma: no cover
+        raise ImportError(
+            'stor-swift and stor-s3 must be installed to use convert-swiftstack command')
     path = stor.Path(path)
-    if utils.is_swift_path(path):
+    if swift_utils.is_swift_path(path):
         if not bucket:
             # TODO (jtratner): print help here
             raise ValueError('--bucket is required for swift paths')
         return swiftstack.swift_to_s3(path, bucket=bucket)
-    elif utils.is_s3_path(path):
+    elif s3_utils.is_s3_path(path):
         return swiftstack.s3_to_swift(path)
-    else:
-        raise ValueError("invalid path for conversion: '%s'" % path)
+    raise ValueError("invalid path for conversion: '%s'" % path)
 
 
 def create_parser():

stor/tests/test_posix.py

@@ -1,4 +1,3 @@
-import mock
 import os
 import tempfile
 import unittest
@@ -7,9 +6,6 @@
 from stor import NamedTemporaryDirectory
 from stor import Path
 from stor import posix
-from stor_s3 import s3
-from stor import settings
-from stor_swift import swift
 from stor import windows
 
 
@@ -27,20 +23,6 @@ def test_w_windows_path(self):
         with self.assertRaisesRegexp(TypeError, 'unsupported operand'):
             posix.PosixPath('my/path') / windows.WindowsPath(r'other\path')
 
-    def test_w_swift_component(self):
-        p = posix.PosixPath('my/path') / swift.SwiftPath('swift://t/c/name').name
-        self.assertEquals(p, posix.PosixPath('my/path/name'))
-        self.assertEquals(stor.join('my/path',
-                                    swift.SwiftPath('swift://t/c/name').name),
-                          p)
-
-    def test_w_s3_component(self):
-        p = posix.PosixPath('my/path') / s3.S3Path('s3://b/name').name
-        self.assertEquals(p, posix.PosixPath('my/path/name'))
-        self.assertEquals(stor.join('my/path',
-                                    s3.S3Path('s3://b/name').name),
-                          p)
-
 
 class TestAdd(unittest.TestCase):
     def test_success(self):
@@ -51,14 +33,6 @@ def test_w_windows_path(self):
         with self.assertRaisesRegexp(TypeError, 'unsupported operand'):
             posix.PosixPath('my/path') + windows.WindowsPath(r'other\path')
 
-    def test_w_swift_component(self):
-        p = posix.PosixPath('my/path') + swift.SwiftPath('swift://t/c/name').name
-        self.assertEquals(p, posix.PosixPath('my/pathname'))
-
-    def test_w_s3_component(self):
-        p = posix.PosixPath('my/path') + s3.S3Path('s3://b/name').name
-        self.assertEquals(p, posix.PosixPath('my/pathname'))
-
     def test_invalid_radd(self):
         with self.assertRaisesRegexp(TypeError, 'unsupported operand'):
             1 + posix.PosixPath('my/path')
@@ -91,65 +65,6 @@ def test_posix_file_destination(self):
             self.assertTrue(dest.exists())
             self.assertEquals(dest.open().read(), '1')
 
-    def test_ambigious_swift_resource_destination(self):
-        with stor.NamedTemporaryDirectory() as tmp_d:
-            source = tmp_d / '1'
-            with open(source, 'w') as tmp_file:
-                tmp_file.write('1')
-
-            dest = 'swift://tenant/container/ambiguous-resource'
-            with self.assertRaisesRegexp(ValueError, 'Swift destination'):
-                stor.copy(source, dest)
-
-    def test_ambigious_swift_container_destination(self):
-        with stor.NamedTemporaryDirectory() as tmp_d:
-            source = tmp_d / '1'
-            with open(source, 'w') as tmp_file:
-                tmp_file.write('1')
-
-            dest = 'swift://tenant/ambiguous-container'
-            with self.assertRaisesRegexp(ValueError, 'Swift destination'):
-                stor.copy(source, dest)
-
-    def test_tenant_swift_destination(self):
-        with stor.NamedTemporaryDirectory() as tmp_d:
-            source = tmp_d / 'source'
-            os.mkdir(source)
-            with open(source / '1.txt', 'w') as tmp_file:
-                tmp_file.write('1')
-            dest = 'swift://tenant/'
-            with self.assertRaisesRegexp(ValueError, 'copy to tenant'):
-                stor.copy(source / '1.txt', dest)
-
-    @mock.patch.object(swift.SwiftPath, 'upload', autospec=True)
-    def test_swift_destination(self, mock_upload):
-        dest = Path('swift://tenant/container/file.txt')
-        with tempfile.NamedTemporaryFile() as tmp_f:
-            Path(tmp_f.name).copy(dest)
-            upload_args = mock_upload.call_args_list[0][0]
-            self.assertEquals(upload_args[0], dest.parent)
-            self.assertEquals(upload_args[1][0].source, tmp_f.name)
-            self.assertEquals(upload_args[1][0].object_name, 'file.txt')
-
-    @mock.patch.object(s3.S3Path, 'upload', autospec=True)
-    def test_s3_destination(self, mock_upload):
-        dest = Path('s3://bucket/key/file.txt')
-        with tempfile.NamedTemporaryFile() as tmp_f:
-            Path(tmp_f.name).copy(dest)
-            upload_args = mock_upload.call_args_list[0][0]
-            self.assertEquals(upload_args[0], dest.parent)
-            self.assertEquals(upload_args[1][0].source, tmp_f.name)
-            self.assertEquals(upload_args[1][0].object_name, 'key/file.txt')
-
-    def test_ambigious_s3_destination(self):
-        with stor.NamedTemporaryDirectory() as tmp_d:
-            source = tmp_d / '1'
-            with open(source, 'w') as tmp_file:
-                tmp_file.write('1')
-            dest = 's3://tenant/ambiguous-container'
-            with self.assertRaisesRegexp(ValueError, 'S3 destination'):
-                stor.copy(source, dest)
-
 
 class TestCopytree(unittest.TestCase):
     def test_posix_destination(self):
@@ -190,38 +105,6 @@ def test_posix_destination_w_error(self):
             with self.assertRaises(OSError):
                 stor.copytree(invalid_source, dest)
 
-    @mock.patch.object(swift.SwiftPath, 'upload', autospec=True)
-    def test_swift_destination(self, mock_upload):
-        source = '.'
-        dest = Path('swift://tenant/container')
-        options = {
-            'swift:upload': {
-                'object_threads': 30,
-                'segment_threads': 40
-            }
-        }
-
-        with settings.use(options):
-            stor.copytree(source, dest)
-        mock_upload.assert_called_once_with(
-            dest,
-            ['.'],
-            condition=None,
-            use_manifest=False,
-            headers=None)
-
-    @mock.patch.object(s3.S3Path, 'upload', autospec=True)
-    def test_s3_destination(self, mock_upload):
-        source = '.'
-        dest = Path('s3://tenant/container')
-        stor.copytree(source, dest)
-        mock_upload.assert_called_once_with(
-            dest,
-            ['.'],
-            condition=None,
-            use_manifest=False,
-            headers=None)
-
 
 class TestOpen(unittest.TestCase):
     def test_functional_open(self):

stor/tests/test_windows.py

@@ -1,6 +1,4 @@
 from stor import posix
-from stor_s3 import s3
-from stor_swift import swift
 from stor import windows
 import unittest
 
@@ -14,14 +12,6 @@ def test_w_posix_path(self):
         with self.assertRaisesRegexp(TypeError, 'unsupported operand'):
             windows.WindowsPath(r'my\path') / posix.PosixPath('other/path')
 
-    def test_w_swift_component(self):
-        with self.assertRaisesRegexp(TypeError, 'unsupported operand'):
-            windows.WindowsPath(r'my\path') / swift.SwiftPath('swift://t/c/name').name
-
-    def test_w_s3_component(self):
-        with self.assertRaisesRegexp(TypeError, 'unsupported operand'):
-            windows.WindowsPath(r'my\path') / s3.S3Path('s3://b/name').name
-
 
 class TestAdd(unittest.TestCase):
     def test_success(self):
@@ -31,11 +21,3 @@ def test_success(self):
     def test_w_posix_path(self):
         with self.assertRaisesRegexp(TypeError, 'unsupported operand'):
             windows.WindowsPath(r'my\path') + posix.PosixPath('other/path')
-
-    def test_w_swift_component(self):
-        with self.assertRaisesRegexp(TypeError, 'unsupported operand'):
-            windows.WindowsPath(r'my\path') + swift.SwiftPath('swift://t/c/name').name
-
-    def test_w_s3_component(self):
-        with self.assertRaisesRegexp(TypeError, 'unsupported operand'):
-            windows.WindowsPath(r'my\path') + s3.S3Path('s3://b/name').name