Make individual modules into separate packages

.coveragerc

@@ -0,0 +1,12 @@
+[run]
+branch = True
+source = stor, stor_dx, stor_s3, stor_swift
+omit = stor/stor/tests/test_posix_path_compat.py,stor/stor/tests/test_integration.py,stor/stor/third_party/backoff.py,stor_dx/stor_dx/tests/test_integration_dx.py,stor_s3/stor_s3/tests/test_integration_s3.py,stor_swift/stor_swift/tests/test_integration_swift.py
+
+[report]
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+    raise NotImplementedError
+show_missing=1
+fail_under=100

.gitignore

@@ -6,12 +6,17 @@
 *.orig
 *.DS_Store
 .tox
+stor*/*.egg-info/*
+stor*/.eggs/
+.*cache/
+stor*/.*cache/
 *.egg/*
 .eggs/
 *.egg-info/*
+stor*/build/*
 build/*
 docs/_build/*
-dist/*
+stor*/dist/*
 
 *.log
 nosetests*.xml

AUTHORS

@@ -3,3 +3,4 @@ Jeff Tratner @jtratner
 Stephanie Huang @phanieste
 Kyle Beauchamp @kyleabeauchamp
 Piotr Kaleta @pkaleta
+Anuj Kumar @akumar

Makefile

@@ -1,6 +1,6 @@
 # Makefile utilities for running tests and publishing the package
 
-PACKAGE_NAME=stor
+PACKAGE_NAMES:=stor/ stor_dx/ stor_swift/ stor_s3/
 TEST_OUTPUT?=nosetests.xml
 PIP_INDEX_URL=https://pypi.python.org/simple/
 PYTHON?=$(shell which python)
@@ -13,7 +13,7 @@ endif
 
 .PHONY: default
 default:
-	python setup.py check build
+	cd stor; python setup.py check build; cd ..
 
 VENV_DIR?=.venv
 VENV_ACTIVATE=$(VENV_DIR)/bin/activate
@@ -23,42 +23,42 @@ WITH_PBR=$(WITH_VENV) PBR_REQUIREMENTS_FILES=requirements-pbr.txt
 .PHONY: venv
 venv: $(VENV_ACTIVATE)
 
-$(VENV_ACTIVATE): requirements*.txt
+$(VENV_ACTIVATE): stor*/requirements*.txt
 	test -f $@ || virtualenv --python=$(PYTHON) $(VENV_DIR)
 	$(WITH_VENV) echo "Within venv, running $$(python --version)"
-	$(WITH_VENV) pip install -r requirements-setup.txt --index-url=${PIP_INDEX_URL}
-	$(WITH_VENV) pip install -e . --index-url=${PIP_INDEX_URL}
-	$(WITH_VENV) pip install -r requirements-dev.txt  --index-url=${PIP_INDEX_URL}
-	$(WITH_VENV) pip install -r requirements-docs.txt --index-url=${PIP_INDEX_URL}
+	$(WITH_VENV) pip install -r stor/requirements-setup.txt --index-url=${PIP_INDEX_URL}
+	$(WITH_VENV) ./run_all.sh 'pip install -e . --index-url=${PIP_INDEX_URL}' $(PACKAGE_NAMES)
+	$(WITH_VENV) pip install -r stor/requirements-dev.txt  --index-url=${PIP_INDEX_URL}
+	$(WITH_VENV) pip install -r stor/requirements-docs.txt --index-url=${PIP_INDEX_URL}
 	touch $@
 
 develop: venv
-	$(WITH_VENV) python setup.py develop
+	$(WITH_VENV) ./run_all.sh 'python setup.py develop' $(PACKAGE_NAMES)
 
 .PHONY: docs
 docs: venv clean-docs
 	$(WITH_VENV) cd docs && make html
 
-
 .PHONY: setup
 setup: ##[setup] Run an arbitrary setup.py command
 setup: venv
 ifdef ARGS
-	$(WITH_PBR) python setup.py ${ARGS}
+	$(WITH_PBR) ./run_all.sh 'python setup.py ${ARGS}' $(PACKAGE_NAMES)
 else
 	@echo "Won't run 'python setup.py ${ARGS}' without ARGS set."
 endif
 
 .PHONY: clean
 clean:
-	$(PYTHON) setup.py clean
-	rm -rf build/
-	rm -rf dist/
-	rm -rf *.egg*/
-	rm -rf __pycache__/
+	./run_all.sh '$(PYTHON) setup.py clean' $(PACKAGE_NAMES)
+	./run_all.sh 'rm -rf *.egg*/' $(PACKAGE_NAMES) .
+	./run_all.sh 'rm -rf dist/' $(PACKAGE_NAMES)
+	./run_all.sh 'rm -rf build/' $(PACKAGE_NAMES)
+	./run_all.sh 'rm -rf __pycache__/' $(PACKAGE_NAMES) .
+	./run_all.sh 'rm -rf .*cache/' $(PACKAGE_NAMES) .
 	rm -f MANIFEST
 	rm -f $(TEST_OUTPUT)
-	find $(PACKAGE_NAME) -type f -name '*.pyc' -delete
+	./run_all.sh 'find . -type f -name '*.pyc' -delete' $(PACKAGE_NAMES) .
 	rm -rf nosetests* "${TEST_OUTPUT}" coverage .coverage
 
 
@@ -73,7 +73,7 @@ teardown:
 
 .PHONY: lint
 lint: venv
-	$(WITH_VENV) flake8 $(PACKAGE_NAME)/
+	$(WITH_VENV) flake8 $(PACKAGE_NAMES)
 
 .PHONY: unit-test
 unit-test: venv
@@ -99,14 +99,15 @@ endif
 .PHONY: travis-test
 travis-test: venv
 	$(WITH_VENV) \
-	coverage erase; \
-	coverage run setup.py test; \
+	./run_all.sh 'coverage erase' $(PACKAGE_NAMES) .; \
+	./run_all.sh 'coverage run setup.py test' $(PACKAGE_NAMES); \
+	coverage combine stor*/.coverage; \
 	status=$$?; \
 	coverage report && exit $$status;
 
 # Distribution
 
-VERSION=$(shell $(WITH_PBR) python setup.py --version | sed 's/\([0-9]*\.[0-9]*\.[0-9]*\).*$$/\1/')
+VERSION=$(shell $(WITH_PBR) cd stor; python setup.py --version | sed 's/\([0-9]*\.[0-9]*\.[0-9]*\).*$$/\1/'; cd ..)
 
 .PHONY: tag
 tag: ##[distribution] Tag the release.
@@ -117,7 +118,7 @@ tag: venv
 
 .PHONY: dist
 dist: venv fullname
-	$(WITH_VENV) python setup.py sdist
+	$(WITH_VENV) ./run_all.sh 'python setup.py sdist' $(PACKAGE_NAMES)
 
 .PHONY: publish-docs
 publish-docs:
@@ -133,4 +134,4 @@ version:
 
 .PHONY: fullname
 fullname:
-	$(PYTHON) setup.py --fullname
+	./run_all.sh '$(PYTHON) setup.py --fullname' $(PACKAGE_NAMES)

docs/settings.rst

@@ -47,7 +47,7 @@ Settings can be configured in the following ways in order of precedence:
 Default Settings
 ----------------
 
-.. literalinclude:: ../stor/default.cfg
+.. literalinclude:: ../stor/stor/default.cfg
 
 Settings API
 ------------

run_all.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+CMD=$1
+shift
+DIRS=$*
+for directory in $DIRS; do
+    cd $directory; ${CMD}; cd ..
+done

setup.py

@@ -1,6 +0,0 @@
-from setuptools import setup
-
-setup(
-    pbr=True,
-    setup_requires=['pbr'],
-)

stor/requirements.txt

@@ -0,0 +1,4 @@
+# have to explicitly ban certain requests versions to match keystoneauth1 package for swift
+requests!=2.12.2,!=2.13.0,>=2.10.0
+six
+python-swiftclient

setup.cfg → stor/setup.cfg

@@ -23,7 +23,7 @@ classifier =
 [coverage:run]
 branch = True
 source = stor
-omit = stor/tests/test_posix_path_compat.py,stor_swift/tests/test_integration_swift.py,stor_s3/tests/test_integration_s3.py,stor/tests/test_integration.py,stor/third_party/backoff.py,stor_dx/tests/test_integration_dx.py
+omit = stor/tests/test_posix_path_compat.py,stor/tests/test_integration.py,stor/third_party/backoff.py
 
 [coverage:report]
 exclude_lines =

stor/setup.py

@@ -0,0 +1,6 @@
+from setuptools import setup
+
+setup(
+    pbr=True,
+    setup_requires=['pbr'],
+)

stor/base.py → stor/stor/base.py

@@ -1,8 +1,10 @@
 import errno
 import fnmatch
 import glob
-import os
+import logging
 import ntpath
+import os
+import pkg_resources
 import posixpath
 import shutil
 import sys
@@ -16,6 +18,30 @@
 from stor import utils
 
 
+logger = logging.getLogger(__name__)
+
+
+def get_modules():
+    modules = {}
+    for entry_point in pkg_resources.iter_entry_points('stor.providers'):
+        try:
+            modules.update({entry_point.name: entry_point.load()})
+        except pkg_resources.DistributionNotFound as e:  # pragma: no cover
+            warnings.warn('Ignoring {entry_point} module as the requirement(s) '
+                          'for the module are not installed. Exception : {exception}'
+                          .format(entry_point=entry_point.name, exception=e))
+            pass
+    return modules
+
+
+def find_cls_for_path(path):
+    module_map = get_modules()
+    for k, v in module_map.items():
+        if path.startswith(k + '://'):
+            return v(k, path)
+    return None, None
+
+
 class TreeWalkWarning(Warning):
     pass
 
@@ -41,32 +67,26 @@ class Path(text_type):
     """
 
     def __new__(cls, path):
-        from stor_swift import utils as swift_utils
-        from stor_s3 import utils as s3_utils
-        from stor_dx import utils as dx_utils
         if cls is Path:
             if not hasattr(path, 'startswith'):
                 raise TypeError('must be a string like')
-            if dx_utils.is_dx_path(path):
-                cls = dx_utils.find_dx_class(path)
-            elif swift_utils.is_swift_path(path):
-                from stor_swift.swift import SwiftPath
-
-                cls = SwiftPath
-            elif s3_utils.is_s3_path(path):
-                from stor_s3.s3 import S3Path
-
-                cls = S3Path
-            elif os.path == ntpath:
-                from stor.windows import WindowsPath
-
-                cls = WindowsPath
-            elif os.path == posixpath:
-                from stor.posix import PosixPath
-
-                cls = PosixPath
-            else:  # pragma: no cover
-                assert False, 'path is not compatible with stor'
+            cls, new_p = find_cls_for_path(path)
+            if cls is None:
+                if os.path == ntpath:
+                    from stor.windows import WindowsPath
+
+                    cls = WindowsPath
+                elif os.path == posixpath:
+                    from stor.posix import PosixPath
+
+                    cls = PosixPath
+                else:  # pragma: no cover
+                    assert False, 'path is not compatible with stor'
+            else:
+                if path != new_p:  # pragma: no cover
+                    path = new_p
+                    cls.path_converted = True
+                    cls.new_path = new_p
         return text_type.__new__(cls, path)
 
     def __init__(self, path):

stor/obs.py → stor/stor/obs.py

@@ -67,6 +67,9 @@ def __init__(self, pth):
                 can be any of s3 or swift.
                 The ``(drive)://`` prefix is required in the path.
         """
+        if hasattr(self, 'path_converted') and self.path_converted \
+                and hasattr(self, 'new_path'):  # pragma: no cover
+            pth = self.new_path
         if not hasattr(pth, 'startswith') or not pth.startswith(self.drive):
             raise ValueError('path must have %s (got %r)' % (self.drive, pth))
         return super(OBSPath, self).__init__(pth)

stor/tests/test_posix.py → stor/stor/tests/test_posix.py

@@ -7,6 +7,14 @@
 from stor import Path
 from stor import posix
 from stor import windows
+from stor.base import find_cls_for_path
+
+
+class TestInit(unittest.TestCase):
+    def test_swift_init(self):
+        cls, path = find_cls_for_path('some/posix/path')
+        self.assertEqual(cls, None)
+        self.assertEqual(path, None)
 
 
 class TestDiv(unittest.TestCase):

stor/utils.py → stor/stor/utils.py

@@ -6,6 +6,7 @@
 import tempfile
 
 from stor import exceptions
+import stor
 
 logger = logging.getLogger(__name__)
 
@@ -144,7 +145,6 @@ def generate_and_save_data_manifest(manifest_dir, data_manifest_contents):
         data_manifest_contents (List[str]): The list of all objects that will
             be part of the manifest.
     """
-    import stor
     from stor import Path
 
     manifest_file_name = Path(manifest_dir) / DATA_MANIFEST_FILE_NAME
@@ -155,7 +155,6 @@ def generate_and_save_data_manifest(manifest_dir, data_manifest_contents):
 
 def get_data_manifest_contents(manifest_dir):
     """Reads the manifest file and returns a set of expected files"""
-    import stor
 
     manifest = manifest_dir / DATA_MANIFEST_FILE_NAME
     with stor.open(manifest, 'r') as manifest_file:
@@ -182,10 +181,11 @@ def is_filesystem_path(p):
     Returns:
         bool: True if p is a Windows path, False otherwise.
     """
-    from stor_swift.utils import is_swift_path
-    from stor_s3.utils import is_s3_path
-    from stor_dx.utils import is_dx_path
-    return not (is_swift_path(p) or is_s3_path(p) or is_dx_path(p))
+    module_map = stor.base.get_modules()
+    for k, v in module_map.items():
+        if p.startswith(k + '://'):
+            return False
+    return True
 
 
 def is_obs_path(p):

stor_dx/requirements.txt

@@ -0,0 +1,4 @@
+cached-property
+contextlib2
+dxpy>=0.265.0; python_version <= '2.7'
+dxpy3; python_version > '3.0'