Consolidating separating individual modules into their packages

.coveragerc

@@ -0,0 +1,12 @@
+[run]
+branch = True
+source = stor, stor_dx, stor_s3, stor_swift
+omit = stor/stor/tests/test_posix_path_compat.py,stor/stor/tests/test_integration.py,stor/stor/third_party/backoff.py,stor_dx/stor_dx/tests/test_integration_dx.py,stor_s3/stor_s3/tests/test_integration_s3.py,stor_swift/stor_swift/tests/test_integration_swift.py
+
+[report]
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+    raise NotImplementedError
+show_missing=1
+fail_under=100

.gitignore

@@ -6,12 +6,17 @@
 *.orig
 *.DS_Store
 .tox
+stor*/*.egg-info/*
+stor*/.eggs/
+.*cache/
+stor*/.*cache/
 *.egg/*
 .eggs/
 *.egg-info/*
+stor*/build/*
 build/*
 docs/_build/*
-dist/*
+stor*/dist/*
 
 *.log
 nosetests*.xml

AUTHORS

@@ -3,3 +3,4 @@ Jeff Tratner @jtratner
 Stephanie Huang @phanieste
 Kyle Beauchamp @kyleabeauchamp
 Piotr Kaleta @pkaleta
+Anuj Kumar @anujkumar93

Makefile

@@ -1,6 +1,6 @@
 # Makefile utilities for running tests and publishing the package
 
-PACKAGE_NAME=stor
+PACKAGE_NAMES:=stor/ stor_dx/ stor_swift/ stor_s3/
 TEST_OUTPUT?=nosetests.xml
 PIP_INDEX_URL=https://pypi.python.org/simple/
 PYTHON?=$(shell which python)
@@ -13,7 +13,7 @@ endif
 
 .PHONY: default
 default:
-	python setup.py check build
+	cd stor; python setup.py check build; cd ..
 
 VENV_DIR?=.venv
 VENV_ACTIVATE=$(VENV_DIR)/bin/activate
@@ -23,42 +23,42 @@ WITH_PBR=$(WITH_VENV) PBR_REQUIREMENTS_FILES=requirements-pbr.txt
 .PHONY: venv
 venv: $(VENV_ACTIVATE)
 
-$(VENV_ACTIVATE): requirements*.txt
+$(VENV_ACTIVATE): stor*/requirements*.txt
 	test -f $@ || virtualenv --python=$(PYTHON) $(VENV_DIR)
 	$(WITH_VENV) echo "Within venv, running $$(python --version)"
-	$(WITH_VENV) pip install -r requirements-setup.txt --index-url=${PIP_INDEX_URL}
-	$(WITH_VENV) pip install -e . --index-url=${PIP_INDEX_URL}
-	$(WITH_VENV) pip install -r requirements-dev.txt  --index-url=${PIP_INDEX_URL}
-	$(WITH_VENV) pip install -r requirements-docs.txt --index-url=${PIP_INDEX_URL}
+	$(WITH_VENV) pip install -r stor/requirements-setup.txt --index-url=${PIP_INDEX_URL}
+	$(WITH_VENV) ./run_all.sh 'pip install -e . --index-url=${PIP_INDEX_URL}' $(PACKAGE_NAMES)
+	$(WITH_VENV) pip install -r stor/requirements-dev.txt  --index-url=${PIP_INDEX_URL}
+	$(WITH_VENV) pip install -r stor/requirements-docs.txt --index-url=${PIP_INDEX_URL}
 	touch $@
 
 develop: venv
-	$(WITH_VENV) python setup.py develop
+	$(WITH_VENV) ./run_all.sh 'python setup.py develop' $(PACKAGE_NAMES)
 
 .PHONY: docs
 docs: venv clean-docs
 	$(WITH_VENV) cd docs && make html
 
-
 .PHONY: setup
 setup: ##[setup] Run an arbitrary setup.py command
 setup: venv
 ifdef ARGS
-	$(WITH_PBR) python setup.py ${ARGS}
+	$(WITH_PBR) ./run_all.sh 'python setup.py ${ARGS}' $(PACKAGE_NAMES)
 else
 	@echo "Won't run 'python setup.py ${ARGS}' without ARGS set."
 endif
 
 .PHONY: clean
 clean:
-	$(PYTHON) setup.py clean
-	rm -rf build/
-	rm -rf dist/
-	rm -rf *.egg*/
-	rm -rf __pycache__/
+	./run_all.sh '$(PYTHON) setup.py clean' $(PACKAGE_NAMES)
+	./run_all.sh 'rm -rf *.egg*/' $(PACKAGE_NAMES) .
+	./run_all.sh 'rm -rf dist/' $(PACKAGE_NAMES)
+	./run_all.sh 'rm -rf build/' $(PACKAGE_NAMES)
+	./run_all.sh 'rm -rf __pycache__/' $(PACKAGE_NAMES) .
+	./run_all.sh 'rm -rf .*cache/' $(PACKAGE_NAMES) .
 	rm -f MANIFEST
 	rm -f $(TEST_OUTPUT)
-	find $(PACKAGE_NAME) -type f -name '*.pyc' -delete
+	./run_all.sh 'find . -type f -name '*.pyc' -delete' $(PACKAGE_NAMES) .
 	rm -rf nosetests* "${TEST_OUTPUT}" coverage .coverage
 
 
@@ -73,7 +73,7 @@ teardown:
 
 .PHONY: lint
 lint: venv
-	$(WITH_VENV) flake8 $(PACKAGE_NAME)/
+	$(WITH_VENV) ./run_all.sh 'flake8 .' $(PACKAGE_NAMES)
 
 .PHONY: unit-test
 unit-test: venv
@@ -99,14 +99,15 @@ endif
 .PHONY: travis-test
 travis-test: venv
 	$(WITH_VENV) \
-	coverage erase; \
-	coverage run setup.py test; \
+	./run_all.sh 'coverage erase' $(PACKAGE_NAMES) .; \
+	./run_all.sh 'coverage run setup.py test' $(PACKAGE_NAMES); \
+	coverage combine stor*/.coverage; \
 	status=$$?; \
 	coverage report && exit $$status;
 
 # Distribution
 
-VERSION=$(shell $(WITH_PBR) python setup.py --version | sed 's/\([0-9]*\.[0-9]*\.[0-9]*\).*$$/\1/')
+VERSION=$(shell $(WITH_PBR) cd stor; python setup.py --version | sed 's/\([0-9]*\.[0-9]*\.[0-9]*\).*$$/\1/'; cd ..)
 
 .PHONY: tag
 tag: ##[distribution] Tag the release.
@@ -117,7 +118,7 @@ tag: venv
 
 .PHONY: dist
 dist: venv fullname
-	$(WITH_VENV) python setup.py sdist
+	$(WITH_VENV) ./run_all.sh 'python setup.py sdist' $(PACKAGE_NAMES)
 
 .PHONY: publish-docs
 publish-docs:
@@ -133,4 +134,4 @@ version:
 
 .PHONY: fullname
 fullname:
-	$(PYTHON) setup.py --fullname
+	./run_all.sh '$(PYTHON) setup.py --fullname' $(PACKAGE_NAMES)

docs/package_split.rst

@@ -0,0 +1,45 @@
+Stor Modularization
+===================
+
+The legacy version of stor has been split into 4 packages: ``stor``, ``stor_dx``, ``stor_swift``, and
+``stor_s3``. ``stor_swift``, ``stor_dx``, ``stor_s3`` are modular packages implemented to be used with
+the core package ``stor`` directly. The core package ``stor`` now only supports Posix and Windows
+filesystems, apart from supporting extra plugins.
+
+
+Implementation
+--------------
+
+Each of the modular packages work by registering themselves onto pkg_resources with an entry_point
+``stor.providers``. This entry point should be a function which takes in a prefix and a path, and
+returns the cls that the path should be instantiated to, as well as the path that should be initialized.
+Typically, this function is called ``class_for_path`` in ``stor_dx``, ``stor_s3``, and ``stor_swift``.
+Each plugin module currently raise an error if the prefix passed is not the prefix it supports. For
+example, ``stor_dx.class_for_path`` errors if the prefix is not ``dx``. ``get_class_for_path`` in each
+plugin module may assume that the prefix is the true prefix to the path argument as this is guaranteed
+by the core ``stor`` package.
+
+
+Code Changes
+------------
+
+`stor.copy`, `stor.copytree` and ``stor.open`` which were earlier present in the core ``stor.utils`` and
+``stor.obs`` have been split according to their individual functionalities into the three packages.
+These functions in the core package now only deal with posix/windows paths while the three plugins
+implement the finer aspects of the logic individual to each platform. The only external effect of
+these changes is that `stor.copy` and `stor.copytree` now don't support a ``source`` kwarg, instead
+expect the first argument to be a ``Path | str``, which is then taken to be the source to be copied from.
+
+``is_swift_path`` has been removed from the core `stor` package. Thus, using ``stor.is_swift_path`` will
+fail. This is because the plugins determine the prefix they support and the core package cannot know in
+advance if ``swift://`` is a supported path. In cases where ``stor.is_swift_path`` was being used,
+``stor.is_obs_path`` is a possible substitution. Thus, each individual plugin ``stor_dx``, ``stor_s3`` and
+``stor_swift`` is now responsible for supporting ``is_dx_path``, ``is_s3_path`` and ``is_swift_path`` resp.
+
+
+Versioning
+----------
+Since the core stor package and the plugins live on the same github repo, and PBR is used for semantic
+versioning, they will have the same published version at any given time. However, it is possible to
+install an older version of ``stor`` with a newer version of the plugins ``stor_dx``, etc. The behavior is
+undefined in this case. The core ``stor`` package has been implemented to require the extra plugins for now.

docs/release_notes.rst

@@ -1,6 +1,27 @@
 Release Notes
 =============
 
+v4.0.0
+------
+
+Package Split
+^^^^^^^^^^^^^
+
+* To ease the installation of dependencies, stor is broken down into 4 packages: ``stor``, ``stor_dx``,
+  ``stor_swift``, and ``stor_s3``.
+* ``stor_dx``, ``stor_s3``, and ``stor_swift`` are meant to be plugins to stor. The core stor package,
+  responsible for posix and windows paths, is still required to be able to use stor.
+* ``stor_dx``, ``stor_s3``, and ``stor_swift`` are responsible for manipulating resources on the DNAnexus,
+  Amazon S3 and Swift cloud platforms respectively.
+
+API Breaks
+^^^^^^^^^^
+* The function signature of ``stor.copy`` has changed slightly, from ``stor.copy(source='some/source',
+  dest='some/dest')`` to ``stor.copy('some/source', dest='some/dest')``. ``stor.copy`` needs a ``Path|str``
+  as its first argument (which is taken to be the source). The source kwarg is no longer supported.
+* ``is_swift_path`` has been removed from the core ``stor`` package. Hence, ``stor.is_swift_path`` is no
+  longer supported. Can use `stor.is_obs_path` in such cases after consideration.
+
 v3.0.0
 ------
 

docs/settings.rst

@@ -47,7 +47,7 @@ Settings can be configured in the following ways in order of precedence:
 Default Settings
 ----------------
 
-.. literalinclude:: ../stor/default.cfg
+.. literalinclude:: ../stor/stor/default.cfg
 
 Settings API
 ------------

docs/toc.rst

@@ -18,5 +18,6 @@ Table of Contents
    settings
    extensions
    contributing
+   package_split
    roadmap
    release_notes

run_all.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+CMD=$1
+shift
+DIRS=$*
+for directory in $DIRS; do
+    cd $directory; ${CMD}; cd ..
+done

setup.py

@@ -1,6 +0,0 @@
-from setuptools import setup
-
-setup(
-    pbr=True,
-    setup_requires=['pbr'],
-)

stor/requirements.txt

@@ -0,0 +1,4 @@
+# have to explicitly ban certain requests versions to match keystoneauth1 package for swift
+requests!=2.12.2,!=2.13.0,>=2.10.0
+six
+python-swiftclient

setup.cfg → stor/setup.cfg

@@ -23,7 +23,7 @@ classifier =
 [coverage:run]
 branch = True
 source = stor
-omit = stor/tests/test_posix_path_compat.py,stor_swift/tests/test_integration_swift.py,stor_s3/tests/test_integration_s3.py,stor/tests/test_integration.py,stor/third_party/backoff.py,stor_dx/tests/test_integration_dx.py
+omit = stor/tests/test_posix_path_compat.py,stor/tests/test_integration.py,stor/third_party/backoff.py
 
 [coverage:report]
 exclude_lines =

stor/setup.py

@@ -0,0 +1,6 @@
+from setuptools import setup
+
+setup(
+    pbr=True,
+    setup_requires=['pbr'],
+)

stor/base.py → stor/stor/base.py

@@ -1,8 +1,10 @@
 import errno
 import fnmatch
 import glob
-import os
+import logging
 import ntpath
+import os
+import pkg_resources
 import posixpath
 import shutil
 import sys
@@ -16,6 +18,30 @@
 from stor import utils
 
 
+logger = logging.getLogger(__name__)
+
+
+def get_modules():
+    modules = {}
+    for entry_point in pkg_resources.iter_entry_points('stor.providers'):
+        try:
+            modules.update({entry_point.name: entry_point.load()})
+        except pkg_resources.DistributionNotFound as e:  # pragma: no cover
+            warnings.warn('Ignoring {entry_point} module as the requirement(s) '
+                          'for the module are not installed. Exception : {exception}'
+                          .format(entry_point=entry_point.name, exception=e))
+            pass
+    return modules
+
+
+def get_class_for_path(path):
+    module_map = get_modules()
+    for k, v in module_map.items():
+        if path.startswith(k + '://'):
+            return v(k, path)
+    return None, None
+
+
 class TreeWalkWarning(Warning):
     pass
 
@@ -41,32 +67,23 @@ class Path(text_type):
     """
 
     def __new__(cls, path):
-        from stor_swift import utils as swift_utils
-        from stor_s3 import utils as s3_utils
-        from stor_dx import utils as dx_utils
         if cls is Path:
             if not hasattr(path, 'startswith'):
                 raise TypeError('must be a string like')
-            if dx_utils.is_dx_path(path):
-                cls = dx_utils.find_dx_class(path)
-            elif swift_utils.is_swift_path(path):
-                from stor_swift.swift import SwiftPath
-
-                cls = SwiftPath
-            elif s3_utils.is_s3_path(path):
-                from stor_s3.s3 import S3Path
-
-                cls = S3Path
-            elif os.path == ntpath:
-                from stor.windows import WindowsPath
-
-                cls = WindowsPath
-            elif os.path == posixpath:
-                from stor.posix import PosixPath
-
-                cls = PosixPath
-            else:  # pragma: no cover
-                assert False, 'path is not compatible with stor'
+            cls, obs_p = get_class_for_path(path)
+            if cls is None:
+                if os.path == ntpath:
+                    from stor.windows import WindowsPath
+
+                    cls = WindowsPath
+                elif os.path == posixpath:
+                    from stor.posix import PosixPath
+
+                    cls = PosixPath
+                else:  # pragma: no cover
+                    assert False, 'path is not compatible with stor'
+            else:
+                path = obs_p
         return text_type.__new__(cls, path)
 
     def __init__(self, path):