From 65607764cdb184cf483e82fccaf2a8eaf80efbe9 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Thu, 6 Jan 2022 22:32:37 -0800 Subject: [PATCH 01/16] Guard dxpy imports so dnanexus can actually be optional dxpy pulls in a wide array of (often legacy) packages, skipping the import means we only need to pull it in as necessary which hopefully makes library more useful to others. --- stor/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stor/utils.py b/stor/utils.py index 01a6f073..d6a3bb46 100644 --- a/stor/utils.py +++ b/stor/utils.py @@ -8,9 +8,6 @@ from subprocess import check_call import tempfile -from dxpy.bindings import verify_string_dxid -from dxpy.exceptions import DXError - from stor import exceptions logger = logging.getLogger(__name__) @@ -257,6 +254,9 @@ def is_valid_dxid(dxid, expected_classes): Returns bool: Whether given dxid is a valid path of one of expected_classes """ + from dxpy.bindings import verify_string_dxid + from dxpy.exceptions import DXError + try: return verify_string_dxid(dxid, expected_classes) is None except DXError: From f83718f8dd3de4d7fac85a9c3763495e46082d77 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Thu, 6 Jan 2022 22:40:57 -0800 Subject: [PATCH 02/16] Guard more imports --- docs/release_notes.rst | 5 +++++ pyproject.toml | 2 +- stor/obs.py | 19 +++++++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 987cda8f..19e8ab65 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -1,6 +1,11 @@ Release Notes ============= +v4.0.3 +------ +* Guard dxpy imports in utils so you can use stor without dxpy installed. + + v4.0.2 ------ * fix DeprecationWarning (#140) diff --git a/pyproject.toml b/pyproject.toml index 6a22f0ca..f57e22cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "stor" -version = "4.0.2" +version = "4.0.3" description = "Cross-compatible API for accessing Posix and OBS storage systems" authors = ["Counsyl Inc. "] license = "MIT" diff --git a/stor/obs.py b/stor/obs.py index a071f4d8..b9622d38 100644 --- a/stor/obs.py +++ b/stor/obs.py @@ -3,10 +3,6 @@ import posixpath import sys -import dxpy -from swiftclient.service import SwiftError -from swiftclient.service import SwiftUploadObject - from stor.base import Path from stor.posix import PosixPath from stor import utils @@ -28,6 +24,17 @@ def wrapper(self, *args, **kwargs): return wrapper +try: + from swiftclient.service import SwiftUploadObject +except ImportError: + class SwiftUploadObject(object): + """Give 90% of the utility of SwiftUploadObject class without swiftclient!""" + def __init__(self, source, object_name=None, options=None): + self.source = source + self.object_name = object_name + self.options = options + + class OBSUploadObject(SwiftUploadObject): """ An upload object similar to swiftclient's SwiftUploadObject that allows the user @@ -41,6 +48,8 @@ def __init__(self, source, object_name, options=None): source (str): A path that specifies a source file. dest (str): A path that specifies a destination file name (full key) """ + from swiftclient.service import SwiftError + try: super(OBSUploadObject, self).__init__(source, object_name=object_name, options=options) except SwiftError as exc: @@ -452,6 +461,8 @@ def close(self): self.closed = True def _wait_on_close(self): + import dxpy + if isinstance(self._path, stor.dx.DXPath): wait_on_close = stor.settings.get()['dx']['wait_on_close'] if wait_on_close: From 2a31c3059a2228afa495e6e250e823ec7c74d540 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Thu, 6 Jan 2022 22:46:14 -0800 Subject: [PATCH 03/16] Hardcode drives so imports work --- stor/utils.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/stor/utils.py b/stor/utils.py index d6a3bb46..2ef4de8f 100644 --- a/stor/utils.py +++ b/stor/utils.py @@ -187,8 +187,7 @@ def is_swift_path(p): Returns: bool: True if p is a Swift path, False otherwise. """ - from stor.swift import SwiftPath - return p.startswith(SwiftPath.drive) + return p.startswith('swift://') def is_filesystem_path(p): @@ -214,8 +213,7 @@ def is_s3_path(p): Returns bool: True if p is a S3 path, False otherwise. """ - from stor.s3 import S3Path - return p.startswith(S3Path.drive) + return p.startswith('s3://') def is_obs_path(p): @@ -241,8 +239,7 @@ def is_dx_path(p): Returns bool: True if p is a DX path, False otherwise. """ - from stor.dx import DXPath - return p.startswith(DXPath.drive) + return p.startswith('dx://') def is_valid_dxid(dxid, expected_classes): From 932d85a6007923eec618c38291e1d2d87c298769 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Thu, 6 Jan 2022 22:48:24 -0800 Subject: [PATCH 04/16] Skip covering importerror hack --- stor/obs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stor/obs.py b/stor/obs.py index b9622d38..7c30f6a8 100644 --- a/stor/obs.py +++ b/stor/obs.py @@ -26,7 +26,7 @@ def wrapper(self, *args, **kwargs): try: from swiftclient.service import SwiftUploadObject -except ImportError: +except ImportError: # pragma: no cover class SwiftUploadObject(object): """Give 90% of the utility of SwiftUploadObject class without swiftclient!""" def __init__(self, source, object_name=None, options=None): From c9c5bb3d77e67d7009acbb94129791b9d138f1d8 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Thu, 18 Apr 2024 18:15:19 -0700 Subject: [PATCH 05/16] Update pyproject.toml --- pyproject.toml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 97f97082..93096087 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,11 @@ requests = ">=2.20.0" boto3 = ">=1.7.0" cached-property = ">=1.5.1" dxpy = ">=0.278.0" -python-keystoneclient = ">=1.8.1" -python-swiftclient = ">=3.6.0" +python-keystoneclient = {version = ">=1.8.1", optional = true} +python-swiftclient = {version = ">=3.6.0", optional = true} + +[tool.poetry.extras] +swift = ["python-keystoneclient", "python-swiftclient"] [tool.poetry.dev-dependencies] flake8 = "^3.7.9" From b22b4bbae3326a35c3aec429942cc1f56cc30698 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Thu, 18 Apr 2024 18:16:08 -0700 Subject: [PATCH 06/16] Update pyproject.toml make dxpy into an extra as well --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 93096087..fff7072c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,12 +18,13 @@ python = "^3.6" requests = ">=2.20.0" boto3 = ">=1.7.0" cached-property = ">=1.5.1" -dxpy = ">=0.278.0" +dxpy = {version = ">=0.278.0", optional = true } python-keystoneclient = {version = ">=1.8.1", optional = true} python-swiftclient = {version = ">=3.6.0", optional = true} [tool.poetry.extras] swift = ["python-keystoneclient", "python-swiftclient"] +dxpy = ["dxpy"] [tool.poetry.dev-dependencies] flake8 = "^3.7.9" From 9ef644112726ff46a85914d526aabb9db2416c66 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Thu, 18 Apr 2024 18:18:01 -0700 Subject: [PATCH 07/16] Allow specifying poetry extras in makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 646f837a..049ab072 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ WITH_PBR=$(WITH_VENV) PBR_REQUIREMENTS_FILES=requirements-pbr.txt venv: $(VENV_ACTIVATE) $(VENV_ACTIVATE): poetry.lock - poetry install + poetry install $(POETRY_EXTRAS) touch $@ develop: venv From 1565891a35097f8d36958afe9e0008d54acf8518 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Thu, 18 Apr 2024 18:21:08 -0700 Subject: [PATCH 08/16] Update pythonpackage.yml include with and without dxpy --- .github/workflows/pythonpackage.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 7fcde7ab..8b0498ff 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -11,6 +11,7 @@ jobs: strategy: matrix: python-version: [3.6, 3.7, 3.8] + POETRY_EXTRAS: ["-E swift -E dxpy", ""] steps: - uses: actions/checkout@v2 @@ -30,7 +31,9 @@ jobs: pip install poetry==${{env.POETRY_VERSION}} poetry config --local virtualenvs.in-project true - name: Install venv - run: make venv + run: | + export POETRY_EXTRAS=${{matrix.POETRY_EXTRAS}} + make venv - name: List versions - pip run: | mkdir htmlcov From 81b939f0b937cc447f946b1f10f5b4a895f13a19 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Thu, 18 Apr 2024 18:21:30 -0700 Subject: [PATCH 09/16] Update pythonpackage.yml --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 8b0498ff..bc1e3936 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: python-version: [3.6, 3.7, 3.8] - POETRY_EXTRAS: ["-E swift -E dxpy", ""] + POETRY_EXTRAS: ["-E swift", "-E dxpy", "", "-E swift -E dxpy"] steps: - uses: actions/checkout@v2 From 8c1eb586fb68f7798a8d4c77fead9755812b0188 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Thu, 18 Apr 2024 18:24:30 -0700 Subject: [PATCH 10/16] Update pythonpackage.yml --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index bc1e3936..8274a55f 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.7, 3.8, 3.9] POETRY_EXTRAS: ["-E swift", "-E dxpy", "", "-E swift -E dxpy"] steps: From c2096518b7b6b2571c1ce91b1c80f4f4ac71288a Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Thu, 18 Apr 2024 18:25:39 -0700 Subject: [PATCH 11/16] Update pythonpackage.yml --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 8274a55f..cf716d69 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -32,7 +32,7 @@ jobs: poetry config --local virtualenvs.in-project true - name: Install venv run: | - export POETRY_EXTRAS=${{matrix.POETRY_EXTRAS}} + export POETRY_EXTRAS="${{matrix.POETRY_EXTRAS}}" make venv - name: List versions - pip run: | From 94fff49d7e6dea5aad104d24e2986d9f621e2148 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Tue, 23 Apr 2024 13:08:33 -0700 Subject: [PATCH 12/16] Update release_notes.rst --- docs/release_notes.rst | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 96f21fb7..4dfa6e2d 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -1,9 +1,23 @@ Release Notes ============= -v4.2.0 ------- -* Guard dxpy imports in utils so you can use stor without either dxpy or swift installed. +v5.0 - Optional dxpy and swift! +------------------------------- + +This release is almost entirely backwards compatible, but removes `dxpy` and `swiftclient` related requirements as direct dependencies. +For most users, this should be a transparent change (because they already depend on `dxpy` or `swiftclient` separately). However, this +would break workflows where stor was passively pulling in `dxpy` and `swiftclient`. To simplify updates, simply change your `requirements.txt` +file to explicitly specify the dxpy and swift extras: + +``` +# Before +stor < 5 +# After +stor[dxpy,swift] >= 5 +``` + + +* Guard dxpy imports in utils so you can use stor without either dxpy or swift installed. (#150) v4.1.0 ------ From ba6cf6a43e05f2e7e57ff7c4aa12fe9a28910626 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Tue, 23 Apr 2024 20:18:45 +0000 Subject: [PATCH 13/16] Update README to install extras by default Sem-Ver: api-break --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0e963eee..631a2abb 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,7 @@ Quickstart :: - pip install stor + pip install stor[dxpy, swift] ``stor`` provides both a CLI and a Python library for manipulating Posix and OBS with a single, cross-compatible API. From 74efbda981a31fc64a8a1071cba0fd172291a92a Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Tue, 23 Apr 2024 21:05:36 +0000 Subject: [PATCH 14/16] Add Salika to AUTHORS --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index a3534f6d..7987eee1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,5 +1,6 @@ Wes Kendall @wesleykendall (wesleykendall@gmail.com) -Jeff Tratner @jtratner +Jeff Tratner @jtratner (jeffrey.tratner@gmail.com) Stephanie Huang @phanieste Kyle Beauchamp @kyleabeauchamp Piotr Kaleta @pkaleta +Salika Dunatunga @scdunatun \ No newline at end of file From 7cfe9d88a447813473a359d2478320a0b1e50617 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Tue, 23 Apr 2024 21:05:47 +0000 Subject: [PATCH 15/16] Generate a useful exception on ImportError Rather than leave the user hanging, show something helpful to the user so they can resolve the situation by installing an additional package. Left this as a separate function so it's easy to test the exception generation fucntion doesn't break without needing to do painful testing contortions. (which should be caught later on by test matrix) --- .github/workflows/pythonpackage.yml | 2 +- README.rst | 2 +- pyproject.toml | 2 +- stor/dx.py | 11 ++++++++--- stor/swift.py | 12 ++++++++---- stor/tests/test_utils.py | 12 ++++++++++++ stor/utils.py | 21 +++++++++++++++++++++ 7 files changed, 52 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index cf716d69..71ee4625 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: python-version: [3.7, 3.8, 3.9] - POETRY_EXTRAS: ["-E swift", "-E dxpy", "", "-E swift -E dxpy"] + POETRY_EXTRAS: ["-E swift", "-E dx", "", "-E swift -E dx"] steps: - uses: actions/checkout@v2 diff --git a/README.rst b/README.rst index 631a2abb..853f0c2e 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,7 @@ Quickstart :: - pip install stor[dxpy, swift] + pip install stor[dx,swift] ``stor`` provides both a CLI and a Python library for manipulating Posix and OBS with a single, cross-compatible API. diff --git a/pyproject.toml b/pyproject.toml index fff7072c..287bec4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ python-swiftclient = {version = ">=3.6.0", optional = true} [tool.poetry.extras] swift = ["python-keystoneclient", "python-swiftclient"] -dxpy = ["dxpy"] +dx = ["dxpy"] [tool.poetry.dev-dependencies] flake8 = "^3.7.9" diff --git a/stor/dx.py b/stor/dx.py index abdb97dc..b2a42ff8 100644 --- a/stor/dx.py +++ b/stor/dx.py @@ -6,9 +6,6 @@ from cached_property import cached_property from contextlib import contextmanager -import dxpy -from dxpy.exceptions import DXError -from dxpy.exceptions import DXSearchError from stor import exceptions as stor_exceptions from stor import Path @@ -20,6 +17,14 @@ import stor.settings +try: + import dxpy + from dxpy.exceptions import DXError + from dxpy.exceptions import DXSearchError +except ImportError as e: + raise utils.missing_storage_library_exception("dx", e) from e + + logger = logging.getLogger(__name__) progress_logger = logging.getLogger('%s.progress' % __name__) diff --git a/stor/swift.py b/stor/swift.py index 71dfb00d..4efb25e8 100644 --- a/stor/swift.py +++ b/stor/swift.py @@ -42,10 +42,6 @@ import threading from urllib import parse -from swiftclient import exceptions as swift_exceptions -from swiftclient import service as swift_service -from swiftclient import client as swift_client -from swiftclient.utils import generate_temp_url from stor import exceptions as stor_exceptions from stor import is_swift_path @@ -57,6 +53,14 @@ from stor.posix import PosixPath from stor.third_party.backoff import with_backoff +try: + from swiftclient import exceptions as swift_exceptions + from swiftclient import service as swift_service + from swiftclient import client as swift_client + from swiftclient.utils import generate_temp_url +except ImportError as e: + raise utils.missing_storage_library_exception("swift", e) from e + logger = logging.getLogger(__name__) progress_logger = logging.getLogger('%s.progress' % __name__) diff --git a/stor/tests/test_utils.py b/stor/tests/test_utils.py index aade0f14..dcc69ae9 100644 --- a/stor/tests/test_utils.py +++ b/stor/tests/test_utils.py @@ -460,3 +460,15 @@ def test_path_no_perms(self): self.mock_copy.side_effect = stor.exceptions.FailedUploadError('foo') self.assertFalse(utils.is_writeable('s3://stor-test/foo/bar')) self.assertFalse(self.mock_remove.called) + +class TestStorageLibraryException: + def test_generates_appropriate_text(self): + try: + import thisshouldnotexistatall + except ImportError as e: + exc = utils.missing_storage_library_exception("dx", e) + else: + assert False, "this should have raised an import error" + assert "dx" in str(exc) + assert "thisshouldnotexistatall" in str(exc) + assert "To use a 'dx' path, stor needs an additional python library" in str(exc) \ No newline at end of file diff --git a/stor/utils.py b/stor/utils.py index 2ef4de8f..c0997cfc 100644 --- a/stor/utils.py +++ b/stor/utils.py @@ -7,6 +7,7 @@ import shutil from subprocess import check_call import tempfile +import warnings from stor import exceptions @@ -741,3 +742,23 @@ def add_result(self, result): progress_msg = self.get_progress_message() if progress_msg: # pragma: no cover self.logger.log(self.level, progress_msg) + +def missing_storage_library_exception(module: str, exc: Exception): + """Generate a helpful error for user about why their import failed. + + Meant to be used as: + + try: + ... + except Import Error as e: + raise missing_storage_library_exception('dx') from e + """ + return ImportError( + f"{type(exc).__name__}: {exc}\n" + f"To use a '{module}' path, stor needs an additional python library. " + f"Please specify it as an extra in the installation.\n" + f"i.e.,: `pip install stor[{module}]` or `stor[{module}] >= 5` " + f"in requirements.txt or `poetry add stor[{module}]`.\n" + f"Alternatively, change an existing " + f'pyproject.toml file to specify `stor = {{version="5.0", extras = {module}}}`' + ) \ No newline at end of file From b7a9223f5fdbfbda79d5ec58082aad07f08dcdb8 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Tue, 23 Apr 2024 21:26:41 +0000 Subject: [PATCH 16/16] Slightly more documentation --- docs/installation.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 12ddc48f..62a64fa6 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -3,12 +3,13 @@ Installation To install the latest release, type:: - pip install stor + pip install stor[dx,swift] To install the latest code directly from source, type:: pip install git+git://github.com/counsyl/stor.git +You can omit the ``dx`` or ``swift`` extras if you do not need to use those storage providers. .. _cli_tab_completion_installation: