From 482e74b55572514ced6f8beaee70d9b1009156aa Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Thu, 6 Jul 2017 19:24:53 -0700 Subject: [PATCH 01/23] Enable CLI multiple SOURCES for ``stor cp`` New usage: stor cp [-h] [-r] SOURCE [SOURCE ...] DEST Also adds ``stor cpto`` which reverses DEST and SOURCE(s). Enable Swift integration to be more flexible by adding more environment variables. Sem-Ver: feature --- README.md | 4 +++- docs/contributing.rst | 2 +- stor/__init__.py | 4 ++++ stor/cli.py | 34 +++++++++++++++++++++------ stor/tests/test_cli.py | 33 +++++++++++++++++++------- stor/tests/test_integration.py | 35 ++++++++++++++++++++++++++++ stor/tests/test_integration_swift.py | 14 ++++++----- stor/utils.py | 24 +++++++++++++++++++ tox.ini | 4 +++- 9 files changed, 129 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 6aa7bc29..77bff308 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,9 @@ positional arguments: list List contents using the path as a prefix. ls List path as a directory. - cp Alias for copy. + cp Copy a source to a destination path. + cpto Copy a source to a destination path. Note that here + DEST comes before SOURCE(s). rm Remove file at a path. walkfiles List all files under a path that match an optional pattern. diff --git a/docs/contributing.rst b/docs/contributing.rst index aa9ae28f..f041d66f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -8,7 +8,7 @@ To run all tests, type:: make test -In order to run swift integration tests, create a swift tenant called ``AUTH_swft_test`` and provide environment variables for a user that has permissions to write to that tenant (``SWIFT_TEST_USERNAME`` and ``SWIFT_TEST_PASSWORD``). Also set the swift auth url environment variable (``OS_AUTH_URL``) +In order to run swift integration tests, create a swift tenant called ``AUTH_swft_test`` (override with the ``SWIFT_TEST_TENANT`` environment variable) and provide environment variables for a user that has permissions to write to that tenant (``SWIFT_TEST_USERNAME`` and ``SWIFT_TEST_PASSWORD``). Also set the swift auth url environment variable (``OS_AUTH_URL``). You can set the prefix of the temporary test container via the ``SWIFT_TEST_CONTAINER_PREFIX`` environment variable. In order to run S3 integration tests, create a ``stor-test-bucket`` S3 bucket and provide environment variables for an AWS user that has permissions to write to it (``AWS_TEST_ACCESS_KEY_ID`` and ``AWS_ACCESS_KEY_ID``). diff --git a/stor/__init__.py b/stor/__init__.py index b08dc907..16a0f583 100644 --- a/stor/__init__.py +++ b/stor/__init__.py @@ -23,7 +23,9 @@ import pkg_resources from stor.utils import copy +from stor.utils import copy_multiple from stor.utils import copytree +from stor.utils import copytree_multiple from stor.utils import is_filesystem_path from stor.utils import is_swift_path from stor.utils import NamedTemporaryDirectory @@ -132,7 +134,9 @@ def path(pth): # pragma: no cover 'ismount', 'getsize', 'copy', + 'copy_multiple', 'copytree', + 'copytree_multiple', 'remove', 'rmtree', 'walkfiles', diff --git a/stor/cli.py b/stor/cli.py index bfa564da..34f4e3ff 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -125,7 +125,7 @@ def _make_stdin_action(func, err_msg): """ class StdinAction(argparse._StoreAction): def __call__(self, parser, namespace, values, option_string=None): - if values == '-': + if '-' in values: if namespace.func == func: raise argparse.ArgumentError(self, err_msg) else: @@ -300,8 +300,7 @@ def create_parser(): parser_ls.add_argument('path', type=get_path, metavar='PATH') parser_ls.set_defaults(func=stor.listdir) - cp_msg = 'Copy a source to a destination path.' - cp_msg = 'Alias for copy.' + cp_msg = 'Copy source(s) to a destination path.' parser_cp = subparsers.add_parser('cp', # noqa help=cp_msg, description='%s\n \'-\' is a special character that allows' @@ -311,15 +310,36 @@ def create_parser(): ' Must be specified before any other flags.', action='store_const', dest='func', - const=stor.copytree, - default=stor.copy) - parser_cp.add_argument('source', + const=stor.copytree_multiple, + default=stor.copy_multiple) + parser_cp.add_argument('sources', type=get_path, metavar='SOURCE', - action=_make_stdin_action(stor.copytree, + nargs='+', + action=_make_stdin_action(stor.copytree_multiple, '- cannot be used with -r')) parser_cp.add_argument('dest', type=get_path, metavar='DEST') + parser_cpto = subparsers.add_parser( + 'cpto', + help=cp_msg + ' Note that here DEST comes before SOURCE(s).', + description='%s\n \'-\' is a special character that allows' + ' for using stdin as the source.' % cp_msg) + parser_cpto.add_argument('-r', + help='Copy a directory and its subtree to the destination directory.' + ' Must be specified before any other flags.', + action='store_const', + dest='func', + const=stor.copytree_multiple, + default=stor.copy_multiple) + parser_cpto.add_argument('dest', type=get_path, metavar='DEST') + parser_cpto.add_argument('sources', + type=get_path, + metavar='SOURCE', + nargs='+', + action=_make_stdin_action(stor.copytree_multiple, + '- cannot be used with -r')) + rm_msg = 'Remove file at a path.' parser_rm = subparsers.add_parser('rm', help=rm_msg, diff --git a/stor/tests/test_cli.py b/stor/tests/test_cli.py index 697c675d..58e1cbb8 100644 --- a/stor/tests/test_cli.py +++ b/stor/tests/test_cli.py @@ -31,7 +31,7 @@ def test_cli_error(self, mock_list, mock_stderr): @mock.patch.dict('stor.settings._global_settings', {}, clear=True) @mock.patch.dict(os.environ, {}, clear=True) - @mock.patch('stor.copytree', autospec=True) + @mock.patch('stor.copytree_multiple', autospec=True) @mock.patch('stor.settings.USER_CONFIG_FILE', '') def test_cli_config(self, mock_copytree): expected_settings = { @@ -78,7 +78,7 @@ def test_cli_config(self, mock_copytree): self.parse_args('stor --config %s cp -r source dest' % filename) self.assertEquals(settings._global_settings, expected_settings) - @mock.patch('stor.copy', autospec=True) + @mock.patch('stor.copy_multiple', autospec=True) @mock.patch('sys.stderr', new=six.StringIO()) def test_not_implemented_error(self, mock_copy): mock_copy.side_effect = NotImplementedError @@ -326,15 +326,21 @@ def test_listdir_swift(self, mock_listdir): mock_listdir.assert_called_once_with(SwiftPath('swift://t/c')) -@mock.patch('stor.copy', autospec=True) +@mock.patch('stor.copy_multiple', autospec=True) class TestCopy(BaseCliTest): - def mock_copy_source(self, source, dest, *args, **kwargs): - with open(dest, 'w') as outfile, open(source) as infile: + def mock_copy_source(self, sources, dest, *args, **kwargs): + print sources, dest + with open(dest, 'w') as outfile, open(sources) as infile: outfile.write(infile.read()) def test_copy(self, mock_copy): self.parse_args('stor cp s3://bucket/file.txt ./file1') - mock_copy.assert_called_once_with(source='s3://bucket/file.txt', dest='./file1') + mock_copy.assert_called_once_with(sources=['s3://bucket/file.txt'], dest='./file1') + + def test_copy_multiple(self, mock_copy): + self.parse_args('stor cp s3://bucket/file.txt ./file2.txt ./dir1/') + mock_copy.assert_called_once_with(sources=['s3://bucket/file.txt', './file2.txt'], + dest='./dir1/') @mock.patch('sys.stdin', new=six.StringIO('some stdin input\n')) def test_copy_stdin(self, mock_copy): @@ -343,17 +349,26 @@ def test_copy_stdin(self, mock_copy): test_file = ntf.name self.parse_args('stor cp - %s' % test_file) self.assertEquals(open(test_file).read(), 'some stdin input\n') - temp_file = mock_copy.call_args_list[0][1]['source'] + print mock_copy.call_args_list + temp_file = mock_copy.call_args_list[0][1]['sources'] self.assertFalse(os.path.exists(temp_file)) os.remove(test_file) self.assertFalse(os.path.exists(test_file)) -@mock.patch('stor.copytree', autospec=True) +@mock.patch('stor.copytree_multiple', autospec=True) class TestCopytree(BaseCliTest): def test_copytree(self, mock_copytree): self.parse_args('stor cp -r s3://bucket .') - mock_copytree.assert_called_once_with(source='s3://bucket', dest='.') + mock_copytree.assert_called_once_with(sources=['s3://bucket'], dest='.') + + def test_copytree_dash(self, mock_copytree): + self.parse_args('stor cp -r s3://bucket-with-dash .') + mock_copytree.assert_called_once_with(sources=['s3://bucket-with-dash'], dest='.') + + def test_copytree_multiple(self, mock_copytree): + self.parse_args('stor cp -r s3://bucket directory .') + mock_copytree.assert_called_once_with(sources=['s3://bucket', 'directory'], dest='.') @mock.patch('sys.stderr', new=six.StringIO()) def test_copytree_stdin_error(self, mock_copytree): diff --git a/stor/tests/test_integration.py b/stor/tests/test_integration.py index fe0a8273..57a89dbf 100644 --- a/stor/tests/test_integration.py +++ b/stor/tests/test_integration.py @@ -55,6 +55,18 @@ def test_copy_to_from_dir(self): stor.copy(obj_path, 'copied_file') self.assertCorrectObjectContents('copied_file', which_obj, min_obj_size) + def test_copy_multiple_to_from_dir(self): + num_test_objs = 5 + min_obj_size = 100 + with NamedTemporaryDirectory(change_dir=True) as tmp_d: + self.create_dataset(tmp_d, num_test_objs, min_obj_size) + objs = self.get_dataset_obj_names(num_test_objs) + stor.copy_multiple(objs, self.test_dir / "") + for which_obj in objs: + obj_path = stor.join(self.test_dir, which_obj) + stor.copy(obj_path, 'copied_file') + self.assertCorrectObjectContents('copied_file', which_obj, min_obj_size) + def test_copytree_to_from_dir(self): num_test_objs = 10 test_obj_size = 100 @@ -74,6 +86,29 @@ def test_copytree_to_from_dir(self): obj_path = Path('test') / which_obj self.assertCorrectObjectContents(obj_path, which_obj, test_obj_size) + def test_copytree_multiple_to_from_dir(self): + num_test_dirs = 10 + num_test_objs = 2 + test_obj_size = 100 + # Create mock dataset and copy all directories + with NamedTemporaryDirectory(change_dir=True) as tmp_d: + test_dirs = self.get_dataset_obj_names(num_test_dirs) + for dirname in test_dirs: + stor.utils.make_dest_dir(dirname) + self.create_dataset(dirname, num_test_objs, test_obj_size) + stor.copytree_multiple(test_dirs, self.test_dir / "") + # Copy the files back to get contents + with NamedTemporaryDirectory(change_dir=True) as tmp_d: + for dirname in test_dirs: + (self.test_dir / dirname).copytree( + 'test-%s' % dirname, + condition=lambda results: len(results) == num_test_objs) + + # Verify contents of all downloaded test objects + for which_obj in self.get_dataset_obj_names(num_test_objs): + obj_path = Path('test-%s' % dirname) / which_obj + self.assertCorrectObjectContents(obj_path, which_obj, test_obj_size) + def test_hidden_file_nested_dir_copytree(self): with NamedTemporaryDirectory(change_dir=True): open('.hidden_file', 'w').close() diff --git a/stor/tests/test_integration_swift.py b/stor/tests/test_integration_swift.py index 624dc2e7..593933ac 100644 --- a/stor/tests/test_integration_swift.py +++ b/stor/tests/test_integration_swift.py @@ -32,12 +32,14 @@ def setUp(self): settings.update({ 'swift': { + 'auth_url': os.environ.get('OS_AUTH_URL'), 'username': os.environ.get('SWIFT_TEST_USERNAME'), 'password': os.environ.get('SWIFT_TEST_PASSWORD'), 'num_retries': 5 }}) - - self.test_container = Path('swift://%s/%s' % ('AUTH_swft_test', uuid.uuid4())) + self.test_tenant = os.environ.get('SWIFT_TEST_TENANT', 'AUTH_swft_test') + self.test_container = Path('swift://%s/%s%s' % ( + self.test_tenant, os.environ.get('SWIFT_TEST_CONTAINER_PREFIX', ''), uuid.uuid4())) if self.test_container.exists(): raise ValueError('test container %s already exists.' % self.test_container) @@ -59,13 +61,13 @@ def test_cached_auth_and_auth_invalidation(self): with mock.patch('swiftclient.client.get_auth_keystone', autospec=True) as mock_get_ks: mock_get_ks.side_effect = real_get_keystone s = Path(self.test_container).stat() - self.assertEquals(s['Account'], 'AUTH_swft_test') + self.assertEquals(s['Account'], self.test_tenant) self.assertEquals(len(mock_get_ks.call_args_list), 1) # The keystone auth should not be called on another stat mock_get_ks.reset_mock() s = Path(self.test_container).stat() - self.assertEquals(s['Account'], 'AUTH_swft_test') + self.assertEquals(s['Account'], self.test_tenant) self.assertEquals(len(mock_get_ks.call_args_list), 0) # Set the auth cache to something bad. The auth keystone should @@ -73,9 +75,9 @@ def test_cached_auth_and_auth_invalidation(self): # when retrying auth (with the bad token) and then called by us without # a token after the swiftclient raises an authorization error. mock_get_ks.reset_mock() - swift._cached_auth_token_map['AUTH_swft_test']['creds']['os_auth_token'] = 'bad_auth' + swift._cached_auth_token_map[self.test_tenant]['creds']['os_auth_token'] = 'bad_auth' s = Path(self.test_container).stat() - self.assertEquals(s['Account'], 'AUTH_swft_test') + self.assertEquals(s['Account'], self.test_tenant) self.assertEquals(len(mock_get_ks.call_args_list), 2) # Note that the auth_token is passed into the keystone client but then popped # from the kwargs. Assert that an auth token is no longer part of the retry calls diff --git a/stor/utils.py b/stor/utils.py index 92211aa3..390da38b 100644 --- a/stor/utils.py +++ b/stor/utils.py @@ -379,6 +379,23 @@ def copy(source, dest, swift_retry_options=None): **swift_retry_options) +def copy_multiple(sources, dest, swift_retry_options=None): + """Copies many sources to the same destination. + + See ``stor.copy``. + + Args: + source (list of path|str): The list of source files. + dest (path|str): The destination directory to copy to. + swift_retry_options (dict): Optional retry arguments to use for swift + upload or download. View the + `swift module-level documentation ` for more + information on retry arguments + """ + for source in sources: + copy(source, dest, swift_retry_options=swift_retry_options) + + def copytree(source, dest, copy_cmd=None, use_manifest=False, headers=None, condition=None, **retry_args): """Copies a source directory to a destination directory. Assumes that @@ -472,6 +489,13 @@ def copytree(source, dest, copy_cmd=None, use_manifest=False, headers=None, condition=condition, **retry_args) +def copytree_multiple(sources, dest, copy_cmd=None, use_manifest=False, + headers=None, condition=None, **retry_args): + for source in sources: + copytree(source, dest, copy_cmd=copy_cmd, use_manifest=use_manifest, + headers=headers, condition=condition, **retry_args) + + def _safe_get_size(name): """Get the size of a file, handling weird edge cases like broken symlinks by returning None""" diff --git a/tox.ini b/tox.ini index 7483f22a..74463dfd 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,8 @@ deps = whitelist_externals = make bash nosetests -passenv = SWIFT_TEST_USERNAME SWIFT_TEST_PASSWORD OS_TEMP_URL_KEY +passenv = SWIFT_TEST_USERNAME SWIFT_TEST_PASSWORD + SWIFT_TEST_TENANT SWIFT_TEST_CONTAINER_PREFIX + OS_TEMP_URL_KEY AWS_TEST_ACCESS_KEY_ID AWS_DEFAULT_REGION AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY OS_AUTH_URL From 5ac32dbb505ae9b458beb212151ab3a2a1af79f8 Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Fri, 7 Jul 2017 16:47:04 -0700 Subject: [PATCH 02/23] Let `stor cat` take multiple PATHs Sem-Ver: feature --- stor/cli.py | 9 ++++++--- stor/tests/test_cli.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 34f4e3ff..492363b0 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -201,9 +201,12 @@ def _clear_env(service=None): _env_chdir(name + '://') -def _cat(pth): +def _cat(pths): """Return the contents of a given path.""" - return pth.open().read() + contents = [] + for pth in pths: + contents.append(pth.open().read()) + return "".join(contents) def _obs_relpath_service(pth): @@ -365,7 +368,7 @@ def create_parser(): cat_msg = 'Output file contents to stdout.' parser_cat = subparsers.add_parser('cat', help=cat_msg, description=cat_msg) - parser_cat.add_argument('path', type=partial(get_path, mode='r'), metavar='PATH') + parser_cat.add_argument('path', type=partial(get_path, mode='r'), metavar='PATH', nargs='+') parser_cat.set_defaults(func=_cat) cd_msg = 'Change directory to a given OBS path.' diff --git a/stor/tests/test_cli.py b/stor/tests/test_cli.py index 58e1cbb8..c46215fa 100644 --- a/stor/tests/test_cli.py +++ b/stor/tests/test_cli.py @@ -329,7 +329,6 @@ def test_listdir_swift(self, mock_listdir): @mock.patch('stor.copy_multiple', autospec=True) class TestCopy(BaseCliTest): def mock_copy_source(self, sources, dest, *args, **kwargs): - print sources, dest with open(dest, 'w') as outfile, open(sources) as infile: outfile.write(infile.read()) @@ -349,7 +348,6 @@ def test_copy_stdin(self, mock_copy): test_file = ntf.name self.parse_args('stor cp - %s' % test_file) self.assertEquals(open(test_file).read(), 'some stdin input\n') - print mock_copy.call_args_list temp_file = mock_copy.call_args_list[0][1]['sources'] self.assertFalse(os.path.exists(temp_file)) os.remove(test_file) @@ -474,6 +472,14 @@ def test_cat_swift(self, mock_read): self.assertEquals(sys.stdout.getvalue(), 'hello world\n') mock_read.assert_called_once_with(SwiftPath('swift://some/test/file')) + @mock.patch.object(SwiftPath, 'read_object', autospec=True) + def test_cat_swift_multiple(self, mock_read): + mock_read.side_effect = ['hello world A', 'hello world B', 'hello world A'] + self.parse_args('stor cat swift://some/test/fileA' + ' swift://some/test/fileB swift://some/test/fileA') + self.assertEquals(sys.stdout.getvalue(), + 'hello world Ahello world Bhello world A\n') + class TestCd(BaseCliTest): def setUp(self): From 19c01e7dd10ce02465a93b69cc0c6ffa3eba67f0 Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Fri, 7 Jul 2017 17:01:11 -0700 Subject: [PATCH 03/23] Let `stor rm` take multiple PATHs Sem-Ver: feature --- stor/__init__.py | 4 ++++ stor/cli.py | 6 +++--- stor/tests/test_cli.py | 28 ++++++++++++++++++++-------- stor/utils.py | 12 ++++++++++++ 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/stor/__init__.py b/stor/__init__.py index 16a0f583..f1bb46ef 100644 --- a/stor/__init__.py +++ b/stor/__init__.py @@ -26,6 +26,8 @@ from stor.utils import copy_multiple from stor.utils import copytree from stor.utils import copytree_multiple +from stor.utils import remove_multiple +from stor.utils import rmtree_multiple from stor.utils import is_filesystem_path from stor.utils import is_swift_path from stor.utils import NamedTemporaryDirectory @@ -138,7 +140,9 @@ def path(pth): # pragma: no cover 'copytree', 'copytree_multiple', 'remove', + 'remove_multiple', 'rmtree', + 'rmtree_multiple', 'walkfiles', 'is_filesystem_path', 'is_swift_path', diff --git a/stor/cli.py b/stor/cli.py index 492363b0..8dd5b8c9 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -351,9 +351,9 @@ def create_parser(): help='Remove a path and all its contents.', action='store_const', dest='func', - const=stor.rmtree, - default=stor.remove) - parser_rm.add_argument('path', type=get_path, metavar='PATH') + const=stor.rmtree_multiple, + default=stor.remove_multiple) + parser_rm.add_argument('path', type=get_path, metavar='PATH', nargs="+") walkfiles_msg = 'List all files under a path that match an optional pattern.' parser_walkfiles = subparsers.add_parser('walkfiles', diff --git a/stor/tests/test_cli.py b/stor/tests/test_cli.py index c46215fa..50f63724 100644 --- a/stor/tests/test_cli.py +++ b/stor/tests/test_cli.py @@ -376,25 +376,37 @@ def test_copytree_stdin_error(self, mock_copytree): class TestRemove(BaseCliTest): - @mock.patch.object(S3Path, 'remove', autospec=True) + @mock.patch('stor.remove_multiple', autospec=True) def test_remove_s3(self, mock_remove): self.parse_args('stor rm s3://bucket/file1') - mock_remove.assert_called_once_with(S3Path('s3://bucket/file1')) + mock_remove.assert_called_once_with([S3Path('s3://bucket/file1')]) - @mock.patch.object(SwiftPath, 'remove', autospec=True) + @mock.patch('stor.remove_multiple', autospec=True) def test_remove_swift(self, mock_remove): self.parse_args('stor rm swift://t/c/file1') - mock_remove.assert_called_once_with(SwiftPath('swift://t/c/file1')) + mock_remove.assert_called_once_with([SwiftPath('swift://t/c/file1')]) - @mock.patch.object(S3Path, 'rmtree', autospec=True) + @mock.patch('stor.remove_multiple', autospec=True) + def test_remove_multiple_swift(self, mock_remove): + self.parse_args('stor rm swift://t/c/file1 swift://t/c/file2') + mock_remove.assert_called_once_with([SwiftPath('swift://t/c/file1'), + SwiftPath('swift://t/c/file2')]) + + @mock.patch('stor.rmtree_multiple', autospec=True) def test_rmtree_s3(self, mock_rmtree): self.parse_args('stor rm -r s3://bucket/dir') - mock_rmtree.assert_called_once_with(S3Path('s3://bucket/dir')) + mock_rmtree.assert_called_once_with([S3Path('s3://bucket/dir')]) - @mock.patch.object(SwiftPath, 'rmtree', autospec=True) + @mock.patch('stor.rmtree_multiple', autospec=True) def test_rmtree_swift(self, mock_rmtree): self.parse_args('stor rm -r swift://t/c/dir') - mock_rmtree.assert_called_once_with(SwiftPath('swift://t/c/dir')) + mock_rmtree.assert_called_once_with([SwiftPath('swift://t/c/dir')]) + + @mock.patch('stor.rmtree_multiple', autospec=True) + def test_rmtree_multiple_swift(self, mock_rmtree): + self.parse_args('stor rm -r swift://t/c/dir1 swift://t/c/dir2') + mock_rmtree.assert_called_once_with([SwiftPath('swift://t/c/dir1'), + SwiftPath('swift://t/c/dir2')]) class TestWalkfiles(BaseCliTest): diff --git a/stor/utils.py b/stor/utils.py index 390da38b..e97f0a65 100644 --- a/stor/utils.py +++ b/stor/utils.py @@ -496,6 +496,18 @@ def copytree_multiple(sources, dest, copy_cmd=None, use_manifest=False, headers=headers, condition=condition, **retry_args) +def remove_multiple(paths): + from stor import Path + for path in paths: + Path(path).remove() + + +def rmtree_multiple(paths): + from stor import Path + for path in paths: + Path(path).rmtree() + + def _safe_get_size(name): """Get the size of a file, handling weird edge cases like broken symlinks by returning None""" From c69313028ce1910b1a89c4f065396052abe82d82 Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Fri, 7 Jul 2017 17:17:17 -0700 Subject: [PATCH 04/23] ``stor ls`` defaults to ./ Sem-Ver: feature --- stor/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stor/cli.py b/stor/cli.py index 8dd5b8c9..e546fc1b 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -300,7 +300,7 @@ def create_parser(): parser_ls = subparsers.add_parser('ls', # noqa help=ls_msg, description=ls_msg) - parser_ls.add_argument('path', type=get_path, metavar='PATH') + parser_ls.add_argument('path', default='./', nargs='?', type=get_path, metavar='PATH') parser_ls.set_defaults(func=stor.listdir) cp_msg = 'Copy source(s) to a destination path.' From c1b400b5bb88d5f927a357ea5e374c66253c5106 Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Fri, 7 Jul 2017 17:42:44 -0700 Subject: [PATCH 05/23] initial pass at ``stor ls`` enhancements --- stor/cli.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index e546fc1b..4b4ee133 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -91,7 +91,7 @@ from stor import Path from stor import utils -PRINT_CMDS = ('list', 'listdir', 'ls', 'cat', 'pwd', 'walkfiles') +PRINT_CMDS = ('list', 'listdir', 'cat', 'pwd', 'walkfiles') SERVICES = ('s3', 'swift') ENV_FILE = os.path.expanduser('~/.stor-cli.env') @@ -301,7 +301,29 @@ def create_parser(): help=ls_msg, description=ls_msg) parser_ls.add_argument('path', default='./', nargs='?', type=get_path, metavar='PATH') - parser_ls.set_defaults(func=stor.listdir) + parser_ls.add_argument('-l', '--long-format', + help='Show long format listing', + action='store_true') + parser_ls.add_argument('-H', '--human-readable', + help='Use human-readable file sizes with long format', + action='store_true') + parser_ls.add_argument('-S', '--sort-by-file-size', + help='Sort files by size, smallest first', + action='store_true') + parser_ls.add_argument('-t', '--sort-by-time', + help='Sort files by modification time, newest first', + action='store_true') + parser_ls.add_argument('-U', '--sort-by-directory-order', + help=("Don't apply a particular sorting. With no sorting " + "flags, sorting is alphabetical"), + action='store_true') + parser_ls.add_argument('-r', '--reverse', + help='Reverse the order', + action='store_true') + parser_ls.add_argument('-u', '--url', + help='Display URL rather than filename', + action='store_true') + parser_ls.set_defaults(func=_print_ls_output) cp_msg = 'Copy source(s) to a destination path.' parser_cp = subparsers.add_parser('cp', # noqa @@ -434,6 +456,51 @@ def print_results(results): sys.stdout.write('%s\n' % str(result)) +def _human_size(size_bytes): + size_kib = size_bytes / 1024. + if size_kib < 1: + return "{} B".format(size_bytes) + size_mib = size_kib / 1024. + if size_mib < 1: + return "{:.1f} KiB".format(size_kib) + size_gib = size_mib / 1024. + if size_gib < 1: + return "{:.1f} MiB".format(size_mib) + size_tib = size_gib / 1024. + if size_tib < 1: + return "{:.1f} GiB".format(size_gib) + return "{:.1f} TiB".format(size_tib) + + +def _print_ls_output(path, long_format=False, human_readable=False, + sort_by_file_size=False, sort_by_time=False, sort_by_directory_order=False, + reverse=False, url=False): + out_lines = [] + paths = Path(path).listdir() + if sort_by_directory_order: + pass + elif sort_by_file_size: + paths = sorted(paths, key=lambda p: p.getsize()) + elif sort_by_time: + pass + else: + paths = sorted(paths) + if reverse: + paths = paths[::-1] + for path in paths: + if long_format: + size = path.getsize() + out_lines.append("{size}\t{mod}\t{ctype}\t{name}".format( + size=size if not human_readable else _human_size(size), + mod="TODO", + ctype="TODO", + name=str(path) if not url else "TODO")) + else: + out_lines.append(str(path)) + sys.stdout.write('\n'.join(out_lines)) + sys.stdout.write('\n') + + def main(): handler = logging.StreamHandler(sys.stdout) handler.setLevel(logging.INFO) From 7c22686f5095ef9fe8d0665e8956bfe2306bb6ef Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sat, 8 Jul 2017 00:12:54 -0700 Subject: [PATCH 06/23] grabbing stat info for swift and local --- stor/cli.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 4b4ee133..48b16258 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -81,6 +81,8 @@ import signal import sys import tempfile +from datetime import datetime as dt +import mimetypes import six from six.moves import configparser @@ -457,6 +459,8 @@ def print_results(results): def _human_size(size_bytes): + if size_bytes is None: # e.g. a directory + return "" size_kib = size_bytes / 1024. if size_kib < 1: return "{} B".format(size_bytes) @@ -472,31 +476,83 @@ def _human_size(size_bytes): return "{:.1f} TiB".format(size_tib) +def _get_swift_metadata_factory(auth_url): + swift_prefix = auth_url.split('auth')[0] + def get_metadata(path): + metadata = { + 'last_modified': None, + 'url': "{}v1/{}/{}/{}".format( + swift_prefix, path.tenant, path.container, path.resource), + 'size_bytes': None, + 'ctype': '', + } + if path.isfile(): + info = path.stat() + metadata['last_modified'] = dt.fromtimestamp(float(info['headers']['x-object-meta-mtime'])) + metadata['size_bytes'] = int(info['Content-Length']) + metadata['ctype'] = info['Content-Type'] + return metadata + return get_metadata + + +def _get_s3_metadata(path): + # TODO new_path = stor.join('s3://', pth.container, pth.resource) + metadata = { + 'last_modified': None, # TODO + 'url': str(path), # TODO + 'size_bytes': 0, # TODO + 'ctype': '', # TODO + } + return metadata + + +def _get_file_metadata(path): + info = os.stat(path) + isfile = path.isfile() + metadata = { + 'last_modified': dt.fromtimestamp(info.st_mtime) if isfile else None, + 'url': "file://" + str(path.abspath()).replace("\\", "/"), # TODO + 'size_bytes': info.st_size if isfile else None, + 'ctype': mimetypes.guess_type(path)[0] if isfile else None, + } + return metadata + + def _print_ls_output(path, long_format=False, human_readable=False, sort_by_file_size=False, sort_by_time=False, sort_by_directory_order=False, reverse=False, url=False): + path = Path(path) out_lines = [] - paths = Path(path).listdir() + paths = path.listdir() + if utils.is_swift_path(path): + get_metadata = _get_swift_metadata_factory(settings.get()['swift']['auth_url']) + elif utils.is_s3_path(path): + get_metadata = _get_s3_metadata + else: + get_metadata = _get_file_metadata + metadata = dict((p, get_metadata(p)) for p in paths) if sort_by_directory_order: + # no particular ordering pass elif sort_by_file_size: - paths = sorted(paths, key=lambda p: p.getsize()) + paths = sorted(paths, key=lambda p: metadata[p]['size_bytes']) elif sort_by_time: - pass + paths = sorted(paths, key=lambda p: metadata[p]['last_modified']) else: paths = sorted(paths) if reverse: paths = paths[::-1] - for path in paths: + for p in paths: if long_format: - size = path.getsize() + m = metadata[p] + mod = m['last_modified'].strftime('%Y-%m-%d %H:%M:%S') if m['last_modified'] else '' out_lines.append("{size}\t{mod}\t{ctype}\t{name}".format( - size=size if not human_readable else _human_size(size), - mod="TODO", - ctype="TODO", - name=str(path) if not url else "TODO")) + size=m['size_bytes'] if not human_readable else _human_size(m['size_bytes']), + mod=mod, + ctype=m['ctype'] or '', + name=str(p) if not url else m['url'])) else: - out_lines.append(str(path)) + out_lines.append(str(p)) sys.stdout.write('\n'.join(out_lines)) sys.stdout.write('\n') From c91a0e5f732159e7cfc9b3091c3891b7d1d8da24 Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sat, 8 Jul 2017 00:17:40 -0700 Subject: [PATCH 07/23] one fewer stat() call for swift? --- stor/cli.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 48b16258..06859c9d 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -486,8 +486,12 @@ def get_metadata(path): 'size_bytes': None, 'ctype': '', } - if path.isfile(): + try: info = path.stat() + isfile = path.resource and 'directory' not in info.get('Content-Type', '') + except exceptions.NotFoundError: + isfile = False + if isfile: metadata['last_modified'] = dt.fromtimestamp(float(info['headers']['x-object-meta-mtime'])) metadata['size_bytes'] = int(info['Content-Length']) metadata['ctype'] = info['Content-Type'] @@ -511,7 +515,7 @@ def _get_file_metadata(path): isfile = path.isfile() metadata = { 'last_modified': dt.fromtimestamp(info.st_mtime) if isfile else None, - 'url': "file://" + str(path.abspath()).replace("\\", "/"), # TODO + 'url': "file://" + str(path.abspath()).replace("\\", "/"), # TODO Windows paths? 'size_bytes': info.st_size if isfile else None, 'ctype': mimetypes.guess_type(path)[0] if isfile else None, } From a68a0bb99ae5e9b0fb07922de04499d6159ea66e Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sat, 8 Jul 2017 00:22:11 -0700 Subject: [PATCH 08/23] stat for s3? --- stor/cli.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 06859c9d..b67b2705 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -478,13 +478,14 @@ def _human_size(size_bytes): def _get_swift_metadata_factory(auth_url): swift_prefix = auth_url.split('auth')[0] + def get_metadata(path): metadata = { 'last_modified': None, 'url': "{}v1/{}/{}/{}".format( swift_prefix, path.tenant, path.container, path.resource), 'size_bytes': None, - 'ctype': '', + 'ctype': None, } try: info = path.stat() @@ -492,7 +493,8 @@ def get_metadata(path): except exceptions.NotFoundError: isfile = False if isfile: - metadata['last_modified'] = dt.fromtimestamp(float(info['headers']['x-object-meta-mtime'])) + metadata['last_modified'] = dt.fromtimestamp(float( + info['headers']['x-object-meta-mtime'])) metadata['size_bytes'] = int(info['Content-Length']) metadata['ctype'] = info['Content-Type'] return metadata @@ -500,13 +502,19 @@ def get_metadata(path): def _get_s3_metadata(path): - # TODO new_path = stor.join('s3://', pth.container, pth.resource) metadata = { - 'last_modified': None, # TODO - 'url': str(path), # TODO - 'size_bytes': 0, # TODO - 'ctype': '', # TODO + 'last_modified': None, + 'url': stor.join('s3://', path.container, path.resource), + 'size_bytes': None, + 'ctype': None, } + try: + info = path.stat() + metadata['last_modified'] = info['LastModified'] + metadata['size_bytes'] = info['ContentLength'] + metadata['ctype'] = info['ContentType'] + except exceptions.NotFoundError: + pass return metadata From fae9e6dfdfc1c701881569dbffc3c22767dc4c5d Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sat, 8 Jul 2017 00:25:08 -0700 Subject: [PATCH 09/23] sorting failed when dirs present, fixed --- stor/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stor/cli.py b/stor/cli.py index b67b2705..262b323b 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -549,7 +549,7 @@ def _print_ls_output(path, long_format=False, human_readable=False, elif sort_by_file_size: paths = sorted(paths, key=lambda p: metadata[p]['size_bytes']) elif sort_by_time: - paths = sorted(paths, key=lambda p: metadata[p]['last_modified']) + paths = sorted(paths, key=lambda p: metadata[p]['last_modified'] or dt.min) else: paths = sorted(paths) if reverse: From f4eb09b941e1d0baebf48ed23221e27cbb363f8c Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sat, 8 Jul 2017 00:39:06 -0700 Subject: [PATCH 10/23] add stor swift get-X --- README.md | 1 + stor/cli.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 77bff308..e647fe34 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ positional arguments: pwd Get the present working directory of a service or all current directories. clear Clear current directories of a specified service. + swift Get Swift-specific information. ``` You can `ls` local and remote directories diff --git a/stor/cli.py b/stor/cli.py index 262b323b..808d0f61 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -93,7 +93,7 @@ from stor import Path from stor import utils -PRINT_CMDS = ('list', 'listdir', 'cat', 'pwd', 'walkfiles') +PRINT_CMDS = ('list', 'listdir', 'cat', 'pwd', 'walkfiles', 'swift') SERVICES = ('s3', 'swift') ENV_FILE = os.path.expanduser('~/.stor-cli.env') @@ -412,6 +412,13 @@ def create_parser(): parser_clear.add_argument('service', nargs='?', type=str, metavar='SERVICE') parser_clear.set_defaults(func=_clear_env) + swift_msg = 'Get Swift-specific information.' + parser_swift = subparsers.add_parser('swift', help=swift_msg, description=swift_msg) + parser_swift.add_argument('get', choices=['get-tenant', 'get-container', 'get-object', 'get-resource'], + help="Which part of swift://tenant/container/object-or-resource to return") + parser_swift.add_argument('path', type=get_path, metavar='PATH') + parser_swift.set_defaults(func=_swift) + return parser @@ -458,6 +465,16 @@ def print_results(results): sys.stdout.write('%s\n' % str(result)) +def _swift(path, get): + if get == 'get-tenant': + return str(path.tenant) + if get == 'get-container': + return str(path.container) + if get == 'get-object' or get == 'get-resource': + return str(path.resource) + raise RuntimeError("Invalid thing to get for a swift path: {}".format(get)) + + def _human_size(size_bytes): if size_bytes is None: # e.g. a directory return "" From 06d0c84fc46395c1a3ed3b6430c05a9c2f1750e8 Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sat, 8 Jul 2017 00:43:54 -0700 Subject: [PATCH 11/23] stor swift get-url --- stor/cli.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 808d0f61..15f8e283 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -414,8 +414,11 @@ def create_parser(): swift_msg = 'Get Swift-specific information.' parser_swift = subparsers.add_parser('swift', help=swift_msg, description=swift_msg) - parser_swift.add_argument('get', choices=['get-tenant', 'get-container', 'get-object', 'get-resource'], - help="Which part of swift://tenant/container/object-or-resource to return") + parser_swift.add_argument( + 'get', choices=['get-tenant', 'get-container', 'get-object', 'get-resource', + 'get-url'], + help=("Which part of swift://tenant/container/object-or-resource to return; " + "get-url returns the https:// URL")) parser_swift.add_argument('path', type=get_path, metavar='PATH') parser_swift.set_defaults(func=_swift) @@ -472,6 +475,10 @@ def _swift(path, get): return str(path.container) if get == 'get-object' or get == 'get-resource': return str(path.resource) + if get == 'get-url': + auth_url = settings.get()['swift']['auth_url'] + swift_prefix = auth_url.split('auth')[0] + return _swift_url(swift_prefix, path) raise RuntimeError("Invalid thing to get for a swift path: {}".format(get)) @@ -493,14 +500,18 @@ def _human_size(size_bytes): return "{:.1f} TiB".format(size_tib) +def _swift_url(swift_prefix, path): + return "{}v1/{}/{}/{}".format( + swift_prefix, path.tenant, path.container, path.resource) + + def _get_swift_metadata_factory(auth_url): swift_prefix = auth_url.split('auth')[0] def get_metadata(path): metadata = { 'last_modified': None, - 'url': "{}v1/{}/{}/{}".format( - swift_prefix, path.tenant, path.container, path.resource), + 'url': _swift_url(swift_prefix, path), 'size_bytes': None, 'ctype': None, } From e795e6ecf86caaaa24bc584d0e096fc4313b8d4f Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sat, 8 Jul 2017 00:45:25 -0700 Subject: [PATCH 12/23] flake8 --- stor/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 15f8e283..967e671c 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -417,8 +417,8 @@ def create_parser(): parser_swift.add_argument( 'get', choices=['get-tenant', 'get-container', 'get-object', 'get-resource', 'get-url'], - help=("Which part of swift://tenant/container/object-or-resource to return; " - "get-url returns the https:// URL")) + help=("Which part of swift://tenant/container/object-or-resource to return; " + "get-url returns the https:// URL")) parser_swift.add_argument('path', type=get_path, metavar='PATH') parser_swift.set_defaults(func=_swift) From 7858500d32db84d44722ea74c6505734524f0b02 Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sat, 8 Jul 2017 10:48:38 -0700 Subject: [PATCH 13/23] show total size of listed files if outputting to terminal --- stor/cli.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 967e671c..72f45413 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -482,9 +482,11 @@ def _swift(path, get): raise RuntimeError("Invalid thing to get for a swift path: {}".format(get)) -def _human_size(size_bytes): +def _format_size(size_bytes, human_readable=True): if size_bytes is None: # e.g. a directory return "" + if not human_readable: + return str(size_bytes) size_kib = size_bytes / 1024. if size_kib < 1: return "{} B".format(size_bytes) @@ -582,17 +584,21 @@ def _print_ls_output(path, long_format=False, human_readable=False, paths = sorted(paths) if reverse: paths = paths[::-1] + total_bytes = 0 for p in paths: if long_format: m = metadata[p] mod = m['last_modified'].strftime('%Y-%m-%d %H:%M:%S') if m['last_modified'] else '' out_lines.append("{size}\t{mod}\t{ctype}\t{name}".format( - size=m['size_bytes'] if not human_readable else _human_size(m['size_bytes']), + size=_format_size(m['size_bytes'], human_readable), mod=mod, ctype=m['ctype'] or '', name=str(p) if not url else m['url'])) + total_bytes += m['size_bytes'] or 0 else: out_lines.append(str(p)) + if long_format and sys.stdout.isatty(): # mimic ls + out_lines = ['Total: {}'.format(_format_size(total_bytes, human_readable))] + out_lines sys.stdout.write('\n'.join(out_lines)) sys.stdout.write('\n') From 7eab5e8153f5ca13228812f88cf8fbff01ff643a Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sat, 8 Jul 2017 11:02:08 -0700 Subject: [PATCH 14/23] Add longform capability to list, walkfiles Note: changes ``stor list -l`` (--limit) to ``stor list -L`` because ``-l`` is now the ``--long-format`` flag. Sem-Ver: feature --- stor/cli.py | 87 ++++++++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 72f45413..91ec0a8e 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -93,7 +93,7 @@ from stor import Path from stor import utils -PRINT_CMDS = ('list', 'listdir', 'cat', 'pwd', 'walkfiles', 'swift') +PRINT_CMDS = ('cat', 'pwd', 'swift') SERVICES = ('s3', 'swift') ENV_FILE = os.path.expanduser('~/.stor-cli.env') @@ -267,6 +267,33 @@ def get_path(pth, mode=None): return prefix / path_part.split(rel_part, depth)[depth].lstrip('/') +def add_listing_parser_args(subparser): + subparser.add_argument('-l', '--long-format', + help='Show long format listing', + action='store_true') + subparser.add_argument('-H', '--human-readable', + help='Use human-readable file sizes with long format', + action='store_true') + subparser.add_argument('-S', '--sort-by-file-size', + help='Sort files by size, smallest first', + action='store_true') + subparser.add_argument('-t', '--sort-by-time', + help='Sort files by modification time, newest first', + action='store_true') + subparser.add_argument('-U', '--sort-by-directory-order', + help=("Don't apply a particular sorting. With no sorting " + "flags, sorting is alphabetical"), + action='store_true') + subparser.add_argument('-r', '--reverse', + help='Reverse the order', + action='store_true') + subparser.add_argument('-u', '--url', + help='Display URL rather than filename', + action='store_true') + subparser.add_argument('path', default='./', nargs='?', type=get_path, metavar='PATH') + subparser.set_defaults(func=_print_ls_output) + + def create_parser(): parser = argparse.ArgumentParser(description='A command line interface for stor.') @@ -286,46 +313,35 @@ def create_parser(): parser_list = subparsers.add_parser('list', help=list_msg, description=list_msg) - parser_list.add_argument('path', type=get_path, metavar='PATH') parser_list.add_argument('-s', '--starts-with', help='Append an additional path to the search path.', type=str, dest='starts_with', metavar='PREFIX') - parser_list.add_argument('-l', '--limit', + parser_list.add_argument('-L', '--limit', help='Limit the amount of results returned.', type=int, metavar='INT') - parser_list.set_defaults(func=stor.list) + add_listing_parser_args(parser_list) + parser_list.set_defaults(list_func=stor.list) ls_msg = 'List path as a directory.' parser_ls = subparsers.add_parser('ls', # noqa help=ls_msg, description=ls_msg) - parser_ls.add_argument('path', default='./', nargs='?', type=get_path, metavar='PATH') - parser_ls.add_argument('-l', '--long-format', - help='Show long format listing', - action='store_true') - parser_ls.add_argument('-H', '--human-readable', - help='Use human-readable file sizes with long format', - action='store_true') - parser_ls.add_argument('-S', '--sort-by-file-size', - help='Sort files by size, smallest first', - action='store_true') - parser_ls.add_argument('-t', '--sort-by-time', - help='Sort files by modification time, newest first', - action='store_true') - parser_ls.add_argument('-U', '--sort-by-directory-order', - help=("Don't apply a particular sorting. With no sorting " - "flags, sorting is alphabetical"), - action='store_true') - parser_ls.add_argument('-r', '--reverse', - help='Reverse the order', - action='store_true') - parser_ls.add_argument('-u', '--url', - help='Display URL rather than filename', - action='store_true') - parser_ls.set_defaults(func=_print_ls_output) + add_listing_parser_args(parser_ls) + parser_ls.set_defaults(list_func=stor.listdir) + + walkfiles_msg = 'List all files under a path that match an optional pattern.' + parser_walkfiles = subparsers.add_parser('walkfiles', + help=walkfiles_msg, + description=walkfiles_msg) + parser_walkfiles.add_argument('-p', '--pattern', + help='A regex pattern to match file names on.', + type=str, + metavar='REGEX') + add_listing_parser_args(parser_walkfiles) + parser_walkfiles.set_defaults(list_func=stor.walkfiles) cp_msg = 'Copy source(s) to a destination path.' parser_cp = subparsers.add_parser('cp', # noqa @@ -379,17 +395,6 @@ def create_parser(): default=stor.remove_multiple) parser_rm.add_argument('path', type=get_path, metavar='PATH', nargs="+") - walkfiles_msg = 'List all files under a path that match an optional pattern.' - parser_walkfiles = subparsers.add_parser('walkfiles', - help=walkfiles_msg, - description=walkfiles_msg) - parser_walkfiles.add_argument('-p', '--pattern', - help='A regex pattern to match file names on.', - type=str, - metavar='REGEX') - parser_walkfiles.add_argument('path', type=get_path, metavar='PATH') - parser_walkfiles.set_defaults(func=stor.walkfiles) - cat_msg = 'Output file contents to stdout.' parser_cat = subparsers.add_parser('cat', help=cat_msg, description=cat_msg) parser_cat.add_argument('path', type=partial(get_path, mode='r'), metavar='PATH', nargs='+') @@ -562,10 +567,10 @@ def _get_file_metadata(path): def _print_ls_output(path, long_format=False, human_readable=False, sort_by_file_size=False, sort_by_time=False, sort_by_directory_order=False, - reverse=False, url=False): + reverse=False, url=False, list_func=stor.listdir, **kwargs): path = Path(path) out_lines = [] - paths = path.listdir() + paths = list(list_func(path, **kwargs)) if utils.is_swift_path(path): get_metadata = _get_swift_metadata_factory(settings.get()['swift']['auth_url']) elif utils.is_s3_path(path): From f100eafa684132ae548625d601979e84e0e4722f Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sat, 8 Jul 2017 21:03:35 -0700 Subject: [PATCH 15/23] I don't actually know how to make s3 http urls --- stor/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stor/cli.py b/stor/cli.py index 91ec0a8e..207c6365 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -539,7 +539,7 @@ def get_metadata(path): def _get_s3_metadata(path): metadata = { 'last_modified': None, - 'url': stor.join('s3://', path.container, path.resource), + 'url': "TODO", # TODO 'size_bytes': None, 'ctype': None, } From b88c37b5981e84686ac0f962c81477252e2cbd37 Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sun, 9 Jul 2017 11:10:24 -0700 Subject: [PATCH 16/23] --tabs and --relative-time for stor ls Use fixed-width columns by default but add --tabs flag to get previous behavior back. Add --relative-time flag to give times relative to now, e.g. "2.5 days [ago]". Sem-Ver: feature --- stor/cli.py | 69 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 207c6365..1c22192d 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -267,7 +267,7 @@ def get_path(pth, mode=None): return prefix / path_part.split(rel_part, depth)[depth].lstrip('/') -def add_listing_parser_args(subparser): +def _add_listing_parser_args(subparser): subparser.add_argument('-l', '--long-format', help='Show long format listing', action='store_true') @@ -290,6 +290,12 @@ def add_listing_parser_args(subparser): subparser.add_argument('-u', '--url', help='Display URL rather than filename', action='store_true') + subparser.add_argument('-T', '--tabs', + help='Use tabs as separators rather than fixed-width columns', + action='store_true') + subparser.add_argument('--relative-time', + help='In long format use relative time', + action='store_true') subparser.add_argument('path', default='./', nargs='?', type=get_path, metavar='PATH') subparser.set_defaults(func=_print_ls_output) @@ -322,14 +328,14 @@ def create_parser(): help='Limit the amount of results returned.', type=int, metavar='INT') - add_listing_parser_args(parser_list) + _add_listing_parser_args(parser_list) parser_list.set_defaults(list_func=stor.list) ls_msg = 'List path as a directory.' parser_ls = subparsers.add_parser('ls', # noqa help=ls_msg, description=ls_msg) - add_listing_parser_args(parser_ls) + _add_listing_parser_args(parser_ls) parser_ls.set_defaults(list_func=stor.listdir) walkfiles_msg = 'List all files under a path that match an optional pattern.' @@ -340,7 +346,7 @@ def create_parser(): help='A regex pattern to match file names on.', type=str, metavar='REGEX') - add_listing_parser_args(parser_walkfiles) + _add_listing_parser_args(parser_walkfiles) parser_walkfiles.set_defaults(list_func=stor.walkfiles) cp_msg = 'Copy source(s) to a destination path.' @@ -565,9 +571,39 @@ def _get_file_metadata(path): return metadata +def _format_time(timestamp, relative_to=None): + if timestamp is None: + return '' + if relative_to is not None: + secs = (relative_to - timestamp).total_seconds() + mins = secs / 60. + if mins < 1: + return "{:.0f} s".format(secs) + hours = mins / 60. + if hours < 1: + return "{:.2g} mins".format(mins) + days = hours / 24. + if days < 1: + return "{:.2g} hrs".format(hours) + weeks = days / 7. + if weeks < 1: + return "{:.2g} d".format(days) + months = weeks / 4. + if months < 1: + return "{:.2g} w".format(weeks) + years = months / 12. + if years < 1: + return "{:.2g} mo".format(months) + return "{:.2g} yrs".format(years) + return timestamp.strftime('%Y-%m-%d %H:%M:%S') + + def _print_ls_output(path, long_format=False, human_readable=False, sort_by_file_size=False, sort_by_time=False, sort_by_directory_order=False, - reverse=False, url=False, list_func=stor.listdir, **kwargs): + reverse=False, url=False, tabs=False, relative_time=False, + list_func=stor.listdir, **kwargs): + # TODO: --tree + # TODO: --depth path = Path(path) out_lines = [] paths = list(list_func(path, **kwargs)) @@ -590,18 +626,29 @@ def _print_ls_output(path, long_format=False, human_readable=False, if reverse: paths = paths[::-1] total_bytes = 0 + now = dt.now() for p in paths: if long_format: m = metadata[p] - mod = m['last_modified'].strftime('%Y-%m-%d %H:%M:%S') if m['last_modified'] else '' - out_lines.append("{size}\t{mod}\t{ctype}\t{name}".format( - size=_format_size(m['size_bytes'], human_readable), - mod=mod, - ctype=m['ctype'] or '', - name=str(p) if not url else m['url'])) + mod = _format_time(m['last_modified'], now if relative_time else None) + out_lines.append({'size': _format_size(m['size_bytes'], human_readable), + 'mod': mod, + 'ctype': m['ctype'] or '', + 'name': str(p) if not url else m['url']}) total_bytes += m['size_bytes'] or 0 else: out_lines.append(str(p)) + if long_format: + if tabs: + fmt = "{size}\t{mod}\t{ctype}\t{name}" + else: + max_lens = {'size': 0, 'mod': 0, 'ctype': 0} + for line in out_lines: + for k in max_lens: + max_lens[k] = max(max_lens[k], len(line[k])) + fmt = "{{size: >{}}} {{mod: >{}}} {{ctype: >{}}} {{name}}".format( + max_lens['size'], max_lens['mod'], max_lens['ctype']) + out_lines = [fmt.format(**line) for line in out_lines] if long_format and sys.stdout.isatty(): # mimic ls out_lines = ['Total: {}'.format(_format_size(total_bytes, human_readable))] + out_lines sys.stdout.write('\n'.join(out_lines)) From ad1ec6b589417dd9ff69b7948256e014a81c440c Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Sun, 9 Jul 2017 22:55:24 -0700 Subject: [PATCH 17/23] flake8 --- stor/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 1c22192d..7527bf00 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -598,7 +598,7 @@ def _format_time(timestamp, relative_to=None): return timestamp.strftime('%Y-%m-%d %H:%M:%S') -def _print_ls_output(path, long_format=False, human_readable=False, +def _print_ls_output(path, long_format=False, human_readable=False, # noqa: C901 sort_by_file_size=False, sort_by_time=False, sort_by_directory_order=False, reverse=False, url=False, tabs=False, relative_time=False, list_func=stor.listdir, **kwargs): @@ -649,8 +649,8 @@ def _print_ls_output(path, long_format=False, human_readable=False, fmt = "{{size: >{}}} {{mod: >{}}} {{ctype: >{}}} {{name}}".format( max_lens['size'], max_lens['mod'], max_lens['ctype']) out_lines = [fmt.format(**line) for line in out_lines] - if long_format and sys.stdout.isatty(): # mimic ls - out_lines = ['Total: {}'.format(_format_size(total_bytes, human_readable))] + out_lines + if sys.stdout.isatty(): # mimic ls + out_lines = ['Total: {}'.format(_format_size(total_bytes, human_readable))] + out_lines sys.stdout.write('\n'.join(out_lines)) sys.stdout.write('\n') From 93d71c76f202943694df6ef97b79a6923c45ed6d Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Mon, 10 Jul 2017 11:50:50 -0700 Subject: [PATCH 18/23] default to long human readable format add -l/--simple-list to show the single column version and -b/--use-bytes to display size in bytes --- stor/cli.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 7527bf00..95aef1fa 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -268,11 +268,11 @@ def get_path(pth, mode=None): def _add_listing_parser_args(subparser): - subparser.add_argument('-l', '--long-format', - help='Show long format listing', + subparser.add_argument('-l', '--simple-list', + help='Show short, simple list format rather than the long format', action='store_true') - subparser.add_argument('-H', '--human-readable', - help='Use human-readable file sizes with long format', + subparser.add_argument('-b', '--use-bytes', + help='In long format use bytes rather than human readable file sizes', action='store_true') subparser.add_argument('-S', '--sort-by-file-size', help='Sort files by size, smallest first', @@ -598,7 +598,7 @@ def _format_time(timestamp, relative_to=None): return timestamp.strftime('%Y-%m-%d %H:%M:%S') -def _print_ls_output(path, long_format=False, human_readable=False, # noqa: C901 +def _print_ls_output(path, simple_list=False, use_bytes=False, # noqa: C901 sort_by_file_size=False, sort_by_time=False, sort_by_directory_order=False, reverse=False, url=False, tabs=False, relative_time=False, list_func=stor.listdir, **kwargs): @@ -628,17 +628,17 @@ def _print_ls_output(path, long_format=False, human_readable=False, # noqa: C90 total_bytes = 0 now = dt.now() for p in paths: - if long_format: + if simple_list: + out_lines.append(str(p)) + else: m = metadata[p] mod = _format_time(m['last_modified'], now if relative_time else None) - out_lines.append({'size': _format_size(m['size_bytes'], human_readable), + out_lines.append({'size': _format_size(m['size_bytes'], not use_bytes), 'mod': mod, 'ctype': m['ctype'] or '', 'name': str(p) if not url else m['url']}) total_bytes += m['size_bytes'] or 0 - else: - out_lines.append(str(p)) - if long_format: + if not simple_list: if tabs: fmt = "{size}\t{mod}\t{ctype}\t{name}" else: @@ -650,7 +650,7 @@ def _print_ls_output(path, long_format=False, human_readable=False, # noqa: C90 max_lens['size'], max_lens['mod'], max_lens['ctype']) out_lines = [fmt.format(**line) for line in out_lines] if sys.stdout.isatty(): # mimic ls - out_lines = ['Total: {}'.format(_format_size(total_bytes, human_readable))] + out_lines + out_lines = ['Total: {}'.format(_format_size(total_bytes, not use_bytes))] + out_lines sys.stdout.write('\n'.join(out_lines)) sys.stdout.write('\n') From d164be3237c5e640489c213fbf329782f26dae00 Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Mon, 10 Jul 2017 11:57:58 -0700 Subject: [PATCH 19/23] use DIR for non-file mimetype --- stor/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 95aef1fa..6beaaf1c 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -538,6 +538,8 @@ def get_metadata(path): info['headers']['x-object-meta-mtime'])) metadata['size_bytes'] = int(info['Content-Length']) metadata['ctype'] = info['Content-Type'] + else: + metadata['ctype'] = 'DIR' return metadata return get_metadata @@ -547,7 +549,7 @@ def _get_s3_metadata(path): 'last_modified': None, 'url': "TODO", # TODO 'size_bytes': None, - 'ctype': None, + 'ctype': 'DIR', } try: info = path.stat() @@ -566,7 +568,7 @@ def _get_file_metadata(path): 'last_modified': dt.fromtimestamp(info.st_mtime) if isfile else None, 'url': "file://" + str(path.abspath()).replace("\\", "/"), # TODO Windows paths? 'size_bytes': info.st_size if isfile else None, - 'ctype': mimetypes.guess_type(path)[0] if isfile else None, + 'ctype': mimetypes.guess_type(path)[0] if isfile else 'DIR', } return metadata From adbfcda69d7fe9d82f7467c25b6f5e387aefa237 Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Mon, 10 Jul 2017 12:26:10 -0700 Subject: [PATCH 20/23] Allow different Path types to have their formats For local files, show last modified / size / mimetype / name For Swift files, also show hash / ETag For s3 files, also show storage class --- stor/cli.py | 108 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 73 insertions(+), 35 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index 6beaaf1c..74c5eb1f 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -82,6 +82,7 @@ import sys import tempfile from datetime import datetime as dt +from collections import defaultdict import mimetypes import six @@ -278,7 +279,7 @@ def _add_listing_parser_args(subparser): help='Sort files by size, smallest first', action='store_true') subparser.add_argument('-t', '--sort-by-time', - help='Sort files by modification time, newest first', + help='Sort files by modification time, oldest first', action='store_true') subparser.add_argument('-U', '--sort-by-directory-order', help=("Don't apply a particular sorting. With no sorting " @@ -521,12 +522,14 @@ def _swift_url(swift_prefix, path): def _get_swift_metadata_factory(auth_url): swift_prefix = auth_url.split('auth')[0] - def get_metadata(path): - metadata = { - 'last_modified': None, + def get_metadata(path, use_bytes, url, relative_to): + raw_metadata = { + 'name': str(path), 'url': _swift_url(swift_prefix, path), 'size_bytes': None, 'ctype': None, + 'last_modified': None, + 'hash': None, } try: info = path.stat() @@ -534,43 +537,67 @@ def get_metadata(path): except exceptions.NotFoundError: isfile = False if isfile: - metadata['last_modified'] = dt.fromtimestamp(float( + raw_metadata['last_modified'] = dt.fromtimestamp(float( info['headers']['x-object-meta-mtime'])) - metadata['size_bytes'] = int(info['Content-Length']) - metadata['ctype'] = info['Content-Type'] + raw_metadata['size_bytes'] = int(info['Content-Length']) + raw_metadata['ctype'] = info['Content-Type'] + raw_metadata['hash'] = info.get('ETag', None) else: - metadata['ctype'] = 'DIR' - return metadata + raw_metadata['ctype'] = 'DIR' + display_metadata = { + 'name': raw_metadata['url'] if url else raw_metadata['name'], + 'size': _format_size(raw_metadata['size_bytes'], not use_bytes), + 'ctype': raw_metadata['ctype'] or '', + 'last_modified': _format_time(raw_metadata['last_modified'], relative_to), + 'hash': raw_metadata['hash'] or '', + } + return {'raw': raw_metadata, 'display': display_metadata} return get_metadata -def _get_s3_metadata(path): - metadata = { +def _get_s3_metadata(path, use_bytes, url, relative_to): + raw_metadata = { 'last_modified': None, 'url': "TODO", # TODO 'size_bytes': None, 'ctype': 'DIR', + 'storage_class': None, } try: info = path.stat() - metadata['last_modified'] = info['LastModified'] - metadata['size_bytes'] = info['ContentLength'] - metadata['ctype'] = info['ContentType'] + raw_metadata['last_modified'] = info['LastModified'] + raw_metadata['size_bytes'] = info['ContentLength'] + raw_metadata['ctype'] = info['ContentType'] + raw_metadata['storage_class'] = info['StorageClass'] except exceptions.NotFoundError: pass - return metadata + display_metadata = { + 'name': raw_metadata['url'] if url else raw_metadata['name'], + 'size': _format_size(raw_metadata['size_bytes'], not use_bytes), + 'ctype': raw_metadata['ctype'] or '', + 'last_modified': _format_time(raw_metadata['last_modified'], relative_to), + 'storage_class': raw_metadata['storage_class'] + } + return {'raw': raw_metadata, 'display': display_metadata} -def _get_file_metadata(path): +def _get_file_metadata(path, use_bytes, url, relative_to): info = os.stat(path) isfile = path.isfile() - metadata = { - 'last_modified': dt.fromtimestamp(info.st_mtime) if isfile else None, - 'url': "file://" + str(path.abspath()).replace("\\", "/"), # TODO Windows paths? + raw_metadata = { + 'name': str(path), + 'url': "file://" + str(path.abspath()).replace("\\", "/"), 'size_bytes': info.st_size if isfile else None, 'ctype': mimetypes.guess_type(path)[0] if isfile else 'DIR', + 'last_modified': dt.fromtimestamp(info.st_mtime) if isfile else None, + } + display_metadata = { + 'name': raw_metadata['url'] if url else raw_metadata['name'], + 'size': _format_size(raw_metadata['size_bytes'], not use_bytes), + 'ctype': raw_metadata['ctype'] or '', + 'last_modified': _format_time(raw_metadata['last_modified'], relative_to), } - return metadata + return {'raw': raw_metadata, 'display': display_metadata} def _format_time(timestamp, relative_to=None): @@ -611,18 +638,35 @@ def _print_ls_output(path, simple_list=False, use_bytes=False, # noqa: C901 paths = list(list_func(path, **kwargs)) if utils.is_swift_path(path): get_metadata = _get_swift_metadata_factory(settings.get()['swift']['auth_url']) + tabs_fmt = "{size}\t{last_modified}\t{ctype}\t{hash}\t{name}" + fixed_fmt = ("{{size: >{max_lens[size]}}} " + "{{last_modified: >{max_lens[last_modified]}}} " + "{{ctype: >{max_lens[ctype]}}} " + "{{hash: >{max_lens[hash]}}} " + "{{name}}") elif utils.is_s3_path(path): get_metadata = _get_s3_metadata + tabs_fmt = "{size}\t{last_modified}\t{ctype}\t{storage_class}\t{name}" + fixed_fmt = ("{{size: >{max_lens[size]}}} " + "{{last_modified: >{max_lens[last_modified]}}} " + "{{ctype: >{max_lens[ctype]}}} " + "{{storage_class: >{max_lens[storage_class]}}} " + "{{name}}") else: get_metadata = _get_file_metadata - metadata = dict((p, get_metadata(p)) for p in paths) + tabs_fmt = "{size}\t{last_modified}\t{ctype}\t{name}" + fixed_fmt = ("{{size: >{max_lens[size]}}} " + "{{last_modified: >{max_lens[last_modified]}}} " + "{{ctype: >{max_lens[ctype]}}} " + "{{name}}") + metadata = dict((p, get_metadata(p, use_bytes, url, dt.now() if relative_time else None)) for p in paths) if sort_by_directory_order: # no particular ordering pass elif sort_by_file_size: - paths = sorted(paths, key=lambda p: metadata[p]['size_bytes']) + paths = sorted(paths, key=lambda p: metadata[p]['raw']['size_bytes']) elif sort_by_time: - paths = sorted(paths, key=lambda p: metadata[p]['last_modified'] or dt.min) + paths = sorted(paths, key=lambda p: metadata[p]['raw']['last_modified'] or dt.min) else: paths = sorted(paths) if reverse: @@ -633,23 +677,17 @@ def _print_ls_output(path, simple_list=False, use_bytes=False, # noqa: C901 if simple_list: out_lines.append(str(p)) else: - m = metadata[p] - mod = _format_time(m['last_modified'], now if relative_time else None) - out_lines.append({'size': _format_size(m['size_bytes'], not use_bytes), - 'mod': mod, - 'ctype': m['ctype'] or '', - 'name': str(p) if not url else m['url']}) - total_bytes += m['size_bytes'] or 0 + out_lines.append(metadata[p]['display']) + total_bytes += metadata[p]['raw']['size_bytes'] or 0 if not simple_list: if tabs: - fmt = "{size}\t{mod}\t{ctype}\t{name}" + fmt = tabs_fmt else: - max_lens = {'size': 0, 'mod': 0, 'ctype': 0} + max_lens = defaultdict(int) for line in out_lines: - for k in max_lens: + for k in line: max_lens[k] = max(max_lens[k], len(line[k])) - fmt = "{{size: >{}}} {{mod: >{}}} {{ctype: >{}}} {{name}}".format( - max_lens['size'], max_lens['mod'], max_lens['ctype']) + fmt = fixed_fmt.format(max_lens=max_lens) out_lines = [fmt.format(**line) for line in out_lines] if sys.stdout.isatty(): # mimic ls out_lines = ['Total: {}'.format(_format_size(total_bytes, not use_bytes))] + out_lines From 4070ed02f6c713d676a2baa197eb08040b4576bb Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Mon, 10 Jul 2017 12:30:38 -0700 Subject: [PATCH 21/23] add ETag/hash for s3 listings --- stor/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/stor/cli.py b/stor/cli.py index 74c5eb1f..bbb88344 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -562,6 +562,7 @@ def _get_s3_metadata(path, use_bytes, url, relative_to): 'size_bytes': None, 'ctype': 'DIR', 'storage_class': None, + 'hash': None, } try: info = path.stat() @@ -569,6 +570,7 @@ def _get_s3_metadata(path, use_bytes, url, relative_to): raw_metadata['size_bytes'] = info['ContentLength'] raw_metadata['ctype'] = info['ContentType'] raw_metadata['storage_class'] = info['StorageClass'] + raw_metadata['hash'] = info['ETag'] except exceptions.NotFoundError: pass display_metadata = { @@ -576,7 +578,8 @@ def _get_s3_metadata(path, use_bytes, url, relative_to): 'size': _format_size(raw_metadata['size_bytes'], not use_bytes), 'ctype': raw_metadata['ctype'] or '', 'last_modified': _format_time(raw_metadata['last_modified'], relative_to), - 'storage_class': raw_metadata['storage_class'] + 'storage_class': raw_metadata['storage_class'], + 'hash': raw_metadata['hash'] or '', } return {'raw': raw_metadata, 'display': display_metadata} @@ -651,6 +654,7 @@ def _print_ls_output(path, simple_list=False, use_bytes=False, # noqa: C901 "{{last_modified: >{max_lens[last_modified]}}} " "{{ctype: >{max_lens[ctype]}}} " "{{storage_class: >{max_lens[storage_class]}}} " + "{{hash: >{max_lens[hash]}}} " "{{name}}") else: get_metadata = _get_file_metadata From 9625dc36dc83037c103847277aa37a6922dc13de Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Mon, 10 Jul 2017 14:54:47 -0700 Subject: [PATCH 22/23] misc tweaks --- stor/cli.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/stor/cli.py b/stor/cli.py index bbb88344..338cb350 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -544,6 +544,7 @@ def get_metadata(path, use_bytes, url, relative_to): raw_metadata['hash'] = info.get('ETag', None) else: raw_metadata['ctype'] = 'DIR' + raw_metadata['isfile'] = isfile display_metadata = { 'name': raw_metadata['url'] if url else raw_metadata['name'], 'size': _format_size(raw_metadata['size_bytes'], not use_bytes), @@ -563,6 +564,7 @@ def _get_s3_metadata(path, use_bytes, url, relative_to): 'ctype': 'DIR', 'storage_class': None, 'hash': None, + 'isfile': False, } try: info = path.stat() @@ -571,6 +573,7 @@ def _get_s3_metadata(path, use_bytes, url, relative_to): raw_metadata['ctype'] = info['ContentType'] raw_metadata['storage_class'] = info['StorageClass'] raw_metadata['hash'] = info['ETag'] + raw_metadata['isfile'] = True except exceptions.NotFoundError: pass display_metadata = { @@ -593,6 +596,7 @@ def _get_file_metadata(path, use_bytes, url, relative_to): 'size_bytes': info.st_size if isfile else None, 'ctype': mimetypes.guess_type(path)[0] if isfile else 'DIR', 'last_modified': dt.fromtimestamp(info.st_mtime) if isfile else None, + 'isfile': isfile, } display_metadata = { 'name': raw_metadata['url'] if url else raw_metadata['name'], @@ -663,7 +667,8 @@ def _print_ls_output(path, simple_list=False, use_bytes=False, # noqa: C901 "{{last_modified: >{max_lens[last_modified]}}} " "{{ctype: >{max_lens[ctype]}}} " "{{name}}") - metadata = dict((p, get_metadata(p, use_bytes, url, dt.now() if relative_time else None)) for p in paths) + metadata = dict((p, get_metadata(p, use_bytes, url, dt.now() + if relative_time else None)) for p in paths) if sort_by_directory_order: # no particular ordering pass @@ -676,14 +681,12 @@ def _print_ls_output(path, simple_list=False, use_bytes=False, # noqa: C901 if reverse: paths = paths[::-1] total_bytes = 0 - now = dt.now() for p in paths: - if simple_list: - out_lines.append(str(p)) - else: - out_lines.append(metadata[p]['display']) - total_bytes += metadata[p]['raw']['size_bytes'] or 0 - if not simple_list: + out_lines.append(metadata[p]['display']) + total_bytes += metadata[p]['raw']['size_bytes'] or 0 + if simple_list: + out_lines = ["{}".format(line['name']) for line in out_lines] + else: if tabs: fmt = tabs_fmt else: From 401aa5785ebd15dd58da24c56038baa97c5dfc26 Mon Sep 17 00:00:00 2001 From: kristjaneerik Date: Mon, 11 Dec 2017 15:05:59 -0800 Subject: [PATCH 23/23] I believe this was a typo --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index f041d66f..54538e45 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -10,7 +10,7 @@ To run all tests, type:: In order to run swift integration tests, create a swift tenant called ``AUTH_swft_test`` (override with the ``SWIFT_TEST_TENANT`` environment variable) and provide environment variables for a user that has permissions to write to that tenant (``SWIFT_TEST_USERNAME`` and ``SWIFT_TEST_PASSWORD``). Also set the swift auth url environment variable (``OS_AUTH_URL``). You can set the prefix of the temporary test container via the ``SWIFT_TEST_CONTAINER_PREFIX`` environment variable. -In order to run S3 integration tests, create a ``stor-test-bucket`` S3 bucket and provide environment variables for an AWS user that has permissions to write to it (``AWS_TEST_ACCESS_KEY_ID`` and ``AWS_ACCESS_KEY_ID``). +In order to run S3 integration tests, create a ``stor-test-bucket`` S3 bucket and provide environment variables for an AWS user that has permissions to write to it (``AWS_TEST_ACCESS_KEY_ID`` and ``AWS_TEST_SECRET_ACCESS_KEY``). Code Quality ------------