diff --git a/src/rosdep2/installers.py b/src/rosdep2/installers.py index 20a996d06..70d266935 100644 --- a/src/rosdep2/installers.py +++ b/src/rosdep2/installers.py @@ -259,14 +259,14 @@ def is_installed(self, resolved_item): """ raise NotImplementedError('is_installed', resolved_item) - def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False): + def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False, oneshot=[]): """ :param resolved: list of resolved installation items, ``[opaque]`` :param interactive: If `False`, disable interactive prompts, e.g. Pass through ``-y`` or equivalant to package manager. :param reinstall: If `True`, install everything even if already installed """ - raise NotImplementedError('get_package_install_command', resolved, interactive, reinstall, quiet) + raise NotImplementedError('get_package_install_command', resolved, interactive, reinstall, quiet, oneshot) def get_depends(self, rosdep_args): """ @@ -388,8 +388,8 @@ def get_version_strings(self): """ raise NotImplementedError('subclasses must implement get_version_strings method') - def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False): - raise NotImplementedError('subclasses must implement', resolved, interactive, reinstall, quiet) + def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False, oneshot=[]): + raise NotImplementedError('subclasses must implement', resolved, interactive, reinstall, quiet, oneshot) def get_depends(self, rosdep_args): """ @@ -467,7 +467,7 @@ def get_uninstalled(self, resources, implicit=False, verbose=False): return uninstalled, errors - def install(self, uninstalled, interactive=True, simulate=False, + def install(self, uninstalled, interactive=True, simulate=False, oneshot=[], continue_on_error=False, reinstall=False, verbose=False, quiet=False): """ Install the uninstalled rosdeps. This API is for the bulk @@ -507,20 +507,18 @@ def install(self, uninstalled, interactive=True, simulate=False, print('install: uninstalled keys are %s' % ', '.join(uninstalled_list)) # Squash uninstalled again, in case some dependencies were already installed - squashed_uninstalled = [] - previous_installer_key = None + squashed_uninstalled = {} for installer_key, resolved in uninstalled: - if previous_installer_key != installer_key: - squashed_uninstalled.append((installer_key, [])) - previous_installer_key = installer_key - squashed_uninstalled[-1][1].extend(resolved) + if installer_key not in squashed_uninstalled: + squashed_uninstalled[installer_key] = [] + squashed_uninstalled[installer_key].extend(resolved) failures = [] - for installer_key, resolved in squashed_uninstalled: + for installer_key, resolved in squashed_uninstalled.items(): try: self.install_resolved(installer_key, resolved, simulate=simulate, interactive=interactive, reinstall=reinstall, continue_on_error=continue_on_error, - verbose=verbose, quiet=quiet) + verbose=verbose, quiet=quiet, oneshot=oneshot) except InstallFailed as e: if not continue_on_error: raise @@ -530,7 +528,7 @@ def install(self, uninstalled, interactive=True, simulate=False, if failures: raise InstallFailed(failures=failures) - def install_resolved(self, installer_key, resolved, simulate=False, interactive=True, + def install_resolved(self, installer_key, resolved, simulate=False, interactive=True, oneshot=[], reinstall=False, continue_on_error=False, verbose=False, quiet=False): """ Lower-level API for installing a rosdep dependency. The @@ -550,7 +548,7 @@ def install_resolved(self, installer_key, resolved, simulate=False, interactive= """ installer_context = self.installer_context installer = installer_context.get_installer(installer_key) - command = installer.get_install_command(resolved, interactive=interactive, reinstall=reinstall, quiet=quiet) + command = installer.get_install_command(resolved, interactive=interactive, reinstall=reinstall, quiet=quiet, oneshot=oneshot) if not command: if verbose: print('#No packages to install') diff --git a/src/rosdep2/main.py b/src/rosdep2/main.py index 080331f3f..0402e50f6 100644 --- a/src/rosdep2/main.py +++ b/src/rosdep2/main.py @@ -374,6 +374,13 @@ def _rosdep_main(args): 'is in the provided list. The option can be supplied ' 'multiple times. A space separated list of installers can also ' 'be passed as a string. Example: `--filter-for-installers "apt pip"`') + parser.add_option('--one-shot', action='append', default=['pip', 'apt'], + metavar='INSTALLER_KEY', + help="Affects the 'install' verb. If supplied, each installer " + 'that supports it, will use a single invocation to install ' + 'its packages. The option can be supplied ' + 'multiple times. A space separated list of installers can also ' + 'be passed as a string. Example: `--one-shot "apt pip"`') parser.add_option('--from-paths', dest='from_paths', default=False, action='store_true', help="Affects the 'check', 'keys', and 'install' verbs. " @@ -439,7 +446,8 @@ def _rosdep_main(args): print('No installers with versions available found.') sys.exit(0) - # flatten list of skipped keys, filter-for-installers, and dependency types + # flatten list of one-shot installers, skipped keys, filter-for-installers, and dependency types + options.one_shot = [inst for s in options.one_shot for inst in s.split(' ')] options.skip_keys = [key for s in options.skip_keys for key in s.split(' ')] options.filter_for_installers = [inst for s in options.filter_for_installers for inst in s.split(' ')] options.dependency_types = [dep for s in options.dependency_types for dep in s.split(' ')] @@ -771,7 +779,7 @@ def error_to_human_readable(error): def command_install(lookup, packages, options): # map options install_options = dict(interactive=not options.default_yes, verbose=options.verbose, - reinstall=options.reinstall, + reinstall=options.reinstall, oneshot=options.one_shot, continue_on_error=options.robust, simulate=options.simulate, quiet=options.quiet) # setup installer diff --git a/src/rosdep2/platforms/debian.py b/src/rosdep2/platforms/debian.py index de79a43ea..d73beca97 100644 --- a/src/rosdep2/platforms/debian.py +++ b/src/rosdep2/platforms/debian.py @@ -262,16 +262,13 @@ def dpkg_detect(pkgs, exec_fn=None): def _iterate_packages(packages, reinstall): - for entry in _read_apt_cache_showpkg(packages): - p, is_virtual, providers = entry + for p, is_virtual, providers in _read_apt_cache_showpkg(packages): if is_virtual: - installed = [] if reinstall: installed = dpkg_detect(providers) if len(installed) > 0: - for i in installed: - yield i - continue # don't ouput providers + yield from installed + continue yield providers else: yield p @@ -292,16 +289,7 @@ def get_version_strings(self): version = output.splitlines()[0].split(b' ')[1].decode() return ['apt-get {}'.format(version)] - def _get_install_commands_for_package(self, base_cmd, package_or_list): - def pkg_command(p): - return self.elevate_priv(base_cmd + [p]) - - if isinstance(package_or_list, list): - return [pkg_command(p) for p in package_or_list] - else: - return pkg_command(package_or_list) - - def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False): + def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False, oneshot=[]): packages = self.get_packages_to_install(resolved, reinstall=reinstall) if not packages: return [] @@ -311,4 +299,16 @@ def get_install_command(self, resolved, interactive=True, reinstall=False, quiet if quiet: base_cmd.append('-qq') - return [self._get_install_commands_for_package(base_cmd, p) for p in _iterate_packages(packages, reinstall)] + packages_single = [] + packages_virtual = [] + for p in _iterate_packages(packages, reinstall): + (packages_virtual if isinstance(p, list) else packages_single).append(p) + # sort to make the output deterministic + if 'apt' in oneshot: + # sort to make the output deterministic + commands_single = [self.elevate_priv(base_cmd + sorted(packages_single))] + else: + commands_single = [self.elevate_priv(base_cmd + [p]) for p in sorted(packages_single)] + + commands_virtual = [[self.elevate_priv(base_cmd + [p]) for p in providers] for providers in packages_virtual] + return commands_single + commands_virtual diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 1d4a19572..ebec6740d 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -206,7 +206,7 @@ def get_version_strings(self): ] return version_strings - def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False): + def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False, oneshot=[]): pip_cmd = get_pip_command() if not pip_cmd: raise InstallFailed((PIP_INSTALLER, 'pip is not installed')) @@ -220,4 +220,6 @@ def get_install_command(self, resolved, interactive=True, reinstall=False, quiet cmd.append('-q') if reinstall: cmd.append('-I') + if 'pip' in oneshot: + return [self.elevate_priv(cmd + sorted(packages))] return [self.elevate_priv(cmd + [p]) for p in packages] diff --git a/test/test_rosdep_debian.py b/test/test_rosdep_debian.py index e8d4a43a7..9c3d3ea2a 100644 --- a/test/test_rosdep_debian.py +++ b/test/test_rosdep_debian.py @@ -64,7 +64,6 @@ def test_dpkg_detect(): val = dpkg_detect(['apt', 'tinyxml-dev', 'python']) assert val == ['apt', 'python'], val assert mock_read_stdout.call_count == 2 - print(mock_read_stdout.call_args_list[0]) assert mock_read_stdout.call_args_list[1] == call(['apt-cache', 'showpkg', 'tinyxml-dev']) # test version lock code (should be filtered out w/o validation) @@ -126,12 +125,19 @@ def test(expected_prefix, mock_get_packages_to_install, mock_read_stdout): expected = [expected_prefix + ['apt-get', 'install', '-y', 'a'], expected_prefix + ['apt-get', 'install', '-y', 'b']] val = installer.get_install_command(['whatever'], interactive=False) - print('VAL', val) assert val == expected, val expected = [expected_prefix + ['apt-get', 'install', 'a'], expected_prefix + ['apt-get', 'install', 'b']] val = installer.get_install_command(['whatever'], interactive=True) assert val == expected, val + + # oneshot + expected = [expected_prefix + ['apt-get', 'install', '-y', 'a', 'b']] + val = installer.get_install_command(['whatever'], interactive=False, oneshot=["apt"]) + assert val == expected, val + expected = [expected_prefix + ['apt-get', 'install', 'a', 'b']] + val = installer.get_install_command(['whatever'], interactive=True, oneshot=["apt"]) + assert val == expected, val try: if hasattr(os, 'geteuid'): with patch('rosdep2.installers.os.geteuid', return_value=1): diff --git a/test/test_rosdep_installers.py b/test/test_rosdep_installers.py index a19dcf2f4..92d01860d 100644 --- a/test/test_rosdep_installers.py +++ b/test/test_rosdep_installers.py @@ -616,7 +616,6 @@ def test_RosdepInstaller_install_resolved(mock_geteuid): raise raise SkipTest('targets ubuntu systems only') stdout_lines = [x.strip() for x in stdout.getvalue().split('\n') if x.strip()] - assert len(stdout_lines) == 3 + assert len(stdout_lines) == 2 assert stdout_lines[0] == '#[apt] Installation commands:' - assert 'sudo -H apt-get install rosdep-fake1' in stdout_lines, 'stdout_lines: %s' % stdout_lines - assert 'sudo -H apt-get install rosdep-fake2' in stdout_lines, 'stdout_lines: %s' % stdout_lines + assert 'sudo -H apt-get install rosdep-fake1 rosdep-fake2' in stdout_lines, 'stdout_lines: %s' % stdout_lines diff --git a/test/test_rosdep_main.py b/test/test_rosdep_main.py index 78bde7ef0..27d933eca 100644 --- a/test/test_rosdep_main.py +++ b/test/test_rosdep_main.py @@ -187,23 +187,40 @@ def read_stdout(cmd, capture_stderr=False): rosdep_main(['install', 'python_dep', '-r'] + cmd_extras) stdout, stderr = b assert 'All required rosdeps installed' in stdout.getvalue(), stdout.getvalue() + # with fakeout() as b: + # rosdep_main([ + # 'install', '-s', '-i', + # '--os', 'ubuntu:lucid', + # '--rosdistro', 'fuerte', + # '--from-paths', catkin_tree + # ] + cmd_extras) + # stdout, stderr = b + # expected = [ + # '#[apt] Installation commands:', + # ' sudo -H apt-get install ros-fuerte-catkin', + # ' sudo -H apt-get install libboost1.40-all-dev', + # ' sudo -H apt-get install libeigen3-dev', + # ' sudo -H apt-get install libtinyxml-dev', + # ' sudo -H apt-get install libltdl-dev', + # ' sudo -H apt-get install libtool', + # ' sudo -H apt-get install libcurl4-openssl-dev', + # ] + # lines = stdout.getvalue().splitlines() + # assert set(lines) == set(expected), lines with fakeout() as b: rosdep_main([ 'install', '-s', '-i', + # '--one-shot', "apt", # oneshot is default now '--os', 'ubuntu:lucid', '--rosdistro', 'fuerte', '--from-paths', catkin_tree ] + cmd_extras) stdout, stderr = b + # the output is sorted in ascending order expected = [ '#[apt] Installation commands:', - ' sudo -H apt-get install ros-fuerte-catkin', - ' sudo -H apt-get install libboost1.40-all-dev', - ' sudo -H apt-get install libeigen3-dev', - ' sudo -H apt-get install libtinyxml-dev', - ' sudo -H apt-get install libltdl-dev', - ' sudo -H apt-get install libtool', - ' sudo -H apt-get install libcurl4-openssl-dev', + ' sudo -H apt-get install libboost1.40-all-dev libcurl4-openssl-dev' + ' libeigen3-dev libltdl-dev libtinyxml-dev libtool ros-fuerte-catkin', ] lines = stdout.getvalue().splitlines() assert set(lines) == set(expected), lines diff --git a/test/test_rosdep_pip.py b/test/test_rosdep_pip.py index 5209b48b9..19293cc1d 100644 --- a/test/test_rosdep_pip.py +++ b/test/test_rosdep_pip.py @@ -156,6 +156,7 @@ def test(expected_prefix, mock_method, mock_get_pip_command): # no interactive option with PIP mock_method.return_value = ['a', 'b'] + expected = [expected_prefix + ['mock-pip', 'install', '-U', 'a'], expected_prefix + ['mock-pip', 'install', '-U', 'b']] val = installer.get_install_command(['whatever'], interactive=False) @@ -164,6 +165,14 @@ def test(expected_prefix, mock_method, mock_get_pip_command): expected_prefix + ['mock-pip', 'install', '-U', 'b']] val = installer.get_install_command(['whatever'], interactive=True) assert val == expected, val + + # oneshot + expected = [expected_prefix + ['mock-pip', 'install', '-U', 'a', 'b']] + val = installer.get_install_command(['whatever'], interactive=False, oneshot=['pip']) + assert val == expected, val + expected = [expected_prefix + ['mock-pip', 'install', '-U', 'a', 'b']] + val = installer.get_install_command(['whatever'], interactive=True, oneshot=['pip']) + assert val == expected, val try: if hasattr(os, 'geteuid'): with patch('rosdep2.installers.os.geteuid', return_value=1):