Single install command

src/rosdep2/installers.py

@@ -259,14 +259,14 @@ def is_installed(self, resolved_item):
         """
         raise NotImplementedError('is_installed', resolved_item)
 
-    def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False):
+    def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False, oneshot=[]):
         """
         :param resolved: list of resolved installation items, ``[opaque]``
         :param interactive: If `False`, disable interactive prompts,
           e.g. Pass through ``-y`` or equivalant to package manager.
         :param reinstall: If `True`, install everything even if already installed
         """
-        raise NotImplementedError('get_package_install_command', resolved, interactive, reinstall, quiet)
+        raise NotImplementedError('get_package_install_command', resolved, interactive, reinstall, quiet, oneshot)
 
     def get_depends(self, rosdep_args):
         """
@@ -388,8 +388,8 @@ def get_version_strings(self):
         """
         raise NotImplementedError('subclasses must implement get_version_strings method')
 
-    def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False):
-        raise NotImplementedError('subclasses must implement', resolved, interactive, reinstall, quiet)
+    def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False, oneshot=[]):
+        raise NotImplementedError('subclasses must implement', resolved, interactive, reinstall, quiet, oneshot)
 
     def get_depends(self, rosdep_args):
         """
@@ -467,7 +467,7 @@ def get_uninstalled(self, resources, implicit=False, verbose=False):
 
         return uninstalled, errors
 
-    def install(self, uninstalled, interactive=True, simulate=False,
+    def install(self, uninstalled, interactive=True, simulate=False, oneshot=[],
                 continue_on_error=False, reinstall=False, verbose=False, quiet=False):
         """
         Install the uninstalled rosdeps.  This API is for the bulk
@@ -507,20 +507,18 @@ def install(self, uninstalled, interactive=True, simulate=False,
             print('install: uninstalled keys are %s' % ', '.join(uninstalled_list))
 
         # Squash uninstalled again, in case some dependencies were already installed
-        squashed_uninstalled = []
-        previous_installer_key = None
+        squashed_uninstalled = {}
         for installer_key, resolved in uninstalled:
-            if previous_installer_key != installer_key:
-                squashed_uninstalled.append((installer_key, []))
-                previous_installer_key = installer_key
-            squashed_uninstalled[-1][1].extend(resolved)
+            if installer_key not in squashed_uninstalled:
+                squashed_uninstalled[installer_key] = []
+            squashed_uninstalled[installer_key].extend(resolved)
 
         failures = []
-        for installer_key, resolved in squashed_uninstalled:
+        for installer_key, resolved in squashed_uninstalled.items():
             try:
                 self.install_resolved(installer_key, resolved, simulate=simulate,
                                       interactive=interactive, reinstall=reinstall, continue_on_error=continue_on_error,
-                                      verbose=verbose, quiet=quiet)
+                                      verbose=verbose, quiet=quiet, oneshot=oneshot)
             except InstallFailed as e:
                 if not continue_on_error:
                     raise
@@ -530,7 +528,7 @@ def install(self, uninstalled, interactive=True, simulate=False,
         if failures:
             raise InstallFailed(failures=failures)
 
-    def install_resolved(self, installer_key, resolved, simulate=False, interactive=True,
+    def install_resolved(self, installer_key, resolved, simulate=False, interactive=True, oneshot=[],
                          reinstall=False, continue_on_error=False, verbose=False, quiet=False):
         """
         Lower-level API for installing a rosdep dependency.  The
@@ -550,7 +548,7 @@ def install_resolved(self, installer_key, resolved, simulate=False, interactive=
         """
         installer_context = self.installer_context
         installer = installer_context.get_installer(installer_key)
-        command = installer.get_install_command(resolved, interactive=interactive, reinstall=reinstall, quiet=quiet)
+        command = installer.get_install_command(resolved, interactive=interactive, reinstall=reinstall, quiet=quiet, oneshot=oneshot)
         if not command:
             if verbose:
                 print('#No packages to install')

src/rosdep2/main.py

@@ -374,6 +374,13 @@ def _rosdep_main(args):
                            'is in the provided list. The option can be supplied '
                            'multiple times. A space separated list of installers can also '
                            'be passed as a string. Example: `--filter-for-installers "apt pip"`')
+    parser.add_option('--one-shot', action='append', default=['pip', 'apt'],
+                      metavar='INSTALLER_KEY',
+                      help="Affects the 'install' verb. If supplied, each installer "
+                           'that supports it, will use a single invocation to install '
+                           'its packages. The option can be supplied '
+                           'multiple times. A space separated list of installers can also '
+                           'be passed as a string. Example: `--one-shot "apt pip"`')
     parser.add_option('--from-paths', dest='from_paths',
                       default=False, action='store_true',
                       help="Affects the 'check', 'keys', and 'install' verbs. "
@@ -439,7 +446,8 @@ def _rosdep_main(args):
             print('No installers with versions available found.')
         sys.exit(0)
 
-    # flatten list of skipped keys, filter-for-installers, and dependency types
+    # flatten list of one-shot installers, skipped keys, filter-for-installers, and dependency types
+    options.one_shot = [inst for s in options.one_shot for inst in s.split(' ')]
     options.skip_keys = [key for s in options.skip_keys for key in s.split(' ')]
     options.filter_for_installers = [inst for s in options.filter_for_installers for inst in s.split(' ')]
     options.dependency_types = [dep for s in options.dependency_types for dep in s.split(' ')]
@@ -771,7 +779,7 @@ def error_to_human_readable(error):
 def command_install(lookup, packages, options):
     # map options
     install_options = dict(interactive=not options.default_yes, verbose=options.verbose,
-                           reinstall=options.reinstall,
+                           reinstall=options.reinstall, oneshot=options.one_shot,
                            continue_on_error=options.robust, simulate=options.simulate, quiet=options.quiet)
 
     # setup installer

src/rosdep2/platforms/debian.py

@@ -262,16 +262,13 @@ def dpkg_detect(pkgs, exec_fn=None):
 
 
 def _iterate_packages(packages, reinstall):
-    for entry in _read_apt_cache_showpkg(packages):
-        p, is_virtual, providers = entry
+    for p, is_virtual, providers in _read_apt_cache_showpkg(packages):
         if is_virtual:
-            installed = []
             if reinstall:
                 installed = dpkg_detect(providers)
                 if len(installed) > 0:
-                    for i in installed:
-                        yield i
-                    continue  # don't ouput providers
+                    yield from installed
+                    continue
             yield providers
         else:
             yield p
@@ -292,16 +289,7 @@ def get_version_strings(self):
         version = output.splitlines()[0].split(b' ')[1].decode()
         return ['apt-get {}'.format(version)]
 
-    def _get_install_commands_for_package(self, base_cmd, package_or_list):
-        def pkg_command(p):
-            return self.elevate_priv(base_cmd + [p])
-
-        if isinstance(package_or_list, list):
-            return [pkg_command(p) for p in package_or_list]
-        else:
-            return pkg_command(package_or_list)
-
-    def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False):
+    def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False, oneshot=[]):
         packages = self.get_packages_to_install(resolved, reinstall=reinstall)
         if not packages:
             return []
@@ -311,4 +299,16 @@ def get_install_command(self, resolved, interactive=True, reinstall=False, quiet
         if quiet:
             base_cmd.append('-qq')
 
-        return [self._get_install_commands_for_package(base_cmd, p) for p in _iterate_packages(packages, reinstall)]
+        packages_single = []
+        packages_virtual = []
+        for p in _iterate_packages(packages, reinstall):
+            (packages_virtual if isinstance(p, list) else packages_single).append(p)
+        # sort to make the output deterministic
+        if 'apt' in oneshot:
+            # sort to make the output deterministic
+            commands_single = [self.elevate_priv(base_cmd + sorted(packages_single))]
+        else:
+            commands_single = [self.elevate_priv(base_cmd + [p]) for p in sorted(packages_single)]
+
+        commands_virtual = [[self.elevate_priv(base_cmd + [p]) for p in providers] for providers in packages_virtual]
+        return commands_single + commands_virtual

src/rosdep2/platforms/pip.py

@@ -206,7 +206,7 @@ def get_version_strings(self):
         ]
         return version_strings
 
-    def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False):
+    def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False, oneshot=[]):
         pip_cmd = get_pip_command()
         if not pip_cmd:
             raise InstallFailed((PIP_INSTALLER, 'pip is not installed'))
@@ -220,4 +220,6 @@ def get_install_command(self, resolved, interactive=True, reinstall=False, quiet
             cmd.append('-q')
         if reinstall:
             cmd.append('-I')
+        if 'pip' in oneshot:
+            return [self.elevate_priv(cmd + sorted(packages))]
         return [self.elevate_priv(cmd + [p]) for p in packages]

test/test_rosdep_debian.py

@@ -64,7 +64,6 @@ def test_dpkg_detect():
         val = dpkg_detect(['apt', 'tinyxml-dev', 'python'])
         assert val == ['apt', 'python'], val
         assert mock_read_stdout.call_count == 2
-        print(mock_read_stdout.call_args_list[0])
         assert mock_read_stdout.call_args_list[1] == call(['apt-cache', 'showpkg', 'tinyxml-dev'])
 
     # test version lock code (should be filtered out w/o validation)
@@ -126,12 +125,19 @@ def test(expected_prefix, mock_get_packages_to_install, mock_read_stdout):
         expected = [expected_prefix + ['apt-get', 'install', '-y', 'a'],
                     expected_prefix + ['apt-get', 'install', '-y', 'b']]
         val = installer.get_install_command(['whatever'], interactive=False)
-        print('VAL', val)
         assert val == expected, val
         expected = [expected_prefix + ['apt-get', 'install', 'a'],
                     expected_prefix + ['apt-get', 'install', 'b']]
         val = installer.get_install_command(['whatever'], interactive=True)
         assert val == expected, val
+
+        # oneshot
+        expected = [expected_prefix + ['apt-get', 'install', '-y', 'a', 'b']]
+        val = installer.get_install_command(['whatever'], interactive=False, oneshot=["apt"])
+        assert val == expected, val
+        expected = [expected_prefix + ['apt-get', 'install', 'a', 'b']]
+        val = installer.get_install_command(['whatever'], interactive=True, oneshot=["apt"])
+        assert val == expected, val
     try:
         if hasattr(os, 'geteuid'):
             with patch('rosdep2.installers.os.geteuid', return_value=1):

test/test_rosdep_installers.py

@@ -616,7 +616,6 @@ def test_RosdepInstaller_install_resolved(mock_geteuid):
                 raise
             raise SkipTest('targets ubuntu systems only')
     stdout_lines = [x.strip() for x in stdout.getvalue().split('\n') if x.strip()]
-    assert len(stdout_lines) == 3
+    assert len(stdout_lines) == 2
     assert stdout_lines[0] == '#[apt] Installation commands:'
-    assert 'sudo -H apt-get install rosdep-fake1' in stdout_lines, 'stdout_lines: %s' % stdout_lines
-    assert 'sudo -H apt-get install rosdep-fake2' in stdout_lines, 'stdout_lines: %s' % stdout_lines
+    assert 'sudo -H apt-get install rosdep-fake1 rosdep-fake2' in stdout_lines, 'stdout_lines: %s' % stdout_lines

test/test_rosdep_main.py

@@ -187,23 +187,40 @@ def read_stdout(cmd, capture_stderr=False):
                 rosdep_main(['install', 'python_dep', '-r'] + cmd_extras)
                 stdout, stderr = b
                 assert 'All required rosdeps installed' in stdout.getvalue(), stdout.getvalue()
+            # with fakeout() as b:
+            #     rosdep_main([
+            #         'install', '-s', '-i',
+            #         '--os', 'ubuntu:lucid',
+            #         '--rosdistro', 'fuerte',
+            #         '--from-paths', catkin_tree
+            #     ] + cmd_extras)
+            #     stdout, stderr = b
+            #     expected = [
+            #         '#[apt] Installation commands:',
+            #         '  sudo -H apt-get install ros-fuerte-catkin',
+            #         '  sudo -H apt-get install libboost1.40-all-dev',
+            #         '  sudo -H apt-get install libeigen3-dev',
+            #         '  sudo -H apt-get install libtinyxml-dev',
+            #         '  sudo -H apt-get install libltdl-dev',
+            #         '  sudo -H apt-get install libtool',
+            #         '  sudo -H apt-get install libcurl4-openssl-dev',
+            #     ]
+            #     lines = stdout.getvalue().splitlines()
+            #     assert set(lines) == set(expected), lines
             with fakeout() as b:
                 rosdep_main([
                     'install', '-s', '-i',
+                    # '--one-shot', "apt", # oneshot is default now
                     '--os', 'ubuntu:lucid',
                     '--rosdistro', 'fuerte',
                     '--from-paths', catkin_tree
                 ] + cmd_extras)
                 stdout, stderr = b
+                # the output is sorted in ascending order
                 expected = [
                     '#[apt] Installation commands:',
-                    '  sudo -H apt-get install ros-fuerte-catkin',
-                    '  sudo -H apt-get install libboost1.40-all-dev',
-                    '  sudo -H apt-get install libeigen3-dev',
-                    '  sudo -H apt-get install libtinyxml-dev',
-                    '  sudo -H apt-get install libltdl-dev',
-                    '  sudo -H apt-get install libtool',
-                    '  sudo -H apt-get install libcurl4-openssl-dev',
+                    '  sudo -H apt-get install libboost1.40-all-dev libcurl4-openssl-dev'
+                    ' libeigen3-dev libltdl-dev libtinyxml-dev libtool ros-fuerte-catkin',
                 ]
                 lines = stdout.getvalue().splitlines()
                 assert set(lines) == set(expected), lines

test/test_rosdep_pip.py

@@ -156,6 +156,7 @@ def test(expected_prefix, mock_method, mock_get_pip_command):
 
         # no interactive option with PIP
         mock_method.return_value = ['a', 'b']
+
         expected = [expected_prefix + ['mock-pip', 'install', '-U', 'a'],
                     expected_prefix + ['mock-pip', 'install', '-U', 'b']]
         val = installer.get_install_command(['whatever'], interactive=False)
@@ -164,6 +165,14 @@ def test(expected_prefix, mock_method, mock_get_pip_command):
                     expected_prefix + ['mock-pip', 'install', '-U', 'b']]
         val = installer.get_install_command(['whatever'], interactive=True)
         assert val == expected, val
+
+        # oneshot
+        expected = [expected_prefix + ['mock-pip', 'install', '-U', 'a', 'b']]
+        val = installer.get_install_command(['whatever'], interactive=False, oneshot=['pip'])
+        assert val == expected, val
+        expected = [expected_prefix + ['mock-pip', 'install', '-U', 'a', 'b']]
+        val = installer.get_install_command(['whatever'], interactive=True, oneshot=['pip'])
+        assert val == expected, val
     try:
         if hasattr(os, 'geteuid'):
             with patch('rosdep2.installers.os.geteuid', return_value=1):