From d3994437f80daebf06125c40ee0224622776f938 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 26 Feb 2026 13:58:00 -0700 Subject: [PATCH 01/36] uman: Embed u_boot_pylib to remove external dependency Uman imports u_boot_pylib from $UBOOT_TOOLS (default ~/u/tools), but that directory points to whatever U-Boot tree the user has checked out. Older trees may lack features uman needs, or the directory may not exist at all. Copy u_boot_pylib into the uman repository so it always works regardless of the U-Boot tree. Add UMAN_EXTERNAL_PYLIB=1 env var to override and use the external version for testing newer library versions. Update CLAUDE.md test commands to drop the PYTHONPATH prefix. Co-developed-by: Claude Opus 4.6 --- CLAUDE.md | 4 +- README.rst | 9 +- u_boot_pylib/__init__.py | 4 + u_boot_pylib/command.py | 222 ++++++++ u_boot_pylib/cros_subprocess.py | 401 +++++++++++++++ u_boot_pylib/gitutil.py | 886 ++++++++++++++++++++++++++++++++ u_boot_pylib/terminal.py | 350 +++++++++++++ u_boot_pylib/test_util.py | 229 +++++++++ u_boot_pylib/tools.py | 612 ++++++++++++++++++++++ u_boot_pylib/tout.py | 190 +++++++ uman_pkg/__main__.py | 10 +- uman_pkg/ftest.py | 24 +- 12 files changed, 2911 insertions(+), 30 deletions(-) create mode 100644 u_boot_pylib/__init__.py create mode 100644 u_boot_pylib/command.py create mode 100644 u_boot_pylib/cros_subprocess.py create mode 100644 u_boot_pylib/gitutil.py create mode 100644 u_boot_pylib/terminal.py create mode 100644 u_boot_pylib/test_util.py create mode 100644 u_boot_pylib/tools.py create mode 100644 u_boot_pylib/tout.py diff --git a/CLAUDE.md b/CLAUDE.md index a3145bd..47c30b8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,8 +8,8 @@ ## Testing -- Run tests with: `PYTHONPATH=~/u/tools python -m pytest uman_pkg/ftest.py -v` -- Run pylint with: `PYTHONPATH=~/u/tools python3 -m pylint uman_pkg/ftest.py` +- Run tests with: `python -m pytest uman_pkg/ftest.py -v` +- Run pylint with: `python3 -m pylint uman_pkg/ftest.py` ### Test conventions diff --git a/README.rst b/README.rst index daa5d83..f9f01b5 100644 --- a/README.rst +++ b/README.rst @@ -847,18 +847,19 @@ Settings are stored in ``~/.uman`` (created on first run):: Environment Variables ~~~~~~~~~~~~~~~~~~~~~ +``UMAN_EXTERNAL_PYLIB`` + Set to ``1`` to use u_boot_pylib from ``UBOOT_TOOLS`` instead of the + embedded copy. Useful for testing against a newer version of the library. + ``UBOOT_TOOLS`` Path to U-Boot tools directory containing Python libraries (u_boot_pylib, - patman, buildman, etc.). This is used for importing Python modules. + patman, buildman, etc.). Only used when ``UMAN_EXTERNAL_PYLIB=1``. Default: ``~/u/tools`` ``USRC`` Path to U-Boot source tree to work in. If not set, uman expects to be run from within a U-Boot source tree. -These are separate: ``UBOOT_TOOLS`` specifies where to find Python imports, -while ``USRC`` specifies the U-Boot source tree to build/test. - Self-testing ------------ diff --git a/u_boot_pylib/__init__.py b/u_boot_pylib/__init__.py new file mode 100644 index 0000000..807a62e --- /dev/null +++ b/u_boot_pylib/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0+ + +__all__ = ['command', 'cros_subprocess', 'gitutil', 'terminal', 'test_util', + 'tools', 'tout'] diff --git a/u_boot_pylib/command.py b/u_boot_pylib/command.py new file mode 100644 index 0000000..6b3f9fe --- /dev/null +++ b/u_boot_pylib/command.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: GPL-2.0+ +""" +Shell command ease-ups for Python + +Copyright (c) 2011 The Chromium OS Authors. +""" + +import subprocess + +from u_boot_pylib import cros_subprocess + +# This permits interception of RunPipe for test purposes. If it is set to +# a function, then that function is called with the pipe list being +# executed. Otherwise, it is assumed to be a CommandResult object, and is +# returned as the result for every run_pipe() call. +# When this value is None, commands are executed as normal. +TEST_RESULT = None + + +class CommandExc(Exception): + """Reports an exception to the caller""" + def __init__(self, msg, result): + """Set up a new exception object + + Args: + result (CommandResult): Execution result so far + """ + super().__init__(msg) + self.result = result + + +class CommandResult: + """A class which captures the result of executing a command. + + Members: + stdout (bytes): stdout obtained from command, as a string + stderr (bytes): stderr obtained from command, as a string + combined (bytes): stdout and stderr interleaved + return_code (int): Return code from command + exception (Exception): Exception received, or None if all ok + output (str or None): Returns output as a single line if requested + """ + def __init__(self, stdout='', stderr='', combined='', return_code=0, + exception=None): + self.stdout = stdout + self.stderr = stderr + self.combined = combined + self.return_code = return_code + self.exception = exception + self.output = None + + def to_output(self, binary): + """Converts binary output to its final form + + Args: + binary (bool): True to report binary output, False to use strings + Returns: + self + """ + if not binary: + self.stdout = self.stdout.decode('utf-8') + self.stderr = self.stderr.decode('utf-8') + self.combined = self.combined.decode('utf-8') + return self + + +def run_pipe(pipe_list, infile=None, outfile=None, capture=False, + capture_stderr=False, oneline=False, raise_on_error=True, cwd=None, + binary=False, output_func=None, **kwargs): + """ + Perform a command pipeline, with optional input/output filenames. + + Args: + pipe_list (list of list): List of command lines to execute. Each command + line is piped into the next, and is itself a list of strings. For + example [ ['ls', '.git'] ['wc'] ] will pipe the output of + 'ls .git' into 'wc'. + infile (str): File to provide stdin to the pipeline + outfile (str): File to store stdout + capture (bool): True to capture output + capture_stderr (bool): True to capture stderr + oneline (bool): True to strip newline chars from output + raise_on_error (bool): True to raise on an error, False to return it in + the CommandResult + cwd (str or None): Directory to run the command in + binary (bool): True to report binary output, False to use strings + output_func (function): Output function to call with each output + fragment (if it returns True the function terminates) + **kwargs: Additional keyword arguments to cros_subprocess.Popen() + Returns: + CommandResult object + Raises: + CommandExc if an exception happens + """ + if TEST_RESULT: + if hasattr(TEST_RESULT, '__call__'): + # pylint: disable=E1102 + result = TEST_RESULT(pipe_list=pipe_list) + if result: + return result + else: + return TEST_RESULT + # No result: fall through to normal processing + result = CommandResult(b'', b'', b'') + last_pipe = None + pipeline = list(pipe_list) + user_pipestr = '|'.join([' '.join(pipe) for pipe in pipe_list]) + kwargs['stdout'] = None + kwargs['stderr'] = None + while pipeline: + cmd = pipeline.pop(0) + if last_pipe is not None: + kwargs['stdin'] = last_pipe.stdout + elif infile: + kwargs['stdin'] = open(infile, 'rb') + if pipeline or capture: + kwargs['stdout'] = cros_subprocess.PIPE + elif outfile: + kwargs['stdout'] = open(outfile, 'wb') + if capture_stderr: + kwargs['stderr'] = cros_subprocess.PIPE + + try: + last_pipe = cros_subprocess.Popen(cmd, cwd=cwd, **kwargs) + except Exception as err: + result.exception = err + if raise_on_error: + raise CommandExc(f"Error running '{user_pipestr}': {err}", + result) from err + result.return_code = 255 + return result.to_output(binary) + + if capture: + result.stdout, result.stderr, result.combined = ( + last_pipe.communicate_filter(output_func)) + if result.stdout and oneline: + result.output = result.stdout.rstrip(b'\r\n') + result.return_code = last_pipe.wait() + result = result.to_output(binary) + if raise_on_error and result.return_code: + raise CommandExc(f"Error running '{user_pipestr}'", result) + return result + + +def output(*cmd, **kwargs): + """Run a command and return its output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + str: command output + """ + kwargs['raise_on_error'] = kwargs.get('raise_on_error', True) + return run_pipe([cmd], capture=True, **kwargs).stdout + + +def output_one_line(*cmd, **kwargs): + """Run a command and output it as a single-line string + + The command is expected to produce a single line of output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + str: output of command with all newlines removed + """ + raise_on_error = kwargs.pop('raise_on_error', True) + result = run_pipe([cmd], capture=True, oneline=True, + raise_on_error=raise_on_error, **kwargs).stdout.strip() + return result + + +def run(*cmd, **kwargs): + """Run a command + + Note that you must add 'capture' to kwargs to obtain non-empty output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + str: output of command + """ + return run_pipe([cmd], **kwargs).stdout + + +def run_one(*cmd, **kwargs): + """Run a single command + + Note that you must add 'capture' to kwargs to obtain non-empty output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + CommandResult: output of command + """ + return run_pipe([cmd], **kwargs) + + +def run_list(cmd, **kwargs): + """Run a command and return its output + + Args: + cmd (list of str): Command to run + + Returns: + str: output of command + **kwargs (dict of args): Extra arguments to pass in + """ + return run_pipe([cmd], capture=True, **kwargs).stdout + + +def stop_all(): + """Stop all subprocesses initiated with cros_subprocess""" + cros_subprocess.stay_alive = False diff --git a/u_boot_pylib/cros_subprocess.py b/u_boot_pylib/cros_subprocess.py new file mode 100644 index 0000000..cd614f3 --- /dev/null +++ b/u_boot_pylib/cros_subprocess.py @@ -0,0 +1,401 @@ +# Copyright (c) 2012 The Chromium OS Authors. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# Copyright (c) 2003-2005 by Peter Astrand +# Licensed to PSF under a Contributor Agreement. +# See http://www.python.org/2.4/license for licensing details. + +"""Subprocess execution + +This module holds a subclass of subprocess.Popen with our own required +features, mainly that we get access to the subprocess output while it +is running rather than just at the end. This makes it easier to show +progress information and filter output in real time. +""" + +import errno +import os +import pty +import select +import subprocess +import sys +import unittest + + +# Import these here so the caller does not need to import subprocess also. +PIPE = subprocess.PIPE +STDOUT = subprocess.STDOUT +PIPE_PTY = -3 # Pipe output through a pty +stay_alive = True + + +class Popen(subprocess.Popen): + """Like subprocess.Popen with ptys and incremental output + + This class deals with running a child process and filtering its output on + both stdout and stderr while it is running. We do this so we can monitor + progress, and possibly relay the output to the user if requested. + + The class is similar to subprocess.Popen, the equivalent is something like: + + Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + But this class has many fewer features, and two enhancement: + + 1. Rather than getting the output data only at the end, this class sends it + to a provided operation as it arrives. + 2. We use pseudo terminals so that the child will hopefully flush its output + to us as soon as it is produced, rather than waiting for the end of a + line. + + Use communicate_filter() to handle output from the subprocess. + + """ + + def __init__(self, args, stdin=None, stdout=PIPE_PTY, stderr=PIPE_PTY, + shell=False, cwd=None, env=None, **kwargs): + """Cut-down constructor + + Args: + args: Program and arguments for subprocess to execute. + stdin: See subprocess.Popen() + stdout: See subprocess.Popen(), except that we support the sentinel + value of cros_subprocess.PIPE_PTY. + stderr: See subprocess.Popen(), except that we support the sentinel + value of cros_subprocess.PIPE_PTY. + shell: See subprocess.Popen() + cwd: Working directory to change to for subprocess, or None if none. + env: Environment to use for this subprocess, or None to inherit parent. + kwargs: No other arguments are supported at the moment. Passing other + arguments will cause a ValueError to be raised. + """ + stdout_pty = None + stderr_pty = None + + if stdout == PIPE_PTY: + stdout_pty = pty.openpty() + stdout = os.fdopen(stdout_pty[1]) + if stderr == PIPE_PTY: + stderr_pty = pty.openpty() + stderr = os.fdopen(stderr_pty[1]) + + super(Popen, self).__init__(args, stdin=stdin, + stdout=stdout, stderr=stderr, shell=shell, cwd=cwd, env=env, + **kwargs) + + # If we're on a PTY, we passed the slave half of the PTY to the subprocess. + # We want to use the master half on our end from now on. Setting this here + # does make some assumptions about the implementation of subprocess, but + # those assumptions are pretty minor. + + # Note that if stderr is STDOUT, then self.stderr will be set to None by + # this constructor. + if stdout_pty is not None: + self.stdout = os.fdopen(stdout_pty[0]) + if stderr_pty is not None: + self.stderr = os.fdopen(stderr_pty[0]) + + # Insist that unit tests exist for other arguments we don't support. + if kwargs: + raise ValueError("Unit tests do not test extra args - please add tests") + + def convert_data(self, data): + """Convert stdout/stderr data to the correct format for output + + Args: + data: Data to convert, or None for '' + + Returns: + Converted data, as bytes + """ + if data is None: + return b'' + return data + + def communicate_filter(self, output, input_buf=''): + """Interact with process: Read data from stdout and stderr. + + This method runs until end-of-file is reached, then waits for the + subprocess to terminate. + + The output function is sent all output from the subprocess and must be + defined like this: + + def output([self,] stream, data) + Args: + stream: the stream the output was received on, which will be + sys.stdout or sys.stderr. + data: a string containing the data + + Returns: + True to terminate the process + + Note: The data read is buffered in memory, so do not use this + method if the data size is large or unlimited. + + Args: + output: Function to call with each fragment of output. + + Returns: + A tuple (stdout, stderr, combined) which is the data received on + stdout, stderr and the combined data (interleaved stdout and stderr). + + Note that the interleaved output will only be sensible if you have + set both stdout and stderr to PIPE or PIPE_PTY. Even then it depends on + the timing of the output in the subprocess. If a subprocess flips + between stdout and stderr quickly in succession, by the time we come to + read the output from each we may see several lines in each, and will read + all the stdout lines, then all the stderr lines. So the interleaving + may not be correct. In this case you might want to pass + stderr=cros_subprocess.STDOUT to the constructor. + + This feature is still useful for subprocesses where stderr is + rarely used and indicates an error. + + Note also that if you set stderr to STDOUT, then stderr will be empty + and the combined output will just be the same as stdout. + """ + + read_set = [] + write_set = [] + stdout = None # Return + stderr = None # Return + + if self.stdin: + # Flush stdio buffer. This might block, if the user has + # been writing to .stdin in an uncontrolled fashion. + self.stdin.flush() + if input_buf: + write_set.append(self.stdin) + else: + self.stdin.close() + if self.stdout: + read_set.append(self.stdout) + stdout = bytearray() + if self.stderr and self.stderr != self.stdout: + read_set.append(self.stderr) + stderr = bytearray() + combined = bytearray() + + stop_now = False + input_offset = 0 + while read_set or write_set: + try: + rlist, wlist, _ = select.select(read_set, write_set, [], 0.2) + except select.error as e: + if e.args[0] == errno.EINTR: + continue + raise + + if not stay_alive: + self.terminate() + + if self.stdin in wlist: + # When select has indicated that the file is writable, + # we can write up to PIPE_BUF bytes without risk + # blocking. POSIX defines PIPE_BUF >= 512 + chunk = input_buf[input_offset : input_offset + 512] + bytes_written = os.write(self.stdin.fileno(), chunk) + input_offset += bytes_written + if input_offset >= len(input_buf): + self.stdin.close() + write_set.remove(self.stdin) + + if self.stdout in rlist: + data = b'' + # We will get an error on read if the pty is closed + try: + data = os.read(self.stdout.fileno(), 1024) + except OSError: + pass + if not len(data): + self.stdout.close() + read_set.remove(self.stdout) + else: + stdout += data + combined += data + if output: + stop_now = output(sys.stdout, data) + if self.stderr in rlist: + data = b'' + # We will get an error on read if the pty is closed + try: + data = os.read(self.stderr.fileno(), 1024) + except OSError: + pass + if not len(data): + self.stderr.close() + read_set.remove(self.stderr) + else: + stderr += data + combined += data + if output: + stop_now = output(sys.stderr, data) + if stop_now: + self.terminate() + + # All data exchanged. Translate lists into strings. + stdout = self.convert_data(stdout) + stderr = self.convert_data(stderr) + combined = self.convert_data(combined) + + self.wait() + return (stdout, stderr, combined) + + +# Just being a unittest.TestCase gives us 14 public methods. Unless we +# disable this, we can only have 6 tests in a TestCase. That's not enough. +# +# pylint: disable=R0904 + +class TestSubprocess(unittest.TestCase): + """Our simple unit test for this module""" + + class MyOperation: + """Provides a operation that we can pass to Popen""" + def __init__(self, input_to_send=None): + """Constructor to set up the operation and possible input. + + Args: + input_to_send: a text string to send when we first get input. We will + add \r\n to the string. + """ + self.stdout_data = '' + self.stderr_data = '' + self.combined_data = '' + self.stdin_pipe = None + self._input_to_send = input_to_send + if input_to_send: + pipe = os.pipe() + self.stdin_read_pipe = pipe[0] + self._stdin_write_pipe = os.fdopen(pipe[1], 'w') + + def output(self, stream, data): + """Output handler for Popen. Stores the data for later comparison""" + if stream == sys.stdout: + self.stdout_data += data + if stream == sys.stderr: + self.stderr_data += data + self.combined_data += data + + # Output the input string if we have one. + if self._input_to_send: + self._stdin_write_pipe.write(self._input_to_send + '\r\n') + self._stdin_write_pipe.flush() + + def _basic_check(self, plist, oper): + """Basic checks that the output looks sane.""" + self.assertEqual(plist[0], oper.stdout_data) + self.assertEqual(plist[1], oper.stderr_data) + self.assertEqual(plist[2], oper.combined_data) + + # The total length of stdout and stderr should equal the combined length + self.assertEqual(len(plist[0]) + len(plist[1]), len(plist[2])) + + def test_simple(self): + """Simple redirection: Get process list""" + oper = TestSubprocess.MyOperation() + plist = Popen(['ps']).communicate_filter(oper.output) + self._basic_check(plist, oper) + + def test_stderr(self): + """Check stdout and stderr""" + oper = TestSubprocess.MyOperation() + cmd = 'echo fred >/dev/stderr && false || echo bad' + plist = Popen([cmd], shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], 'bad\r\n') + self.assertEqual(plist [1], 'fred\r\n') + + def test_shell(self): + """Check with and without shell works""" + oper = TestSubprocess.MyOperation() + cmd = 'echo test >/dev/stderr' + self.assertRaises(OSError, Popen, [cmd], shell=False) + plist = Popen([cmd], shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(len(plist [0]), 0) + self.assertEqual(plist [1], 'test\r\n') + + def test_list_args(self): + """Check with and without shell works using list arguments""" + oper = TestSubprocess.MyOperation() + cmd = ['echo', 'test', '>/dev/stderr'] + plist = Popen(cmd, shell=False).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], ' '.join(cmd[1:]) + '\r\n') + self.assertEqual(len(plist [1]), 0) + + oper = TestSubprocess.MyOperation() + + # this should be interpreted as 'echo' with the other args dropped + cmd = ['echo', 'test', '>/dev/stderr'] + plist = Popen(cmd, shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], '\r\n') + + def test_cwd(self): + """Check we can change directory""" + for shell in (False, True): + oper = TestSubprocess.MyOperation() + plist = Popen('pwd', shell=shell, cwd='/tmp').communicate_filter( + oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], '/tmp\r\n') + + def test_env(self): + """Check we can change environment""" + for add in (False, True): + oper = TestSubprocess.MyOperation() + env = os.environ + if add: + env ['FRED'] = 'fred' + cmd = 'echo $FRED' + plist = Popen(cmd, shell=True, env=env).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], add and 'fred\r\n' or '\r\n') + + def test_extra_args(self): + """Check we can't add extra arguments""" + self.assertRaises(ValueError, Popen, 'true', close_fds=False) + + def test_basic_input(self): + """Check that incremental input works + + We set up a subprocess which will prompt for name. When we see this prompt + we send the name as input to the process. It should then print the name + properly to stdout. + """ + oper = TestSubprocess.MyOperation('Flash') + prompt = 'What is your name?: ' + cmd = 'echo -n "%s"; read name; echo Hello $name' % prompt + plist = Popen([cmd], stdin=oper.stdin_read_pipe, + shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(len(plist [1]), 0) + self.assertEqual(plist [0], prompt + 'Hello Flash\r\r\n') + + def test_isatty(self): + """Check that ptys appear as terminals to the subprocess""" + oper = TestSubprocess.MyOperation() + cmd = ('if [ -t %d ]; then echo "terminal %d" >&%d; ' + 'else echo "not %d" >&%d; fi;') + both_cmds = '' + for fd in (1, 2): + both_cmds += cmd % (fd, fd, fd, fd, fd) + plist = Popen(both_cmds, shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], 'terminal 1\r\n') + self.assertEqual(plist [1], 'terminal 2\r\n') + + # Now try with PIPE and make sure it is not a terminal + oper = TestSubprocess.MyOperation() + plist = Popen(both_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], 'not 1\n') + self.assertEqual(plist [1], 'not 2\n') + +if __name__ == '__main__': + unittest.main() diff --git a/u_boot_pylib/gitutil.py b/u_boot_pylib/gitutil.py new file mode 100644 index 0000000..34b4dbb --- /dev/null +++ b/u_boot_pylib/gitutil.py @@ -0,0 +1,886 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2011 The Chromium OS Authors. +# + +"""Basic utilities for running the git command-line tool from Python""" + +import os +import sys + +from u_boot_pylib import command +from u_boot_pylib import terminal + +# True to use --no-decorate - we check this in setup() +USE_NO_DECORATE = True + + +def log_cmd(commit_range, git_dir=None, oneline=False, reverse=False, + count=None, decorate=False): + """Create a command to perform a 'git log' + + Args: + commit_range (str): Range expression to use for log, None for none + git_dir (str): Path to git repository (None to use default) + oneline (bool): True to use --oneline, else False + reverse (bool): True to reverse the log (--reverse) + count (int or None): Number of commits to list, or None for no limit + decorate (bool): True to use --decorate + + Return: + List containing command and arguments to run + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['--no-pager', 'log', '--no-color'] + if oneline: + cmd.append('--oneline') + if USE_NO_DECORATE and not decorate: + cmd.append('--no-decorate') + if decorate: + cmd.append('--decorate') + if reverse: + cmd.append('--reverse') + if count is not None: + cmd.append(f'-n{count}') + if commit_range: + cmd.append(commit_range) + + # Add this in case we have a branch with the same name as a directory. + # This avoids messages like this, for example: + # fatal: ambiguous argument 'test': both revision and filename + cmd.append('--') + return cmd + + +def count_commits_to_branch(branch, git_dir=None, end=None): + """Returns number of commits between HEAD and the tracking branch. + + This looks back to the tracking branch and works out the number of commits + since then. + + Args: + branch (str or None): Branch to count from (None for current branch) + git_dir (str): Path to git repository (None to use default) + end (str): End commit to stop before + + Return: + Number of patches that exist on top of the branch + """ + if end: + rev_range = f'{end}..{branch}' + elif branch: + us, msg = get_upstream(git_dir or '.git', branch) + if not us: + raise ValueError(msg) + rev_range = f'{us}..{branch}' + else: + rev_range = '@{upstream}..' + cmd = log_cmd(rev_range, git_dir=git_dir, oneline=True) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + oneline=True, raise_on_error=False) + if result.return_code: + raise ValueError( + f'Failed to determine upstream: {result.stderr.strip()}') + patch_count = len(result.stdout.splitlines()) + return patch_count + + +def name_revision(commit_hash): + """Gets the revision name for a commit + + Args: + commit_hash (str): Commit hash to look up + + Return: + Name of revision, if any, else None + """ + stdout = command.output_one_line('git', 'name-rev', commit_hash) + if not stdout: + return None + + # We expect a commit, a space, then a revision name + name = stdout.split()[1].strip() + return name + + +def guess_upstream(git_dir, branch): + """Tries to guess the upstream for a branch + + This lists out top commits on a branch and tries to find a suitable + upstream. It does this by looking for the first commit where + 'git name-rev' returns a plain branch name, with no ! or ^ modifiers. + + Args: + git_dir (str): Git directory containing repo + branch (str): Name of branch + + Returns: + Tuple: + Name of upstream branch (e.g. 'upstream/master') or None if none + Warning/error message, or None if none + """ + cmd = log_cmd(branch, git_dir=git_dir, oneline=True, count=100, + decorate=True) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + raise_on_error=False) + if result.return_code: + return None, f"Branch '{branch}' not found" + for line in result.stdout.splitlines()[1:]: + parts = line.split(maxsplit=1) + if len(parts) >= 2 and parts[1].startswith('('): + commit_hash = parts[0] + name = name_revision(commit_hash) + if '~' not in name and '^' not in name: + if name.startswith('remotes/'): + name = name[8:] + return name, f"Guessing upstream as '{name}'" + return None, f"Cannot find a suitable upstream for branch '{branch}'" + + +def get_upstream(git_dir, branch): + """Returns the name of the upstream for a branch + + Args: + git_dir (str): Git directory containing repo + branch (str): Name of branch + + Returns: + Tuple: + Name of upstream branch (e.g. 'upstream/master') or None if none + Warning/error message, or None if none + """ + try: + remote = command.output_one_line('git', '--git-dir', git_dir, 'config', + f'branch.{branch}.remote') + merge = command.output_one_line('git', '--git-dir', git_dir, 'config', + f'branch.{branch}.merge') + except command.CommandExc: + upstream, msg = guess_upstream(git_dir, branch) + return upstream, msg + + if remote == '.': + return merge, None + if remote and merge: + # Drop the initial refs/heads from merge + leaf = merge.split('/', maxsplit=2)[2:] + return f'{remote}/{"/".join(leaf)}', None + raise ValueError("Cannot determine upstream branch for branch " + f"'{branch}' remote='{remote}', merge='{merge}'") + + +def get_range_in_branch(git_dir, branch, include_upstream=False): + """Returns an expression for the commits in the given branch. + + Args: + git_dir (str): Directory containing git repo + branch (str): Name of branch + include_upstream (bool): Include the upstream commit as well + Return: + Expression in the form 'upstream..branch' which can be used to + access the commits. If the branch does not exist, returns None. + """ + upstream, msg = get_upstream(git_dir, branch) + if not upstream: + return None, msg + rstr = f"{upstream}{'~' if include_upstream else ''}..{branch}" + return rstr, msg + + +def count_commits_in_range(git_dir, range_expr): + """Returns the number of commits in the given range. + + Args: + git_dir (str): Directory containing git repo + range_expr (str): Range to check + Return: + Number of patches that exist in the supplied range or None if none + were found + """ + cmd = log_cmd(range_expr, git_dir=git_dir, oneline=True) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + raise_on_error=False) + if result.return_code: + return None, f"Range '{range_expr}' not found or is invalid" + patch_count = len(result.stdout.splitlines()) + return patch_count, None + + +def count_commits_in_branch(git_dir, branch, include_upstream=False): + """Returns the number of commits in the given branch. + + Args: + git_dir (str): Directory containing git repo + branch (str): Name of branch + include_upstream (bool): Include the upstream commit as well + Return: + Number of patches that exist on top of the branch, or None if the + branch does not exist. + """ + range_expr, msg = get_range_in_branch(git_dir, branch, include_upstream) + if not range_expr: + return None, msg + return count_commits_in_range(git_dir, range_expr) + + +def count_commits(commit_range): + """Returns the number of commits in the given range. + + Args: + commit_range (str): Range of commits to count (e.g. 'HEAD..base') + Return: + Number of patches that exist on top of the branch + """ + pipe = [log_cmd(commit_range, oneline=True), + ['wc', '-l']] + stdout = command.run_pipe(pipe, capture=True, oneline=True).stdout + patch_count = int(stdout) + return patch_count + + +def checkout(commit_hash, git_dir=None, work_tree=None, force=False): + """Checkout the selected commit for this build + + Args: + commit_hash (str): Commit hash to check out + git_dir (str): Directory containing git repo, or None for current dir + work_tree (str): Git worktree to use, or None if none + force (bool): True to force the checkout (git checkout -f) + """ + pipe = ['git'] + if git_dir: + pipe.extend(['--git-dir', git_dir]) + if work_tree: + pipe.extend(['--work-tree', work_tree]) + pipe.append('checkout') + if force: + pipe.append('-f') + pipe.append(commit_hash) + result = command.run_pipe([pipe], capture=True, raise_on_error=False, + capture_stderr=True) + if result.return_code != 0: + raise OSError(f'git checkout ({pipe}): {result.stderr}') + + +def clone(repo, output_dir): + """Clone a repo + + Args: + repo (str): Repo to clone (e.g. web address) + output_dir (str): Directory to close into + """ + result = command.run_one('git', 'clone', repo, '.', capture=True, + cwd=output_dir, capture_stderr=True) + if result.return_code != 0: + raise OSError(f'git clone: {result.stderr}') + + +def fetch(git_dir=None, work_tree=None): + """Fetch from the origin repo + + Args: + git_dir (str): Directory containing git repo, or None for current dir + work_tree (str or None): Git worktree to use, or None if none + """ + cmd = ['git'] + if git_dir: + cmd.extend(['--git-dir', git_dir]) + if work_tree: + cmd.extend(['--work-tree', work_tree]) + cmd.append('fetch') + result = command.run_one(*cmd, capture=True, capture_stderr=True) + if result.return_code != 0: + raise OSError(f'git fetch: {result.stderr}') + + +def check_worktree_is_available(git_dir): + """Check if git-worktree functionality is available + + Args: + git_dir (str): The repository to test in + + Returns: + True if git-worktree commands will work, False otherwise. + """ + result = command.run_one('git', '--git-dir', git_dir, 'worktree', 'list', + capture=True, capture_stderr=True, + raise_on_error=False) + return result.return_code == 0 + + +def add_worktree(git_dir, output_dir, commit_hash=None): + """Create and checkout a new git worktree for this build + + Args: + git_dir (str): The repository to checkout the worktree from + output_dir (str): Path for the new worktree + commit_hash (str): Commit hash to checkout + """ + # We need to pass --detach to avoid creating a new branch + cmd = ['git', '--git-dir', git_dir, 'worktree', 'add', '.', '--detach'] + if commit_hash: + cmd.append(commit_hash) + result = command.run_one(*cmd, capture=True, cwd=output_dir, + capture_stderr=True) + if result.return_code != 0: + raise OSError(f'git worktree add: {result.stderr}') + + +def prune_worktrees(git_dir): + """Remove administrative files for deleted worktrees + + Args: + git_dir (str): The repository whose deleted worktrees should be pruned + """ + result = command.run_one('git', '--git-dir', git_dir, 'worktree', 'prune', + capture=True, capture_stderr=True) + if result.return_code != 0: + raise OSError(f'git worktree prune: {result.stderr}') + + +def create_patches(branch, start, count, ignore_binary, series, signoff=True, + git_dir=None, cwd=None): + """Create a series of patches from the top of the current branch. + + The patch files are written to the current directory using + git format-patch. + + Args: + branch (str): Branch to create patches from (None for current branch) + start (int): Commit to start from: 0=HEAD, 1=next one, etc. + count (int): number of commits to include + ignore_binary (bool): Don't generate patches for binary files + series (Series): Series object for this series (set of patches) + signoff (bool): True to add signoff lines automatically + git_dir (str): Path to git repository (None to use default) + cwd (str): Path to use for git operations + Return: + Filename of cover letter (None if none) + List of filenames of patch files + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['format-patch', '-M'] + if signoff: + cmd.append('--signoff') + if ignore_binary: + cmd.append('--no-binary') + if series.get('cover'): + cmd.append('--cover-letter') + prefix = series.GetPatchPrefix() + if prefix: + cmd += [f'--subject-prefix={prefix}'] + brname = branch or 'HEAD' + cmd += [f'{brname}~{start + count}..{brname}~{start}'] + + stdout = command.run_list(cmd, cwd=cwd) + files = stdout.splitlines() + + # We have an extra file if there is a cover letter + if series.get('cover'): + return files[0], files[1:] + return None, files + + +def build_email_list(in_list, alias, tag=None, warn_on_error=True): + """Build a list of email addresses based on an input list. + + Takes a list of email addresses and aliases, and turns this into a list + of only email address, by resolving any aliases that are present. + + If the tag is given, then each email address is prepended with this + tag and a space. If the tag starts with a minus sign (indicating a + command line parameter) then the email address is quoted. + + Args: + in_list (list of str): List of aliases/email addresses + alias (dict): Alias dictionary: + key: alias + value: list of aliases or email addresses + tag (str): Text to put before each address + warn_on_error (bool): True to raise an error when an alias fails to + match, False to just print a message. + + Returns: + List of email addresses + + >>> alias = {} + >>> alias['fred'] = ['f.bloggs@napier.co.nz'] + >>> alias['john'] = ['j.bloggs@napier.co.nz'] + >>> alias['mary'] = ['Mary Poppins '] + >>> alias['boys'] = ['fred', ' john'] + >>> alias['all'] = ['fred ', 'john', ' mary '] + >>> build_email_list(['john', 'mary'], alias, None) + ['j.bloggs@napier.co.nz', 'Mary Poppins '] + >>> build_email_list(['john', 'mary'], alias, '--to') + ['--to "j.bloggs@napier.co.nz"', \ +'--to "Mary Poppins "'] + >>> build_email_list(['john', 'mary'], alias, 'Cc') + ['Cc j.bloggs@napier.co.nz', 'Cc Mary Poppins '] + """ + raw = [] + for item in in_list: + raw += lookup_email(item, alias, warn_on_error=warn_on_error) + result = [] + for item in raw: + if item not in result: + result.append(item) + if tag: + return [x for email in result for x in (tag, email)] + return result + + +def check_suppress_cc_config(): + """Check if sendemail.suppresscc is configured correctly. + + Returns: + bool: True if the option is configured correctly, False otherwise. + """ + suppresscc = command.output_one_line( + 'git', 'config', 'sendemail.suppresscc', raise_on_error=False) + + # Other settings should be fine. + if suppresscc in ('all', 'cccmd'): + col = terminal.Color() + + print(col.build(col.RED, 'error') + + f': git config sendemail.suppresscc set to {suppresscc}\n' + + ' patman needs --cc-cmd to be run to set the cc list.\n' + + ' Please run:\n' + + ' git config --unset sendemail.suppresscc\n' + + ' Or read the man page:\n' + + ' git send-email --help\n' + + ' and set an option that runs --cc-cmd\n') + return False + + return True + + +def email_patches(series, cover_fname, args, dry_run, warn_on_error, cc_fname, + alias, self_only=False, in_reply_to=None, thread=False, + smtp_server=None, cwd=None): + """Email a patch series. + + Args: + series (Series): Series object containing destination info + cover_fname (str or None): filename of cover letter + args (list of str): list of filenames of patch files + dry_run (bool): Just return the command that would be run + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + cc_fname (str): Filename of Cc file for per-commit Cc + alias (dict): Alias dictionary: + key: alias + value: list of aliases or email addresses + self_only (bool): True to just email to yourself as a test + in_reply_to (str or None): If set we'll pass this to git as + --in-reply-to - should be a message ID that this is in reply to. + thread (bool): True to add --thread to git send-email (make + all patches reply to cover-letter or first patch in series) + smtp_server (str or None): SMTP server to use to send patches + cwd (str): Path to use for patch files (None to use current dir) + + Returns: + Git command that was/would be run + + # For the duration of this doctest pretend that we ran patman with ./patman + >>> _old_argv0 = sys.argv[0] + >>> sys.argv[0] = './patman' + + >>> alias = {} + >>> alias['fred'] = ['f.bloggs@napier.co.nz'] + >>> alias['john'] = ['j.bloggs@napier.co.nz'] + >>> alias['mary'] = ['m.poppins@cloud.net'] + >>> alias['boys'] = ['fred', ' john'] + >>> alias['all'] = ['fred ', 'john', ' mary '] + >>> alias[os.getenv('USER')] = ['this-is-me@me.com'] + >>> series = {} + >>> series['to'] = ['fred'] + >>> series['cc'] = ['mary'] + >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ + False, alias) + 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ +"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" cover p1 p2' + >>> email_patches(series, None, ['p1'], True, True, 'cc-fname', False, \ + alias) + 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ +"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" p1' + >>> series['cc'] = ['all'] + >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ + True, alias) + 'git send-email --annotate --to "this-is-me@me.com" --cc-cmd "./patman \ +send --cc-cmd cc-fname" cover p1 p2' + >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ + False, alias) + 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ +"f.bloggs@napier.co.nz" --cc "j.bloggs@napier.co.nz" --cc \ +"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" cover p1 p2' + + # Restore argv[0] since we clobbered it. + >>> sys.argv[0] = _old_argv0 + """ + to = build_email_list(series.get('to'), alias, '--to', warn_on_error) + if not to: + if not command.output('git', 'config', 'sendemail.to', + raise_on_error=False): + print("No recipient.\n" + "Please add something like this to a commit\n" + "Series-to: Fred Bloggs \n" + "Or do something like this\n" + "git config sendemail.to u-boot@lists.denx.de") + return None + cc = build_email_list(list(set(series.get('cc')) - set(series.get('to'))), + alias, '--cc', warn_on_error) + if self_only: + to = build_email_list([os.getenv('USER')], '--to', alias, + warn_on_error) + cc = [] + cmd = ['git', 'send-email', '--annotate'] + if smtp_server: + cmd.append(f'--smtp-server={smtp_server}') + if in_reply_to: + cmd.append(f'--in-reply-to="{in_reply_to}"') + if thread: + cmd.append('--thread') + + cmd += to + cmd += cc + cmd += ['--cc-cmd', f'{sys.argv[0]} send --cc-cmd {cc_fname}'] + if cover_fname: + cmd.append(cover_fname) + cmd += args + if not dry_run: + command.run(*cmd, capture=False, capture_stderr=False, cwd=cwd) + return' '.join([f'"{x}"' if ' ' in x and '"' not in x else x + for x in cmd]) + + +def lookup_email(lookup_name, alias, warn_on_error=True, level=0): + """If an email address is an alias, look it up and return the full name + + TODO: Why not just use git's own alias feature? + + Args: + lookup_name (str): Alias or email address to look up + alias (dict): Alias dictionary + key: alias + value: list of aliases or email addresses + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + level (int): Depth of alias stack, used to detect recusion/loops + + Returns: + tuple: + list containing a list of email addresses + + Raises: + OSError if a recursive alias reference was found + ValueError if an alias was not found + + >>> alias = {} + >>> alias['fred'] = ['f.bloggs@napier.co.nz'] + >>> alias['john'] = ['j.bloggs@napier.co.nz'] + >>> alias['mary'] = ['m.poppins@cloud.net'] + >>> alias['boys'] = ['fred', ' john', 'f.bloggs@napier.co.nz'] + >>> alias['all'] = ['fred ', 'john', ' mary '] + >>> alias['loop'] = ['other', 'john', ' mary '] + >>> alias['other'] = ['loop', 'john', ' mary '] + >>> lookup_email('mary', alias) + ['m.poppins@cloud.net'] + >>> lookup_email('arthur.wellesley@howe.ro.uk', alias) + ['arthur.wellesley@howe.ro.uk'] + >>> lookup_email('boys', alias) + ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz'] + >>> lookup_email('all', alias) + ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz', 'm.poppins@cloud.net'] + >>> lookup_email('odd', alias) + Alias 'odd' not found + [] + >>> lookup_email('loop', alias) + Traceback (most recent call last): + ... + OSError: Recursive email alias at 'other' + >>> lookup_email('odd', alias, warn_on_error=False) + [] + >>> # In this case the loop part will effectively be ignored. + >>> lookup_email('loop', alias, warn_on_error=False) + Recursive email alias at 'other' + Recursive email alias at 'john' + Recursive email alias at 'mary' + ['j.bloggs@napier.co.nz', 'm.poppins@cloud.net'] + """ + lookup_name = lookup_name.strip() + if '@' in lookup_name: # Perhaps a real email address + return [lookup_name] + + lookup_name = lookup_name.lower() + col = terminal.Color() + + out_list = [] + if level > 10: + msg = f"Recursive email alias at '{lookup_name}'" + if warn_on_error: + raise OSError(msg) + print(col.build(col.RED, msg)) + return out_list + + if lookup_name: + if lookup_name not in alias: + msg = f"Alias '{lookup_name}' not found" + if warn_on_error: + print(col.build(col.RED, msg)) + return out_list + for item in alias[lookup_name]: + todo = lookup_email(item, alias, warn_on_error, level + 1) + for new_item in todo: + if new_item not in out_list: + out_list.append(new_item) + + return out_list + + +def get_top_level(): + """Return name of top-level directory for this git repo. + + Returns: + str: Full path to git top-level directory, or None if not found + + This test makes sure that we are running tests in the right subdir + + >>> os.path.realpath(os.path.dirname(__file__)) == \ + os.path.join(get_top_level(), 'tools', 'patman') + True + """ + result = command.run_one( + 'git', 'rev-parse', '--show-toplevel', oneline=True, capture=True, + capture_stderr=True, raise_on_error=False) + if result.return_code: + return None + return result.stdout.strip() + + +def get_alias_file(): + """Gets the name of the git alias file. + + Returns: + str: Filename of git alias file, or None if none + """ + fname = command.output_one_line('git', 'config', 'sendemail.aliasesfile', + raise_on_error=False) + if not fname: + return None + + fname = os.path.expanduser(fname.strip()) + if os.path.isabs(fname): + return fname + + return os.path.join(get_top_level() or '', fname) + + +def get_default_user_name(): + """Gets the user.name from .gitconfig file. + + Returns: + User name found in .gitconfig file, or None if none + """ + uname = command.output_one_line('git', 'config', '--global', '--includes', + 'user.name') + return uname + + +def get_default_user_email(): + """Gets the user.email from the global .gitconfig file. + + Returns: + User's email found in .gitconfig file, or None if none + """ + uemail = command.output_one_line('git', 'config', '--global', '--includes', + 'user.email') + return uemail + + +def get_default_subject_prefix(): + """Gets the format.subjectprefix from local .git/config file. + + Returns: + Subject prefix found in local .git/config file, or None if none + """ + sub_prefix = command.output_one_line( + 'git', 'config', 'format.subjectprefix', raise_on_error=False) + + return sub_prefix + + +def setup(): + """setup() - Set up git utils, by reading the alias files.""" + # Check for a git alias file also + global USE_NO_DECORATE + + cmd = log_cmd(None, count=0) + USE_NO_DECORATE = (command.run_one(*cmd, raise_on_error=False) + .return_code == 0) + + +def get_hash(spec, git_dir=None): + """Get the hash of a commit + + Args: + spec (str): Git commit to show, e.g. 'my-branch~12' + git_dir (str): Path to git repository (None to use default) + + Returns: + str: Hash of commit + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['show', '-s', '--pretty=format:%H', spec] + return command.output_one_line(*cmd) + + +def get_head(): + """Get the hash of the current HEAD + + Returns: + Hash of HEAD + """ + return get_hash('HEAD') + + +def get_branch(git_dir=None): + """Get the branch we are currently on + + Return: + str: branch name, or None if none + git_dir (str): Path to git repository (None to use default) + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['rev-parse', '--abbrev-ref', 'HEAD'] + out = command.output_one_line(*cmd, raise_on_error=False) + if out == 'HEAD': + return None + return out + + +def check_dirty(git_dir=None, work_tree=None): + """Check if the tree is dirty + + Args: + git_dir (str): Path to git repository (None to use default) + work_tree (str): Git worktree to use, or None if none + + Return: + str: List of dirty filenames and state + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + if work_tree: + cmd += ['--work-tree', work_tree] + cmd += ['status', '--porcelain', '--untracked-files=no'] + return command.output(*cmd).splitlines() + + +def check_branch(name, git_dir=None): + """Check if a branch exists + + Args: + name (str): Name of the branch to check + git_dir (str): Path to git repository (None to use default) + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['branch', '--list', name] + + # This produces ' ' or '* ' + out = command.output(*cmd).rstrip() + return out[2:] == name + + +def rename_branch(old_name, name, git_dir=None): + """Check if a branch exists + + Args: + old_name (str): Name of the branch to rename + name (str): New name for the branch + git_dir (str): Path to git repository (None to use default) + + Return: + str: Output from command + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['branch', '--move', old_name, name] + + # This produces ' ' or '* ' + return command.output(*cmd).rstrip() + + +def get_commit_message(commit, git_dir=None): + """Gets the commit message for a commit + + Args: + commit (str): commit to check + git_dir (str): Path to git repository (None to use default) + + Return: + list of str: Lines from the commit message + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['show', '--quiet', commit] + + out = command.output(*cmd) + # the header is followed by a blank line + lines = out.splitlines() + empty = lines.index('') + msg = lines[empty + 1:] + unindented = [line[4:] for line in msg] + + return unindented + + +def show_commit(commit, msg=True, diffstat=False, patch=False, colour=True, + git_dir=None): + """Runs 'git show' and returns the output + + Args: + commit (str): commit to check + msg (bool): Show the commit message + diffstat (bool): True to include the diffstat + patch (bool): True to include the patch + colour (bool): True to force use of colour + git_dir (str): Path to git repository (None to use default) + + Return: + list of str: Lines from the commit message + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['show'] + if colour: + cmd.append('--color') + if not msg: + cmd.append('--oneline') + if diffstat: + cmd.append('--stat') + else: + cmd.append('--quiet') + if patch: + cmd.append('--patch') + cmd.append(commit) + + return command.output(*cmd) + + +if __name__ == "__main__": + import doctest + + doctest.testmod() diff --git a/u_boot_pylib/terminal.py b/u_boot_pylib/terminal.py new file mode 100644 index 0000000..e62fa16 --- /dev/null +++ b/u_boot_pylib/terminal.py @@ -0,0 +1,350 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2011 The Chromium OS Authors. +# + +"""Terminal utilities + +This module handles terminal interaction including ANSI color codes. +""" + +from contextlib import contextmanager +from io import StringIO +import os +import re +import shutil +import subprocess +import sys + +# Selection of when we want our output to be colored +COLOR_IF_TERMINAL, COLOR_ALWAYS, COLOR_NEVER = range(3) + +# Initially, we are set up to print to the terminal +print_test_mode = False +print_test_list = [] + +# The length of the last line printed without a newline +last_print_len = None + +# credit: +# stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python +ansi_escape = re.compile(r'\x1b(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + +# True if we are capturing console output +CAPTURING = False + +# Set this to False to disable output-capturing globally +USE_CAPTURE = True + + +class PrintLine: + """A line of text output + + Members: + text: Text line that was printed + newline: True to output a newline after the text + colour: Text colour to use + """ + def __init__(self, text, colour, newline=True, bright=True): + self.text = text + self.newline = newline + self.colour = colour + self.bright = bright + + def __eq__(self, other): + return (self.text == other.text and + self.newline == other.newline and + self.colour == other.colour and + self.bright == other.bright) + + def __str__(self): + return ("newline=%s, colour=%s, bright=%d, text='%s'" % + (self.newline, self.colour, self.bright, self.text)) + + +def calc_ascii_len(text): + """Calculate the length of a string, ignoring any ANSI sequences + + When displayed on a terminal, ANSI sequences don't take any space, so we + need to ignore them when calculating the length of a string. + + Args: + text: Text to check + + Returns: + Length of text, after skipping ANSI sequences + + >>> col = Color(COLOR_ALWAYS) + >>> text = col.build(Color.RED, 'abc') + >>> len(text) + 14 + >>> calc_ascii_len(text) + 3 + >>> + >>> text += 'def' + >>> calc_ascii_len(text) + 6 + >>> text += col.build(Color.RED, 'abc') + >>> calc_ascii_len(text) + 9 + """ + result = ansi_escape.sub('', text) + return len(result) + +def trim_ascii_len(text, size): + """Trim a string containing ANSI sequences to the given ASCII length + + The string is trimmed with ANSI sequences being ignored for the length + calculation. + + >>> col = Color(COLOR_ALWAYS) + >>> text = col.build(Color.RED, 'abc') + >>> len(text) + 14 + >>> calc_ascii_len(trim_ascii_len(text, 4)) + 3 + >>> calc_ascii_len(trim_ascii_len(text, 2)) + 2 + >>> text += 'def' + >>> calc_ascii_len(trim_ascii_len(text, 4)) + 4 + >>> text += col.build(Color.RED, 'ghi') + >>> calc_ascii_len(trim_ascii_len(text, 7)) + 7 + """ + if calc_ascii_len(text) < size: + return text + pos = 0 + out = '' + left = size + + # Work through each ANSI sequence in turn + for m in ansi_escape.finditer(text): + # Find the text before the sequence and add it to our string, making + # sure it doesn't overflow + before = text[pos:m.start()] + toadd = before[:left] + out += toadd + + # Figure out how much non-ANSI space we have left + left -= len(toadd) + + # Add the ANSI sequence and move to the position immediately after it + out += m.group() + pos = m.start() + len(m.group()) + + # Deal with text after the last ANSI sequence + after = text[pos:] + toadd = after[:left] + out += toadd + + return out + + +def tprint(text='', newline=True, colour=None, limit_to_line=False, + bright=True, back=None, col=None, stderr=False): + """Handle a line of output to the terminal. + + In test mode this is recorded in a list. Otherwise it is output to the + terminal. + + Args: + text: Text to print + newline: True to add a new line at the end of the text + colour: Colour to use for the text + stderr: True to print to stderr instead of stdout + """ + global last_print_len + + if print_test_mode: + print_test_list.append(PrintLine(text, colour, newline, bright)) + else: + if colour is not None: + if not col: + col = Color() + text = col.build(colour, text, bright=bright, back=back) + + file = sys.stderr if stderr else sys.stdout + + if newline: + print(text, file=file) + last_print_len = None + else: + if limit_to_line: + cols = shutil.get_terminal_size().columns + text = trim_ascii_len(text, cols) + print(text, end='', flush=True, file=file) + last_print_len = calc_ascii_len(text) + +def print_clear(): + """Clear a previously line that was printed with no newline""" + global last_print_len + + if last_print_len: + if print_test_mode: + print_test_list.append(PrintLine(None, None, None, None)) + else: + print('\r%s\r' % (' '* last_print_len), end='', flush=True) + last_print_len = None + +def set_print_test_mode(enable=True): + """Go into test mode, where all printing is recorded""" + global print_test_mode + + print_test_mode = enable + get_print_test_lines() + +def get_print_test_lines(): + """Get a list of all lines output through tprint() + + Returns: + A list of PrintLine objects + """ + global print_test_list + + ret = print_test_list + print_test_list = [] + return ret + +def echo_print_test_lines(): + """Print out the text lines collected""" + for line in print_test_list: + if line.colour: + col = Color() + print(col.build(line.colour, line.text), end='') + else: + print(line.text, end='') + if line.newline: + print() + +def have_terminal(): + """Check if we have an interactive terminal or not + + Returns: + bool: true if an interactive terminal is attached + """ + return os.isatty(sys.stdout.fileno()) + + +class Color(): + """Conditionally wraps text in ANSI color escape sequences.""" + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) + BOLD = -1 + BRIGHT_START = '\033[1;%d%sm' + NORMAL_START = '\033[22;%d%sm' + BOLD_START = '\033[1m' + BACK_EXTRA = ';%d' + RESET = '\033[0m' + + def __init__(self, colored=COLOR_IF_TERMINAL): + """Create a new Color object, optionally disabling color output. + + Args: + enabled: True if color output should be enabled. If False then this + class will not add color codes at all. + """ + try: + self._enabled = (colored == COLOR_ALWAYS or + (colored == COLOR_IF_TERMINAL and + os.isatty(sys.stdout.fileno()))) + except: + self._enabled = False + + def enabled(self): + """Check if colour is enabled + + Return: True if enabled, else False + """ + return self._enabled + + def start(self, color, bright=True, back=None): + """Returns a start color code. + + Args: + color: Color to use, .e.g BLACK, RED, etc. + + Returns: + If color is enabled, returns an ANSI sequence to start the given + color, otherwise returns empty string + """ + if self._enabled: + if color == self.BOLD: + return self.BOLD_START + base = self.BRIGHT_START if bright else self.NORMAL_START + extra = self.BACK_EXTRA % (back + 40) if back else '' + return base % (color + 30, extra) + return '' + + def stop(self): + """Returns a stop color code. + + Returns: + If color is enabled, returns an ANSI color reset sequence, + otherwise returns empty string + """ + if self._enabled: + return self.RESET + return '' + + def build(self, color, text, bright=True, back=None): + """Returns text with conditionally added color escape sequences. + + Keyword arguments: + color: Text color -- one of the color constants defined in this + class. + text: The text to color. + + Returns: + If self._enabled is False, returns the original text. If it's True, + returns text with color escape sequences based on the value of + color. + """ + if not self._enabled: + return text + return self.start(color, bright, back) + text + self.RESET + + +# Use this to suppress stdout/stderr output: +# with terminal.capture() as (stdout, stderr) +# ...do something... +@contextmanager +def capture(): + global CAPTURING + + capture_out, capture_err = StringIO(), StringIO() + old_out, old_err = sys.stdout, sys.stderr + try: + CAPTURING = True + sys.stdout, sys.stderr = capture_out, capture_err + yield capture_out, capture_err + finally: + sys.stdout, sys.stderr = old_out, old_err + CAPTURING = False + if not USE_CAPTURE: + sys.stdout.write(capture_out.getvalue()) + sys.stderr.write(capture_err.getvalue()) + + +@contextmanager +def pager(): + """Simple pager for outputting lots of text + + Usage: + with terminal.pager(): + print(...) + """ + proc = None + old_stdout = None + try: + less = os.getenv('PAGER') + if not CAPTURING and less != 'none' and have_terminal(): + if not less: + less = 'less -R --quit-if-one-screen' + proc = subprocess.Popen(less, stdin=subprocess.PIPE, text=True, + shell=True) + old_stdout = sys.stdout + sys.stdout = proc.stdin + yield + finally: + if proc: + sys.stdout = old_stdout + proc.communicate() diff --git a/u_boot_pylib/test_util.py b/u_boot_pylib/test_util.py new file mode 100644 index 0000000..b1c8740 --- /dev/null +++ b/u_boot_pylib/test_util.py @@ -0,0 +1,229 @@ +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright (c) 2016 Google, Inc +# + +import doctest +import glob +import multiprocessing +import os +import re +import sys +import unittest + +from u_boot_pylib import command +from u_boot_pylib import terminal + +use_concurrent = True +try: + from concurrencytest import ConcurrentTestSuite + from concurrencytest import fork_for_tests +except: + use_concurrent = False + + +def run_test_coverage(prog, filter_fname, exclude_list, build_dir, + required=None, extra_args=None, single_thread='-P1', + args=None, allow_failures=None): + """Run tests and check that we get 100% coverage + + Args: + prog: Program to run (with be passed a '-t' argument to run tests + filter_fname: Normally all *.py files in the program's directory will + be included. If this is not None, then it is used to filter the + list so that only filenames that don't contain filter_fname are + included. + exclude_list: List of file patterns to exclude from the coverage + calculation + build_dir: Build directory, used to locate libfdt.py + required: Set of modules which must be in the coverage report + extra_args (str): Extra arguments to pass to the tool before the -t/test + arg + single_thread (str): Argument string to make the tests run + single-threaded. This is necessary to get proper coverage results. + The default is '-P0' + args (list of str): List of tests to run, or None to run all + + Raises: + ValueError if the code coverage is not 100% + """ + # This uses the build output from sandbox_spl to get _libfdt.so + path = os.path.dirname(prog) + if filter_fname: + glob_list = glob.glob(os.path.join(path, '*.py')) + glob_list = [fname for fname in glob_list if filter_fname in fname] + else: + glob_list = [] + glob_list += exclude_list + glob_list += ['*libfdt.py', '*/site-packages/*', '*/dist-packages/*'] + glob_list += ['*concurrencytest*'] + use_test = 'binman' in prog or 'patman' in prog or 'pickman' in prog + test_cmd = 'test' if use_test else '-t' + prefix = '' + if build_dir: + prefix = 'PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools ' % build_dir + + # Detect a Python sandbox and use 'coverage' instead + covtool = ('python3-coverage' if sys.prefix == sys.base_prefix else + 'coverage') + + cmd = ('%s%s run ' + '--omit "%s" %s %s %s %s %s' % (prefix, covtool, ','.join(glob_list), + prog, extra_args or '', test_cmd, + single_thread or '-P1', + ' '.join(args) if args else '')) + os.system(cmd) + stdout = command.output(covtool, 'report') + lines = stdout.splitlines() + if required: + # Convert '/path/to/name.py' just the module name 'name' + test_set = set([os.path.splitext(os.path.basename(line.split()[0]))[0] + for line in lines if '/etype/' in line]) + missing_list = required + missing_list.discard('__init__') + missing_list.difference_update(test_set) + if missing_list: + print('Missing tests for %s' % (', '.join(missing_list))) + print(stdout) + ok = False + + coverage = lines[-1].split(' ')[-1] + ok = True + print(coverage) + if coverage != '100%': + print(stdout) + print("To get a report in 'htmlcov/index.html', type: " + "python3-coverage html") + print('Coverage error: %s, but should be 100%%' % coverage) + ok = False + if not ok: + if allow_failures: + lines = [re.match(r'^(tools/.*py) *\d+ *(\d+) *\d+%$', line) + for line in stdout.splitlines()] + bad = [] + for mat in lines: + if mat and mat.group(2) != '0': + fname = mat.group(1) + if fname not in allow_failures: + bad.append(fname) + if not bad: + return + raise ValueError('Test coverage failure') + + +class FullTextTestResult(unittest.TextTestResult): + """A test result class that can print extended text results to a stream + + This is meant to be used by a TestRunner as a result class. Like + TextTestResult, this prints out the names of tests as they are run, + errors as they occur, and a summary of the results at the end of the + test run. Beyond those, this prints information about skipped tests, + expected failures and unexpected successes. + + Args: + stream: A file-like object to write results to + descriptions (bool): True to print descriptions with test names + verbosity (int): Detail of printed output per test as they run + Test stdout and stderr always get printed when buffering + them is disabled by the test runner. In addition to that, + 0: Print nothing + 1: Print a dot per test + 2: Print test names + """ + def __init__(self, stream, descriptions, verbosity): + self.verbosity = verbosity + super().__init__(stream, descriptions, verbosity) + + def printErrors(self): + "Called by TestRunner after test run to summarize the tests" + # The parent class doesn't keep unexpected successes in the same + # format as the rest. Adapt it to what printErrorList expects. + unexpected_successes = [ + (test, 'Test was expected to fail, but succeeded.\n') + for test in self.unexpectedSuccesses + ] + + super().printErrors() # FAIL and ERROR + self.printErrorList('SKIP', self.skipped) + self.printErrorList('XFAIL', self.expectedFailures) + self.printErrorList('XPASS', unexpected_successes) + + def addSkip(self, test, reason): + """Called when a test is skipped.""" + # Add empty line to keep spacing consistent with other results + if not reason.endswith('\n'): + reason += '\n' + super().addSkip(test, reason) + + +def run_test_suites(toolname, debug, verbosity, no_capture, test_preserve_dirs, + processes, test_name, toolpath, class_and_module_list): + """Run a series of test suites and collect the results + + Args: + toolname: Name of the tool that ran the tests + debug: True to enable debugging, which shows a full stack trace on error + verbosity: Verbosity level to use (0-4) + test_preserve_dirs: True to preserve the input directory used by tests + so that it can be examined afterwards (only useful for debugging + tests). If a single test is selected (in args[0]) it also preserves + the output directory for this test. Both directories are displayed + on the command line. + processes: Number of processes to use to run tests (None=same as #CPUs) + test_name: Name of test to run, or None for all + toolpath: List of paths to use for tools + class_and_module_list: List of test classes (type class) and module + names (type str) to run + """ + sys.argv = [sys.argv[0]] + if debug: + sys.argv.append('-D') + if verbosity: + sys.argv.append('-v%d' % verbosity) + if no_capture: + sys.argv.append('-N') + terminal.USE_CAPTURE = False + if toolpath: + for path in toolpath: + sys.argv += ['--toolpath', path] + + suite = unittest.TestSuite() + loader = unittest.TestLoader() + runner = unittest.TextTestRunner( + stream=sys.stdout, + verbosity=(1 if verbosity is None else verbosity), + resultclass=FullTextTestResult, + ) + + if use_concurrent and processes != 1 and not test_name: + suite = ConcurrentTestSuite(suite, + fork_for_tests(processes or multiprocessing.cpu_count())) + + for module in class_and_module_list: + if isinstance(module, str) and (not test_name or test_name == module): + suite.addTests(doctest.DocTestSuite(module)) + + for module in class_and_module_list: + if isinstance(module, str): + continue + # Test the test module about our arguments, if it is interested + if hasattr(module, 'setup_test_args'): + setup_test_args = getattr(module, 'setup_test_args') + setup_test_args(preserve_indir=test_preserve_dirs, + preserve_outdirs=test_preserve_dirs and test_name is not None, + toolpath=toolpath, verbosity=verbosity, no_capture=no_capture) + if test_name: + # Since Python v3.5 If an ImportError or AttributeError occurs + # while traversing a name then a synthetic test that raises that + # error when run will be returned. Check that the requested test + # exists, otherwise these errors are included in the results. + if test_name in loader.getTestCaseNames(module): + suite.addTests(loader.loadTestsFromName(test_name, module)) + else: + suite.addTests(loader.loadTestsFromTestCase(module)) + + print(f" Running {toolname} tests ".center(70, "=")) + result = runner.run(suite) + print() + + return result diff --git a/u_boot_pylib/tools.py b/u_boot_pylib/tools.py new file mode 100644 index 0000000..1afd289 --- /dev/null +++ b/u_boot_pylib/tools.py @@ -0,0 +1,612 @@ +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright (c) 2016 Google, Inc +# + +import glob +import os +import shlex +import shutil +import sys +import tempfile +import urllib.request + +from u_boot_pylib import command +from u_boot_pylib import tout + +# Output directly (generally this is temporary) +outdir = None + +# True to keep the output directory around after exiting +preserve_outdir = False + +# Path to the Chrome OS chroot, if we know it +chroot_path = None + +# Search paths to use for filename(), used to find files +search_paths = [] + +tool_search_paths = [] + +# Tools and the packages that contain them, on debian +packages = { + 'lz4': 'liblz4-tool', + } + +# List of paths to use when looking for an input file +indir = [] + +def prepare_output_dir(dirname, preserve=False): + """Select an output directory, ensuring it exists. + + This either creates a temporary directory or checks that the one supplied + by the user is valid. For a temporary directory, it makes a note to + remove it later if required. + + Args: + dirname: a string, name of the output directory to use to store + intermediate and output files. If is None - create a temporary + directory. + preserve: a Boolean. If outdir above is None and preserve is False, the + created temporary directory will be destroyed on exit. + + Raises: + OSError: If it cannot create the output directory. + """ + global outdir, preserve_outdir + + preserve_outdir = dirname or preserve + if dirname: + outdir = dirname + if not os.path.isdir(outdir): + try: + os.makedirs(outdir) + except OSError as err: + raise ValueError( + f"Cannot make output directory 'outdir': 'err.strerror'") + tout.debug("Using output directory '%s'" % outdir) + else: + outdir = tempfile.mkdtemp(prefix='binman.') + tout.debug("Using temporary directory '%s'" % outdir) + +def _remove_output_dir(): + global outdir + + shutil.rmtree(outdir) + tout.debug("Deleted temporary directory '%s'" % outdir) + outdir = None + +def finalise_output_dir(): + global outdir, preserve_outdir + + """Tidy up: delete output directory if temporary and not preserved.""" + if outdir and not preserve_outdir: + _remove_output_dir() + outdir = None + +def get_output_filename(fname): + """Return a filename within the output directory. + + Args: + fname: Filename to use for new file + + Returns: + The full path of the filename, within the output directory + """ + return os.path.join(outdir, fname) + +def get_output_dir(): + """Return the current output directory + + Returns: + str: The output directory + """ + return outdir + +def _finalise_for_test(): + """Remove the output directory (for use by tests)""" + global outdir + + if outdir: + _remove_output_dir() + outdir = None + +def set_input_dirs(dirname): + """Add a list of input directories, where input files are kept. + + Args: + dirname: a list of paths to input directories to use for obtaining + files needed by binman to place in the image. + """ + global indir + + indir = dirname + tout.debug("Using input directories %s" % indir) + +def append_input_dirs(dirname): + """Append a list of input directories to the current list of input + directories + + Args: + dirname: a list of paths to input directories to use for obtaining + files needed by binman to place in the image. + """ + global indir + + for dir in dirname: + if dirname not in indir: + indir.append(dirname) + + tout.debug("Updated input directories %s" % indir) + +def get_input_filename(fname, allow_missing=False): + """Return a filename for use as input. + + Args: + fname: Filename to use for new file + allow_missing: True if the filename can be missing + + Returns: + fname, if indir is None; + full path of the filename, within the input directory; + None, if file is missing and allow_missing is True + + Raises: + ValueError if file is missing and allow_missing is False + """ + if not indir or fname[:1] == '/': + return fname + for dirname in indir: + pathname = os.path.join(dirname, fname) + if os.path.exists(pathname): + return pathname + + if allow_missing: + return None + raise ValueError("Filename '%s' not found in input path (%s) (cwd='%s')" % + (fname, ','.join(indir), os.getcwd())) + +def get_input_filename_glob(pattern): + """Return a list of filenames for use as input. + + Args: + pattern: Filename pattern to search for + + Returns: + A list of matching files in all input directories + """ + if not indir: + return glob.glob(pattern) + files = [] + for dirname in indir: + pathname = os.path.join(dirname, pattern) + files += glob.glob(pathname) + return sorted(files) + +def align(pos, align): + if align: + mask = align - 1 + pos = (pos + mask) & ~mask + return pos + +def not_power_of_two(num): + return num and (num & (num - 1)) + +def set_tool_paths(toolpaths): + """Set the path to search for tools + + Args: + toolpaths: List of paths to search for tools executed by run() + """ + global tool_search_paths + + tool_search_paths = toolpaths + +def path_has_file(path_spec, fname): + """Check if a given filename is in the PATH + + Args: + path_spec: Value of PATH variable to check + fname: Filename to check + + Returns: + True if found, False if not + """ + for dir in path_spec.split(':'): + if os.path.exists(os.path.join(dir, fname)): + return True + return False + +def get_host_compile_tool(env, name): + """Get the host-specific version for a compile tool + + This checks the environment variables that specify which version of + the tool should be used (e.g. ${HOSTCC}). + + The following table lists the host-specific versions of the tools + this function resolves to: + + Compile Tool | Host version + --------------+---------------- + as | ${HOSTAS} + ld | ${HOSTLD} + cc | ${HOSTCC} + cpp | ${HOSTCPP} + c++ | ${HOSTCXX} + ar | ${HOSTAR} + nm | ${HOSTNM} + ldr | ${HOSTLDR} + strip | ${HOSTSTRIP} + objcopy | ${HOSTOBJCOPY} + objdump | ${HOSTOBJDUMP} + dtc | ${HOSTDTC} + + Args: + name: Command name to run + + Returns: + host_name: Exact command name to run instead + extra_args: List of extra arguments to pass + """ + host_name = None + extra_args = [] + if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', + 'objcopy', 'objdump', 'dtc'): + host_name, *host_args = env.get('HOST' + name.upper(), '').split(' ') + elif name == 'c++': + host_name, *host_args = env.get('HOSTCXX', '').split(' ') + + if host_name: + return host_name, extra_args + return name, [] + +def get_target_compile_tool(name, cross_compile=None): + """Get the target-specific version for a compile tool + + This first checks the environment variables that specify which + version of the tool should be used (e.g. ${CC}). If those aren't + specified, it checks the CROSS_COMPILE variable as a prefix for the + tool with some substitutions (e.g. "${CROSS_COMPILE}gcc" for cc). + + The following table lists the target-specific versions of the tools + this function resolves to: + + Compile Tool | First choice | Second choice + --------------+----------------+---------------------------- + as | ${AS} | ${CROSS_COMPILE}as + ld | ${LD} | ${CROSS_COMPILE}ld.bfd + | | or ${CROSS_COMPILE}ld + cc | ${CC} | ${CROSS_COMPILE}gcc + cpp | ${CPP} | ${CROSS_COMPILE}gcc -E + c++ | ${CXX} | ${CROSS_COMPILE}g++ + ar | ${AR} | ${CROSS_COMPILE}ar + nm | ${NM} | ${CROSS_COMPILE}nm + ldr | ${LDR} | ${CROSS_COMPILE}ldr + strip | ${STRIP} | ${CROSS_COMPILE}strip + objcopy | ${OBJCOPY} | ${CROSS_COMPILE}objcopy + objdump | ${OBJDUMP} | ${CROSS_COMPILE}objdump + dtc | ${DTC} | (no CROSS_COMPILE version) + + Args: + name: Command name to run + + Returns: + target_name: Exact command name to run instead + extra_args: List of extra arguments to pass + """ + env = dict(os.environ) + + target_name = None + extra_args = [] + if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', + 'objcopy', 'objdump', 'dtc'): + target_name, *extra_args = env.get(name.upper(), '').split(' ') + elif name == 'c++': + target_name, *extra_args = env.get('CXX', '').split(' ') + + if target_name: + return target_name, extra_args + + if cross_compile is None: + cross_compile = env.get('CROSS_COMPILE', '') + + if name in ('as', 'ar', 'nm', 'ldr', 'strip', 'objcopy', 'objdump'): + target_name = cross_compile + name + elif name == 'ld': + try: + if run(cross_compile + 'ld.bfd', '-v'): + target_name = cross_compile + 'ld.bfd' + except: + target_name = cross_compile + 'ld' + elif name == 'cc': + target_name = cross_compile + 'gcc' + elif name == 'cpp': + target_name = cross_compile + 'gcc' + extra_args = ['-E'] + elif name == 'c++': + target_name = cross_compile + 'g++' + else: + target_name = name + return target_name, extra_args + +def get_env_with_path(): + """Get an updated environment with the PATH variable set correctly + + If there are any search paths set, these need to come first in the PATH so + that these override any other version of the tools. + + Returns: + dict: New environment with PATH updated, or None if there are not search + paths + """ + if tool_search_paths: + env = dict(os.environ) + env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH'] + return env + +def run_result(name, *args, **kwargs): + """Run a tool with some arguments + + This runs a 'tool', which is a program used by binman to process files and + perhaps produce some output. Tools can be located on the PATH or in a + search path. + + Args: + name: Command name to run + args: Arguments to the tool + for_host: True to resolve the command to the version for the host + for_target: False to run the command as-is, without resolving it + to the version for the compile target + raise_on_error: Raise an error if the command fails (True by default) + + Returns: + CommandResult object + """ + try: + binary = kwargs.get('binary') + for_host = kwargs.get('for_host', False) + for_target = kwargs.get('for_target', not for_host) + raise_on_error = kwargs.get('raise_on_error', True) + env = get_env_with_path() + if for_target: + name, extra_args = get_target_compile_tool(name) + args = tuple(extra_args) + args + elif for_host: + name, extra_args = get_host_compile_tool(env, name) + args = tuple(extra_args) + args + name = os.path.expanduser(name) # Expand paths containing ~ + all_args = (name,) + args + result = command.run_one(*all_args, capture=True, capture_stderr=True, + env=env, raise_on_error=False, binary=binary) + if result.return_code: + if raise_on_error: + raise ValueError("Error %d running '%s': %s" % + (result.return_code,' '.join(all_args), + result.stderr or result.stdout)) + return result + except ValueError: + if env and not path_has_file(env['PATH'], name): + msg = "Please install tool '%s'" % name + package = packages.get(name) + if package: + msg += " (e.g. from package '%s')" % package + raise ValueError(msg) + raise + +def tool_find(name): + """Search the current path for a tool + + This uses both PATH and any value from set_tool_paths() to search for a tool + + Args: + name (str): Name of tool to locate + + Returns: + str: Full path to tool if found, else None + """ + name = os.path.expanduser(name) # Expand paths containing ~ + paths = [] + pathvar = os.environ.get('PATH') + if pathvar: + paths = pathvar.split(':') + if tool_search_paths: + paths += tool_search_paths + for path in paths: + fname = os.path.join(path, name) + if os.path.isfile(fname) and os.access(fname, os.X_OK): + return fname + +def run(name, *args, **kwargs): + """Run a tool with some arguments + + This runs a 'tool', which is a program used by binman to process files and + perhaps produce some output. Tools can be located on the PATH or in a + search path. + + Args: + name: Command name to run + args: Arguments to the tool + for_host: True to resolve the command to the version for the host + for_target: False to run the command as-is, without resolving it + to the version for the compile target + + Returns: + CommandResult object + """ + result = run_result(name, *args, **kwargs) + if result is not None: + return result.stdout + +def filename(fname): + """Resolve a file path to an absolute path. + + If fname starts with ##/ and chroot is available, ##/ gets replaced with + the chroot path. If chroot is not available, this file name can not be + resolved, `None' is returned. + + If fname is not prepended with the above prefix, and is not an existing + file, the actual file name is retrieved from the passed in string and the + search_paths directories (if any) are searched to for the file. If found - + the path to the found file is returned, `None' is returned otherwise. + + Args: + fname: a string, the path to resolve. + + Returns: + Absolute path to the file or None if not found. + """ + if fname.startswith('##/'): + if chroot_path: + fname = os.path.join(chroot_path, fname[3:]) + else: + return None + + # Search for a pathname that exists, and return it if found + if fname and not os.path.exists(fname): + for path in search_paths: + pathname = os.path.join(path, os.path.basename(fname)) + if os.path.exists(pathname): + return pathname + + # If not found, just return the standard, unchanged path + return fname + +def read_file(fname, binary=True): + """Read and return the contents of a file. + + Args: + fname: path to filename to read, where ## signifiies the chroot. + + Returns: + data read from file, as a string. + """ + with open(filename(fname), binary and 'rb' or 'r') as fd: + data = fd.read() + #self._out.Info("Read file '%s' size %d (%#0x)" % + #(fname, len(data), len(data))) + return data + +def write_file(fname, data, binary=True): + """Write data into a file. + + Args: + fname: path to filename to write + data: data to write to file, as a string + """ + #self._out.Info("Write file '%s' size %d (%#0x)" % + #(fname, len(data), len(data))) + with open(filename(fname), binary and 'wb' or 'w') as fd: + fd.write(data) + +def get_bytes(byte, size): + """Get a string of bytes of a given size + + Args: + byte: Numeric byte value to use + size: Size of bytes/string to return + + Returns: + A bytes type with 'byte' repeated 'size' times + """ + return bytes([byte]) * size + +def to_bytes(string): + """Convert a str type into a bytes type + + Args: + string: string to convert + + Returns: + A bytes type + """ + return string.encode('utf-8') + +def to_string(bval): + """Convert a bytes type into a str type + + Args: + bval: bytes value to convert + + Returns: + Python 3: A bytes type + Python 2: A string type + """ + return bval.decode('utf-8') + +def to_hex(val): + """Convert an integer value (or None) to a string + + Returns: + hex value, or 'None' if the value is None + """ + return 'None' if val is None else '%#x' % val + +def to_hex_size(val): + """Return the size of an object in hex + + Returns: + hex value of size, or 'None' if the value is None + """ + return 'None' if val is None else '%#x' % len(val) + +def print_full_help(fname): + """Print the full help message for a tool using an appropriate pager. + + Args: + fname: Path to a file containing the full help message + """ + pager = shlex.split(os.getenv('PAGER', '')) + if not pager: + lesspath = shutil.which('less') + pager = [lesspath] if lesspath else None + if not pager: + pager = ['more'] + command.run(*pager, fname) + +def download(url, tmpdir_pattern='.patman'): + """Download a file to a temporary directory + + Args: + url (str): URL to download + tmpdir_pattern (str): pattern to use for the temporary directory + + Returns: + Tuple: + Full path to the downloaded archive file in that directory, + or None if there was an error while downloading + Temporary directory name + """ + print('- downloading: %s' % url) + leaf = url.split('/')[-1] + tmpdir = tempfile.mkdtemp(tmpdir_pattern) + response = urllib.request.urlopen(url) + fname = os.path.join(tmpdir, leaf) + fd = open(fname, 'wb') + meta = response.info() + size = int(meta.get('Content-Length')) + done = 0 + block_size = 1 << 16 + status = '' + + # Read the file in chunks and show progress as we go + while True: + buffer = response.read(block_size) + if not buffer: + print(chr(8) * (len(status) + 1), '\r', end=' ') + break + + done += len(buffer) + fd.write(buffer) + status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024, + done * 100 // size) + status = status + chr(8) * (len(status) + 1) + print(status, end=' ') + sys.stdout.flush() + print('\r', end='') + sys.stdout.flush() + fd.close() + if done != size: + print('Error, failed to download') + os.remove(fname) + fname = None + return fname, tmpdir diff --git a/u_boot_pylib/tout.py b/u_boot_pylib/tout.py new file mode 100644 index 0000000..c608806 --- /dev/null +++ b/u_boot_pylib/tout.py @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2016 Google, Inc +# +# Terminal output logging. +# + +import sys + +from u_boot_pylib import terminal + +# Output verbosity levels that we support +FATAL, ERROR, WARNING, NOTICE, INFO, DETAIL, DEBUG = range(7) + +""" +This class handles output of progress and other useful information +to the user. It provides for simple verbosity level control and can +output nothing but errors at verbosity zero. + +The idea is that modules set up an Output object early in their years and pass +it around to other modules that need it. This keeps the output under control +of a single class. + +Public properties: + verbose: Verbosity level: 0=silent, 1=progress, 3=full, 4=debug +""" +def __enter__(): + return + +def __exit__(unused1, unused2, unused3): + """Clean up and remove any progress message.""" + clear_progress() + return False + +def user_is_present(): + """This returns True if it is likely that a user is present. + + Sometimes we want to prompt the user, but if no one is there then this + is a waste of time, and may lock a script which should otherwise fail. + + Returns: + True if it thinks the user is there, and False otherwise + """ + return stdout_is_tty and verbose > ERROR + +def clear_progress(): + """Clear any active progress message on the terminal.""" + if verbose > ERROR and stdout_is_tty: + terminal.print_clear() + +def progress(msg, warning=False, trailer='...'): + """Display progress information. + + Args: + msg: Message to display. + warning: True if this is a warning.""" + clear_progress() + if verbose > ERROR: + _progress = msg + trailer + if stdout_is_tty: + col = _color.YELLOW if warning else _color.GREEN + terminal.tprint('\r' + _progress, newline=False, colour=col, col=_color) + else: + terminal.tprint(_progress) + +def _output(level, msg, color=None, newline=True): + """Output a message to the terminal. + + Args: + level: Verbosity level for this message. It will only be displayed if + this as high as the currently selected level. + msg: Message to display. + color: Colour to use for the text, None for default. + newline: True to add a newline at the end. + """ + if verbose >= level: + clear_progress() + if level <= WARNING: + terminal.tprint(msg, newline=newline, colour=color, col=_color, + stderr=True) + else: + terminal.tprint(msg, newline=newline, colour=color, col=_color) + if level == FATAL: + sys.exit(1) + +def do_output(level, msg, newline=True): + """Output a message to the terminal. + + Args: + level: Verbosity level for this message. It will only be displayed if + this as high as the currently selected level. + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(level, msg, newline=newline) + +def fatal(msg): + """Display an error message and exit + + Args: + msg: Message to display. + """ + _output(FATAL, msg, _color.RED) + +def error(msg, newline=True): + """Display an error message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(ERROR, msg, _color.RED, newline=newline) + +def warning(msg, newline=True): + """Display a warning message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(WARNING, msg, _color.YELLOW, newline=newline) + +def notice(msg, newline=True): + """Display an important infomation message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(NOTICE, msg, newline=newline) + +def info(msg, newline=True): + """Display an infomation message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(INFO, msg, newline=newline) + +def detail(msg, newline=True): + """Display a detailed message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(DETAIL, msg, newline=newline) + +def debug(msg, newline=True): + """Display a debug message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(DEBUG, msg, newline=newline) + +def user_output(msg, newline=True): + """Display a message regardless of the current output level. + + This is used when the output was specifically requested by the user. + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(ERROR, msg, newline=newline) + +def init(_verbose=WARNING, stdout=sys.stdout, allow_colour=True): + """Initialize a new output object. + + Args: + verbose: Verbosity level (0-6). + stdout: File to use for stdout. + """ + global verbose, _progress, _color, _stdout, stdout_is_tty + + verbose = _verbose + _progress = '' # Our last progress message + _color = terminal.Color(terminal.COLOR_IF_TERMINAL if allow_colour + else terminal.COLOR_NEVER) + _stdout = stdout + + # TODO(sjg): Move this into Chromite libraries when we have them + stdout_is_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() + stderr_is_tty = hasattr(sys.stderr, 'isatty') and sys.stderr.isatty() + +def uninit(): + clear_progress() + +init() diff --git a/uman_pkg/__main__.py b/uman_pkg/__main__.py index 3e31898..04679c4 100755 --- a/uman_pkg/__main__.py +++ b/uman_pkg/__main__.py @@ -13,10 +13,12 @@ parent_path = os.path.dirname(our_path) sys.path.append(parent_path) -# Get U-Boot tools path from UBOOT_TOOLS env var (default: ~/u/tools) -# This is separate from USRC which specifies the U-Boot source to work in -uboot_tools = os.path.expanduser(os.environ.get('UBOOT_TOOLS', '~/u/tools')) -sys.path.append(uboot_tools) +# Use embedded u_boot_pylib by default; set UMAN_EXTERNAL_PYLIB=1 to +# use the version from UBOOT_TOOLS instead (for testing newer versions) +if os.environ.get('UMAN_EXTERNAL_PYLIB'): + uboot_tools = os.path.expanduser( + os.environ.get('UBOOT_TOOLS', '~/u/tools')) + sys.path.insert(0, uboot_tools) # pylint: disable=import-error,wrong-import-position from uman_pkg import cmdline diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index 0aa360b..fd45a88 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -4780,32 +4780,16 @@ def setUpClass(cls): """Get the uman package directory""" cls.uman_dir = os.path.dirname(os.path.dirname(__file__)) - def test_uboot_tools_env_var(self): - """Test UBOOT_TOOLS env var overrides default path""" - # With invalid UBOOT_TOOLS, uman should fail to import u_boot_pylib - # Clear PYTHONPATH to ensure only UBOOT_TOOLS is used + def test_embedded_pylib(self): + """Test uman works without UBOOT_TOOLS using embedded u_boot_pylib""" env = os.environ.copy() env['UBOOT_TOOLS'] = '/nonexistent/path' env.pop('PYTHONPATH', None) + env.pop('UMAN_EXTERNAL_PYLIB', None) result = subprocess.run( ['python3', '-m', 'uman_pkg', '--help'], capture_output=True, env=env, cwd=self.uman_dir, check=False) - # Should fail with import error - self.assertNotEqual(0, result.returncode) - self.assertIn(b'ModuleNotFoundError', result.stderr) - - def test_uboot_tools_tilde_expansion(self): - """Test UBOOT_TOOLS expands ~ in path""" - # Verify tilde is expanded (not passed literally) - # Clear PYTHONPATH to ensure only UBOOT_TOOLS is used - env = os.environ.copy() - env['UBOOT_TOOLS'] = '~/nonexistent' - env.pop('PYTHONPATH', None) - result = subprocess.run( - ['python3', '-m', 'uman_pkg', '--help'], - capture_output=True, env=env, cwd=self.uman_dir, check=False) - # Error should show expanded path, not literal ~ - self.assertNotIn(b"'~/nonexistent'", result.stderr) + self.assertEqual(0, result.returncode) class TestUtil(TestBase): From a549a772e4d91c6d6ed5d93c03e1db99e1fcdab4 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 8 Mar 2026 13:16:28 -0600 Subject: [PATCH 02/36] uman: Always use the embedded u_boot_pylib by default When running inside a U-Boot tree, an older u_boot_pylib from the tree's tools/ directory can end up on sys.path and take priority over uman's embedded copy. This causes failures like missing CommandExc. Use sys.path.insert(0, ...) instead of sys.path.append() so the embedded u_boot_pylib always wins. The UMAN_EXTERNAL_PYLIB override still works for testing newer versions. Co-developed-by: Claude Opus 4.6 --- uman_pkg/__main__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uman_pkg/__main__.py b/uman_pkg/__main__.py index 04679c4..c76f00d 100755 --- a/uman_pkg/__main__.py +++ b/uman_pkg/__main__.py @@ -8,17 +8,18 @@ import os import sys -# Allow imports to work when run as module +# Use the embedded u_boot_pylib by putting uman's parent first on +# sys.path, so it takes priority over any older version in the U-Boot +# tree's tools/ directory. Set UMAN_EXTERNAL_PYLIB=1 to use the +# UBOOT_TOOLS version instead (for testing newer versions). our_path = os.path.dirname(os.path.realpath(__file__)) parent_path = os.path.dirname(our_path) -sys.path.append(parent_path) - -# Use embedded u_boot_pylib by default; set UMAN_EXTERNAL_PYLIB=1 to -# use the version from UBOOT_TOOLS instead (for testing newer versions) if os.environ.get('UMAN_EXTERNAL_PYLIB'): uboot_tools = os.path.expanduser( os.environ.get('UBOOT_TOOLS', '~/u/tools')) sys.path.insert(0, uboot_tools) +else: + sys.path.insert(0, parent_path) # pylint: disable=import-error,wrong-import-position from uman_pkg import cmdline From 96e857e72afefaea5aee21b7b6b3476c8b77638f Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 5 Mar 2026 07:52:07 -0700 Subject: [PATCH 03/36] uman: Add UBOOT_TOOLS to sys.path for pickman import The pickman module lives in the U-Boot tools directory but this path is not on sys.path when using the embedded u_boot_pylib, since the UMAN_EXTERNAL_PYLIB path addition in __main__.py is skipped. Add the UBOOT_TOOLS path (defaulting to ~/u/tools) to sys.path in do_merge_request() before importing pickman, so that 'um ci -sm' works without needing UMAN_EXTERNAL_PYLIB=1. Co-developed-by: Claude Opus 4.6 --- uman_pkg/control.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/uman_pkg/control.py b/uman_pkg/control.py index 2341f71..a0334a9 100644 --- a/uman_pkg/control.py +++ b/uman_pkg/control.py @@ -8,6 +8,7 @@ the features of uman. """ +import os import sys # pylint: disable=import-error @@ -419,6 +420,11 @@ def do_merge_request(args): # pylint: disable=too-many-locals """ # pylint: disable=import-outside-toplevel import gitlab + + uboot_tools = os.path.expanduser( + os.environ.get('UBOOT_TOOLS', '~/u/tools')) + if uboot_tools not in sys.path: + sys.path.insert(0, uboot_tools) from pickman import gitlab_api from u_boot_pylib import gitutil From 06ad60804d66bd7975f79e14883caef23fcee4d9 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 6 Mar 2026 11:04:00 -0700 Subject: [PATCH 04/36] uman: Rename 'dh' alias to 'di' to avoid shadowing Debian dh The 'dh' alias for diff-head shadows /usr/bin/dh, the Debian build helper command. Rename it to 'di' to avoid the conflict. Co-developed-by: Claude Opus 4.6 --- README.rst | 2 +- uman_pkg/cmdgit.py | 4 ++-- uman_pkg/ftest.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index f9f01b5..e19973e 100644 --- a/README.rst +++ b/README.rst @@ -293,7 +293,7 @@ making it easier to step through commits during development. - ``cms`` / ``commit-signoff``: Commit with signoff (git commit --signoff) - ``co`` / ``checkout``: Checkout (switch branches or restore files) - ``db`` / ``diff-branch`` [BRANCH]: Diff current commit files against upstream (or BRANCH) -- ``dh`` / ``diff-head`` [N] [FILES...]: Show diff using difftool (git difftool HEAD~ or HEAD~N) +- ``di`` / ``diff-head`` [N] [FILES...]: Show diff using difftool (git difftool HEAD~ or HEAD~N) - ``eg`` / ``errno-grep`` PATTERN: Search include/linux/errno.h for error codes - ``et`` / ``edit-todo``: Edit the rebase todo list - ``fa`` / ``find-all`` [N]: Check all branches against us/master (default 5 commits each) diff --git a/uman_pkg/cmdgit.py b/uman_pkg/cmdgit.py index 2c92ae7..5442bd2 100644 --- a/uman_pkg/cmdgit.py +++ b/uman_pkg/cmdgit.py @@ -1354,7 +1354,7 @@ def do_ust(_args): GitAction('cms', 'commit-signoff', 'Commit with signoff', do_cms), GitAction('co', 'checkout', 'Checkout (switch branches/restore)', do_co), GitAction('db', 'diff-branch', 'Diff commit files against upstream', do_db), - GitAction('dh', 'diff-head', 'Show diff of top commit', do_dh), + GitAction('di', 'diff-head', 'Show diff of top commit', do_dh), GitAction('eg', 'errno-grep', 'Search errno.h for error codes', do_eg), GitAction('et', 'edit-todo', 'Edit rebase todo list', do_et), GitAction('g', 'status', 'Show short status', do_g), @@ -1407,7 +1407,7 @@ def do_ust(_args): 'cms': 'git commit -s', 'co': 'git checkout', 'cs': 'git show', - 'dh': 'git difftool HEAD~', + 'di': 'git difftool HEAD~', 'g': 'git status -sb', 'gb': 'git branch', 'gba': 'git branch -a', diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index fd45a88..98f1a6e 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -1745,7 +1745,7 @@ def test_do_db_with_branch(self): def test_do_dh(self): """Test do_dh runs git difftool HEAD~""" - args = cmdline.parse_args(['git', 'dh']) + args = cmdline.parse_args(['git', 'di']) with mock.patch('u_boot_pylib.command.run_one') as mock_run: mock_run.return_value = mock.Mock(return_code=0) result = cmdgit.do_dh(args) @@ -1755,7 +1755,7 @@ def test_do_dh(self): def test_do_dh_with_count(self): """Test do_dh N runs git difftool HEAD~N""" - args = cmdline.parse_args(['git', 'dh', '3']) + args = cmdline.parse_args(['git', 'di', '3']) with mock.patch('u_boot_pylib.command.run_one') as mock_run: mock_run.return_value = mock.Mock(return_code=0) result = cmdgit.do_dh(args) @@ -1765,7 +1765,7 @@ def test_do_dh_with_count(self): def test_do_dh_with_file(self): """Test do_dh with file path passes it to difftool""" - args = cmdline.parse_args(['git', 'dh', 'some/file.c']) + args = cmdline.parse_args(['git', 'di', 'some/file.c']) with mock.patch('u_boot_pylib.command.run_one') as mock_run: mock_run.return_value = mock.Mock(return_code=0) result = cmdgit.do_dh(args) @@ -1776,7 +1776,7 @@ def test_do_dh_with_file(self): def test_do_dh_with_count_and_file(self): """Test do_dh N file passes both to difftool""" - args = cmdline.parse_args(['git', 'dh', '2', 'some/file.c']) + args = cmdline.parse_args(['git', 'di', '2', 'some/file.c']) with mock.patch('u_boot_pylib.command.run_one') as mock_run: mock_run.return_value = mock.Mock(return_code=0) result = cmdgit.do_dh(args) From 8a80f9d2a421876b5b6820b15be0b4b96d6789cd Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 6 Mar 2026 12:46:53 -0700 Subject: [PATCH 05/36] uman: Show error output when rebase fails for an unrecognised reason When a rebase command fails with a return code but the output does not match any known pattern (e.g. 'Could not apply'), the error is silently swallowed. This happens with unstaged changes, where git prints 'error: cannot rebase: You have unstaged changes' but show_rebase_status() has no handler for it. Show the raw error output as a fallback when no pattern matches. Co-developed-by: Claude Opus 4.6 --- uman_pkg/cmdgit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uman_pkg/cmdgit.py b/uman_pkg/cmdgit.py index 5442bd2..39d2163 100644 --- a/uman_pkg/cmdgit.py +++ b/uman_pkg/cmdgit.py @@ -97,6 +97,8 @@ def show_rebase_status(output, return_code=0): label = 'empty commit' tout.notice(f'Rebasing{pos_str} {label} {match.group(1)}... ' f'{match.group(2)}') + else: + tout.error(output.strip()) def show_rb_status(): From b6dd6332ab541237abdc664793bd3b34a1540b3a Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 8 Mar 2026 13:28:16 -0600 Subject: [PATCH 06/36] uman: Use Python instead of sed for GIT_SEQUENCE_EDITOR The sed -i flag has incompatible syntax between GNU sed (Linux) and BSD sed (macOS). GNU sed uses 'sed -i "..."' while BSD sed requires 'sed -i '' "..."'. Replace the sed commands with Python one-liners so the rebase commands (rf, rn, rp, rb) work on both Linux and macOS. Co-developed-by: Claude Opus 4.6 --- uman_pkg/cmdgit.py | 11 +++++++++-- uman_pkg/ftest.py | 44 +++++++++++++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/uman_pkg/cmdgit.py b/uman_pkg/cmdgit.py index 39d2163..ffeac27 100644 --- a/uman_pkg/cmdgit.py +++ b/uman_pkg/cmdgit.py @@ -135,9 +135,16 @@ def seq_edit_env(action, line=1): """ env = os.environ.copy() if action == 'break': - env['GIT_SEQUENCE_EDITOR'] = f'sed -i "{line}i break"' + script = (f"import sys; p=sys.argv[1]; " + f"lines=open(p).readlines(); " + f"lines.insert({line - 1},'break\\n'); " + f"open(p,'w').writelines(lines)") else: # edit - env['GIT_SEQUENCE_EDITOR'] = f'sed -i "{line}s/^pick/edit/"' + script = (f"import sys,re; p=sys.argv[1]; " + f"lines=open(p).readlines(); " + f"lines[{line - 1}]=re.sub(r'^pick','edit',lines[{line - 1}]); " + f"open(p,'w').writelines(lines)") + env['GIT_SEQUENCE_EDITOR'] = f'python3 -c "{script}"' return env diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index 98f1a6e..4a7bd23 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -1152,20 +1152,39 @@ def mock_git_output(*_args): self.assertEqual('', pos) def test_seq_edit_env_break(self): - """Test seq_edit_env creates break command""" + """Test seq_edit_env inserts a break""" env = cmdgit.seq_edit_env('break') self.assertIn('GIT_SEQUENCE_EDITOR', env) - self.assertEqual('sed -i "1i break"', env['GIT_SEQUENCE_EDITOR']) + todo = os.path.join(self.test_dir, 'todo') + with open(todo, 'w', encoding='utf-8') as fil: + fil.write('pick abc First\npick def Second\n') + os.system(f'{env["GIT_SEQUENCE_EDITOR"]} {todo}') + with open(todo, encoding='utf-8') as fil: + self.assertEqual('break\npick abc First\npick def Second\n', + fil.read()) def test_seq_edit_env_edit(self): - """Test seq_edit_env creates edit command""" + """Test seq_edit_env changes pick to edit""" env = cmdgit.seq_edit_env('edit') - self.assertEqual('sed -i "1s/^pick/edit/"', env['GIT_SEQUENCE_EDITOR']) + todo = os.path.join(self.test_dir, 'todo') + with open(todo, 'w', encoding='utf-8') as fil: + fil.write('pick abc First\npick def Second\n') + os.system(f'{env["GIT_SEQUENCE_EDITOR"]} {todo}') + with open(todo, encoding='utf-8') as fil: + self.assertEqual('edit abc First\npick def Second\n', + fil.read()) def test_seq_edit_env_edit_line(self): """Test seq_edit_env with specific line number""" - env = cmdgit.seq_edit_env('edit', 3) - self.assertEqual('sed -i "3s/^pick/edit/"', env['GIT_SEQUENCE_EDITOR']) + env = cmdgit.seq_edit_env('edit', 2) + todo = os.path.join(self.test_dir, 'todo') + with open(todo, 'w', encoding='utf-8') as fil: + fil.write('pick abc First\npick def Second\npick ghi Third\n') + os.system(f'{env["GIT_SEQUENCE_EDITOR"]} {todo}') + with open(todo, encoding='utf-8') as fil: + self.assertEqual( + 'pick abc First\nedit def Second\npick ghi Third\n', + fil.read()) def test_show_rebase_status_success(self): """Test show_rebase_status parses success message""" @@ -2059,7 +2078,7 @@ def mock_git(*args, env=None, dry_run=None): self.assertEqual(('rebase', '-i', 'origin/main'), cap[0]) self.assertIn('GIT_SEQUENCE_EDITOR', cap_env[0]) # rb inserts a break before first commit to stop at upstream - self.assertIn("1i break", cap_env[0]['GIT_SEQUENCE_EDITOR']) + self.assertIn("insert(0,'break", cap_env[0]['GIT_SEQUENCE_EDITOR']) def test_do_rf_no_arg(self): """Test do_rf without arg uses upstream""" @@ -2081,7 +2100,8 @@ def mock_git(*args, env=None, dry_run=None): result = cmdgit.do_rf(args) self.assertEqual(0, result.return_code) self.assertEqual(('rebase', '-i', 'origin/main'), cap[0]) - self.assertIn("1s/^pick/edit/", cap_env[0]['GIT_SEQUENCE_EDITOR']) + self.assertIn("re.sub(r'^pick','edit',lines[0])", + cap_env[0]['GIT_SEQUENCE_EDITOR']) def test_do_rf_with_count(self): """Test do_rf with commit count""" @@ -2168,7 +2188,7 @@ def test_rb_stops_at_upstream(self): self.assertEqual(('rebase', '-i', 'upstream'), call_args[0]) # Check that GIT_SEQUENCE_EDITOR inserts break before first commit env = call_args[1]['env'] - self.assertIn("1i break", env['GIT_SEQUENCE_EDITOR']) + self.assertIn("insert(0,'break", env['GIT_SEQUENCE_EDITOR']) def test_rf_with_count(self): """Test rf N rebases last N commits, stopping at first""" @@ -2181,7 +2201,8 @@ def test_rf_with_count(self): call_args = mock_git.call_args self.assertEqual(('rebase', '-i', 'HEAD~3'), call_args[0]) env = call_args[1]['env'] - self.assertIn("1s/^pick/edit/", env['GIT_SEQUENCE_EDITOR']) + self.assertIn("re.sub(r'^pick','edit',lines[0])", + env['GIT_SEQUENCE_EDITOR']) def test_rp_stops_at_patch_n(self): """Test rp N stops at patch N""" @@ -2194,7 +2215,8 @@ def test_rp_stops_at_patch_n(self): call_args = mock_git.call_args self.assertEqual(('rebase', '-i', 'upstream'), call_args[0]) env = call_args[1]['env'] - self.assertIn("2s/^pick/edit/", env['GIT_SEQUENCE_EDITOR']) + self.assertIn("re.sub(r'^pick','edit',lines[1])", + env['GIT_SEQUENCE_EDITOR']) def test_real_rf_and_rc(self): """Test a real rf followed by rc to complete rebase""" From 7b527febe14d49eed843ede280f05dfad394df54 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 8 Mar 2026 13:17:46 -0600 Subject: [PATCH 07/36] uman: Fix setup aliases when um is not yet in PATH On first run (e.g. on a new machine), um is not in PATH yet so shutil.which() fails. The fallback looks for 'um' in the repo root, which is a symlink that may not exist. Look for uman_pkg/uman first, since that is the actual executable and is always present in the repo. Co-developed-by: Claude Opus 4.6 --- uman_pkg/setup.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/uman_pkg/setup.py b/uman_pkg/setup.py index 7a0c7c8..1aaf7a6 100644 --- a/uman_pkg/setup.py +++ b/uman_pkg/setup.py @@ -66,15 +66,14 @@ def setup_aliases(args): alias_dir = os.path.expanduser(alias_dir) - # Find uman executable - uman_path = shutil.which('um') or shutil.which('uman') + # Find uman executable - prefer the one next to this file + uman_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'uman') + if not os.path.exists(uman_path): + uman_path = shutil.which('um') or shutil.which('uman') if not uman_path: - # Try to find it relative to this file - this_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - uman_path = os.path.join(this_dir, 'um') - if not os.path.exists(uman_path): - tout.error('Cannot find uman executable') - return 1 + tout.error('Cannot find uman executable') + return 1 uman_path = os.path.abspath(uman_path) From b2b886513191ff2b0da2909950e4dd275d665bd9 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 8 Mar 2026 10:17:49 -0600 Subject: [PATCH 08/36] uman: Add 'remote' component to setup for deploying to other machines Add a 'remote' setup component that deploys uman to a remote machine over SSH. It rsyncs the uman directory (excluding .git, caches and build artifacts), creates the ~/bin/um symlink and runs setup aliases on the remote. Usage: um setup remote Co-developed-by: Claude Opus 4.6 --- uman_pkg/cmdline.py | 5 +++- uman_pkg/ftest.py | 34 ++++++++++++++++++++++ uman_pkg/setup.py | 71 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 107 insertions(+), 3 deletions(-) diff --git a/uman_pkg/cmdline.py b/uman_pkg/cmdline.py index 23ae61d..ba6f4a4 100644 --- a/uman_pkg/cmdline.py +++ b/uman_pkg/cmdline.py @@ -300,7 +300,10 @@ def add_setup_subparser(subparsers): 'setup', help='Build firmware blobs needed for testing') setup.add_argument( 'component', type=str, nargs='?', default=None, - help="Component to build (e.g. 'opensbi'), or omit to build all") + help="Component to set up (e.g. 'opensbi', 'remote'), or omit for all") + setup.add_argument( + 'host', type=str, nargs='?', default=None, + help="Hostname for 'remote' component (e.g. user@host)") setup.add_argument( '-d', '--alias-dir', metavar='DIR', help='Directory for aliases symlinks (default: ~/bin)') diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index 4a7bd23..f85bbf5 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -4710,6 +4710,7 @@ def test_setup_components_dict(self): self.assertIn('gcc', setup.SETUP_COMPONENTS) self.assertIn('qemu', setup.SETUP_COMPONENTS) self.assertIn('opensbi', setup.SETUP_COMPONENTS) + self.assertIn('remote', setup.SETUP_COMPONENTS) self.assertIn('tfa', setup.SETUP_COMPONENTS) self.assertIn('xtensa', setup.SETUP_COMPONENTS) @@ -4793,6 +4794,39 @@ def test_setup_aliases_quiet(self): self.assertEqual(0, res) self.assertFalse(out.getvalue()) + def test_setup_remote_parsing(self): + """Test that setup remote subcommand is parsed correctly""" + args = cmdline.parse_args(['setup', 'remote', 'myhost']) + self.assertEqual('setup', args.cmd) + self.assertEqual('remote', args.component) + self.assertEqual('myhost', args.host) + + def test_setup_remote_no_host(self): + """Test setup remote with no hostname""" + args = argparse.Namespace( + cmd='setup', component='remote', host=None, + list_components=False, force=False, dry_run=False, + verbose=False, debug=False, alias_dir=None) + with terminal.capture() as (_, err): + res = setup.do_setup(args) + self.assertEqual(1, res) + self.assertIn('Hostname required', err.getvalue()) + + def test_setup_remote_dry_run(self): + """Test setup remote in dry-run mode""" + args = argparse.Namespace( + cmd='setup', component='remote', host='testhost', + list_components=False, force=False, dry_run=True, + verbose=False, debug=False, alias_dir=None) + with terminal.capture() as (out, _): + res = setup.do_setup(args) + self.assertEqual(0, res) + output = out.getvalue() + self.assertIn('rsync', output) + self.assertIn('testhost', output) + self.assertIn('ln -sf', output) + self.assertIn('setup aliases', output) + class TestMain(TestBase): """Tests for __main__.py""" diff --git a/uman_pkg/setup.py b/uman_pkg/setup.py index 1aaf7a6..bae50f1 100644 --- a/uman_pkg/setup.py +++ b/uman_pkg/setup.py @@ -25,6 +25,7 @@ # Available components for setup command SETUP_COMPONENTS = { 'aliases': 'Create symlinks for git action commands', + 'remote': 'Deploy uman to a remote machine via SSH', 'efi': 'QEMU EFI firmware for ARM, ARM64, RISC-V and x86', 'gcc': 'GCC cross-compiler and build dependencies', 'qemu': 'QEMU emulators for all architectures', @@ -77,7 +78,7 @@ def setup_aliases(args): uman_path = os.path.abspath(uman_path) - aliases = [a.short for a in cmdgit.GIT_ACTIONS] + ['cg'] + aliases = [a.short for a in cmdgit.GIT_ACTIONS] + ['cg', 'uman'] if args.dry_run: tout.notice(f'Would create symlinks in {alias_dir} -> {uman_path}') @@ -521,6 +522,71 @@ def setup_xtensa(blobs_dir, args): return 0 +RSYNC_EXCLUDES = [ + '.git/', '__pycache__/', '*.pyc', '*.pyo', '.pytest_cache/', + '.benchmarks/', '.claude/', '.hypothesis/', 'mmc*.img', 'spi.bin', 'um', +] + +REMOTE_UMAN_DIR = '~/dev/uman' +REMOTE_BIN = '~/bin' + + +def setup_remote(args): + """Deploy uman to a remote machine via SSH + + Rsyncs the uman repo, creates ~/bin/um symlink, and runs + setup aliases on the remote. + + Args: + args (argparse.Namespace): Command line arguments + args.host: SSH hostname (e.g. 'user@host' or 'host') + + Returns: + int: Exit code (0 for success, non-zero for failure) + """ + host = args.host + if not host: + tout.error('Hostname required: um setup remote ') + return 1 + + uman_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # Step 1: rsync the uman repo to the remote + rsync_cmd = ['rsync', '-az', '--delete'] + for excl in RSYNC_EXCLUDES: + rsync_cmd += ['--exclude', excl] + rsync_cmd += [uman_dir + '/', f'{host}:{REMOTE_UMAN_DIR}/'] + + tout.notice(f'Syncing uman to {host}:{REMOTE_UMAN_DIR}...') + if args.dry_run: + tout.notice(f' {" ".join(rsync_cmd)}') + else: + mkdir_cmd = ['ssh', host, + f'mkdir -p {REMOTE_UMAN_DIR} {REMOTE_BIN}'] + command.run_pipe([mkdir_cmd], capture=True) + command.run_pipe([rsync_cmd], capture=False, raise_on_error=True) + + # Step 2: Create ~/bin/um symlink on the remote + link = f'{REMOTE_UMAN_DIR}/uman_pkg/uman' + ln_cmd = ['ssh', host, f'ln -sf {link} {REMOTE_BIN}/um'] + tout.notice(f'Creating symlink {REMOTE_BIN}/um on {host}...') + if args.dry_run: + tout.notice(f' {" ".join(ln_cmd)}') + else: + command.run_pipe([ln_cmd], capture=True) + + # Step 3: Run setup aliases on the remote + setup_cmd = ['ssh', host, f'{REMOTE_BIN}/um setup aliases -f'] + tout.notice(f'Setting up aliases on {host}...') + if args.dry_run: + tout.notice(f' {" ".join(setup_cmd)}') + else: + command.run_pipe([setup_cmd], capture=False, raise_on_error=True) + + tout.notice(f'Remote setup complete on {host}') + return 0 + + def do_setup(args): """Handle setup command - build firmware blobs @@ -546,7 +612,7 @@ def do_setup(args): return 1 components = [args.component] else: - components = list(SETUP_COMPONENTS.keys()) + components = [c for c in SETUP_COMPONENTS if c != 'remote'] # Dispatch table for component setup functions setup_funcs = { @@ -557,6 +623,7 @@ def do_setup(args): 'opensbi': lambda: setup_opensbi(blobs_dir, args), 'tfa': lambda: setup_tfa(blobs_dir, args), 'xtensa': lambda: setup_xtensa(blobs_dir, args), + 'remote': lambda: setup_remote(args), } # Build each component From 5d9e142cbc72259ae89df35b758519d6a4ed5aed Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 8 Mar 2026 10:19:33 -0600 Subject: [PATCH 09/36] uman: List components when setup is run without arguments Running 'um setup' with no component now lists the available components instead of running them all. Use 'um setup all' to run all components (excluding remote, which needs a hostname). Co-developed-by: Claude Opus 4.6 --- uman_pkg/setup.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/uman_pkg/setup.py b/uman_pkg/setup.py index bae50f1..f396025 100644 --- a/uman_pkg/setup.py +++ b/uman_pkg/setup.py @@ -605,14 +605,21 @@ def do_setup(args): blobs_dir = settings.get('blobs_dir', '~/dev/blobs') # Determine which components to build - if args.component: - if args.component not in SETUP_COMPONENTS: - tout.error(f'Unknown component: {args.component}') - tout.notice('Use --list to see available components') - return 1 - components = [args.component] - else: + if not args.component: + tout.notice('Available components:') + for name, desc in SETUP_COMPONENTS.items(): + tout.notice(f' {name}: {desc}') + tout.notice("Use 'um setup ' or 'um setup all'") + return 0 + + if args.component == 'all': components = [c for c in SETUP_COMPONENTS if c != 'remote'] + elif args.component not in SETUP_COMPONENTS: + tout.error(f'Unknown component: {args.component}') + tout.notice('Use --list to see available components') + return 1 + else: + components = [args.component] # Dispatch table for component setup functions setup_funcs = { From 19e0645dbdb06eb82b8f20e0fe076e3f85883392 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 12 Mar 2026 14:44:44 -0600 Subject: [PATCH 10/36] uman: Group build options under a heading in help output Use an argparse argument group in add_build_opts() so the build-related flags (-a, -f, -F, -j, -L, -o, -T, --no-trace-early) appear under a 'build options:' heading in the help output for the py and t subcommands. Co-developed-by: Claude Opus 4.6 --- uman_pkg/cmdline.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/uman_pkg/cmdline.py b/uman_pkg/cmdline.py index ba6f4a4..556e792 100644 --- a/uman_pkg/cmdline.py +++ b/uman_pkg/cmdline.py @@ -151,28 +151,29 @@ def add_build_opts(parser): Args: parser: Argument parser to add options to """ - parser.add_argument( + group = parser.add_argument_group('build options') + group.add_argument( '-a', '--adjust-cfg', action='append', metavar='CFG', dest='adjust_cfg', help='Adjust Kconfig setting (use with -b; can use multiple times)') - parser.add_argument( + group.add_argument( '-f', '--force-reconfig', action='store_true', help='Force reconfiguration (use with -b)') - parser.add_argument( + group.add_argument( '-F', '--fresh', action='store_true', help='Delete build dir before building (use with -b)') - parser.add_argument( + group.add_argument( '-j', '--jobs', type=int, metavar='JOBS', help='Number of parallel jobs for build (use with -b)') - parser.add_argument( + group.add_argument( '-L', '--lto', action='store_true', help='Enable LTO when building (use with -b)') - parser.add_argument( + group.add_argument( '-o', '--output-dir', metavar='DIR', dest='output_dir', help='Override build directory (use with -b)') - parser.add_argument( + group.add_argument( '-T', '--trace', action='store_true', help='Enable function tracing (use with -b)') - parser.add_argument( + group.add_argument( '--no-trace-early', action='store_true', dest='no_trace_early', help='Disable TRACE_EARLY when using -T (use with -b)') From fd0b9710db48add5df34228d225f2d94c117525f Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 28 Feb 2026 07:55:50 -0700 Subject: [PATCH 11/36] uman: Detect -F flag support from the source tree The -F sandbox flag to skip flat-tree tests is not yet upstream. Running 'um t' against upstream U-Boot fails with "U-Boot does not support -F flag" since build_ut_cmd() unconditionally adds it. Add has_no_flat() to check whether arch/sandbox/cpu/start.c contains the 'noflat' callback, and only pass -F when the source tree supports it. This mirrors how has_no_full() in cmdpy.py detects --no-full support for the pytest path. Co-developed-by: Claude Opus 4.6 --- uman_pkg/cmdtest.py | 18 ++++++++++- uman_pkg/ftest.py | 74 +++++++++++++++++++++++++++++---------------- 2 files changed, 65 insertions(+), 27 deletions(-) diff --git a/uman_pkg/cmdtest.py b/uman_pkg/cmdtest.py index c71f276..f3c4539 100644 --- a/uman_pkg/cmdtest.py +++ b/uman_pkg/cmdtest.py @@ -409,6 +409,22 @@ def validate_specs(sandbox, specs): return unmatched +def has_no_flat(): + """Check whether the U-Boot tree supports the -F sandbox flag + + Looks for 'noflat' in arch/sandbox/cpu/start.c in the current directory. + + Returns: + bool: True if -F is supported + """ + start_c = os.path.join('arch', 'sandbox', 'cpu', 'start.c') + try: + with open(start_c, encoding='utf-8') as inf: + return 'noflat' in inf.read() + except OSError: + return False + + def build_ut_cmd(sandbox, specs, full=False, verbose=False, legacy=False, manual=False): """Build the sandbox command line for running tests @@ -427,7 +443,7 @@ def build_ut_cmd(sandbox, specs, full=False, verbose=False, legacy=False, cmd = [sandbox, '-T'] # Add -F to skip flat-tree tests (live-tree only) unless full mode - if not full: + if not full and has_no_flat(): cmd.append('-F') # Add -v to sandbox to show test output diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index f85bbf5..8297dc9 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -5056,7 +5056,8 @@ def test_do_test_list_tests_by_suite(self): self.assertIn('dm.test_gpio', stdout) self.assertNotIn('env.test_env_basic', stdout) - def test_build_ut_cmd_no_tests(self): + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_no_tests(self, _): """Test build_ut_cmd with all specs""" cmd = cmdtest.build_ut_cmd('/sb', [('all', None)]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E all'], cmd) @@ -5066,7 +5067,8 @@ def test_build_ut_cmd_full(self): cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], full=True) self.assertEqual(['/sb', '-T', '-c', 'ut -E dm'], cmd) - def test_build_ut_cmd_verbose(self): + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_verbose(self, _): """Test build_ut_cmd with verbose flag""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], verbose=True) self.assertEqual(['/sb', '-T', '-F', '-v', '-c', 'ut -E dm'], cmd) @@ -5077,31 +5079,42 @@ def test_build_ut_cmd_all_flags(self): full=True, verbose=True) self.assertEqual(['/sb', '-T', '-v', '-c', 'ut -E dm'], cmd) - def test_build_ut_cmd_suite(self): + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_suite(self, _): """Test build_ut_cmd with suite name""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E dm'], cmd) - def test_build_ut_cmd_specific_test(self): + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_specific_test(self, _): """Test build_ut_cmd with specific test (suite.test)""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', 'test_one')]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E dm test_one'], cmd) - def test_build_ut_cmd_multiple_tests(self): + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_multiple_tests(self, _): """Test build_ut_cmd with multiple test specifications""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None), ('env', None)]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E dm; ut -E env'], cmd) - def test_build_ut_cmd_legacy(self): + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_legacy(self, _): """Test build_ut_cmd with legacy flag omits -E""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], legacy=True) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut dm'], cmd) - def test_build_ut_cmd_manual(self): + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_manual(self, _): """Test build_ut_cmd with manual flag""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], manual=True) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E -m dm'], cmd) + @mock.patch.object(cmdtest, 'has_no_flat', return_value=False) + def test_build_ut_cmd_no_flat_unsupported(self, _): + """Test build_ut_cmd omits -F when source tree lacks support""" + cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)]) + self.assertEqual(['/sb', '-T', '-c', 'ut -E dm'], cmd) + def test_run_tests_basic(self): """Test run_tests executes sandbox correctly""" cap = [] @@ -5113,11 +5126,13 @@ def mock_run(*cmd_args, **_kwargs): args = cmdline.parse_args(['test', 'dm']) col = terminal.Color() - with mock.patch.object(command, 'run_one', mock_run): - with mock.patch.object(cmdtest, 'ensure_dm_init_files', - return_value=True): - with terminal.capture(): - result = cmdtest.run_tests('/sb', [('dm', None)], args, col) + with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): + with mock.patch.object(command, 'run_one', mock_run): + with mock.patch.object(cmdtest, 'ensure_dm_init_files', + return_value=True): + with terminal.capture(): + result = cmdtest.run_tests( + '/sb', [('dm', None)], args, col) self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-F', '-c', 'ut -E dm'), cap[0]) @@ -5151,11 +5166,13 @@ def mock_run(*cmd_args, **_kwargs): args = cmdline.parse_args(['test', '-V', 'dm']) col = terminal.Color() - with mock.patch.object(command, 'run_one', mock_run): - with mock.patch.object(cmdtest, 'ensure_dm_init_files', - return_value=True): - with terminal.capture(): - result = cmdtest.run_tests('/sb', [('dm', None)], args, col) + with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): + with mock.patch.object(command, 'run_one', mock_run): + with mock.patch.object(cmdtest, 'ensure_dm_init_files', + return_value=True): + with terminal.capture(): + result = cmdtest.run_tests( + '/sb', [('dm', None)], args, col) self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-F', '-v', '-c', 'ut -E dm'), cap[0]) @@ -5362,15 +5379,20 @@ def mock_resolve(_sandbox, specs): args = cmdline.parse_args(['test', 'dm']) args.col = terminal.Color() - with mock.patch.object(cmdtest, 'get_sandbox_path', return_value='/sb'): - with mock.patch.object(cmdtest, 'resolve_specs', mock_resolve): - with mock.patch.object(cmdtest, 'validate_specs', - return_value=[]): - with mock.patch.object(cmdtest, 'ensure_dm_init_files', - return_value=True): - with mock.patch.object(command, 'run_one', mock_run): - with terminal.capture(): - result = cmdtest.do_test(args) + with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): + with mock.patch.object(cmdtest, 'get_sandbox_path', + return_value='/sb'): + with mock.patch.object(cmdtest, 'resolve_specs', + mock_resolve): + with mock.patch.object(cmdtest, 'validate_specs', + return_value=[]): + with mock.patch.object(cmdtest, + 'ensure_dm_init_files', + return_value=True): + with mock.patch.object(command, 'run_one', + mock_run): + with terminal.capture(): + result = cmdtest.do_test(args) self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-F', '-c', 'ut -E dm'), cap[0]) From 652f00a6d154404e589f3b8f040400c2db563fed Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 28 Feb 2026 10:03:29 -0700 Subject: [PATCH 12/36] uman: Detect -E flag support from the source tree The -E ut flag to emit per-test result lines is not yet upstream. Running 'um t' against upstream U-Boot fails because build_ut_cmd() unconditionally adds -E, then run_tests() cannot parse results and reports "No results detected". Add has_emit_result() to check whether test/cmd_ut.c contains 'emit_result', and only pass -E when the source tree supports it. When -E is unavailable, automatically fall back to legacy result parsing, removing the need for the -L flag on older trees. Co-developed-by: Claude Opus 4.6 --- uman_pkg/cmdtest.py | 21 ++++++++- uman_pkg/ftest.py | 104 ++++++++++++++++++++++++++------------------ 2 files changed, 81 insertions(+), 44 deletions(-) diff --git a/uman_pkg/cmdtest.py b/uman_pkg/cmdtest.py index f3c4539..6be0738 100644 --- a/uman_pkg/cmdtest.py +++ b/uman_pkg/cmdtest.py @@ -425,6 +425,22 @@ def has_no_flat(): return False +def has_emit_result(): + """Check whether the U-Boot tree supports the -E ut flag + + Looks for 'emit_result' in test/cmd_ut.c in the current directory. + + Returns: + bool: True if -E is supported + """ + cmd_ut = os.path.join('test', 'cmd_ut.c') + try: + with open(cmd_ut, encoding='utf-8') as inf: + return 'emit_result' in inf.read() + except OSError: + return False + + def build_ut_cmd(sandbox, specs, full=False, verbose=False, legacy=False, manual=False): """Build the sandbox command line for running tests @@ -453,7 +469,7 @@ def build_ut_cmd(sandbox, specs, full=False, verbose=False, legacy=False, # Build ut commands from specs; use -E to emit Result: lines # Flags must come before suite name flags = '' - if not legacy: + if not legacy and has_emit_result(): flags += '-E ' if manual: flags += '-m ' @@ -614,8 +630,9 @@ def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 return 1 # Parse results first to check for failures + legacy = args.legacy or not has_emit_result() res = parse_results(result.stdout, show_results=args.results, col=col) - if not res and args.legacy: + if not res and legacy: res = parse_legacy_results(result.stdout, show_results=args.results, col=col) diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index 8297dc9..964e21c 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -5056,43 +5056,50 @@ def test_do_test_list_tests_by_suite(self): self.assertIn('dm.test_gpio', stdout) self.assertNotIn('env.test_env_basic', stdout) + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) - def test_build_ut_cmd_no_tests(self, _): + def test_build_ut_cmd_no_tests(self, *_): """Test build_ut_cmd with all specs""" cmd = cmdtest.build_ut_cmd('/sb', [('all', None)]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E all'], cmd) - def test_build_ut_cmd_full(self): + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + def test_build_ut_cmd_full(self, _): """Test build_ut_cmd with full flag (both tree types)""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], full=True) self.assertEqual(['/sb', '-T', '-c', 'ut -E dm'], cmd) + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) - def test_build_ut_cmd_verbose(self, _): + def test_build_ut_cmd_verbose(self, *_): """Test build_ut_cmd with verbose flag""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], verbose=True) self.assertEqual(['/sb', '-T', '-F', '-v', '-c', 'ut -E dm'], cmd) - def test_build_ut_cmd_all_flags(self): + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + def test_build_ut_cmd_all_flags(self, _): """Test build_ut_cmd with all flags""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], full=True, verbose=True) self.assertEqual(['/sb', '-T', '-v', '-c', 'ut -E dm'], cmd) + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) - def test_build_ut_cmd_suite(self, _): + def test_build_ut_cmd_suite(self, *_): """Test build_ut_cmd with suite name""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E dm'], cmd) + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) - def test_build_ut_cmd_specific_test(self, _): + def test_build_ut_cmd_specific_test(self, *_): """Test build_ut_cmd with specific test (suite.test)""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', 'test_one')]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E dm test_one'], cmd) + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) - def test_build_ut_cmd_multiple_tests(self, _): + def test_build_ut_cmd_multiple_tests(self, *_): """Test build_ut_cmd with multiple test specifications""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None), ('env', None)]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E dm; ut -E env'], cmd) @@ -5103,18 +5110,26 @@ def test_build_ut_cmd_legacy(self, _): cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], legacy=True) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut dm'], cmd) + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) - def test_build_ut_cmd_manual(self, _): + def test_build_ut_cmd_manual(self, *_): """Test build_ut_cmd with manual flag""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], manual=True) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E -m dm'], cmd) + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) @mock.patch.object(cmdtest, 'has_no_flat', return_value=False) - def test_build_ut_cmd_no_flat_unsupported(self, _): + def test_build_ut_cmd_no_flat_unsupported(self, *_): """Test build_ut_cmd omits -F when source tree lacks support""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)]) self.assertEqual(['/sb', '-T', '-c', 'ut -E dm'], cmd) + @mock.patch.object(cmdtest, 'has_emit_result', return_value=False) + def test_build_ut_cmd_no_emit_unsupported(self, _): + """Test build_ut_cmd omits -E when source tree lacks support""" + cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)]) + self.assertEqual(['/sb', '-T', '-c', 'ut dm'], cmd) + def test_run_tests_basic(self): """Test run_tests executes sandbox correctly""" cap = [] @@ -5126,13 +5141,14 @@ def mock_run(*cmd_args, **_kwargs): args = cmdline.parse_args(['test', 'dm']) col = terminal.Color() - with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): - with mock.patch.object(command, 'run_one', mock_run): - with mock.patch.object(cmdtest, 'ensure_dm_init_files', - return_value=True): - with terminal.capture(): - result = cmdtest.run_tests( - '/sb', [('dm', None)], args, col) + with mock.patch.object(cmdtest, 'has_emit_result', return_value=True): + with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): + with mock.patch.object(command, 'run_one', mock_run): + with mock.patch.object(cmdtest, 'ensure_dm_init_files', + return_value=True): + with terminal.capture(): + result = cmdtest.run_tests( + '/sb', [('dm', None)], args, col) self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-F', '-c', 'ut -E dm'), cap[0]) @@ -5147,11 +5163,13 @@ def mock_run(*cmd_args, **_kwargs): args = cmdline.parse_args(['test', '--flattree-too', 'dm']) col = terminal.Color() - with mock.patch.object(command, 'run_one', mock_run): - with mock.patch.object(cmdtest, 'ensure_dm_init_files', - return_value=True): - with terminal.capture(): - result = cmdtest.run_tests('/sb', [('dm', None)], args, col) + with mock.patch.object(cmdtest, 'has_emit_result', return_value=True): + with mock.patch.object(command, 'run_one', mock_run): + with mock.patch.object(cmdtest, 'ensure_dm_init_files', + return_value=True): + with terminal.capture(): + result = cmdtest.run_tests( + '/sb', [('dm', None)], args, col) self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-c', 'ut -E dm'), cap[0]) @@ -5166,13 +5184,14 @@ def mock_run(*cmd_args, **_kwargs): args = cmdline.parse_args(['test', '-V', 'dm']) col = terminal.Color() - with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): - with mock.patch.object(command, 'run_one', mock_run): - with mock.patch.object(cmdtest, 'ensure_dm_init_files', - return_value=True): - with terminal.capture(): - result = cmdtest.run_tests( - '/sb', [('dm', None)], args, col) + with mock.patch.object(cmdtest, 'has_emit_result', return_value=True): + with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): + with mock.patch.object(command, 'run_one', mock_run): + with mock.patch.object(cmdtest, 'ensure_dm_init_files', + return_value=True): + with terminal.capture(): + result = cmdtest.run_tests( + '/sb', [('dm', None)], args, col) self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-F', '-v', '-c', 'ut -E dm'), cap[0]) @@ -5379,20 +5398,21 @@ def mock_resolve(_sandbox, specs): args = cmdline.parse_args(['test', 'dm']) args.col = terminal.Color() - with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): - with mock.patch.object(cmdtest, 'get_sandbox_path', - return_value='/sb'): - with mock.patch.object(cmdtest, 'resolve_specs', - mock_resolve): - with mock.patch.object(cmdtest, 'validate_specs', - return_value=[]): - with mock.patch.object(cmdtest, - 'ensure_dm_init_files', - return_value=True): - with mock.patch.object(command, 'run_one', - mock_run): - with terminal.capture(): - result = cmdtest.do_test(args) + with mock.patch.object(cmdtest, 'has_emit_result', return_value=True): + with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): + with mock.patch.object(cmdtest, 'get_sandbox_path', + return_value='/sb'): + with mock.patch.object(cmdtest, 'resolve_specs', + mock_resolve): + with mock.patch.object(cmdtest, 'validate_specs', + return_value=[]): + with mock.patch.object( + cmdtest, 'ensure_dm_init_files', + return_value=True): + with mock.patch.object(command, 'run_one', + mock_run): + with terminal.capture(): + result = cmdtest.do_test(args) self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-F', '-c', 'ut -E dm'), cap[0]) From 497f6040c129bfd884b4b206f02b4a8011473b7e Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 28 Feb 2026 10:08:39 -0700 Subject: [PATCH 13/36] uman: Parse the 'Tests run:' summary line in test output The upstream sandbox outputs a summary line like 'Tests run: 540, 11595 ms, average: 21 ms, failures: 7' but neither the Result: parser nor the legacy '... ok' parser handles this format. When -E is unavailable, run_tests() reports only 1 test because it falls through to no-results handling. Add parse_summary() to extract pass/fail counts from the 'Tests run:' summary line, and use it as a final fallback in run_tests(). This gives correct totals when running against upstream U-Boot without the -E patch. Co-developed-by: Claude Opus 4.6 --- uman_pkg/cmdtest.py | 23 +++++++++++++++++++++++ uman_pkg/ftest.py | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/uman_pkg/cmdtest.py b/uman_pkg/cmdtest.py index 6be0738..3834439 100644 --- a/uman_pkg/cmdtest.py +++ b/uman_pkg/cmdtest.py @@ -38,6 +38,7 @@ # Patterns for parsing test output RE_TEST_NAME = re.compile(r'Test:\s*(\S+)') RE_RESULT = re.compile(r'Result:\s*(PASS|FAIL|SKIP):?\s+(\S+)') +RE_SUMMARY = re.compile(r'Tests run:\s*(\d+),.*failures:\s*(\d+)') # Unit test flags from include/test/test.h UTF_FLAT_TREE = 0x08 @@ -543,6 +544,26 @@ def parse_legacy_results(output, show_results=False, col=None): return TestCounts(passed, failed, skipped) +def parse_summary(output): + """Parse 'Tests run:' summary line from test output + + Handles the format: Tests run: N, Xms, average: Xms, failures: N + + Args: + output (str): Test output from sandbox + + Returns: + TestCounts or None: Counts of passed/failed/skipped, or None if none + """ + for line in output.splitlines(): + match = RE_SUMMARY.match(line) + if match: + total = int(match.group(1)) + failed = int(match.group(2)) + return TestCounts(total - failed, failed, 0) + return None + + def parse_results(output, show_results=False, col=None): """Parse test output to extract results from Result: lines @@ -635,6 +656,8 @@ def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 if not res and legacy: res = parse_legacy_results(result.stdout, show_results=args.results, col=col) + if not res: + res = parse_summary(result.stdout) # Print output in verbose mode, if there are failures, or no results if result.stdout and not args.results: diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index 964e21c..2a605f2 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -5248,6 +5248,25 @@ def test_parse_results_only_result_lines(self): self.assertEqual(0, res.failed) self.assertEqual(1, res.skipped) + def test_parse_summary_basic(self): + """Test parse_summary with standard output""" + output = 'Tests run: 540, 11595 ms, average: 21 ms, failures: 7' + res = cmdtest.parse_summary(output) + self.assertEqual(533, res.passed) + self.assertEqual(7, res.failed) + self.assertEqual(0, res.skipped) + + def test_parse_summary_no_failures(self): + """Test parse_summary with no failures""" + output = 'Tests run: 1, 4 ms, average: 4 ms, failures: 0' + res = cmdtest.parse_summary(output) + self.assertEqual(1, res.passed) + self.assertEqual(0, res.failed) + + def test_parse_summary_empty(self): + """Test parse_summary with no summary line""" + self.assertIsNone(cmdtest.parse_summary('')) + def test_parse_results_empty(self): """Test parse_results with empty output returns None""" self.assertIsNone(cmdtest.parse_results('')) @@ -5322,7 +5341,6 @@ def test_run_tests_shows_output_when_no_results(self): output = ''' U-Boot banner here Missing required argument 'fs_image' for test 'pxe_test_sysboot' -Tests run: 1, failures: 1 ''' def mock_run(*_args, **_kwargs): @@ -5341,6 +5359,26 @@ def mock_run(*_args, **_kwargs): # Error message should be shown in output self.assertIn('Missing required argument', out.getvalue()) + def test_run_tests_parses_summary(self): + """Test run_tests uses summary line when -E is unavailable""" + output = 'Tests run: 10, 100 ms, average: 10 ms, failures: 2' + + def mock_run(*_args, **_kwargs): + return command.CommandResult(return_code=1, stdout=output) + + args = cmdline.parse_args(['test', 'dm']) + col = terminal.Color() + with mock.patch.object(cmdtest, 'has_emit_result', return_value=False): + with mock.patch.object(command, 'run_one', mock_run): + with mock.patch.object(cmdtest, 'ensure_dm_init_files', + return_value=True): + with terminal.capture() as (out, err): + result = cmdtest.run_tests( + '/sb', [('dm', None)], args, col) + self.assertEqual(1, result) + self.assertIn('8 passed', out.getvalue()) + self.assertIn('2 failed', out.getvalue()) + def test_run_tests_detects_segfault(self): """Test run_tests detects segfault (SIGSEGV) and resets terminal""" col = terminal.Color() From 86acb97d1af97fb591f25860abe0efee4f1bc19f Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 28 Feb 2026 10:16:05 -0700 Subject: [PATCH 14/36] uman: Require 'Test:' prefix in legacy result parsing The legacy parser matches '... ok' anywhere in the output, which incorrectly counts lines like 'Loading Environment from nowhere... OK' as a passing test. This causes wrong results when running against upstream U-Boot without the -E flag. Require a 'Test:' prefix (via RE_TEST_NAME) before checking for the result suffix, so only actual test output lines are counted. Co-developed-by: Claude Opus 4.6 --- uman_pkg/cmdtest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uman_pkg/cmdtest.py b/uman_pkg/cmdtest.py index 3834439..c4f6e4b 100644 --- a/uman_pkg/cmdtest.py +++ b/uman_pkg/cmdtest.py @@ -522,7 +522,9 @@ def parse_legacy_results(output, show_results=False, col=None): for line in output.splitlines(): name_match = RE_TEST_NAME.search(line) - name = name_match.group(1) if name_match else None + if not name_match: + continue + name = name_match.group(1) lower = line.lower() if '... ok' in lower: From 0c92b81258bae81f101102a49894af7bdaf47543 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 28 Feb 2026 10:37:56 -0700 Subject: [PATCH 15/36] uman: Show live test progress during 'um t' runs There is no feedback while tests run, which is unhelpful for large suites like 'dm' with over 500 tests. Add a Progress class that uses the output_func callback in command.run_pipe() to parse sandbox output as it arrives, displaying a running count of passed/failed/skipped tests that updates in place on stderr. With -E, it counts Result: lines; without -E, it counts Test: lines as passes and detects "Test '' failed N times" lines as failures. Progress is only shown when stderr is a TTY. Co-developed-by: Claude Opus 4.6 --- uman_pkg/cmdtest.py | 84 ++++++++++++++++++++++++++++++++++++++++++++- uman_pkg/ftest.py | 51 +++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 1 deletion(-) diff --git a/uman_pkg/cmdtest.py b/uman_pkg/cmdtest.py index c4f6e4b..fc48194 100644 --- a/uman_pkg/cmdtest.py +++ b/uman_pkg/cmdtest.py @@ -13,6 +13,7 @@ import re import shlex import struct +import sys import time # pylint: disable=import-error @@ -39,6 +40,7 @@ RE_TEST_NAME = re.compile(r'Test:\s*(\S+)') RE_RESULT = re.compile(r'Result:\s*(PASS|FAIL|SKIP):?\s+(\S+)') RE_SUMMARY = re.compile(r'Tests run:\s*(\d+),.*failures:\s*(\d+)') +RE_TEST_FAILED = re.compile(r"Test '.+' failed \d+ times") # Unit test flags from include/test/test.h UTF_FLAT_TREE = 0x08 @@ -599,6 +601,77 @@ def parse_results(output, show_results=False, col=None): return TestCounts(passed, failed, skipped) +class Progress: + """Show live test progress on stderr + + Parses sandbox output as it arrives and displays a running count of + passed/failed/skipped tests, updating in place with carriage return. + + With -E (emit_result=True): counts Result: PASS/FAIL/SKIP lines. + Without -E: counts Test: lines as passes, detects failure lines like + "Test '' failed N times" to adjust the count. + """ + + def __init__(self, emit_result): + self.emit = emit_result + self.passed = 0 + self.failed = 0 + self.skipped = 0 + self.buf = '' + self.pending = False # A Test: line seen, not yet resolved + + def _show(self): + """Print the progress line, overwriting the previous one""" + total = self.passed + self.failed + self.skipped + sys.stderr.write( + f'\r {total}: {self.passed} passed, {self.failed} failed, ' + f'{self.skipped} skipped') + sys.stderr.flush() + + def _process_line(self, line): + """Process one complete line of output""" + if self.emit: + match = RE_RESULT.match(line) + if match: + status = match.group(1) + if status == 'PASS': + self.passed += 1 + elif status == 'FAIL': + self.failed += 1 + elif status == 'SKIP': + self.skipped += 1 + self._show() + else: + if RE_TEST_FAILED.search(line): + self.failed += 1 + self.pending = False + self._show() + elif RE_TEST_NAME.match(line): + if self.pending: + self.passed += 1 + self.pending = True + self._show() + + def update(self, _stream, data): # pylint: disable=W9016,W9019 + """output_func callback for command.run_pipe()""" + if isinstance(data, (bytes, bytearray)): + data = data.decode('utf-8', errors='replace') + self.buf += data + while '\n' in self.buf: + line, self.buf = self.buf.split('\n', 1) + self._process_line(line) + + def finish(self): + """Close out the progress line""" + if not self.emit and self.pending: + self.passed += 1 + self.pending = False + if self.passed or self.failed or self.skipped: + self._show() + sys.stderr.write('\n') + sys.stderr.flush() + + def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 """Run sandbox tests @@ -631,9 +704,15 @@ def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 env = os.environ.copy() env['U_BOOT_PERSISTENT_DATA_DIR'] = persist_dir + # Show live progress if stderr is a terminal + emit = has_emit_result() + progress = Progress(emit) if sys.stderr.isatty() else None + output_func = progress.update if progress else None + start_time = time.time() try: - result = command.run_one(*cmd, capture=True, env=env) + result = command.run_one(*cmd, capture=True, env=env, + output_func=output_func) except command.CommandExc as exc: # Tests may fail but still produce parseable output result = exc.result @@ -642,6 +721,9 @@ def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 if not result: tout.error(f'Command failed: {exc}') return 1 + finally: + if progress: + progress.finish() elapsed = time.time() - start_time # Detect old U-Boot that doesn't understand -E or -F flags diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index 2a605f2..18e9c1f 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -5454,6 +5454,57 @@ def mock_resolve(_sandbox, specs): self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-F', '-c', 'ut -E dm'), cap[0]) + def test_progress_with_emit(self): + """Test Progress counts Result: lines with -E""" + prog = cmdtest.Progress(emit_result=True) + prog.update(None, b'Result: PASS test1\n') + prog.update(None, b'Result: FAIL test2\n') + prog.update(None, b'Result: SKIP test3\n') + prog.update(None, b'Result: PASS test4\n') + prog.finish() + self.assertEqual(2, prog.passed) + self.assertEqual(1, prog.failed) + self.assertEqual(1, prog.skipped) + + def test_progress_without_emit(self): + """Test Progress counts Test: and failure lines without -E""" + prog = cmdtest.Progress(emit_result=False) + prog.update(None, b'Test: acpi: acpi.c\n') + prog.update(None, b'Test: gpio: gpio.c\n') + prog.update(None, + b"Test 'gpio' failed 1 times\nTest: host: host.c\n") + prog.finish() + self.assertEqual(2, prog.passed) + self.assertEqual(1, prog.failed) + self.assertEqual(0, prog.skipped) + + def test_progress_partial_lines(self): + """Test Progress handles data split across chunks""" + prog = cmdtest.Progress(emit_result=True) + prog.update(None, b'Result: PA') + prog.update(None, b'SS test1\nResult:') + prog.update(None, b' FAIL test2\n') + prog.finish() + self.assertEqual(1, prog.passed) + self.assertEqual(1, prog.failed) + + def test_progress_no_output(self): + """Test Progress with no test output""" + prog = cmdtest.Progress(emit_result=True) + prog.finish() + self.assertEqual(0, prog.passed) + self.assertEqual(0, prog.failed) + self.assertEqual(0, prog.skipped) + + def test_progress_last_test_passes(self): + """Test Progress counts the last test as passed at finish""" + prog = cmdtest.Progress(emit_result=False) + prog.update(None, b'Test: foo: foo.c\n') + self.assertTrue(prog.pending) + prog.finish() + self.assertEqual(1, prog.passed) + self.assertFalse(prog.pending) + def test_parse_one_test_suite(self): """Test parse_one_test with suite name""" self.assertEqual(('dm', None), cmdtest.parse_one_test('dm')) From 5b956e5e9e3c78c5274ec63a82e3b5ee9e65dc56 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 28 Feb 2026 10:41:01 -0700 Subject: [PATCH 16/36] uman: Add colour and test count to progress display The progress line shows plain text with no indication of how many tests remain. Add colour to the passed/failed/skipped counts (green/red/yellow, matching the final summary) and show the total expected test count obtained from nm symbols. Add count_tests() to count matching tests for the given specs. Co-developed-by: Claude Opus 4.6 --- uman_pkg/cmdtest.py | 47 ++++++++++++++++++++++++++++++++++++++++----- uman_pkg/ftest.py | 30 +++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/uman_pkg/cmdtest.py b/uman_pkg/cmdtest.py index fc48194..e6891fd 100644 --- a/uman_pkg/cmdtest.py +++ b/uman_pkg/cmdtest.py @@ -601,6 +601,28 @@ def parse_results(output, show_results=False, col=None): return TestCounts(passed, failed, skipped) +def count_tests(sandbox, specs): + """Count expected tests for the given specs + + Args: + sandbox (str): Path to sandbox executable + specs (list): List of (suite, pattern) tuples + + Returns: + int: Number of matching tests, or 0 if unknown + """ + total = 0 + for suite, pattern in specs: + if suite == 'all': + return len(get_tests_from_nm(sandbox)) + tests = get_tests_from_nm(sandbox, suite) + if pattern: + total += sum(1 for _, name in tests if name.endswith(pattern)) + else: + total += len(tests) + return total + + class Progress: """Show live test progress on stderr @@ -612,8 +634,9 @@ class Progress: "Test '' failed N times" to adjust the count. """ - def __init__(self, emit_result): + def __init__(self, emit_result, total=0): self.emit = emit_result + self.total = total self.passed = 0 self.failed = 0 self.skipped = 0 @@ -622,10 +645,20 @@ def __init__(self, emit_result): def _show(self): """Print the progress line, overwriting the previous one""" - total = self.passed + self.failed + self.skipped + col = terminal.Color() + grn = col.start(terminal.Color.GREEN) + red = col.start(terminal.Color.RED) + yel = col.start(terminal.Color.YELLOW) + rst = col.stop() + done = self.passed + self.failed + self.skipped + if self.total: + hdr = f'{done}/{self.total}:' + else: + hdr = f'{done}:' sys.stderr.write( - f'\r {total}: {self.passed} passed, {self.failed} failed, ' - f'{self.skipped} skipped') + f'\r {hdr} {grn}{self.passed} passed{rst}, ' + f'{red}{self.failed} failed{rst}, ' + f'{yel}{self.skipped} skipped{rst}') sys.stderr.flush() def _process_line(self, line): @@ -706,7 +739,11 @@ def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 # Show live progress if stderr is a terminal emit = has_emit_result() - progress = Progress(emit) if sys.stderr.isatty() else None + if sys.stderr.isatty(): + total = count_tests(sandbox, specs) + progress = Progress(emit, total) + else: + progress = None output_func = progress.update if progress else None start_time = time.time() diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index 18e9c1f..d918770 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -5454,6 +5454,36 @@ def mock_resolve(_sandbox, specs): self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-F', '-c', 'ut -E dm'), cap[0]) + def test_count_tests_suite(self): + """Test count_tests counts tests in a suite""" + tests = [('dm', 'test_a'), ('dm', 'test_b')] + with mock.patch.object(cmdtest, 'get_tests_from_nm', + return_value=tests) as mock_nm: + result = cmdtest.count_tests('/sb', [('dm', None)]) + self.assertEqual(2, result) + mock_nm.assert_called_once_with('/sb', 'dm') + + def test_count_tests_all(self): + """Test count_tests counts all tests""" + tests = [('dm', 'a'), ('dm', 'b'), ('env', 'c')] + with mock.patch.object(cmdtest, 'get_tests_from_nm', + return_value=tests): + result = cmdtest.count_tests('/sb', [('all', None)]) + self.assertEqual(3, result) + + def test_count_tests_pattern(self): + """Test count_tests counts tests matching a pattern""" + tests = [('dm', 'gpio'), ('dm', 'gpio_irq'), ('dm', 'i2c')] + with mock.patch.object(cmdtest, 'get_tests_from_nm', + return_value=tests): + result = cmdtest.count_tests('/sb', [('dm', 'gpio')]) + self.assertEqual(1, result) + + def test_progress_with_total(self): + """Test Progress shows total in output""" + prog = cmdtest.Progress(emit_result=True, total=10) + self.assertEqual(10, prog.total) + def test_progress_with_emit(self): """Test Progress counts Result: lines with -E""" prog = cmdtest.Progress(emit_result=True) From ef6e2e32f0977963cd4b2632c575658de69573cf Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 7 Mar 2026 10:14:05 -0700 Subject: [PATCH 17/36] uman: Support wildcards in test spec matching The validate_specs() and resolve_specs() functions use endswith() to match test patterns, which does not handle glob wildcards like 'adj*'. This causes 'um t dm_test_adj*' to fail with 'No tests found matching' even though matching tests exist. Use fnmatch to support glob patterns in test specs. Co-developed-by: Claude Opus 4.6 --- README.rst | 3 +++ uman_pkg/cmdtest.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index e19973e..7b88be2 100644 --- a/README.rst +++ b/README.rst @@ -656,6 +656,9 @@ without going through pytest. This is faster for quick iteration on C code. # Run test using pytest-style name (ut__) uman test ut_bootstd_bootflow + # Run tests matching a wildcard pattern + uman test 'dm.adj*' + # List available suites uman test -s diff --git a/uman_pkg/cmdtest.py b/uman_pkg/cmdtest.py index e6891fd..16e334b 100644 --- a/uman_pkg/cmdtest.py +++ b/uman_pkg/cmdtest.py @@ -9,6 +9,7 @@ """ from collections import namedtuple +import fnmatch import os import re import shlex @@ -369,7 +370,7 @@ def resolve_specs(sandbox, specs): all_tests = get_tests_from_nm(sandbox) found = False for test_suite, test_name in all_tests: - if test_name.endswith(pattern): + if fnmatch.fnmatch(test_name, f'*{pattern}'): resolved.append((test_suite, pattern)) found = True break # Only add first match @@ -403,7 +404,7 @@ def validate_specs(sandbox, specs): if pattern is None: found = True break - if test_name.endswith(pattern): + if fnmatch.fnmatch(test_name, f'*{pattern}'): found = True break if not found: From e312975ce950dc0ebbe4ec1d3c020139db522604 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 12 Mar 2026 17:58:38 -0600 Subject: [PATCH 18/36] uman: Add --malloc-dump flag to the t and py subcommands Add a --malloc-dump FILE option that writes a malloc heap dump on exit. For the t command this passes --malloc_dump to the sandbox binary directly. For the py command it passes --malloc-dump to test.py which forwards it to sandbox. A %d placeholder in the filename is expanded to a sequence number, so that each restart of U-Boot writes to a separate file. Co-developed-by: Claude Opus 4.6 --- README.rst | 4 ++++ uman_pkg/cmdline.py | 32 ++++++++++++++++++++++++++++++++ uman_pkg/cmdpy.py | 2 ++ uman_pkg/cmdtest.py | 9 +++++++-- uman_pkg/ftest.py | 1 + 5 files changed, 46 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 7b88be2..c3a3919 100644 --- a/README.rst +++ b/README.rst @@ -528,6 +528,8 @@ hooks to PATH. - ``-T, --trace``: Enable function tracing; adds CONFIG_TRACE and CONFIG_TRACE_EARLY (use with -b) - ``--no-trace-early``: Disable TRACE_EARLY when using -T (use with -b) +- ``--malloc-dump FILE``: Write malloc heap dump on exit; ``%d`` in the filename + is expanded to a sequence number - ``--no-timeout``: Disable test timeout - ``-x, --exitfirst``: Stop on first test failure - ``--pollute TEST``: Find which test pollutes TEST @@ -677,6 +679,8 @@ without going through pytest. This is faster for quick iteration on C code. - ``-l, --list``: List available tests - ``-L, --lto``: Enable LTO when building (use with -b) - ``--legacy``: Use legacy result parsing (for old U-Boot) +- ``--malloc-dump FILE``: Write malloc heap dump on exit; ``%d`` in the filename + is expanded to a sequence number - ``-m, --manual``: Force manual tests to run (tests with _norun suffix) - ``-o, --output-dir DIR``: Override build directory (use with -b) - ``-r, --results``: Show per-test pass/fail status diff --git a/uman_pkg/cmdline.py b/uman_pkg/cmdline.py index 556e792..9676682 100644 --- a/uman_pkg/cmdline.py +++ b/uman_pkg/cmdline.py @@ -145,6 +145,35 @@ def add_selftest_subparser(subparsers): return stest +def add_test_opts(parser, board_help=None, board_default=None): + """Add common test options to a parser + + Args: + parser: Argument parser to add options to + board_help (str or None): Help text for -B flag + board_default (str or None): Default board value + """ + parser.add_argument( + 'test_spec', type=str, nargs='*', + help="Test specification (e.g. 'test_dm', 'not sleep')") + parser.add_argument( + '-B', '--board', metavar='BOARD', default=board_default, + help=board_help or 'Board name to test') + parser.add_argument( + '-g', nargs='?', const='u-boot', default=None, dest='gdb_phase', + metavar='PHASE', + help='Debug with gdbserver (spl, tpl, vpl; default: u-boot)') + parser.add_argument( + '-s', '--show-output', action='store_true', + help='Show all test output in real-time (pytest -s)') + parser.add_argument( + '-x', '--exitfirst', action='store_true', + help='Stop on first test failure') + parser.add_argument( + '--malloc-dump', metavar='FILE', dest='malloc_dump', + help='Write malloc dump to FILE on exit') + + def add_build_opts(parser): """Add common build options to a parser @@ -349,6 +378,9 @@ def add_test_subparser(subparsers): test.add_argument( '-s', '--suites', action='store_true', dest='list_suites', help='List available test suites') + test.add_argument( + '--malloc-dump', metavar='FILE', dest='malloc_dump', + help='Write malloc dump to FILE on exit') test.add_argument( '-V', '--test-verbose', action='store_true', dest='test_verbose', help='Enable verbose test output') diff --git a/uman_pkg/cmdpy.py b/uman_pkg/cmdpy.py index bcbfa3e..876d0b8 100644 --- a/uman_pkg/cmdpy.py +++ b/uman_pkg/cmdpy.py @@ -467,6 +467,8 @@ def build_pytest_cmd(args): cmd.append('-x') if not args.flattree_too and has_no_full(): cmd.append('--no-full') + if args.malloc_dump: + cmd.extend(['--malloc-dump', args.malloc_dump]) # Add extra pytest arguments (after --) if args.extra_args: diff --git a/uman_pkg/cmdtest.py b/uman_pkg/cmdtest.py index 16e334b..8b123dc 100644 --- a/uman_pkg/cmdtest.py +++ b/uman_pkg/cmdtest.py @@ -446,7 +446,7 @@ def has_emit_result(): def build_ut_cmd(sandbox, specs, full=False, verbose=False, legacy=False, - manual=False): + manual=False, malloc_dump=None): """Build the sandbox command line for running tests Args: @@ -456,12 +456,16 @@ def build_ut_cmd(sandbox, specs, full=False, verbose=False, legacy=False, verbose (bool): Enable verbose test output legacy (bool): Legacy mode (don't use -E flag for older U-Boot) manual (bool): Force manual tests to run + malloc_dump (str or None): File to write malloc dump to on exit Returns: list: Command and arguments """ cmd = [sandbox, '-T'] + if malloc_dump: + cmd.extend(['--malloc_dump', malloc_dump]) + # Add -F to skip flat-tree tests (live-tree only) unless full mode if not full and has_no_flat(): cmd.append('-F') @@ -724,7 +728,8 @@ def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 cmd = build_ut_cmd(sandbox, specs, full=args.flattree_too, verbose=args.test_verbose, legacy=args.legacy, - manual=args.manual) + manual=args.manual, + malloc_dump=args.malloc_dump) if args.dry_run: tout.notice(shlex.join(cmd)) diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index d918770..4fafccc 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -137,6 +137,7 @@ def make_args(**kwargs): 'jobs': None, 'list_boards': False, 'lto': False, + 'malloc_dump': None, 'merge': False, 'no_timeout': False, 'null': False, From 6f42b36523217c4536347f1e85c0cef3ecb43521 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 4 Mar 2026 04:27:17 -0700 Subject: [PATCH 19/36] uman: Add docker subcommand for running tests in CI container Add a 'docker' (alias 'd') subcommand that runs U-Boot tests inside the same Docker container used by GitLab CI. This parses .gitlab-ci.yml to determine the Docker image and build/test script, so the local test environment matches CI exactly. The container bind-mounts the U-Boot source at /source and runs as the current user. Flags include -B (board), -a (adjust-cfg), -x (exitfirst), -s (show-output), -I (interactive shell), and -i (image override). Co-developed-by: Claude Opus 4.6 --- README.rst | 51 +++++++++++ uman_pkg/cmddocker.py | 199 ++++++++++++++++++++++++++++++++++++++++++ uman_pkg/cmdline.py | 33 ++++--- uman_pkg/control.py | 4 + uman_pkg/ftest.py | 194 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 467 insertions(+), 14 deletions(-) create mode 100644 uman_pkg/cmddocker.py diff --git a/README.rst b/README.rst index c3a3919..0b46b62 100644 --- a/README.rst +++ b/README.rst @@ -23,6 +23,9 @@ Subcommands ``ci`` Push current branch to GitLab CI with configurable test stages +``docker`` (alias: ``d``) + Run U-Boot tests in the same Docker container used by CI + ``pytest`` (alias: ``py``) Run U-Boot's test.py framework with automatic environment setup @@ -181,6 +184,54 @@ run), not fine-grained selection of specific boards or test specifications. For precise targeting like ``-p coreboot`` or ``-t "test_ofplatdata"``, use regular CI pushes instead of merge requests. +Docker Subcommand +----------------- + +The ``docker`` command (alias ``d``) runs U-Boot tests inside the same Docker +image used by GitLab CI. It parses ``.gitlab-ci.yml`` from the U-Boot tree to +determine the Docker image and build/test script, so the local test environment +matches CI exactly. + +The container bind-mounts the U-Boot source directory (from ``$USRC`` or the +current directory) at ``/source`` and runs as the current user, so build +artefacts have the correct ownership. + +:: + + # Run all sandbox tests (board defaults to sandbox) + uman docker + + # Run specific tests + uman docker test_ofplatdata or test_handoff + + # Test a different board + uman docker -B sandbox_noinst test_ofplatdata + + # Stop on first failure, show output + uman docker -x -s test_dm + + # Adjust Kconfig before building + uman docker -a CONFIG_TRACE test_trace + + # Drop to an interactive shell in the container + uman docker -I + + # Override the Docker image + uman docker -i my-registry/u-boot-ci:latest + + # Dry-run to see the docker command + uman -n docker -B sandbox test_dm + +**Options**: + +- ``test_spec``: Test specification using pytest -k syntax (positional) +- ``-a, --adjust-cfg CFG``: Adjust Kconfig setting (can use multiple times) +- ``-B, --board BOARD``: Board name (default: sandbox) +- ``-i, --image IMAGE``: Override Docker image (default: from .gitlab-ci.yml) +- ``-I, --interactive``: Drop to a bash shell in the container +- ``-s, --show-output``: Show all test output in real-time (pytest -s) +- ``-x, --exitfirst``: Stop on first test failure + CC Subcommand ------------- diff --git a/uman_pkg/cmddocker.py b/uman_pkg/cmddocker.py new file mode 100644 index 0000000..c7dfa34 --- /dev/null +++ b/uman_pkg/cmddocker.py @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2025 Canonical Ltd +# Written by Simon Glass + +"""Docker command for running U-Boot tests in a CI Docker container + +This module handles the 'docker' subcommand which runs U-Boot pytest +tests inside the same Docker image used by CI. +""" + +import os + +import yaml + +# pylint: disable=import-error +from u_boot_pylib import command +from u_boot_pylib import tout + +from uman_pkg import util + + +# Template key in .gitlab-ci.yml +CI_TEMPLATE = '.buildman_and_testpy_template' + +# CI variable substitutions (variable name -> default value) +CI_SUBS = { + 'CI_PROJECT_DIR': '/source', + 'OVERRIDE': '', + 'BUILD_ENV': '', + 'TEST_PY_ID': '', + 'TEST_PY_EXTRA': '', +} + + +def load_ci_yaml(uboot_dir): + """Load and parse .gitlab-ci.yml from a U-Boot tree + + Args: + uboot_dir (str): Path to U-Boot source directory + + Returns: + dict: Parsed YAML data, or None if not found + """ + ci_path = os.path.join(uboot_dir, '.gitlab-ci.yml') + if not os.path.exists(ci_path): + tout.error(f'Cannot find {ci_path}') + return None + + with open(ci_path, 'r', encoding='utf-8') as inf: + return yaml.safe_load(inf) + + +def get_ci_image(data): + """Extract the CI Docker image from parsed .gitlab-ci.yml data + + Looks for the default image and expands ${MIRROR_DOCKER} to + docker.io. + + Args: + data (dict): Parsed YAML data + + Returns: + str: Full Docker image path, or None if not found + """ + image = data.get('image') + if not image: + default = data.get('default', {}) + image = default.get('image') if isinstance(default, dict) else None + if not image: + tout.error('Cannot find image in .gitlab-ci.yml') + return None + + return image.replace('${MIRROR_DOCKER}', 'docker.io') + + +def get_ci_script(data): + """Extract before_script and script from the CI test template + + Args: + data (dict): Parsed YAML data + + Returns: + tuple: (before_script, script) as lists of shell command strings, + or (None, None) if not found + """ + template = data.get(CI_TEMPLATE) + if not template: + tout.error(f'Cannot find {CI_TEMPLATE} in .gitlab-ci.yml') + return None, None + + before = template.get('before_script', []) + script = template.get('script', []) + return before, script + + +def build_script(data, board, test_spec, adjust_cfg=None, + pytest_args=None): + """Generate the shell script to run inside the Docker container + + Parses the before_script and script sections from .gitlab-ci.yml + and applies variable substitutions for the given board and test spec. + + Args: + data (dict): Parsed YAML data + board (str): Board name (e.g. 'sandbox') + test_spec (str or None): pytest -k filter spec + adjust_cfg (list or None): Kconfig adjustments for buildman -a + pytest_args (list or None): Extra pytest flags (e.g. ['-x', '-s']) + + Returns: + str: Shell script to pass to bash -c, or None on error + """ + before, script = get_ci_script(data) + if before is None: + return None + + # Build substitution map; put -a flags into OVERRIDE so they + # are appended to the buildman command line + override = '' + if adjust_cfg: + override = ' '.join(f'-a {cfg}' for cfg in adjust_cfg) + subs = dict(CI_SUBS) + subs['OVERRIDE'] = override + subs['TEST_PY_BD'] = board + subs['TEST_PY_EXTRA'] = ' '.join(pytest_args) if pytest_args else '' + + # Apply substitutions to each command; set test spec as shell + # variables so bash handles ${VAR:+...} expansions natively + spec = test_spec or '' + commands = ['set -e', + 'mkdir -p test/hooks/bin test/hooks/py', + f'export TEST_PY_TEST_SPEC="{spec}"', + f'export TEST_SPEC="{spec}"'] + for cmd in before + script: + for var, val in subs.items(): + cmd = cmd.replace(f'${{{var}}}', val) + commands.append(cmd) + + return '\n'.join(commands) + + +def run(args): + """Run the docker command + + Args: + args (argparse.Namespace): Parsed arguments + + Returns: + int: Exit code (0 for success, non-zero for failure) + """ + uboot_dir = util.get_uboot_dir() + if not uboot_dir: + tout.error('Not in a U-Boot tree and $USRC not set') + return 1 + + # Load CI config + data = load_ci_yaml(uboot_dir) + if not data: + return 1 + + # Determine Docker image + image = args.image + if not image: + image = get_ci_image(data) + if not image: + return 1 + + board = args.board + + uid_gid = command.output_one_line('id', '-u') + ':' + \ + command.output_one_line('id', '-g') + + docker_cmd = ['docker', 'run', '--rm', + '--user', uid_gid, + '-e', 'HOME=/tmp', + '-v', '/etc/passwd:/etc/passwd:ro', + '-v', f'{uboot_dir}:/source', + '-w', '/source', image] + + if args.interactive: + docker_cmd.append('bash') + else: + spec = ' '.join(args.test_spec) if args.test_spec else None + extra = [] + if args.exitfirst: + extra.append('-x') + if args.show_output: + extra.append('-s') + script = build_script(data, board, spec, args.adjust_cfg, + extra or None) + if not script: + return 1 + docker_cmd.extend(['bash', '-c', script]) + + result = util.exec_cmd(docker_cmd, args.dry_run, capture=False) + if result and result.return_code: + return result.return_code + + return 0 diff --git a/uman_pkg/cmdline.py b/uman_pkg/cmdline.py index 9676682..05cee02 100644 --- a/uman_pkg/cmdline.py +++ b/uman_pkg/cmdline.py @@ -39,6 +39,7 @@ def get_git_action_names(): ALIASES = { 'claude-code': ['cc'], 'config': ['cfg'], + 'docker': ['d'], 'git': ['g'], 'selftest': ['st'], 'pytest': ['py'], @@ -97,6 +98,23 @@ def add_claude_code_subparser(subparsers): return cc +def add_docker_subparser(subparsers): + """Add the 'docker' subparser""" + dtest = subparsers.add_parser( + 'docker', aliases=ALIASES['docker'], + help='Run U-Boot tests in CI Docker container') + add_test_opts(dtest, board_help='Board name (default: sandbox)', + board_default='sandbox') + dtest.add_argument('-a', '--adjust-cfg', action='append', + metavar='CFG', dest='adjust_cfg', + help='Adjust Kconfig (can use multiple times)') + dtest.add_argument('-i', '--image', metavar='IMAGE', + help='Override Docker image') + dtest.add_argument('-I', '--interactive', action='store_true', + help='Drop to shell in container') + return dtest + + def add_ci_subparser(subparsers): """Add the 'ci' subparser""" ci = subparsers.add_parser('ci', help='Push current branch to CI') @@ -212,15 +230,11 @@ def add_pytest_subparser(subparsers): pyt = subparsers.add_parser( 'pytest', aliases=ALIASES['pytest'], help='Run pytest tests for U-Boot') - pyt.add_argument( - 'test_spec', type=str, nargs='*', - help="Test specification (e.g. 'test_dm', 'not sleep')") + add_test_opts(pyt, + board_help='Board name to test (required; use -l to list QEMU boards)') pyt.add_argument( '-b', '--build', action='store_true', help='Build U-Boot before running tests') - pyt.add_argument( - '-B', '--board', metavar='BOARD', - help='Board name to test (required; use -l to list QEMU boards)') pyt.add_argument( '-c', '--show-cmd', action='store_true', help='Show QEMU command line without running tests') @@ -248,9 +262,6 @@ def add_pytest_subparser(subparsers): pyt.add_argument( '-q', '--quiet', action='store_true', help='Quiet mode: only show build output, progress, and result') - pyt.add_argument( - '-s', '--show-output', action='store_true', - help='Show all test output in real-time (pytest -s)') pyt.add_argument( '-S', '--setup-only', action='store_true', help='Run only fixture setup (create test images) without tests') @@ -261,9 +272,6 @@ def add_pytest_subparser(subparsers): pyt.add_argument( '--no-timeout', action='store_true', help='Disable test timeout') - pyt.add_argument( - '-x', '--exitfirst', action='store_true', - help='Stop on first test failure') pyt.add_argument( '--pollute', metavar='TEST', help='Find which test pollutes TEST (causes it to fail)') @@ -473,6 +481,7 @@ def setup_parser(): add_claude_code_subparser(subparsers) add_ci_subparser(subparsers) add_config_subparser(subparsers) + add_docker_subparser(subparsers) add_git_subparser(subparsers) add_selftest_subparser(subparsers) add_pytest_subparser(subparsers) diff --git a/uman_pkg/control.py b/uman_pkg/control.py index a0334a9..2936d15 100644 --- a/uman_pkg/control.py +++ b/uman_pkg/control.py @@ -329,6 +329,10 @@ def run_command(args): # pylint: disable=R0911 from uman_pkg import cmdconfig return cmdconfig.run(args) + if args.cmd == 'docker': + from uman_pkg import cmddocker + return cmddocker.run(args) + if args.cmd == 'git': from uman_pkg import cmdgit return cmdgit.run(args) diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index 4fafccc..e37a94d 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -21,8 +21,8 @@ from u_boot_pylib import tout import gitlab -from uman_pkg import (build, cc, cmdconfig, cmdgit, cmdline, cmdpy, cmdtest, - control, gitlab_parser, settings, setup, util) +from uman_pkg import (build, cc, cmdconfig, cmddocker, cmdgit, cmdline, cmdpy, + cmdtest, control, gitlab_parser, settings, setup, util) # Capture stdout and stderr for silent command execution CAPTURE = {'capture': True, 'capture_stderr': True} @@ -6351,3 +6351,193 @@ def test_get_board_config_fallback(self, mock_socket, mock_settings): self.assertEqual('qemu', result['console_impl']) self.assertEqual('qemu-system-arm', result['qemu_binary']) + + +class TestDockerTest(TestBase): + """Test the docker (d) subcommand""" + + GITLAB_CI = '''default: + image: ${MIRROR_DOCKER}/sjg20/u-boot-ci-runner:jammy-20250404-abc123 + +.buildman_and_testpy_template: + before_script: + - git config --global --add safe.directory "${CI_PROJECT_DIR}" + - ln -s travis-ci test/hooks/bin/`hostname` + - ln -s travis-ci test/hooks/py/`hostname` + - python3 -m venv /tmp/venv; + . /tmp/venv/bin/activate; + pip install -r test/py/requirements.txt + script: + - export UBOOT_TRAVIS_BUILD_DIR=/tmp/${TEST_PY_BD} + - tools/buildman/buildman -o ${UBOOT_TRAVIS_BUILD_DIR} -w -E -W -e + --board ${TEST_PY_BD} ${OVERRIDE} + - cp /opt/grub/grub_x86.efi $UBOOT_TRAVIS_BUILD_DIR/ + - if [[ -n "${TEST_SPEC}" ]]; then + SPEC="${TEST_SPEC}"; + else + SPEC="${TEST_PY_TEST_SPEC}"; + fi + - export PATH=/opt/qemu/bin:test/hooks/bin:${PATH}; + ./test/py/test.py -ra --bd ${TEST_PY_BD} + ${SPEC:+"-k ${SPEC}"} + --build-dir "$UBOOT_TRAVIS_BUILD_DIR" + +stages: + - build +''' + + def setUp(self): + super().setUp() + tout.init(tout.NOTICE) + self.orig_cwd = os.getcwd() + self.orig_usrc = os.environ.get('USRC') + if 'USRC' in os.environ: + del os.environ['USRC'] + + # Create a fake U-Boot tree + os.chdir(self.test_dir) + os.makedirs('test/py') + tools.write_file('test/py/test.py', b'# test') + tools.write_file('.gitlab-ci.yml', + self.GITLAB_CI.encode(), binary=True) + + def tearDown(self): + os.chdir(self.orig_cwd) + if self.orig_usrc is not None: + os.environ['USRC'] = self.orig_usrc + elif 'USRC' in os.environ: + del os.environ['USRC'] + super().tearDown() + + def test_load_ci_yaml(self): + """Test loading .gitlab-ci.yml""" + data = cmddocker.load_ci_yaml(self.test_dir) + self.assertIsNotNone(data) + self.assertIn('default', data) + + def test_load_ci_yaml_missing(self): + """Test load_ci_yaml with missing file""" + with terminal.capture() as (out, err): + data = cmddocker.load_ci_yaml(self.orig_cwd) + self.assertIsNone(data) + + def test_get_ci_image(self): + """Test parsing Docker image from parsed YAML""" + data = cmddocker.load_ci_yaml(self.test_dir) + image = cmddocker.get_ci_image(data) + self.assertEqual( + 'docker.io/sjg20/u-boot-ci-runner:jammy-20250404-abc123', + image) + + def test_get_ci_image_no_match(self): + """Test get_ci_image when image is not found""" + with terminal.capture() as (out, err): + image = cmddocker.get_ci_image({'stages': ['build']}) + self.assertIsNone(image) + self.assertEqual('Cannot find image in .gitlab-ci.yml\n', + err.getvalue()) + + def test_get_ci_script(self): + """Test extracting script from parsed YAML""" + data = cmddocker.load_ci_yaml(self.test_dir) + before, script = cmddocker.get_ci_script(data) + self.assertTrue(len(before) > 0) + self.assertTrue(len(script) > 0) + + def test_build_script(self): + """Test script generation without test spec""" + data = cmddocker.load_ci_yaml(self.test_dir) + script = cmddocker.build_script(data, 'sandbox', None) + self.assertIn('--board sandbox', script) + self.assertIn('UBOOT_TRAVIS_BUILD_DIR=/tmp/sandbox', script) + self.assertIn('cp /opt/grub/grub_x86.efi', script) + + def test_build_script_test_spec(self): + """Test script generation with test spec""" + data = cmddocker.load_ci_yaml(self.test_dir) + script = cmddocker.build_script(data, 'sandbox', + 'test_ofplatdata') + self.assertIn('test_ofplatdata', script) + + def test_build_script_board(self): + """Test script generation with a different board""" + data = cmddocker.load_ci_yaml(self.test_dir) + script = cmddocker.build_script(data, 'sandbox_flattree', None) + self.assertIn('--board sandbox_flattree', script) + self.assertIn('UBOOT_TRAVIS_BUILD_DIR=/tmp/sandbox_flattree', + script) + + def test_run_dry_run(self): + """Test dry-run shows docker command""" + args = cmdline.parse_args(['-n', 'docker']) + with terminal.capture() as (out, err): + res = cmddocker.run(args) + self.assertEqual(0, res) + output = out.getvalue() + self.assertIn('docker run --rm', output) + self.assertIn('--user', output) + self.assertIn('HOME=/tmp', output) + self.assertIn('/etc/passwd:/etc/passwd:ro', output) + self.assertIn('docker.io/sjg20/u-boot-ci-runner', output) + self.assertFalse(err.getvalue()) + + def test_run_dry_run_with_spec(self): + """Test dry-run with test spec""" + args = cmdline.parse_args(['-n', 'd', 'test_ofplatdata']) + with terminal.capture() as (out, err): + res = cmddocker.run(args) + self.assertEqual(0, res) + output = out.getvalue() + self.assertIn('test_ofplatdata', output) + self.assertFalse(err.getvalue()) + + def test_run_dry_run_interactive(self): + """Test dry-run in interactive mode""" + args = cmdline.parse_args(['-n', 'docker', '-I']) + with terminal.capture() as (out, err): + res = cmddocker.run(args) + self.assertEqual(0, res) + output = out.getvalue() + self.assertIn('docker run --rm', output) + # Interactive mode should just run bash, not bash -c