diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..d8839c1 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,27 @@ +name: Tests + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10', '3.12'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - run: pip install -r requirements.txt pytest pylint + - run: > + python -m pytest uman_pkg/ftest.py -v -k + 'not TestCcFunctional + and not TestGitRebase + and not TestUmanControl + and not TestUmanMergeRequest' + - run: python3 -m pylint -E uman_pkg/ftest.py diff --git a/CLAUDE.md b/CLAUDE.md index a3145bd..47c30b8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,8 +8,8 @@ ## Testing -- Run tests with: `PYTHONPATH=~/u/tools python -m pytest uman_pkg/ftest.py -v` -- Run pylint with: `PYTHONPATH=~/u/tools python3 -m pylint uman_pkg/ftest.py` +- Run tests with: `python -m pytest uman_pkg/ftest.py -v` +- Run pylint with: `python3 -m pylint uman_pkg/ftest.py` ### Test conventions diff --git a/README.rst b/README.rst index daa5d83..2567499 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,11 @@ uman - U-Boot Manager ===================== +.. note:: + + Parts of this code and documentation were written with AI assistance. + Review all changes carefully before relying on them. + This is a simple tool to handle common tasks when developing U-Boot, including pushing to CI, running tests, and setting up firmware dependencies. @@ -23,6 +28,9 @@ Subcommands ``ci`` Push current branch to GitLab CI with configurable test stages +``docker`` (alias: ``d``) + Run U-Boot tests in the same Docker container used by CI + ``pytest`` (alias: ``py``) Run U-Boot's test.py framework with automatic environment setup @@ -181,6 +189,96 @@ run), not fine-grained selection of specific boards or test specifications. For precise targeting like ``-p coreboot`` or ``-t "test_ofplatdata"``, use regular CI pushes instead of merge requests. +Docker Subcommand +----------------- + +The ``docker`` command (alias ``d``) runs U-Boot tests inside the same Docker +image used by GitLab CI. It parses ``.gitlab-ci.yml`` from the U-Boot tree to +determine the Docker image and build/test script, so the local test environment +matches CI exactly. + +The container bind-mounts the U-Boot source directory (from ``$USRC`` or the +current directory) at ``/source`` and runs as the current user, so build +artefacts have the correct ownership. + +:: + + # Run all sandbox tests (board defaults to sandbox) + uman docker + + # Run specific tests + uman docker test_ofplatdata or test_handoff + + # Test a different board + uman docker -B sandbox_noinst test_ofplatdata + + # Stop on first failure, show output + uman docker -x -s test_dm + + # Adjust Kconfig before building + uman docker -a CONFIG_TRACE test_trace + + # Drop to an interactive shell in the container + uman docker -I + + # Debug u-boot under gdbserver + uman docker -g -B sandbox_noinst test_spl + + # Debug SPL under gdbserver + uman docker --gdb-phase spl -B sandbox_noinst test_spl + + # Override the Docker image + uman docker -i my-registry/u-boot-ci:latest + + # Dry-run to see the docker command + uman -n docker -B sandbox test_dm + +**Options**: + +- ``test_spec``: Test specification using pytest -k syntax (positional) +- ``-a, --adjust-cfg CFG``: Adjust Kconfig setting (can use multiple times) +- ``-B, --board BOARD``: Board name (default: sandbox) +- ``-g``: Debug with gdbserver (u-boot phase; see below) +- ``--gdb-phase PHASE``: Debug a specific phase (spl, tpl, vpl) +- ``-i, --image IMAGE``: Override Docker image (default: from .gitlab-ci.yml) +- ``-I, --interactive``: Drop to a bash shell in the container +- ``-s, --show-output``: Show all test output in real-time (pytest -s) +- ``-x, --exitfirst``: Stop on first test failure + +**Debugging with GDB**: + +The ``-g`` flag enables gdbserver inside the Docker container. It installs +gdbserver (as root), exposes port 1234, and prints instructions for +connecting from another terminal. + +Use ``--gdb-phase`` to select which binary to debug: + +- ``-g``: Debug the main U-Boot binary. A wrapper script replaces ``u-boot`` + so that gdbserver starts only after SPL has finished. SPL runs normally, + then exec's the wrapper which starts gdbserver on the real ``u-boot`` + binary. This avoids gdb's inability to follow exec calls over remote + debugging. + +- ``--gdb-phase spl``: Debug SPL directly. Passes ``--gdbserver`` to test.py + which wraps the initial SPL binary with gdbserver. Use this when the + problem is in SPL itself. + +Workflow:: + + # Terminal 1: start tests with gdbserver + uman docker -g -B sandbox_noinst test_spl + + # Terminal 2: connect gdb (after "Listening on port 1234" appears) + um py -G -B sandbox_noinst + (gdb) c + +The ``-G`` flag in ``um py`` launches ``gdb-multiarch``, loads symbols from +the local build (``/tmp/b//u-boot``), and connects to +``localhost:1234``. SIGUSR2 is automatically silenced since sandbox uses it +internally for coroutine setup. Type ``c`` to continue execution; tests then +proceed normally. Set breakpoints before continuing to catch specific code +paths. + CC Subcommand ------------- @@ -189,6 +287,19 @@ the current directory as a project, installs build tools and Claude Code, and sets up uman aliases inside the container. The container name defaults to the current directory name and is permanent. Use ``-e`` for a throwaway container. +**Prerequisites**: + +LXD must be installed and initialised:: + + sudo snap install lxd + lxd init --minimal + +Add your user to the ``lxd`` group if not already a member:: + + sudo usermod -aG lxd $USER + +Log out and back in for the group change to take effect. + :: # Launch Claude Code (container named after current directory) @@ -215,6 +326,19 @@ current directory name and is permanent. Use ``-e`` for a throwaway container. # Delete a container uman cc -d mybox + # Mount extra host directories + uman cc -m /opt/data + uman cc -m /opt/data:/mnt/data + + # List mounts for a container + uman cc -M mybox + + # Remove a mount (use -M to see device names) + uman cc -u data mybox + + # Enable device-mapper for LUKS encryption tests + uman cc -p + # Dry-run to see what would be executed uman -n cc @@ -230,9 +354,17 @@ idempotent setup steps. - ``-d, --delete``: Delete the named container - ``-e, --ephemeral``: Use a random name and delete on exit - ``-l, --list``: List existing uman containers with project paths +- ``-m, --mount PATH``: Mount a host directory (see **Mounts** below) +- ``-M, --mounts``: List mounts for the container +- ``-o, --output``: Mount ``/tmp/b`` into the container +- ``-O, --no-output``: Remove the ``/tmp/b`` mount +- ``-u, --unmount NAME``: Remove a mount by device name (see ``-M`` for names) - ``-r, --rename NEW``: Rename the named container - ``-R, --restart``: Restart the container before launching - ``-S, --stop``: Stop a running container +- ``-p, --privileged``: Enable privileged mode for device-mapper (e.g. LUKS tests); + if the container is already running, prints a message about restarting +- ``-P, --no-privileged``: Disable privileged mode (auto-restarts and restores uid) - ``-s, --shell [CMD]``: Open interactive shell, or run CMD in container **Console Logging**: @@ -247,6 +379,43 @@ For example: ``~/files/dev/uman-logs/paperman/2026/Feb/log-26.02feb.21-143022.lo The log path is printed at launch. The ``-q`` flag suppresses script's own start/done messages so only the session content is captured. +**Clipboard (Image Paste)**: + +The X11 socket is mounted into the container and ``xclip`` is installed so +that Claude Code can access the clipboard for image paste (Ctrl-V). The host +must allow local X11 connections:: + + xhost +local: + +This is checked at launch and a reminder is printed if not set. Add the +command to ``~/.bashrc`` to make it permanent. For existing containers, +install xclip manually: ``sudo apt-get install -yqq xclip`` + +**Editor Proxy**: + +An editor proxy runs on the host so that Ctrl-G in Claude Code opens the host +``$EDITOR``. For files inside the project, the container path is translated to +the host path. For other files (e.g. temp files created by Ctrl-G for prompt +editing), the content is transferred over the socket and a host temp file is +used. + +**Voice Input**: + +The PulseAudio socket is mounted into the container and ``sox`` is installed +so that Claude Code's ``/voice`` command can access the host microphone. For +existing containers, install sox manually: +``sudo apt-get install -yqq sox libsox-fmt-pulse libasound2-plugins`` + +**GitHub / GitLab CLI**: + +The ``gh`` and ``glab`` CLI tools are installed for managing pull requests and +CI pipelines. Authenticate on first use:: + + gh auth login + glab auth login + +Credentials are stored inside the container and persist across restarts. + **Essential Mounts** (always added): - ``datadir``: Current directory to ``/home/ubuntu/project`` @@ -258,12 +427,30 @@ start/done messages so only the session content is captured. - ``hostbin``: ``~/bin`` for host scripts - ``uman``: Uman install directory (so ``~/bin`` symlinks work) - ``uboottools``: U-Boot tools directory (``$UBOOT_TOOLS`` or ``~/u/tools``) +- ``patman``: ``~/dev/patman`` for patch workflows (if present) +- ``pulse``: PulseAudio socket for voice input (if present) +- ``x11``: ``/tmp/.X11-unix`` for clipboard access (if present) - ``tmpb``: Container ``/tmp/b`` to ``/tmp//b`` on the host - ``buildman``: ``~/.buildman`` (if present) - ``toolchains``: ``~/.buildman-toolchains`` (if present) - ``pbuilder``: ``/var/cache/pbuilder`` (if present), with uid/gid shift - ``dotgit``: If ``.git`` is a symlink, the real target is mounted +**Mounts** (``-m``): + +The ``-m`` flag mounts a host directory into the container. It accepts +``HOST:DEST`` or just ``HOST`` (mounted at the same path). It can be repeated +for multiple mounts:: + + uman cc -m ~/dev/linux # mount at /home/ubuntu/dev/linux + uman cc -m /opt/data:/mnt/data # mount at /mnt/data + +Used alone, ``-m`` adds the mount without entering the container. Combine with +``-s`` to also enter a shell. Tilde (``~``) in the destination expands to the +container home (``/home/ubuntu``) rather than the host home. The device name is +derived from the leaf directory (e.g. ``linux`` for ``~/dev/linux``) and is +shown on success. + **Configuration** (``~/.uman``): Add a ``[claude-code]`` section to configure additional mounts and packages:: @@ -293,7 +480,7 @@ making it easier to step through commits during development. - ``cms`` / ``commit-signoff``: Commit with signoff (git commit --signoff) - ``co`` / ``checkout``: Checkout (switch branches or restore files) - ``db`` / ``diff-branch`` [BRANCH]: Diff current commit files against upstream (or BRANCH) -- ``dh`` / ``diff-head`` [N] [FILES...]: Show diff using difftool (git difftool HEAD~ or HEAD~N) +- ``di`` / ``diff-head`` [N] [FILES...]: Show diff using difftool (git difftool HEAD~ or HEAD~N) - ``eg`` / ``errno-grep`` PATTERN: Search include/linux/errno.h for error codes - ``et`` / ``edit-todo``: Edit the rebase todo list - ``fa`` / ``find-all`` [N]: Check all branches against us/master (default 5 commits each) @@ -516,6 +703,7 @@ hooks to PATH. - ``--force-reconfig``: Force reconfiguration (use with -b) - ``--fresh``: Delete build dir before building (use with -b) - ``-g``: Run sandbox under gdbserver at localhost:1234 +- ``--gdb-phase PHASE``: Debug a specific phase (spl, tpl, vpl) - ``-G, --gdb``: Launch gdb-multiarch and connect to an existing gdbserver - ``-j, --jobs JOBS``: Number of parallel jobs (use with -b) - ``-l, --list``: List available QEMU and sandbox boards @@ -528,6 +716,8 @@ hooks to PATH. - ``-T, --trace``: Enable function tracing; adds CONFIG_TRACE and CONFIG_TRACE_EARLY (use with -b) - ``--no-trace-early``: Disable TRACE_EARLY when using -T (use with -b) +- ``--malloc-dump FILE``: Write malloc heap dump on exit; ``%d`` in the filename + is expanded to a sequence number - ``--no-timeout``: Disable test timeout - ``-x, --exitfirst``: Stop on first test failure - ``--pollute TEST``: Find which test pollutes TEST @@ -656,6 +846,9 @@ without going through pytest. This is faster for quick iteration on C code. # Run test using pytest-style name (ut__) uman test ut_bootstd_bootflow + # Run tests matching a wildcard pattern + uman test 'dm.adj*' + # List available suites uman test -s @@ -674,6 +867,8 @@ without going through pytest. This is faster for quick iteration on C code. - ``-l, --list``: List available tests - ``-L, --lto``: Enable LTO when building (use with -b) - ``--legacy``: Use legacy result parsing (for old U-Boot) +- ``--malloc-dump FILE``: Write malloc heap dump on exit; ``%d`` in the filename + is expanded to a sequence number - ``-m, --manual``: Force manual tests to run (tests with _norun suffix) - ``-o, --output-dir DIR``: Override build directory (use with -b) - ``-r, --results``: Show per-test pass/fail status @@ -847,18 +1042,19 @@ Settings are stored in ``~/.uman`` (created on first run):: Environment Variables ~~~~~~~~~~~~~~~~~~~~~ +``UMAN_EXTERNAL_PYLIB`` + Set to ``1`` to use u_boot_pylib from ``UBOOT_TOOLS`` instead of the + embedded copy. Useful for testing against a newer version of the library. + ``UBOOT_TOOLS`` Path to U-Boot tools directory containing Python libraries (u_boot_pylib, - patman, buildman, etc.). This is used for importing Python modules. + patman, buildman, etc.). Only used when ``UMAN_EXTERNAL_PYLIB=1``. Default: ``~/u/tools`` ``USRC`` Path to U-Boot source tree to work in. If not set, uman expects to be run from within a U-Boot source tree. -These are separate: ``UBOOT_TOOLS`` specifies where to find Python imports, -while ``USRC`` specifies the U-Boot source tree to build/test. - Self-testing ------------ diff --git a/u_boot_pylib/__init__.py b/u_boot_pylib/__init__.py new file mode 100644 index 0000000..807a62e --- /dev/null +++ b/u_boot_pylib/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0+ + +__all__ = ['command', 'cros_subprocess', 'gitutil', 'terminal', 'test_util', + 'tools', 'tout'] diff --git a/u_boot_pylib/command.py b/u_boot_pylib/command.py new file mode 100644 index 0000000..6b3f9fe --- /dev/null +++ b/u_boot_pylib/command.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: GPL-2.0+ +""" +Shell command ease-ups for Python + +Copyright (c) 2011 The Chromium OS Authors. +""" + +import subprocess + +from u_boot_pylib import cros_subprocess + +# This permits interception of RunPipe for test purposes. If it is set to +# a function, then that function is called with the pipe list being +# executed. Otherwise, it is assumed to be a CommandResult object, and is +# returned as the result for every run_pipe() call. +# When this value is None, commands are executed as normal. +TEST_RESULT = None + + +class CommandExc(Exception): + """Reports an exception to the caller""" + def __init__(self, msg, result): + """Set up a new exception object + + Args: + result (CommandResult): Execution result so far + """ + super().__init__(msg) + self.result = result + + +class CommandResult: + """A class which captures the result of executing a command. + + Members: + stdout (bytes): stdout obtained from command, as a string + stderr (bytes): stderr obtained from command, as a string + combined (bytes): stdout and stderr interleaved + return_code (int): Return code from command + exception (Exception): Exception received, or None if all ok + output (str or None): Returns output as a single line if requested + """ + def __init__(self, stdout='', stderr='', combined='', return_code=0, + exception=None): + self.stdout = stdout + self.stderr = stderr + self.combined = combined + self.return_code = return_code + self.exception = exception + self.output = None + + def to_output(self, binary): + """Converts binary output to its final form + + Args: + binary (bool): True to report binary output, False to use strings + Returns: + self + """ + if not binary: + self.stdout = self.stdout.decode('utf-8') + self.stderr = self.stderr.decode('utf-8') + self.combined = self.combined.decode('utf-8') + return self + + +def run_pipe(pipe_list, infile=None, outfile=None, capture=False, + capture_stderr=False, oneline=False, raise_on_error=True, cwd=None, + binary=False, output_func=None, **kwargs): + """ + Perform a command pipeline, with optional input/output filenames. + + Args: + pipe_list (list of list): List of command lines to execute. Each command + line is piped into the next, and is itself a list of strings. For + example [ ['ls', '.git'] ['wc'] ] will pipe the output of + 'ls .git' into 'wc'. + infile (str): File to provide stdin to the pipeline + outfile (str): File to store stdout + capture (bool): True to capture output + capture_stderr (bool): True to capture stderr + oneline (bool): True to strip newline chars from output + raise_on_error (bool): True to raise on an error, False to return it in + the CommandResult + cwd (str or None): Directory to run the command in + binary (bool): True to report binary output, False to use strings + output_func (function): Output function to call with each output + fragment (if it returns True the function terminates) + **kwargs: Additional keyword arguments to cros_subprocess.Popen() + Returns: + CommandResult object + Raises: + CommandExc if an exception happens + """ + if TEST_RESULT: + if hasattr(TEST_RESULT, '__call__'): + # pylint: disable=E1102 + result = TEST_RESULT(pipe_list=pipe_list) + if result: + return result + else: + return TEST_RESULT + # No result: fall through to normal processing + result = CommandResult(b'', b'', b'') + last_pipe = None + pipeline = list(pipe_list) + user_pipestr = '|'.join([' '.join(pipe) for pipe in pipe_list]) + kwargs['stdout'] = None + kwargs['stderr'] = None + while pipeline: + cmd = pipeline.pop(0) + if last_pipe is not None: + kwargs['stdin'] = last_pipe.stdout + elif infile: + kwargs['stdin'] = open(infile, 'rb') + if pipeline or capture: + kwargs['stdout'] = cros_subprocess.PIPE + elif outfile: + kwargs['stdout'] = open(outfile, 'wb') + if capture_stderr: + kwargs['stderr'] = cros_subprocess.PIPE + + try: + last_pipe = cros_subprocess.Popen(cmd, cwd=cwd, **kwargs) + except Exception as err: + result.exception = err + if raise_on_error: + raise CommandExc(f"Error running '{user_pipestr}': {err}", + result) from err + result.return_code = 255 + return result.to_output(binary) + + if capture: + result.stdout, result.stderr, result.combined = ( + last_pipe.communicate_filter(output_func)) + if result.stdout and oneline: + result.output = result.stdout.rstrip(b'\r\n') + result.return_code = last_pipe.wait() + result = result.to_output(binary) + if raise_on_error and result.return_code: + raise CommandExc(f"Error running '{user_pipestr}'", result) + return result + + +def output(*cmd, **kwargs): + """Run a command and return its output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + str: command output + """ + kwargs['raise_on_error'] = kwargs.get('raise_on_error', True) + return run_pipe([cmd], capture=True, **kwargs).stdout + + +def output_one_line(*cmd, **kwargs): + """Run a command and output it as a single-line string + + The command is expected to produce a single line of output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + str: output of command with all newlines removed + """ + raise_on_error = kwargs.pop('raise_on_error', True) + result = run_pipe([cmd], capture=True, oneline=True, + raise_on_error=raise_on_error, **kwargs).stdout.strip() + return result + + +def run(*cmd, **kwargs): + """Run a command + + Note that you must add 'capture' to kwargs to obtain non-empty output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + str: output of command + """ + return run_pipe([cmd], **kwargs).stdout + + +def run_one(*cmd, **kwargs): + """Run a single command + + Note that you must add 'capture' to kwargs to obtain non-empty output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + CommandResult: output of command + """ + return run_pipe([cmd], **kwargs) + + +def run_list(cmd, **kwargs): + """Run a command and return its output + + Args: + cmd (list of str): Command to run + + Returns: + str: output of command + **kwargs (dict of args): Extra arguments to pass in + """ + return run_pipe([cmd], capture=True, **kwargs).stdout + + +def stop_all(): + """Stop all subprocesses initiated with cros_subprocess""" + cros_subprocess.stay_alive = False diff --git a/u_boot_pylib/cros_subprocess.py b/u_boot_pylib/cros_subprocess.py new file mode 100644 index 0000000..cd614f3 --- /dev/null +++ b/u_boot_pylib/cros_subprocess.py @@ -0,0 +1,401 @@ +# Copyright (c) 2012 The Chromium OS Authors. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# Copyright (c) 2003-2005 by Peter Astrand +# Licensed to PSF under a Contributor Agreement. +# See http://www.python.org/2.4/license for licensing details. + +"""Subprocess execution + +This module holds a subclass of subprocess.Popen with our own required +features, mainly that we get access to the subprocess output while it +is running rather than just at the end. This makes it easier to show +progress information and filter output in real time. +""" + +import errno +import os +import pty +import select +import subprocess +import sys +import unittest + + +# Import these here so the caller does not need to import subprocess also. +PIPE = subprocess.PIPE +STDOUT = subprocess.STDOUT +PIPE_PTY = -3 # Pipe output through a pty +stay_alive = True + + +class Popen(subprocess.Popen): + """Like subprocess.Popen with ptys and incremental output + + This class deals with running a child process and filtering its output on + both stdout and stderr while it is running. We do this so we can monitor + progress, and possibly relay the output to the user if requested. + + The class is similar to subprocess.Popen, the equivalent is something like: + + Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + But this class has many fewer features, and two enhancement: + + 1. Rather than getting the output data only at the end, this class sends it + to a provided operation as it arrives. + 2. We use pseudo terminals so that the child will hopefully flush its output + to us as soon as it is produced, rather than waiting for the end of a + line. + + Use communicate_filter() to handle output from the subprocess. + + """ + + def __init__(self, args, stdin=None, stdout=PIPE_PTY, stderr=PIPE_PTY, + shell=False, cwd=None, env=None, **kwargs): + """Cut-down constructor + + Args: + args: Program and arguments for subprocess to execute. + stdin: See subprocess.Popen() + stdout: See subprocess.Popen(), except that we support the sentinel + value of cros_subprocess.PIPE_PTY. + stderr: See subprocess.Popen(), except that we support the sentinel + value of cros_subprocess.PIPE_PTY. + shell: See subprocess.Popen() + cwd: Working directory to change to for subprocess, or None if none. + env: Environment to use for this subprocess, or None to inherit parent. + kwargs: No other arguments are supported at the moment. Passing other + arguments will cause a ValueError to be raised. + """ + stdout_pty = None + stderr_pty = None + + if stdout == PIPE_PTY: + stdout_pty = pty.openpty() + stdout = os.fdopen(stdout_pty[1]) + if stderr == PIPE_PTY: + stderr_pty = pty.openpty() + stderr = os.fdopen(stderr_pty[1]) + + super(Popen, self).__init__(args, stdin=stdin, + stdout=stdout, stderr=stderr, shell=shell, cwd=cwd, env=env, + **kwargs) + + # If we're on a PTY, we passed the slave half of the PTY to the subprocess. + # We want to use the master half on our end from now on. Setting this here + # does make some assumptions about the implementation of subprocess, but + # those assumptions are pretty minor. + + # Note that if stderr is STDOUT, then self.stderr will be set to None by + # this constructor. + if stdout_pty is not None: + self.stdout = os.fdopen(stdout_pty[0]) + if stderr_pty is not None: + self.stderr = os.fdopen(stderr_pty[0]) + + # Insist that unit tests exist for other arguments we don't support. + if kwargs: + raise ValueError("Unit tests do not test extra args - please add tests") + + def convert_data(self, data): + """Convert stdout/stderr data to the correct format for output + + Args: + data: Data to convert, or None for '' + + Returns: + Converted data, as bytes + """ + if data is None: + return b'' + return data + + def communicate_filter(self, output, input_buf=''): + """Interact with process: Read data from stdout and stderr. + + This method runs until end-of-file is reached, then waits for the + subprocess to terminate. + + The output function is sent all output from the subprocess and must be + defined like this: + + def output([self,] stream, data) + Args: + stream: the stream the output was received on, which will be + sys.stdout or sys.stderr. + data: a string containing the data + + Returns: + True to terminate the process + + Note: The data read is buffered in memory, so do not use this + method if the data size is large or unlimited. + + Args: + output: Function to call with each fragment of output. + + Returns: + A tuple (stdout, stderr, combined) which is the data received on + stdout, stderr and the combined data (interleaved stdout and stderr). + + Note that the interleaved output will only be sensible if you have + set both stdout and stderr to PIPE or PIPE_PTY. Even then it depends on + the timing of the output in the subprocess. If a subprocess flips + between stdout and stderr quickly in succession, by the time we come to + read the output from each we may see several lines in each, and will read + all the stdout lines, then all the stderr lines. So the interleaving + may not be correct. In this case you might want to pass + stderr=cros_subprocess.STDOUT to the constructor. + + This feature is still useful for subprocesses where stderr is + rarely used and indicates an error. + + Note also that if you set stderr to STDOUT, then stderr will be empty + and the combined output will just be the same as stdout. + """ + + read_set = [] + write_set = [] + stdout = None # Return + stderr = None # Return + + if self.stdin: + # Flush stdio buffer. This might block, if the user has + # been writing to .stdin in an uncontrolled fashion. + self.stdin.flush() + if input_buf: + write_set.append(self.stdin) + else: + self.stdin.close() + if self.stdout: + read_set.append(self.stdout) + stdout = bytearray() + if self.stderr and self.stderr != self.stdout: + read_set.append(self.stderr) + stderr = bytearray() + combined = bytearray() + + stop_now = False + input_offset = 0 + while read_set or write_set: + try: + rlist, wlist, _ = select.select(read_set, write_set, [], 0.2) + except select.error as e: + if e.args[0] == errno.EINTR: + continue + raise + + if not stay_alive: + self.terminate() + + if self.stdin in wlist: + # When select has indicated that the file is writable, + # we can write up to PIPE_BUF bytes without risk + # blocking. POSIX defines PIPE_BUF >= 512 + chunk = input_buf[input_offset : input_offset + 512] + bytes_written = os.write(self.stdin.fileno(), chunk) + input_offset += bytes_written + if input_offset >= len(input_buf): + self.stdin.close() + write_set.remove(self.stdin) + + if self.stdout in rlist: + data = b'' + # We will get an error on read if the pty is closed + try: + data = os.read(self.stdout.fileno(), 1024) + except OSError: + pass + if not len(data): + self.stdout.close() + read_set.remove(self.stdout) + else: + stdout += data + combined += data + if output: + stop_now = output(sys.stdout, data) + if self.stderr in rlist: + data = b'' + # We will get an error on read if the pty is closed + try: + data = os.read(self.stderr.fileno(), 1024) + except OSError: + pass + if not len(data): + self.stderr.close() + read_set.remove(self.stderr) + else: + stderr += data + combined += data + if output: + stop_now = output(sys.stderr, data) + if stop_now: + self.terminate() + + # All data exchanged. Translate lists into strings. + stdout = self.convert_data(stdout) + stderr = self.convert_data(stderr) + combined = self.convert_data(combined) + + self.wait() + return (stdout, stderr, combined) + + +# Just being a unittest.TestCase gives us 14 public methods. Unless we +# disable this, we can only have 6 tests in a TestCase. That's not enough. +# +# pylint: disable=R0904 + +class TestSubprocess(unittest.TestCase): + """Our simple unit test for this module""" + + class MyOperation: + """Provides a operation that we can pass to Popen""" + def __init__(self, input_to_send=None): + """Constructor to set up the operation and possible input. + + Args: + input_to_send: a text string to send when we first get input. We will + add \r\n to the string. + """ + self.stdout_data = '' + self.stderr_data = '' + self.combined_data = '' + self.stdin_pipe = None + self._input_to_send = input_to_send + if input_to_send: + pipe = os.pipe() + self.stdin_read_pipe = pipe[0] + self._stdin_write_pipe = os.fdopen(pipe[1], 'w') + + def output(self, stream, data): + """Output handler for Popen. Stores the data for later comparison""" + if stream == sys.stdout: + self.stdout_data += data + if stream == sys.stderr: + self.stderr_data += data + self.combined_data += data + + # Output the input string if we have one. + if self._input_to_send: + self._stdin_write_pipe.write(self._input_to_send + '\r\n') + self._stdin_write_pipe.flush() + + def _basic_check(self, plist, oper): + """Basic checks that the output looks sane.""" + self.assertEqual(plist[0], oper.stdout_data) + self.assertEqual(plist[1], oper.stderr_data) + self.assertEqual(plist[2], oper.combined_data) + + # The total length of stdout and stderr should equal the combined length + self.assertEqual(len(plist[0]) + len(plist[1]), len(plist[2])) + + def test_simple(self): + """Simple redirection: Get process list""" + oper = TestSubprocess.MyOperation() + plist = Popen(['ps']).communicate_filter(oper.output) + self._basic_check(plist, oper) + + def test_stderr(self): + """Check stdout and stderr""" + oper = TestSubprocess.MyOperation() + cmd = 'echo fred >/dev/stderr && false || echo bad' + plist = Popen([cmd], shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], 'bad\r\n') + self.assertEqual(plist [1], 'fred\r\n') + + def test_shell(self): + """Check with and without shell works""" + oper = TestSubprocess.MyOperation() + cmd = 'echo test >/dev/stderr' + self.assertRaises(OSError, Popen, [cmd], shell=False) + plist = Popen([cmd], shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(len(plist [0]), 0) + self.assertEqual(plist [1], 'test\r\n') + + def test_list_args(self): + """Check with and without shell works using list arguments""" + oper = TestSubprocess.MyOperation() + cmd = ['echo', 'test', '>/dev/stderr'] + plist = Popen(cmd, shell=False).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], ' '.join(cmd[1:]) + '\r\n') + self.assertEqual(len(plist [1]), 0) + + oper = TestSubprocess.MyOperation() + + # this should be interpreted as 'echo' with the other args dropped + cmd = ['echo', 'test', '>/dev/stderr'] + plist = Popen(cmd, shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], '\r\n') + + def test_cwd(self): + """Check we can change directory""" + for shell in (False, True): + oper = TestSubprocess.MyOperation() + plist = Popen('pwd', shell=shell, cwd='/tmp').communicate_filter( + oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], '/tmp\r\n') + + def test_env(self): + """Check we can change environment""" + for add in (False, True): + oper = TestSubprocess.MyOperation() + env = os.environ + if add: + env ['FRED'] = 'fred' + cmd = 'echo $FRED' + plist = Popen(cmd, shell=True, env=env).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], add and 'fred\r\n' or '\r\n') + + def test_extra_args(self): + """Check we can't add extra arguments""" + self.assertRaises(ValueError, Popen, 'true', close_fds=False) + + def test_basic_input(self): + """Check that incremental input works + + We set up a subprocess which will prompt for name. When we see this prompt + we send the name as input to the process. It should then print the name + properly to stdout. + """ + oper = TestSubprocess.MyOperation('Flash') + prompt = 'What is your name?: ' + cmd = 'echo -n "%s"; read name; echo Hello $name' % prompt + plist = Popen([cmd], stdin=oper.stdin_read_pipe, + shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(len(plist [1]), 0) + self.assertEqual(plist [0], prompt + 'Hello Flash\r\r\n') + + def test_isatty(self): + """Check that ptys appear as terminals to the subprocess""" + oper = TestSubprocess.MyOperation() + cmd = ('if [ -t %d ]; then echo "terminal %d" >&%d; ' + 'else echo "not %d" >&%d; fi;') + both_cmds = '' + for fd in (1, 2): + both_cmds += cmd % (fd, fd, fd, fd, fd) + plist = Popen(both_cmds, shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], 'terminal 1\r\n') + self.assertEqual(plist [1], 'terminal 2\r\n') + + # Now try with PIPE and make sure it is not a terminal + oper = TestSubprocess.MyOperation() + plist = Popen(both_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], 'not 1\n') + self.assertEqual(plist [1], 'not 2\n') + +if __name__ == '__main__': + unittest.main() diff --git a/u_boot_pylib/gitutil.py b/u_boot_pylib/gitutil.py new file mode 100644 index 0000000..34b4dbb --- /dev/null +++ b/u_boot_pylib/gitutil.py @@ -0,0 +1,886 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2011 The Chromium OS Authors. +# + +"""Basic utilities for running the git command-line tool from Python""" + +import os +import sys + +from u_boot_pylib import command +from u_boot_pylib import terminal + +# True to use --no-decorate - we check this in setup() +USE_NO_DECORATE = True + + +def log_cmd(commit_range, git_dir=None, oneline=False, reverse=False, + count=None, decorate=False): + """Create a command to perform a 'git log' + + Args: + commit_range (str): Range expression to use for log, None for none + git_dir (str): Path to git repository (None to use default) + oneline (bool): True to use --oneline, else False + reverse (bool): True to reverse the log (--reverse) + count (int or None): Number of commits to list, or None for no limit + decorate (bool): True to use --decorate + + Return: + List containing command and arguments to run + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['--no-pager', 'log', '--no-color'] + if oneline: + cmd.append('--oneline') + if USE_NO_DECORATE and not decorate: + cmd.append('--no-decorate') + if decorate: + cmd.append('--decorate') + if reverse: + cmd.append('--reverse') + if count is not None: + cmd.append(f'-n{count}') + if commit_range: + cmd.append(commit_range) + + # Add this in case we have a branch with the same name as a directory. + # This avoids messages like this, for example: + # fatal: ambiguous argument 'test': both revision and filename + cmd.append('--') + return cmd + + +def count_commits_to_branch(branch, git_dir=None, end=None): + """Returns number of commits between HEAD and the tracking branch. + + This looks back to the tracking branch and works out the number of commits + since then. + + Args: + branch (str or None): Branch to count from (None for current branch) + git_dir (str): Path to git repository (None to use default) + end (str): End commit to stop before + + Return: + Number of patches that exist on top of the branch + """ + if end: + rev_range = f'{end}..{branch}' + elif branch: + us, msg = get_upstream(git_dir or '.git', branch) + if not us: + raise ValueError(msg) + rev_range = f'{us}..{branch}' + else: + rev_range = '@{upstream}..' + cmd = log_cmd(rev_range, git_dir=git_dir, oneline=True) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + oneline=True, raise_on_error=False) + if result.return_code: + raise ValueError( + f'Failed to determine upstream: {result.stderr.strip()}') + patch_count = len(result.stdout.splitlines()) + return patch_count + + +def name_revision(commit_hash): + """Gets the revision name for a commit + + Args: + commit_hash (str): Commit hash to look up + + Return: + Name of revision, if any, else None + """ + stdout = command.output_one_line('git', 'name-rev', commit_hash) + if not stdout: + return None + + # We expect a commit, a space, then a revision name + name = stdout.split()[1].strip() + return name + + +def guess_upstream(git_dir, branch): + """Tries to guess the upstream for a branch + + This lists out top commits on a branch and tries to find a suitable + upstream. It does this by looking for the first commit where + 'git name-rev' returns a plain branch name, with no ! or ^ modifiers. + + Args: + git_dir (str): Git directory containing repo + branch (str): Name of branch + + Returns: + Tuple: + Name of upstream branch (e.g. 'upstream/master') or None if none + Warning/error message, or None if none + """ + cmd = log_cmd(branch, git_dir=git_dir, oneline=True, count=100, + decorate=True) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + raise_on_error=False) + if result.return_code: + return None, f"Branch '{branch}' not found" + for line in result.stdout.splitlines()[1:]: + parts = line.split(maxsplit=1) + if len(parts) >= 2 and parts[1].startswith('('): + commit_hash = parts[0] + name = name_revision(commit_hash) + if '~' not in name and '^' not in name: + if name.startswith('remotes/'): + name = name[8:] + return name, f"Guessing upstream as '{name}'" + return None, f"Cannot find a suitable upstream for branch '{branch}'" + + +def get_upstream(git_dir, branch): + """Returns the name of the upstream for a branch + + Args: + git_dir (str): Git directory containing repo + branch (str): Name of branch + + Returns: + Tuple: + Name of upstream branch (e.g. 'upstream/master') or None if none + Warning/error message, or None if none + """ + try: + remote = command.output_one_line('git', '--git-dir', git_dir, 'config', + f'branch.{branch}.remote') + merge = command.output_one_line('git', '--git-dir', git_dir, 'config', + f'branch.{branch}.merge') + except command.CommandExc: + upstream, msg = guess_upstream(git_dir, branch) + return upstream, msg + + if remote == '.': + return merge, None + if remote and merge: + # Drop the initial refs/heads from merge + leaf = merge.split('/', maxsplit=2)[2:] + return f'{remote}/{"/".join(leaf)}', None + raise ValueError("Cannot determine upstream branch for branch " + f"'{branch}' remote='{remote}', merge='{merge}'") + + +def get_range_in_branch(git_dir, branch, include_upstream=False): + """Returns an expression for the commits in the given branch. + + Args: + git_dir (str): Directory containing git repo + branch (str): Name of branch + include_upstream (bool): Include the upstream commit as well + Return: + Expression in the form 'upstream..branch' which can be used to + access the commits. If the branch does not exist, returns None. + """ + upstream, msg = get_upstream(git_dir, branch) + if not upstream: + return None, msg + rstr = f"{upstream}{'~' if include_upstream else ''}..{branch}" + return rstr, msg + + +def count_commits_in_range(git_dir, range_expr): + """Returns the number of commits in the given range. + + Args: + git_dir (str): Directory containing git repo + range_expr (str): Range to check + Return: + Number of patches that exist in the supplied range or None if none + were found + """ + cmd = log_cmd(range_expr, git_dir=git_dir, oneline=True) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + raise_on_error=False) + if result.return_code: + return None, f"Range '{range_expr}' not found or is invalid" + patch_count = len(result.stdout.splitlines()) + return patch_count, None + + +def count_commits_in_branch(git_dir, branch, include_upstream=False): + """Returns the number of commits in the given branch. + + Args: + git_dir (str): Directory containing git repo + branch (str): Name of branch + include_upstream (bool): Include the upstream commit as well + Return: + Number of patches that exist on top of the branch, or None if the + branch does not exist. + """ + range_expr, msg = get_range_in_branch(git_dir, branch, include_upstream) + if not range_expr: + return None, msg + return count_commits_in_range(git_dir, range_expr) + + +def count_commits(commit_range): + """Returns the number of commits in the given range. + + Args: + commit_range (str): Range of commits to count (e.g. 'HEAD..base') + Return: + Number of patches that exist on top of the branch + """ + pipe = [log_cmd(commit_range, oneline=True), + ['wc', '-l']] + stdout = command.run_pipe(pipe, capture=True, oneline=True).stdout + patch_count = int(stdout) + return patch_count + + +def checkout(commit_hash, git_dir=None, work_tree=None, force=False): + """Checkout the selected commit for this build + + Args: + commit_hash (str): Commit hash to check out + git_dir (str): Directory containing git repo, or None for current dir + work_tree (str): Git worktree to use, or None if none + force (bool): True to force the checkout (git checkout -f) + """ + pipe = ['git'] + if git_dir: + pipe.extend(['--git-dir', git_dir]) + if work_tree: + pipe.extend(['--work-tree', work_tree]) + pipe.append('checkout') + if force: + pipe.append('-f') + pipe.append(commit_hash) + result = command.run_pipe([pipe], capture=True, raise_on_error=False, + capture_stderr=True) + if result.return_code != 0: + raise OSError(f'git checkout ({pipe}): {result.stderr}') + + +def clone(repo, output_dir): + """Clone a repo + + Args: + repo (str): Repo to clone (e.g. web address) + output_dir (str): Directory to close into + """ + result = command.run_one('git', 'clone', repo, '.', capture=True, + cwd=output_dir, capture_stderr=True) + if result.return_code != 0: + raise OSError(f'git clone: {result.stderr}') + + +def fetch(git_dir=None, work_tree=None): + """Fetch from the origin repo + + Args: + git_dir (str): Directory containing git repo, or None for current dir + work_tree (str or None): Git worktree to use, or None if none + """ + cmd = ['git'] + if git_dir: + cmd.extend(['--git-dir', git_dir]) + if work_tree: + cmd.extend(['--work-tree', work_tree]) + cmd.append('fetch') + result = command.run_one(*cmd, capture=True, capture_stderr=True) + if result.return_code != 0: + raise OSError(f'git fetch: {result.stderr}') + + +def check_worktree_is_available(git_dir): + """Check if git-worktree functionality is available + + Args: + git_dir (str): The repository to test in + + Returns: + True if git-worktree commands will work, False otherwise. + """ + result = command.run_one('git', '--git-dir', git_dir, 'worktree', 'list', + capture=True, capture_stderr=True, + raise_on_error=False) + return result.return_code == 0 + + +def add_worktree(git_dir, output_dir, commit_hash=None): + """Create and checkout a new git worktree for this build + + Args: + git_dir (str): The repository to checkout the worktree from + output_dir (str): Path for the new worktree + commit_hash (str): Commit hash to checkout + """ + # We need to pass --detach to avoid creating a new branch + cmd = ['git', '--git-dir', git_dir, 'worktree', 'add', '.', '--detach'] + if commit_hash: + cmd.append(commit_hash) + result = command.run_one(*cmd, capture=True, cwd=output_dir, + capture_stderr=True) + if result.return_code != 0: + raise OSError(f'git worktree add: {result.stderr}') + + +def prune_worktrees(git_dir): + """Remove administrative files for deleted worktrees + + Args: + git_dir (str): The repository whose deleted worktrees should be pruned + """ + result = command.run_one('git', '--git-dir', git_dir, 'worktree', 'prune', + capture=True, capture_stderr=True) + if result.return_code != 0: + raise OSError(f'git worktree prune: {result.stderr}') + + +def create_patches(branch, start, count, ignore_binary, series, signoff=True, + git_dir=None, cwd=None): + """Create a series of patches from the top of the current branch. + + The patch files are written to the current directory using + git format-patch. + + Args: + branch (str): Branch to create patches from (None for current branch) + start (int): Commit to start from: 0=HEAD, 1=next one, etc. + count (int): number of commits to include + ignore_binary (bool): Don't generate patches for binary files + series (Series): Series object for this series (set of patches) + signoff (bool): True to add signoff lines automatically + git_dir (str): Path to git repository (None to use default) + cwd (str): Path to use for git operations + Return: + Filename of cover letter (None if none) + List of filenames of patch files + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['format-patch', '-M'] + if signoff: + cmd.append('--signoff') + if ignore_binary: + cmd.append('--no-binary') + if series.get('cover'): + cmd.append('--cover-letter') + prefix = series.GetPatchPrefix() + if prefix: + cmd += [f'--subject-prefix={prefix}'] + brname = branch or 'HEAD' + cmd += [f'{brname}~{start + count}..{brname}~{start}'] + + stdout = command.run_list(cmd, cwd=cwd) + files = stdout.splitlines() + + # We have an extra file if there is a cover letter + if series.get('cover'): + return files[0], files[1:] + return None, files + + +def build_email_list(in_list, alias, tag=None, warn_on_error=True): + """Build a list of email addresses based on an input list. + + Takes a list of email addresses and aliases, and turns this into a list + of only email address, by resolving any aliases that are present. + + If the tag is given, then each email address is prepended with this + tag and a space. If the tag starts with a minus sign (indicating a + command line parameter) then the email address is quoted. + + Args: + in_list (list of str): List of aliases/email addresses + alias (dict): Alias dictionary: + key: alias + value: list of aliases or email addresses + tag (str): Text to put before each address + warn_on_error (bool): True to raise an error when an alias fails to + match, False to just print a message. + + Returns: + List of email addresses + + >>> alias = {} + >>> alias['fred'] = ['f.bloggs@napier.co.nz'] + >>> alias['john'] = ['j.bloggs@napier.co.nz'] + >>> alias['mary'] = ['Mary Poppins '] + >>> alias['boys'] = ['fred', ' john'] + >>> alias['all'] = ['fred ', 'john', ' mary '] + >>> build_email_list(['john', 'mary'], alias, None) + ['j.bloggs@napier.co.nz', 'Mary Poppins '] + >>> build_email_list(['john', 'mary'], alias, '--to') + ['--to "j.bloggs@napier.co.nz"', \ +'--to "Mary Poppins "'] + >>> build_email_list(['john', 'mary'], alias, 'Cc') + ['Cc j.bloggs@napier.co.nz', 'Cc Mary Poppins '] + """ + raw = [] + for item in in_list: + raw += lookup_email(item, alias, warn_on_error=warn_on_error) + result = [] + for item in raw: + if item not in result: + result.append(item) + if tag: + return [x for email in result for x in (tag, email)] + return result + + +def check_suppress_cc_config(): + """Check if sendemail.suppresscc is configured correctly. + + Returns: + bool: True if the option is configured correctly, False otherwise. + """ + suppresscc = command.output_one_line( + 'git', 'config', 'sendemail.suppresscc', raise_on_error=False) + + # Other settings should be fine. + if suppresscc in ('all', 'cccmd'): + col = terminal.Color() + + print(col.build(col.RED, 'error') + + f': git config sendemail.suppresscc set to {suppresscc}\n' + + ' patman needs --cc-cmd to be run to set the cc list.\n' + + ' Please run:\n' + + ' git config --unset sendemail.suppresscc\n' + + ' Or read the man page:\n' + + ' git send-email --help\n' + + ' and set an option that runs --cc-cmd\n') + return False + + return True + + +def email_patches(series, cover_fname, args, dry_run, warn_on_error, cc_fname, + alias, self_only=False, in_reply_to=None, thread=False, + smtp_server=None, cwd=None): + """Email a patch series. + + Args: + series (Series): Series object containing destination info + cover_fname (str or None): filename of cover letter + args (list of str): list of filenames of patch files + dry_run (bool): Just return the command that would be run + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + cc_fname (str): Filename of Cc file for per-commit Cc + alias (dict): Alias dictionary: + key: alias + value: list of aliases or email addresses + self_only (bool): True to just email to yourself as a test + in_reply_to (str or None): If set we'll pass this to git as + --in-reply-to - should be a message ID that this is in reply to. + thread (bool): True to add --thread to git send-email (make + all patches reply to cover-letter or first patch in series) + smtp_server (str or None): SMTP server to use to send patches + cwd (str): Path to use for patch files (None to use current dir) + + Returns: + Git command that was/would be run + + # For the duration of this doctest pretend that we ran patman with ./patman + >>> _old_argv0 = sys.argv[0] + >>> sys.argv[0] = './patman' + + >>> alias = {} + >>> alias['fred'] = ['f.bloggs@napier.co.nz'] + >>> alias['john'] = ['j.bloggs@napier.co.nz'] + >>> alias['mary'] = ['m.poppins@cloud.net'] + >>> alias['boys'] = ['fred', ' john'] + >>> alias['all'] = ['fred ', 'john', ' mary '] + >>> alias[os.getenv('USER')] = ['this-is-me@me.com'] + >>> series = {} + >>> series['to'] = ['fred'] + >>> series['cc'] = ['mary'] + >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ + False, alias) + 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ +"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" cover p1 p2' + >>> email_patches(series, None, ['p1'], True, True, 'cc-fname', False, \ + alias) + 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ +"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" p1' + >>> series['cc'] = ['all'] + >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ + True, alias) + 'git send-email --annotate --to "this-is-me@me.com" --cc-cmd "./patman \ +send --cc-cmd cc-fname" cover p1 p2' + >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ + False, alias) + 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ +"f.bloggs@napier.co.nz" --cc "j.bloggs@napier.co.nz" --cc \ +"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" cover p1 p2' + + # Restore argv[0] since we clobbered it. + >>> sys.argv[0] = _old_argv0 + """ + to = build_email_list(series.get('to'), alias, '--to', warn_on_error) + if not to: + if not command.output('git', 'config', 'sendemail.to', + raise_on_error=False): + print("No recipient.\n" + "Please add something like this to a commit\n" + "Series-to: Fred Bloggs \n" + "Or do something like this\n" + "git config sendemail.to u-boot@lists.denx.de") + return None + cc = build_email_list(list(set(series.get('cc')) - set(series.get('to'))), + alias, '--cc', warn_on_error) + if self_only: + to = build_email_list([os.getenv('USER')], '--to', alias, + warn_on_error) + cc = [] + cmd = ['git', 'send-email', '--annotate'] + if smtp_server: + cmd.append(f'--smtp-server={smtp_server}') + if in_reply_to: + cmd.append(f'--in-reply-to="{in_reply_to}"') + if thread: + cmd.append('--thread') + + cmd += to + cmd += cc + cmd += ['--cc-cmd', f'{sys.argv[0]} send --cc-cmd {cc_fname}'] + if cover_fname: + cmd.append(cover_fname) + cmd += args + if not dry_run: + command.run(*cmd, capture=False, capture_stderr=False, cwd=cwd) + return' '.join([f'"{x}"' if ' ' in x and '"' not in x else x + for x in cmd]) + + +def lookup_email(lookup_name, alias, warn_on_error=True, level=0): + """If an email address is an alias, look it up and return the full name + + TODO: Why not just use git's own alias feature? + + Args: + lookup_name (str): Alias or email address to look up + alias (dict): Alias dictionary + key: alias + value: list of aliases or email addresses + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + level (int): Depth of alias stack, used to detect recusion/loops + + Returns: + tuple: + list containing a list of email addresses + + Raises: + OSError if a recursive alias reference was found + ValueError if an alias was not found + + >>> alias = {} + >>> alias['fred'] = ['f.bloggs@napier.co.nz'] + >>> alias['john'] = ['j.bloggs@napier.co.nz'] + >>> alias['mary'] = ['m.poppins@cloud.net'] + >>> alias['boys'] = ['fred', ' john', 'f.bloggs@napier.co.nz'] + >>> alias['all'] = ['fred ', 'john', ' mary '] + >>> alias['loop'] = ['other', 'john', ' mary '] + >>> alias['other'] = ['loop', 'john', ' mary '] + >>> lookup_email('mary', alias) + ['m.poppins@cloud.net'] + >>> lookup_email('arthur.wellesley@howe.ro.uk', alias) + ['arthur.wellesley@howe.ro.uk'] + >>> lookup_email('boys', alias) + ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz'] + >>> lookup_email('all', alias) + ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz', 'm.poppins@cloud.net'] + >>> lookup_email('odd', alias) + Alias 'odd' not found + [] + >>> lookup_email('loop', alias) + Traceback (most recent call last): + ... + OSError: Recursive email alias at 'other' + >>> lookup_email('odd', alias, warn_on_error=False) + [] + >>> # In this case the loop part will effectively be ignored. + >>> lookup_email('loop', alias, warn_on_error=False) + Recursive email alias at 'other' + Recursive email alias at 'john' + Recursive email alias at 'mary' + ['j.bloggs@napier.co.nz', 'm.poppins@cloud.net'] + """ + lookup_name = lookup_name.strip() + if '@' in lookup_name: # Perhaps a real email address + return [lookup_name] + + lookup_name = lookup_name.lower() + col = terminal.Color() + + out_list = [] + if level > 10: + msg = f"Recursive email alias at '{lookup_name}'" + if warn_on_error: + raise OSError(msg) + print(col.build(col.RED, msg)) + return out_list + + if lookup_name: + if lookup_name not in alias: + msg = f"Alias '{lookup_name}' not found" + if warn_on_error: + print(col.build(col.RED, msg)) + return out_list + for item in alias[lookup_name]: + todo = lookup_email(item, alias, warn_on_error, level + 1) + for new_item in todo: + if new_item not in out_list: + out_list.append(new_item) + + return out_list + + +def get_top_level(): + """Return name of top-level directory for this git repo. + + Returns: + str: Full path to git top-level directory, or None if not found + + This test makes sure that we are running tests in the right subdir + + >>> os.path.realpath(os.path.dirname(__file__)) == \ + os.path.join(get_top_level(), 'tools', 'patman') + True + """ + result = command.run_one( + 'git', 'rev-parse', '--show-toplevel', oneline=True, capture=True, + capture_stderr=True, raise_on_error=False) + if result.return_code: + return None + return result.stdout.strip() + + +def get_alias_file(): + """Gets the name of the git alias file. + + Returns: + str: Filename of git alias file, or None if none + """ + fname = command.output_one_line('git', 'config', 'sendemail.aliasesfile', + raise_on_error=False) + if not fname: + return None + + fname = os.path.expanduser(fname.strip()) + if os.path.isabs(fname): + return fname + + return os.path.join(get_top_level() or '', fname) + + +def get_default_user_name(): + """Gets the user.name from .gitconfig file. + + Returns: + User name found in .gitconfig file, or None if none + """ + uname = command.output_one_line('git', 'config', '--global', '--includes', + 'user.name') + return uname + + +def get_default_user_email(): + """Gets the user.email from the global .gitconfig file. + + Returns: + User's email found in .gitconfig file, or None if none + """ + uemail = command.output_one_line('git', 'config', '--global', '--includes', + 'user.email') + return uemail + + +def get_default_subject_prefix(): + """Gets the format.subjectprefix from local .git/config file. + + Returns: + Subject prefix found in local .git/config file, or None if none + """ + sub_prefix = command.output_one_line( + 'git', 'config', 'format.subjectprefix', raise_on_error=False) + + return sub_prefix + + +def setup(): + """setup() - Set up git utils, by reading the alias files.""" + # Check for a git alias file also + global USE_NO_DECORATE + + cmd = log_cmd(None, count=0) + USE_NO_DECORATE = (command.run_one(*cmd, raise_on_error=False) + .return_code == 0) + + +def get_hash(spec, git_dir=None): + """Get the hash of a commit + + Args: + spec (str): Git commit to show, e.g. 'my-branch~12' + git_dir (str): Path to git repository (None to use default) + + Returns: + str: Hash of commit + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['show', '-s', '--pretty=format:%H', spec] + return command.output_one_line(*cmd) + + +def get_head(): + """Get the hash of the current HEAD + + Returns: + Hash of HEAD + """ + return get_hash('HEAD') + + +def get_branch(git_dir=None): + """Get the branch we are currently on + + Return: + str: branch name, or None if none + git_dir (str): Path to git repository (None to use default) + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['rev-parse', '--abbrev-ref', 'HEAD'] + out = command.output_one_line(*cmd, raise_on_error=False) + if out == 'HEAD': + return None + return out + + +def check_dirty(git_dir=None, work_tree=None): + """Check if the tree is dirty + + Args: + git_dir (str): Path to git repository (None to use default) + work_tree (str): Git worktree to use, or None if none + + Return: + str: List of dirty filenames and state + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + if work_tree: + cmd += ['--work-tree', work_tree] + cmd += ['status', '--porcelain', '--untracked-files=no'] + return command.output(*cmd).splitlines() + + +def check_branch(name, git_dir=None): + """Check if a branch exists + + Args: + name (str): Name of the branch to check + git_dir (str): Path to git repository (None to use default) + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['branch', '--list', name] + + # This produces ' ' or '* ' + out = command.output(*cmd).rstrip() + return out[2:] == name + + +def rename_branch(old_name, name, git_dir=None): + """Check if a branch exists + + Args: + old_name (str): Name of the branch to rename + name (str): New name for the branch + git_dir (str): Path to git repository (None to use default) + + Return: + str: Output from command + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['branch', '--move', old_name, name] + + # This produces ' ' or '* ' + return command.output(*cmd).rstrip() + + +def get_commit_message(commit, git_dir=None): + """Gets the commit message for a commit + + Args: + commit (str): commit to check + git_dir (str): Path to git repository (None to use default) + + Return: + list of str: Lines from the commit message + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['show', '--quiet', commit] + + out = command.output(*cmd) + # the header is followed by a blank line + lines = out.splitlines() + empty = lines.index('') + msg = lines[empty + 1:] + unindented = [line[4:] for line in msg] + + return unindented + + +def show_commit(commit, msg=True, diffstat=False, patch=False, colour=True, + git_dir=None): + """Runs 'git show' and returns the output + + Args: + commit (str): commit to check + msg (bool): Show the commit message + diffstat (bool): True to include the diffstat + patch (bool): True to include the patch + colour (bool): True to force use of colour + git_dir (str): Path to git repository (None to use default) + + Return: + list of str: Lines from the commit message + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['show'] + if colour: + cmd.append('--color') + if not msg: + cmd.append('--oneline') + if diffstat: + cmd.append('--stat') + else: + cmd.append('--quiet') + if patch: + cmd.append('--patch') + cmd.append(commit) + + return command.output(*cmd) + + +if __name__ == "__main__": + import doctest + + doctest.testmod() diff --git a/u_boot_pylib/terminal.py b/u_boot_pylib/terminal.py new file mode 100644 index 0000000..e62fa16 --- /dev/null +++ b/u_boot_pylib/terminal.py @@ -0,0 +1,350 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2011 The Chromium OS Authors. +# + +"""Terminal utilities + +This module handles terminal interaction including ANSI color codes. +""" + +from contextlib import contextmanager +from io import StringIO +import os +import re +import shutil +import subprocess +import sys + +# Selection of when we want our output to be colored +COLOR_IF_TERMINAL, COLOR_ALWAYS, COLOR_NEVER = range(3) + +# Initially, we are set up to print to the terminal +print_test_mode = False +print_test_list = [] + +# The length of the last line printed without a newline +last_print_len = None + +# credit: +# stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python +ansi_escape = re.compile(r'\x1b(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + +# True if we are capturing console output +CAPTURING = False + +# Set this to False to disable output-capturing globally +USE_CAPTURE = True + + +class PrintLine: + """A line of text output + + Members: + text: Text line that was printed + newline: True to output a newline after the text + colour: Text colour to use + """ + def __init__(self, text, colour, newline=True, bright=True): + self.text = text + self.newline = newline + self.colour = colour + self.bright = bright + + def __eq__(self, other): + return (self.text == other.text and + self.newline == other.newline and + self.colour == other.colour and + self.bright == other.bright) + + def __str__(self): + return ("newline=%s, colour=%s, bright=%d, text='%s'" % + (self.newline, self.colour, self.bright, self.text)) + + +def calc_ascii_len(text): + """Calculate the length of a string, ignoring any ANSI sequences + + When displayed on a terminal, ANSI sequences don't take any space, so we + need to ignore them when calculating the length of a string. + + Args: + text: Text to check + + Returns: + Length of text, after skipping ANSI sequences + + >>> col = Color(COLOR_ALWAYS) + >>> text = col.build(Color.RED, 'abc') + >>> len(text) + 14 + >>> calc_ascii_len(text) + 3 + >>> + >>> text += 'def' + >>> calc_ascii_len(text) + 6 + >>> text += col.build(Color.RED, 'abc') + >>> calc_ascii_len(text) + 9 + """ + result = ansi_escape.sub('', text) + return len(result) + +def trim_ascii_len(text, size): + """Trim a string containing ANSI sequences to the given ASCII length + + The string is trimmed with ANSI sequences being ignored for the length + calculation. + + >>> col = Color(COLOR_ALWAYS) + >>> text = col.build(Color.RED, 'abc') + >>> len(text) + 14 + >>> calc_ascii_len(trim_ascii_len(text, 4)) + 3 + >>> calc_ascii_len(trim_ascii_len(text, 2)) + 2 + >>> text += 'def' + >>> calc_ascii_len(trim_ascii_len(text, 4)) + 4 + >>> text += col.build(Color.RED, 'ghi') + >>> calc_ascii_len(trim_ascii_len(text, 7)) + 7 + """ + if calc_ascii_len(text) < size: + return text + pos = 0 + out = '' + left = size + + # Work through each ANSI sequence in turn + for m in ansi_escape.finditer(text): + # Find the text before the sequence and add it to our string, making + # sure it doesn't overflow + before = text[pos:m.start()] + toadd = before[:left] + out += toadd + + # Figure out how much non-ANSI space we have left + left -= len(toadd) + + # Add the ANSI sequence and move to the position immediately after it + out += m.group() + pos = m.start() + len(m.group()) + + # Deal with text after the last ANSI sequence + after = text[pos:] + toadd = after[:left] + out += toadd + + return out + + +def tprint(text='', newline=True, colour=None, limit_to_line=False, + bright=True, back=None, col=None, stderr=False): + """Handle a line of output to the terminal. + + In test mode this is recorded in a list. Otherwise it is output to the + terminal. + + Args: + text: Text to print + newline: True to add a new line at the end of the text + colour: Colour to use for the text + stderr: True to print to stderr instead of stdout + """ + global last_print_len + + if print_test_mode: + print_test_list.append(PrintLine(text, colour, newline, bright)) + else: + if colour is not None: + if not col: + col = Color() + text = col.build(colour, text, bright=bright, back=back) + + file = sys.stderr if stderr else sys.stdout + + if newline: + print(text, file=file) + last_print_len = None + else: + if limit_to_line: + cols = shutil.get_terminal_size().columns + text = trim_ascii_len(text, cols) + print(text, end='', flush=True, file=file) + last_print_len = calc_ascii_len(text) + +def print_clear(): + """Clear a previously line that was printed with no newline""" + global last_print_len + + if last_print_len: + if print_test_mode: + print_test_list.append(PrintLine(None, None, None, None)) + else: + print('\r%s\r' % (' '* last_print_len), end='', flush=True) + last_print_len = None + +def set_print_test_mode(enable=True): + """Go into test mode, where all printing is recorded""" + global print_test_mode + + print_test_mode = enable + get_print_test_lines() + +def get_print_test_lines(): + """Get a list of all lines output through tprint() + + Returns: + A list of PrintLine objects + """ + global print_test_list + + ret = print_test_list + print_test_list = [] + return ret + +def echo_print_test_lines(): + """Print out the text lines collected""" + for line in print_test_list: + if line.colour: + col = Color() + print(col.build(line.colour, line.text), end='') + else: + print(line.text, end='') + if line.newline: + print() + +def have_terminal(): + """Check if we have an interactive terminal or not + + Returns: + bool: true if an interactive terminal is attached + """ + return os.isatty(sys.stdout.fileno()) + + +class Color(): + """Conditionally wraps text in ANSI color escape sequences.""" + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) + BOLD = -1 + BRIGHT_START = '\033[1;%d%sm' + NORMAL_START = '\033[22;%d%sm' + BOLD_START = '\033[1m' + BACK_EXTRA = ';%d' + RESET = '\033[0m' + + def __init__(self, colored=COLOR_IF_TERMINAL): + """Create a new Color object, optionally disabling color output. + + Args: + enabled: True if color output should be enabled. If False then this + class will not add color codes at all. + """ + try: + self._enabled = (colored == COLOR_ALWAYS or + (colored == COLOR_IF_TERMINAL and + os.isatty(sys.stdout.fileno()))) + except: + self._enabled = False + + def enabled(self): + """Check if colour is enabled + + Return: True if enabled, else False + """ + return self._enabled + + def start(self, color, bright=True, back=None): + """Returns a start color code. + + Args: + color: Color to use, .e.g BLACK, RED, etc. + + Returns: + If color is enabled, returns an ANSI sequence to start the given + color, otherwise returns empty string + """ + if self._enabled: + if color == self.BOLD: + return self.BOLD_START + base = self.BRIGHT_START if bright else self.NORMAL_START + extra = self.BACK_EXTRA % (back + 40) if back else '' + return base % (color + 30, extra) + return '' + + def stop(self): + """Returns a stop color code. + + Returns: + If color is enabled, returns an ANSI color reset sequence, + otherwise returns empty string + """ + if self._enabled: + return self.RESET + return '' + + def build(self, color, text, bright=True, back=None): + """Returns text with conditionally added color escape sequences. + + Keyword arguments: + color: Text color -- one of the color constants defined in this + class. + text: The text to color. + + Returns: + If self._enabled is False, returns the original text. If it's True, + returns text with color escape sequences based on the value of + color. + """ + if not self._enabled: + return text + return self.start(color, bright, back) + text + self.RESET + + +# Use this to suppress stdout/stderr output: +# with terminal.capture() as (stdout, stderr) +# ...do something... +@contextmanager +def capture(): + global CAPTURING + + capture_out, capture_err = StringIO(), StringIO() + old_out, old_err = sys.stdout, sys.stderr + try: + CAPTURING = True + sys.stdout, sys.stderr = capture_out, capture_err + yield capture_out, capture_err + finally: + sys.stdout, sys.stderr = old_out, old_err + CAPTURING = False + if not USE_CAPTURE: + sys.stdout.write(capture_out.getvalue()) + sys.stderr.write(capture_err.getvalue()) + + +@contextmanager +def pager(): + """Simple pager for outputting lots of text + + Usage: + with terminal.pager(): + print(...) + """ + proc = None + old_stdout = None + try: + less = os.getenv('PAGER') + if not CAPTURING and less != 'none' and have_terminal(): + if not less: + less = 'less -R --quit-if-one-screen' + proc = subprocess.Popen(less, stdin=subprocess.PIPE, text=True, + shell=True) + old_stdout = sys.stdout + sys.stdout = proc.stdin + yield + finally: + if proc: + sys.stdout = old_stdout + proc.communicate() diff --git a/u_boot_pylib/test_util.py b/u_boot_pylib/test_util.py new file mode 100644 index 0000000..b1c8740 --- /dev/null +++ b/u_boot_pylib/test_util.py @@ -0,0 +1,229 @@ +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright (c) 2016 Google, Inc +# + +import doctest +import glob +import multiprocessing +import os +import re +import sys +import unittest + +from u_boot_pylib import command +from u_boot_pylib import terminal + +use_concurrent = True +try: + from concurrencytest import ConcurrentTestSuite + from concurrencytest import fork_for_tests +except: + use_concurrent = False + + +def run_test_coverage(prog, filter_fname, exclude_list, build_dir, + required=None, extra_args=None, single_thread='-P1', + args=None, allow_failures=None): + """Run tests and check that we get 100% coverage + + Args: + prog: Program to run (with be passed a '-t' argument to run tests + filter_fname: Normally all *.py files in the program's directory will + be included. If this is not None, then it is used to filter the + list so that only filenames that don't contain filter_fname are + included. + exclude_list: List of file patterns to exclude from the coverage + calculation + build_dir: Build directory, used to locate libfdt.py + required: Set of modules which must be in the coverage report + extra_args (str): Extra arguments to pass to the tool before the -t/test + arg + single_thread (str): Argument string to make the tests run + single-threaded. This is necessary to get proper coverage results. + The default is '-P0' + args (list of str): List of tests to run, or None to run all + + Raises: + ValueError if the code coverage is not 100% + """ + # This uses the build output from sandbox_spl to get _libfdt.so + path = os.path.dirname(prog) + if filter_fname: + glob_list = glob.glob(os.path.join(path, '*.py')) + glob_list = [fname for fname in glob_list if filter_fname in fname] + else: + glob_list = [] + glob_list += exclude_list + glob_list += ['*libfdt.py', '*/site-packages/*', '*/dist-packages/*'] + glob_list += ['*concurrencytest*'] + use_test = 'binman' in prog or 'patman' in prog or 'pickman' in prog + test_cmd = 'test' if use_test else '-t' + prefix = '' + if build_dir: + prefix = 'PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools ' % build_dir + + # Detect a Python sandbox and use 'coverage' instead + covtool = ('python3-coverage' if sys.prefix == sys.base_prefix else + 'coverage') + + cmd = ('%s%s run ' + '--omit "%s" %s %s %s %s %s' % (prefix, covtool, ','.join(glob_list), + prog, extra_args or '', test_cmd, + single_thread or '-P1', + ' '.join(args) if args else '')) + os.system(cmd) + stdout = command.output(covtool, 'report') + lines = stdout.splitlines() + if required: + # Convert '/path/to/name.py' just the module name 'name' + test_set = set([os.path.splitext(os.path.basename(line.split()[0]))[0] + for line in lines if '/etype/' in line]) + missing_list = required + missing_list.discard('__init__') + missing_list.difference_update(test_set) + if missing_list: + print('Missing tests for %s' % (', '.join(missing_list))) + print(stdout) + ok = False + + coverage = lines[-1].split(' ')[-1] + ok = True + print(coverage) + if coverage != '100%': + print(stdout) + print("To get a report in 'htmlcov/index.html', type: " + "python3-coverage html") + print('Coverage error: %s, but should be 100%%' % coverage) + ok = False + if not ok: + if allow_failures: + lines = [re.match(r'^(tools/.*py) *\d+ *(\d+) *\d+%$', line) + for line in stdout.splitlines()] + bad = [] + for mat in lines: + if mat and mat.group(2) != '0': + fname = mat.group(1) + if fname not in allow_failures: + bad.append(fname) + if not bad: + return + raise ValueError('Test coverage failure') + + +class FullTextTestResult(unittest.TextTestResult): + """A test result class that can print extended text results to a stream + + This is meant to be used by a TestRunner as a result class. Like + TextTestResult, this prints out the names of tests as they are run, + errors as they occur, and a summary of the results at the end of the + test run. Beyond those, this prints information about skipped tests, + expected failures and unexpected successes. + + Args: + stream: A file-like object to write results to + descriptions (bool): True to print descriptions with test names + verbosity (int): Detail of printed output per test as they run + Test stdout and stderr always get printed when buffering + them is disabled by the test runner. In addition to that, + 0: Print nothing + 1: Print a dot per test + 2: Print test names + """ + def __init__(self, stream, descriptions, verbosity): + self.verbosity = verbosity + super().__init__(stream, descriptions, verbosity) + + def printErrors(self): + "Called by TestRunner after test run to summarize the tests" + # The parent class doesn't keep unexpected successes in the same + # format as the rest. Adapt it to what printErrorList expects. + unexpected_successes = [ + (test, 'Test was expected to fail, but succeeded.\n') + for test in self.unexpectedSuccesses + ] + + super().printErrors() # FAIL and ERROR + self.printErrorList('SKIP', self.skipped) + self.printErrorList('XFAIL', self.expectedFailures) + self.printErrorList('XPASS', unexpected_successes) + + def addSkip(self, test, reason): + """Called when a test is skipped.""" + # Add empty line to keep spacing consistent with other results + if not reason.endswith('\n'): + reason += '\n' + super().addSkip(test, reason) + + +def run_test_suites(toolname, debug, verbosity, no_capture, test_preserve_dirs, + processes, test_name, toolpath, class_and_module_list): + """Run a series of test suites and collect the results + + Args: + toolname: Name of the tool that ran the tests + debug: True to enable debugging, which shows a full stack trace on error + verbosity: Verbosity level to use (0-4) + test_preserve_dirs: True to preserve the input directory used by tests + so that it can be examined afterwards (only useful for debugging + tests). If a single test is selected (in args[0]) it also preserves + the output directory for this test. Both directories are displayed + on the command line. + processes: Number of processes to use to run tests (None=same as #CPUs) + test_name: Name of test to run, or None for all + toolpath: List of paths to use for tools + class_and_module_list: List of test classes (type class) and module + names (type str) to run + """ + sys.argv = [sys.argv[0]] + if debug: + sys.argv.append('-D') + if verbosity: + sys.argv.append('-v%d' % verbosity) + if no_capture: + sys.argv.append('-N') + terminal.USE_CAPTURE = False + if toolpath: + for path in toolpath: + sys.argv += ['--toolpath', path] + + suite = unittest.TestSuite() + loader = unittest.TestLoader() + runner = unittest.TextTestRunner( + stream=sys.stdout, + verbosity=(1 if verbosity is None else verbosity), + resultclass=FullTextTestResult, + ) + + if use_concurrent and processes != 1 and not test_name: + suite = ConcurrentTestSuite(suite, + fork_for_tests(processes or multiprocessing.cpu_count())) + + for module in class_and_module_list: + if isinstance(module, str) and (not test_name or test_name == module): + suite.addTests(doctest.DocTestSuite(module)) + + for module in class_and_module_list: + if isinstance(module, str): + continue + # Test the test module about our arguments, if it is interested + if hasattr(module, 'setup_test_args'): + setup_test_args = getattr(module, 'setup_test_args') + setup_test_args(preserve_indir=test_preserve_dirs, + preserve_outdirs=test_preserve_dirs and test_name is not None, + toolpath=toolpath, verbosity=verbosity, no_capture=no_capture) + if test_name: + # Since Python v3.5 If an ImportError or AttributeError occurs + # while traversing a name then a synthetic test that raises that + # error when run will be returned. Check that the requested test + # exists, otherwise these errors are included in the results. + if test_name in loader.getTestCaseNames(module): + suite.addTests(loader.loadTestsFromName(test_name, module)) + else: + suite.addTests(loader.loadTestsFromTestCase(module)) + + print(f" Running {toolname} tests ".center(70, "=")) + result = runner.run(suite) + print() + + return result diff --git a/u_boot_pylib/tools.py b/u_boot_pylib/tools.py new file mode 100644 index 0000000..1afd289 --- /dev/null +++ b/u_boot_pylib/tools.py @@ -0,0 +1,612 @@ +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright (c) 2016 Google, Inc +# + +import glob +import os +import shlex +import shutil +import sys +import tempfile +import urllib.request + +from u_boot_pylib import command +from u_boot_pylib import tout + +# Output directly (generally this is temporary) +outdir = None + +# True to keep the output directory around after exiting +preserve_outdir = False + +# Path to the Chrome OS chroot, if we know it +chroot_path = None + +# Search paths to use for filename(), used to find files +search_paths = [] + +tool_search_paths = [] + +# Tools and the packages that contain them, on debian +packages = { + 'lz4': 'liblz4-tool', + } + +# List of paths to use when looking for an input file +indir = [] + +def prepare_output_dir(dirname, preserve=False): + """Select an output directory, ensuring it exists. + + This either creates a temporary directory or checks that the one supplied + by the user is valid. For a temporary directory, it makes a note to + remove it later if required. + + Args: + dirname: a string, name of the output directory to use to store + intermediate and output files. If is None - create a temporary + directory. + preserve: a Boolean. If outdir above is None and preserve is False, the + created temporary directory will be destroyed on exit. + + Raises: + OSError: If it cannot create the output directory. + """ + global outdir, preserve_outdir + + preserve_outdir = dirname or preserve + if dirname: + outdir = dirname + if not os.path.isdir(outdir): + try: + os.makedirs(outdir) + except OSError as err: + raise ValueError( + f"Cannot make output directory 'outdir': 'err.strerror'") + tout.debug("Using output directory '%s'" % outdir) + else: + outdir = tempfile.mkdtemp(prefix='binman.') + tout.debug("Using temporary directory '%s'" % outdir) + +def _remove_output_dir(): + global outdir + + shutil.rmtree(outdir) + tout.debug("Deleted temporary directory '%s'" % outdir) + outdir = None + +def finalise_output_dir(): + global outdir, preserve_outdir + + """Tidy up: delete output directory if temporary and not preserved.""" + if outdir and not preserve_outdir: + _remove_output_dir() + outdir = None + +def get_output_filename(fname): + """Return a filename within the output directory. + + Args: + fname: Filename to use for new file + + Returns: + The full path of the filename, within the output directory + """ + return os.path.join(outdir, fname) + +def get_output_dir(): + """Return the current output directory + + Returns: + str: The output directory + """ + return outdir + +def _finalise_for_test(): + """Remove the output directory (for use by tests)""" + global outdir + + if outdir: + _remove_output_dir() + outdir = None + +def set_input_dirs(dirname): + """Add a list of input directories, where input files are kept. + + Args: + dirname: a list of paths to input directories to use for obtaining + files needed by binman to place in the image. + """ + global indir + + indir = dirname + tout.debug("Using input directories %s" % indir) + +def append_input_dirs(dirname): + """Append a list of input directories to the current list of input + directories + + Args: + dirname: a list of paths to input directories to use for obtaining + files needed by binman to place in the image. + """ + global indir + + for dir in dirname: + if dirname not in indir: + indir.append(dirname) + + tout.debug("Updated input directories %s" % indir) + +def get_input_filename(fname, allow_missing=False): + """Return a filename for use as input. + + Args: + fname: Filename to use for new file + allow_missing: True if the filename can be missing + + Returns: + fname, if indir is None; + full path of the filename, within the input directory; + None, if file is missing and allow_missing is True + + Raises: + ValueError if file is missing and allow_missing is False + """ + if not indir or fname[:1] == '/': + return fname + for dirname in indir: + pathname = os.path.join(dirname, fname) + if os.path.exists(pathname): + return pathname + + if allow_missing: + return None + raise ValueError("Filename '%s' not found in input path (%s) (cwd='%s')" % + (fname, ','.join(indir), os.getcwd())) + +def get_input_filename_glob(pattern): + """Return a list of filenames for use as input. + + Args: + pattern: Filename pattern to search for + + Returns: + A list of matching files in all input directories + """ + if not indir: + return glob.glob(pattern) + files = [] + for dirname in indir: + pathname = os.path.join(dirname, pattern) + files += glob.glob(pathname) + return sorted(files) + +def align(pos, align): + if align: + mask = align - 1 + pos = (pos + mask) & ~mask + return pos + +def not_power_of_two(num): + return num and (num & (num - 1)) + +def set_tool_paths(toolpaths): + """Set the path to search for tools + + Args: + toolpaths: List of paths to search for tools executed by run() + """ + global tool_search_paths + + tool_search_paths = toolpaths + +def path_has_file(path_spec, fname): + """Check if a given filename is in the PATH + + Args: + path_spec: Value of PATH variable to check + fname: Filename to check + + Returns: + True if found, False if not + """ + for dir in path_spec.split(':'): + if os.path.exists(os.path.join(dir, fname)): + return True + return False + +def get_host_compile_tool(env, name): + """Get the host-specific version for a compile tool + + This checks the environment variables that specify which version of + the tool should be used (e.g. ${HOSTCC}). + + The following table lists the host-specific versions of the tools + this function resolves to: + + Compile Tool | Host version + --------------+---------------- + as | ${HOSTAS} + ld | ${HOSTLD} + cc | ${HOSTCC} + cpp | ${HOSTCPP} + c++ | ${HOSTCXX} + ar | ${HOSTAR} + nm | ${HOSTNM} + ldr | ${HOSTLDR} + strip | ${HOSTSTRIP} + objcopy | ${HOSTOBJCOPY} + objdump | ${HOSTOBJDUMP} + dtc | ${HOSTDTC} + + Args: + name: Command name to run + + Returns: + host_name: Exact command name to run instead + extra_args: List of extra arguments to pass + """ + host_name = None + extra_args = [] + if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', + 'objcopy', 'objdump', 'dtc'): + host_name, *host_args = env.get('HOST' + name.upper(), '').split(' ') + elif name == 'c++': + host_name, *host_args = env.get('HOSTCXX', '').split(' ') + + if host_name: + return host_name, extra_args + return name, [] + +def get_target_compile_tool(name, cross_compile=None): + """Get the target-specific version for a compile tool + + This first checks the environment variables that specify which + version of the tool should be used (e.g. ${CC}). If those aren't + specified, it checks the CROSS_COMPILE variable as a prefix for the + tool with some substitutions (e.g. "${CROSS_COMPILE}gcc" for cc). + + The following table lists the target-specific versions of the tools + this function resolves to: + + Compile Tool | First choice | Second choice + --------------+----------------+---------------------------- + as | ${AS} | ${CROSS_COMPILE}as + ld | ${LD} | ${CROSS_COMPILE}ld.bfd + | | or ${CROSS_COMPILE}ld + cc | ${CC} | ${CROSS_COMPILE}gcc + cpp | ${CPP} | ${CROSS_COMPILE}gcc -E + c++ | ${CXX} | ${CROSS_COMPILE}g++ + ar | ${AR} | ${CROSS_COMPILE}ar + nm | ${NM} | ${CROSS_COMPILE}nm + ldr | ${LDR} | ${CROSS_COMPILE}ldr + strip | ${STRIP} | ${CROSS_COMPILE}strip + objcopy | ${OBJCOPY} | ${CROSS_COMPILE}objcopy + objdump | ${OBJDUMP} | ${CROSS_COMPILE}objdump + dtc | ${DTC} | (no CROSS_COMPILE version) + + Args: + name: Command name to run + + Returns: + target_name: Exact command name to run instead + extra_args: List of extra arguments to pass + """ + env = dict(os.environ) + + target_name = None + extra_args = [] + if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', + 'objcopy', 'objdump', 'dtc'): + target_name, *extra_args = env.get(name.upper(), '').split(' ') + elif name == 'c++': + target_name, *extra_args = env.get('CXX', '').split(' ') + + if target_name: + return target_name, extra_args + + if cross_compile is None: + cross_compile = env.get('CROSS_COMPILE', '') + + if name in ('as', 'ar', 'nm', 'ldr', 'strip', 'objcopy', 'objdump'): + target_name = cross_compile + name + elif name == 'ld': + try: + if run(cross_compile + 'ld.bfd', '-v'): + target_name = cross_compile + 'ld.bfd' + except: + target_name = cross_compile + 'ld' + elif name == 'cc': + target_name = cross_compile + 'gcc' + elif name == 'cpp': + target_name = cross_compile + 'gcc' + extra_args = ['-E'] + elif name == 'c++': + target_name = cross_compile + 'g++' + else: + target_name = name + return target_name, extra_args + +def get_env_with_path(): + """Get an updated environment with the PATH variable set correctly + + If there are any search paths set, these need to come first in the PATH so + that these override any other version of the tools. + + Returns: + dict: New environment with PATH updated, or None if there are not search + paths + """ + if tool_search_paths: + env = dict(os.environ) + env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH'] + return env + +def run_result(name, *args, **kwargs): + """Run a tool with some arguments + + This runs a 'tool', which is a program used by binman to process files and + perhaps produce some output. Tools can be located on the PATH or in a + search path. + + Args: + name: Command name to run + args: Arguments to the tool + for_host: True to resolve the command to the version for the host + for_target: False to run the command as-is, without resolving it + to the version for the compile target + raise_on_error: Raise an error if the command fails (True by default) + + Returns: + CommandResult object + """ + try: + binary = kwargs.get('binary') + for_host = kwargs.get('for_host', False) + for_target = kwargs.get('for_target', not for_host) + raise_on_error = kwargs.get('raise_on_error', True) + env = get_env_with_path() + if for_target: + name, extra_args = get_target_compile_tool(name) + args = tuple(extra_args) + args + elif for_host: + name, extra_args = get_host_compile_tool(env, name) + args = tuple(extra_args) + args + name = os.path.expanduser(name) # Expand paths containing ~ + all_args = (name,) + args + result = command.run_one(*all_args, capture=True, capture_stderr=True, + env=env, raise_on_error=False, binary=binary) + if result.return_code: + if raise_on_error: + raise ValueError("Error %d running '%s': %s" % + (result.return_code,' '.join(all_args), + result.stderr or result.stdout)) + return result + except ValueError: + if env and not path_has_file(env['PATH'], name): + msg = "Please install tool '%s'" % name + package = packages.get(name) + if package: + msg += " (e.g. from package '%s')" % package + raise ValueError(msg) + raise + +def tool_find(name): + """Search the current path for a tool + + This uses both PATH and any value from set_tool_paths() to search for a tool + + Args: + name (str): Name of tool to locate + + Returns: + str: Full path to tool if found, else None + """ + name = os.path.expanduser(name) # Expand paths containing ~ + paths = [] + pathvar = os.environ.get('PATH') + if pathvar: + paths = pathvar.split(':') + if tool_search_paths: + paths += tool_search_paths + for path in paths: + fname = os.path.join(path, name) + if os.path.isfile(fname) and os.access(fname, os.X_OK): + return fname + +def run(name, *args, **kwargs): + """Run a tool with some arguments + + This runs a 'tool', which is a program used by binman to process files and + perhaps produce some output. Tools can be located on the PATH or in a + search path. + + Args: + name: Command name to run + args: Arguments to the tool + for_host: True to resolve the command to the version for the host + for_target: False to run the command as-is, without resolving it + to the version for the compile target + + Returns: + CommandResult object + """ + result = run_result(name, *args, **kwargs) + if result is not None: + return result.stdout + +def filename(fname): + """Resolve a file path to an absolute path. + + If fname starts with ##/ and chroot is available, ##/ gets replaced with + the chroot path. If chroot is not available, this file name can not be + resolved, `None' is returned. + + If fname is not prepended with the above prefix, and is not an existing + file, the actual file name is retrieved from the passed in string and the + search_paths directories (if any) are searched to for the file. If found - + the path to the found file is returned, `None' is returned otherwise. + + Args: + fname: a string, the path to resolve. + + Returns: + Absolute path to the file or None if not found. + """ + if fname.startswith('##/'): + if chroot_path: + fname = os.path.join(chroot_path, fname[3:]) + else: + return None + + # Search for a pathname that exists, and return it if found + if fname and not os.path.exists(fname): + for path in search_paths: + pathname = os.path.join(path, os.path.basename(fname)) + if os.path.exists(pathname): + return pathname + + # If not found, just return the standard, unchanged path + return fname + +def read_file(fname, binary=True): + """Read and return the contents of a file. + + Args: + fname: path to filename to read, where ## signifiies the chroot. + + Returns: + data read from file, as a string. + """ + with open(filename(fname), binary and 'rb' or 'r') as fd: + data = fd.read() + #self._out.Info("Read file '%s' size %d (%#0x)" % + #(fname, len(data), len(data))) + return data + +def write_file(fname, data, binary=True): + """Write data into a file. + + Args: + fname: path to filename to write + data: data to write to file, as a string + """ + #self._out.Info("Write file '%s' size %d (%#0x)" % + #(fname, len(data), len(data))) + with open(filename(fname), binary and 'wb' or 'w') as fd: + fd.write(data) + +def get_bytes(byte, size): + """Get a string of bytes of a given size + + Args: + byte: Numeric byte value to use + size: Size of bytes/string to return + + Returns: + A bytes type with 'byte' repeated 'size' times + """ + return bytes([byte]) * size + +def to_bytes(string): + """Convert a str type into a bytes type + + Args: + string: string to convert + + Returns: + A bytes type + """ + return string.encode('utf-8') + +def to_string(bval): + """Convert a bytes type into a str type + + Args: + bval: bytes value to convert + + Returns: + Python 3: A bytes type + Python 2: A string type + """ + return bval.decode('utf-8') + +def to_hex(val): + """Convert an integer value (or None) to a string + + Returns: + hex value, or 'None' if the value is None + """ + return 'None' if val is None else '%#x' % val + +def to_hex_size(val): + """Return the size of an object in hex + + Returns: + hex value of size, or 'None' if the value is None + """ + return 'None' if val is None else '%#x' % len(val) + +def print_full_help(fname): + """Print the full help message for a tool using an appropriate pager. + + Args: + fname: Path to a file containing the full help message + """ + pager = shlex.split(os.getenv('PAGER', '')) + if not pager: + lesspath = shutil.which('less') + pager = [lesspath] if lesspath else None + if not pager: + pager = ['more'] + command.run(*pager, fname) + +def download(url, tmpdir_pattern='.patman'): + """Download a file to a temporary directory + + Args: + url (str): URL to download + tmpdir_pattern (str): pattern to use for the temporary directory + + Returns: + Tuple: + Full path to the downloaded archive file in that directory, + or None if there was an error while downloading + Temporary directory name + """ + print('- downloading: %s' % url) + leaf = url.split('/')[-1] + tmpdir = tempfile.mkdtemp(tmpdir_pattern) + response = urllib.request.urlopen(url) + fname = os.path.join(tmpdir, leaf) + fd = open(fname, 'wb') + meta = response.info() + size = int(meta.get('Content-Length')) + done = 0 + block_size = 1 << 16 + status = '' + + # Read the file in chunks and show progress as we go + while True: + buffer = response.read(block_size) + if not buffer: + print(chr(8) * (len(status) + 1), '\r', end=' ') + break + + done += len(buffer) + fd.write(buffer) + status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024, + done * 100 // size) + status = status + chr(8) * (len(status) + 1) + print(status, end=' ') + sys.stdout.flush() + print('\r', end='') + sys.stdout.flush() + fd.close() + if done != size: + print('Error, failed to download') + os.remove(fname) + fname = None + return fname, tmpdir diff --git a/u_boot_pylib/tout.py b/u_boot_pylib/tout.py new file mode 100644 index 0000000..c608806 --- /dev/null +++ b/u_boot_pylib/tout.py @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2016 Google, Inc +# +# Terminal output logging. +# + +import sys + +from u_boot_pylib import terminal + +# Output verbosity levels that we support +FATAL, ERROR, WARNING, NOTICE, INFO, DETAIL, DEBUG = range(7) + +""" +This class handles output of progress and other useful information +to the user. It provides for simple verbosity level control and can +output nothing but errors at verbosity zero. + +The idea is that modules set up an Output object early in their years and pass +it around to other modules that need it. This keeps the output under control +of a single class. + +Public properties: + verbose: Verbosity level: 0=silent, 1=progress, 3=full, 4=debug +""" +def __enter__(): + return + +def __exit__(unused1, unused2, unused3): + """Clean up and remove any progress message.""" + clear_progress() + return False + +def user_is_present(): + """This returns True if it is likely that a user is present. + + Sometimes we want to prompt the user, but if no one is there then this + is a waste of time, and may lock a script which should otherwise fail. + + Returns: + True if it thinks the user is there, and False otherwise + """ + return stdout_is_tty and verbose > ERROR + +def clear_progress(): + """Clear any active progress message on the terminal.""" + if verbose > ERROR and stdout_is_tty: + terminal.print_clear() + +def progress(msg, warning=False, trailer='...'): + """Display progress information. + + Args: + msg: Message to display. + warning: True if this is a warning.""" + clear_progress() + if verbose > ERROR: + _progress = msg + trailer + if stdout_is_tty: + col = _color.YELLOW if warning else _color.GREEN + terminal.tprint('\r' + _progress, newline=False, colour=col, col=_color) + else: + terminal.tprint(_progress) + +def _output(level, msg, color=None, newline=True): + """Output a message to the terminal. + + Args: + level: Verbosity level for this message. It will only be displayed if + this as high as the currently selected level. + msg: Message to display. + color: Colour to use for the text, None for default. + newline: True to add a newline at the end. + """ + if verbose >= level: + clear_progress() + if level <= WARNING: + terminal.tprint(msg, newline=newline, colour=color, col=_color, + stderr=True) + else: + terminal.tprint(msg, newline=newline, colour=color, col=_color) + if level == FATAL: + sys.exit(1) + +def do_output(level, msg, newline=True): + """Output a message to the terminal. + + Args: + level: Verbosity level for this message. It will only be displayed if + this as high as the currently selected level. + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(level, msg, newline=newline) + +def fatal(msg): + """Display an error message and exit + + Args: + msg: Message to display. + """ + _output(FATAL, msg, _color.RED) + +def error(msg, newline=True): + """Display an error message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(ERROR, msg, _color.RED, newline=newline) + +def warning(msg, newline=True): + """Display a warning message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(WARNING, msg, _color.YELLOW, newline=newline) + +def notice(msg, newline=True): + """Display an important infomation message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(NOTICE, msg, newline=newline) + +def info(msg, newline=True): + """Display an infomation message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(INFO, msg, newline=newline) + +def detail(msg, newline=True): + """Display a detailed message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(DETAIL, msg, newline=newline) + +def debug(msg, newline=True): + """Display a debug message + + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(DEBUG, msg, newline=newline) + +def user_output(msg, newline=True): + """Display a message regardless of the current output level. + + This is used when the output was specifically requested by the user. + Args: + msg: Message to display. + newline: True to add a newline at the end. + """ + _output(ERROR, msg, newline=newline) + +def init(_verbose=WARNING, stdout=sys.stdout, allow_colour=True): + """Initialize a new output object. + + Args: + verbose: Verbosity level (0-6). + stdout: File to use for stdout. + """ + global verbose, _progress, _color, _stdout, stdout_is_tty + + verbose = _verbose + _progress = '' # Our last progress message + _color = terminal.Color(terminal.COLOR_IF_TERMINAL if allow_colour + else terminal.COLOR_NEVER) + _stdout = stdout + + # TODO(sjg): Move this into Chromite libraries when we have them + stdout_is_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() + stderr_is_tty = hasattr(sys.stderr, 'isatty') and sys.stderr.isatty() + +def uninit(): + clear_progress() + +init() diff --git a/uman_pkg/__main__.py b/uman_pkg/__main__.py index 3e31898..c76f00d 100755 --- a/uman_pkg/__main__.py +++ b/uman_pkg/__main__.py @@ -8,15 +8,18 @@ import os import sys -# Allow imports to work when run as module +# Use the embedded u_boot_pylib by putting uman's parent first on +# sys.path, so it takes priority over any older version in the U-Boot +# tree's tools/ directory. Set UMAN_EXTERNAL_PYLIB=1 to use the +# UBOOT_TOOLS version instead (for testing newer versions). our_path = os.path.dirname(os.path.realpath(__file__)) parent_path = os.path.dirname(our_path) -sys.path.append(parent_path) - -# Get U-Boot tools path from UBOOT_TOOLS env var (default: ~/u/tools) -# This is separate from USRC which specifies the U-Boot source to work in -uboot_tools = os.path.expanduser(os.environ.get('UBOOT_TOOLS', '~/u/tools')) -sys.path.append(uboot_tools) +if os.environ.get('UMAN_EXTERNAL_PYLIB'): + uboot_tools = os.path.expanduser( + os.environ.get('UBOOT_TOOLS', '~/u/tools')) + sys.path.insert(0, uboot_tools) +else: + sys.path.insert(0, parent_path) # pylint: disable=import-error,wrong-import-position from uman_pkg import cmdline diff --git a/uman_pkg/cc.py b/uman_pkg/cc.py index 923727b..47a12b7 100644 --- a/uman_pkg/cc.py +++ b/uman_pkg/cc.py @@ -10,7 +10,10 @@ import os import random +import socket as socket_mod import string +import subprocess +import threading import time # pylint: disable=import-error @@ -27,8 +30,18 @@ # Project mount point inside the container PROJECT_DEST = f'{UBUNTU_HOME}/project' +# Socket filename for editor proxy (in project directory) +EDITOR_SOCK = '.uman-editor.sock' + +# Fixed mount point for PulseAudio socket inside the container +PULSE_DEST = '/tmp/pulse-native' + +# ALSA config that routes through PulseAudio (suppresses hardware errors) +ALSA_PULSE_CONF = '/etc/alsa-pulse.conf' + # Default packages to install in containers -DEFAULT_PACKAGES = 'build-essential pylint' +DEFAULT_PACKAGES = ('build-essential gh glab libasound2-plugins' + ' libsox-fmt-pulse pylint sox xclip') def get_log_path(name): @@ -83,6 +96,20 @@ def get_essential_mounts(project_src): os.environ.get('UBOOT_TOOLS', '~/u/tools'))), f'{UBUNTU_HOME}/u/tools'), ] + patman_dir = os.path.join(home, 'dev', 'patman') + if os.path.isdir(patman_dir): + mounts.append(('patman', patman_dir, f'{UBUNTU_HOME}/dev/patman')) + + # X11 socket for clipboard access (image paste in Claude Code) + x11_dir = '/tmp/.X11-unix' + if os.path.isdir(x11_dir): + mounts.append(('x11', x11_dir, x11_dir)) + + # PulseAudio socket for voice input (/voice in Claude Code) + pulse_sock = f'/run/user/{os.getuid()}/pulse/native' + if os.path.exists(pulse_sock): + mounts.append(('pulse', pulse_sock, PULSE_DEST)) + for fname, mname in [('.gitconfig', 'gitconfig'), ('.buildman', 'buildman'), ('.buildman-toolchains', 'toolchains')]: @@ -120,6 +147,50 @@ def get_config_mounts(): return mounts +def get_cli_mounts(mount_args): + """Parse -m/--mount command-line arguments into mount triples + + Supports HOST:DEST or just HOST (mounted at the same path). + + Args: + mount_args (list of str): Mount arguments from command line + + Returns: + list of tuple: (name, src, dst) triples + """ + if not mount_args: + return [] + + home = os.path.expanduser('~') + mounts = [] + for i, arg in enumerate(mount_args): + parts = arg.split(':') + if len(parts) == 2: + src, dst = parts + elif len(parts) == 1: + src = dst = parts[0] + else: + tout.warning(f'Ignoring malformed mount: {arg}') + continue + src = os.path.expandvars(os.path.expanduser(src)) + src = os.path.realpath(src) + # Expand ~ to the container home, not the host home + dst = os.path.expandvars(dst) + if dst.startswith('~'): + dst = UBUNTU_HOME + dst[1:] + elif dst.startswith(home): + dst = UBUNTU_HOME + dst[len(home):] + # Use the leaf directory as the device name, with a suffix if needed + leaf = os.path.basename(dst) or f'cli{i}' + name = leaf + suffix = 2 + while any(m[0] == name for m in mounts): + name = f'{leaf}{suffix}' + suffix += 1 + mounts.append((name, src, dst)) + return mounts + + def get_git_symlink_mount(project_src): """Handle .git symlink by creating a mount for the real target @@ -248,6 +319,21 @@ def create_container(name, base, dry_run=False): f'{proc.stderr.decode("utf-8", errors="replace")}') +def is_privileged(name): + """Check whether a container has privileged mode enabled + + Args: + name (str): Container name + + Returns: + bool: True if security.privileged is set to true + """ + result = exec_cmd( + ['lxc', 'config', 'get', name, 'security.privileged'], + dry_run=False) + return result is not None and result.stdout.strip() == 'true' + + def has_mount(name, mount_name): """Check whether a container already has a named device @@ -288,6 +374,24 @@ def add_mount(name, mount_name, source, path, dry_run=False, shift=False): *args, dry_run=dry_run) +def remove_mount(name, mount_name, dry_run=False): + """Remove a disk device from a container + + Args: + name (str): Container name + mount_name (str): Device name + dry_run (bool): If True, just show command + + Returns: + bool: True if removed successfully + """ + if not dry_run and not has_mount(name, mount_name): + tout.error(f'No device {mount_name!r} on container {name}') + return False + lxc('config', 'device', 'remove', name, mount_name, dry_run=dry_run) + return True + + def wait_for_user(name, dry_run=False): """Wait until the ubuntu user exists in the container @@ -318,6 +422,10 @@ def setup_container(name, dry_run=False): """ lxc_exec(name, 'chown ubuntu:ubuntu /home/ubuntu', dry_run=dry_run) + # Suppress the Ubuntu sudo hint message + lxc_exec(name, 'touch /home/ubuntu/.sudo_as_admin_successful', + dry_run=dry_run, user='ubuntu') + # Install terminfo from host if not dry_run: import subprocess # pylint: disable=import-outside-toplevel @@ -397,10 +505,13 @@ def setup_uman(name, uboot_tools=None, dry_run=False): uman_dir = get_uman_dir() um_path = os.path.join(uman_dir, 'um') uman_bin = os.path.join(uman_dir, 'uman_pkg', 'uman') + patman = f'{UBUNTU_HOME}/dev/patman/tools/patman/patman' lxc_exec(name, f'mkdir -p ~/.local/bin && ' f'ln -sf {uman_bin} {um_path} && ' - f'ln -sf {uman_bin} ~/.local/bin/um', + f'ln -sf {uman_bin} ~/.local/bin/um && ' + f'test -e {patman} && ln -sf {patman} ~/.local/bin/patman ' + f'|| true', dry_run=dry_run, user='ubuntu') setup_cmd = ( f'export PATH="$HOME/.local/bin:$HOME/bin:$PATH" && ' @@ -408,13 +519,68 @@ def setup_uman(name, uboot_tools=None, dry_run=False): f'{um_path} -q setup aliases -d ~/.local/bin -f') lxc_exec(name, setup_cmd, dry_run=dry_run, user='ubuntu') + # Write editor proxy script + editor_script = ( + '#!/usr/bin/env python3\n' + 'import json, os, socket, sys\n' + 'path = os.path.abspath(sys.argv[1]) if len(sys.argv) > 1' + ' else sys.exit(1)\n' + f'sock_path = "{PROJECT_DEST}/{EDITOR_SOCK}"\n' + 's = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n' + 'try:\n' + ' s.connect(sock_path)\n' + 'except OSError:\n' + ' sys.exit("editor proxy not running")\n' + f'if path.startswith("{PROJECT_DEST}/"):\n' + ' s.sendall((path + "\\n").encode())\n' + ' resp = s.recv(4096).decode().strip()\n' + 'else:\n' + ' content = open(path).read() if os.path.exists(path) else ""\n' + ' ext = os.path.splitext(path)[1]\n' + ' msg = json.dumps({"content": content, "ext": ext}) + "\\n"\n' + ' s.sendall(msg.encode())\n' + ' resp_raw = b""\n' + ' while True:\n' + ' chunk = s.recv(65536)\n' + ' if not chunk:\n' + ' break\n' + ' resp_raw += chunk\n' + ' resp_data = json.loads(resp_raw.decode())\n' + ' if resp_data.get("error"):\n' + ' print(resp_data["error"], file=sys.stderr)\n' + ' sys.exit(1)\n' + ' open(path, "w").write(resp_data["content"])\n' + ' resp = "done"\n' + 's.close()\n' + 'if resp != "done":\n' + ' print(resp, file=sys.stderr)\n' + ' sys.exit(1)\n') + editor_path = f'{UBUNTU_HOME}/.local/bin/uman-editor' + write_editor = ( + f"cat > {editor_path} <<'EDEOF'\n{editor_script}EDEOF\n" + f"chmod +x {editor_path}") + lxc_exec(name, write_editor, dry_run=dry_run, user='ubuntu') + + # Write ALSA config that routes through PulseAudio + alsa_conf = 'pcm.!default { type pulse }\nctl.!default { type pulse }\n' + lxc_exec(name, + f"cat > {ALSA_PULSE_CONF} <<'ALSAEOF'\n{alsa_conf}ALSAEOF", + dry_run=dry_run) + # Write ~/.uman_env with the full environment block + display = os.environ.get('DISPLAY', ':0') env_block = ( '# uman setup — sourced by ~/.bashrc, ~/.profile and BASH_ENV\n' '[ "$_UMAN_ENV_LOADED" = 1 ] && return\n' '_UMAN_ENV_LOADED=1\n' 'export PATH="$HOME/bin:$HOME/.local/bin:$PATH"\n' f'export UBOOT_TOOLS="{uboot_tools}"\n' + f'export DISPLAY="{display}"\n' + f'export EDITOR="{editor_path}"\n' + f'export PULSE_SERVER="unix:{PULSE_DEST}"\n' + 'export AUDIODRIVER=pulseaudio\n' + f'export ALSA_CONFIG_PATH="{ALSA_PULSE_CONF}"\n' + 'export PULSE_LATENCY_MSEC=50\n' 'um() { b="$b" USRC="$USRC" command um "$@"; }\n' 'eval "$(um git -a)"\n' 'export BASH_ENV=~/.uman_env\n') @@ -431,6 +597,106 @@ def setup_uman(name, uboot_tools=None, dry_run=False): lxc_exec(name, add_cmd, dry_run=dry_run, user='ubuntu') +def editor_listen(sock_path, project_src, host_editor, ready=None): + """Listen for editor requests from the container + + Runs in a daemon thread. Accepts connections on a Unix socket. + For project paths, translates to host paths and opens the editor. + For non-project paths (e.g. /tmp), receives file content as JSON, + writes a temp file on the host, opens the editor, and sends back + the edited content. + + Args: + sock_path (str): Path to the Unix socket file + project_src (str): Host-side project directory + host_editor (str): Host editor command + ready (threading.Event or None): Set when socket is bound + """ + import json + import tempfile + + sock = socket_mod.socket(socket_mod.AF_UNIX, socket_mod.SOCK_STREAM) + sock.bind(sock_path) + # Make socket world-writable so container user can connect + os.chmod(sock_path, 0o777) + sock.listen(1) + sock.settimeout(2) + if ready: + ready.set() + while True: + try: + conn, _ = sock.accept() + except socket_mod.timeout: + continue + except OSError: + break + try: + data = conn.recv(4096).decode().strip() + if not data: + continue + + # Translate container path to host path + if data.startswith(PROJECT_DEST + '/'): + rel = data[len(PROJECT_DEST) + 1:] + host_path = os.path.join(project_src, rel) + subprocess.run([host_editor, host_path], check=False) + conn.sendall(b'done\n') + elif data.startswith(PROJECT_DEST): + subprocess.run([host_editor, project_src], check=False) + conn.sendall(b'done\n') + elif data.startswith('{'): + msg = json.loads(data) + ext = msg.get('ext', '.txt') + with tempfile.NamedTemporaryFile( + mode='w', suffix=ext, delete=False) as tmp: + tmp.write(msg['content']) + tmp_path = tmp.name + try: + subprocess.run([host_editor, tmp_path], check=False) + with open(tmp_path) as fh: + edited = fh.read() + resp = json.dumps({'content': edited}) + conn.sendall(resp.encode()) + finally: + os.unlink(tmp_path) + else: + conn.sendall(b'error: path outside project\n') + except OSError: + pass + finally: + conn.close() + + +def start_editor_proxy(project_src, dry_run=False): + """Start the editor proxy listener in a background thread + + Args: + project_src (str): Host-side project directory + dry_run (bool): If True, just show what would happen + + Returns: + str: Path to the socket file + """ + sock_path = os.path.join(project_src, EDITOR_SOCK) + if dry_run: + tout.notice(f'# editor proxy: {sock_path}') + return sock_path + + # Remove stale socket + if os.path.exists(sock_path): + os.unlink(sock_path) + + host_editor = os.environ.get('EDITOR', 'vi') + ready = threading.Event() + thread = threading.Thread(target=editor_listen, + args=(sock_path, project_src, host_editor, + ready), + daemon=True) + thread.start() + ready.wait() + return sock_path + + def launch_shell(name, shell_command=None, dry_run=False, log_file=None): """Open an interactive shell or run a command in the container @@ -519,7 +785,7 @@ def list_containers(): """List uman containers (those with a datadir device) Returns: - list of tuple: (name, status, project) triples + list of tuple: (name, status, project, privileged) 4-tuples """ result = exec_cmd(['lxc', 'list', '--format', 'csv', '-c', 'ns'], dry_run=False) @@ -534,18 +800,23 @@ def list_containers(): if len(parts) >= 2: project = get_project(parts[0]) if project: - containers.append((parts[0], parts[1], project)) + priv = is_privileged(parts[0]) + containers.append((parts[0], parts[1], project, priv)) return containers -def add_all_mounts(name, project_src, dry_run=False): - """Add all mounts (essential, git symlink, and config) to a container +def add_all_mounts(name, project_src, mount_args=None, output=False, + no_output=False, dry_run=False): + """Add all mounts (essential, git symlink, config, CLI) to a container Skips any devices that already exist. Args: name (str): Container name project_src (str): Absolute path to the project source directory + mount_args (list of str): Mount arguments from -m flag + output (bool): If True, mount /tmp/b into the container + no_output (bool): If True, remove /tmp/b mount dry_run (bool): If True, just show commands """ for mname, source, dest in get_essential_mounts(project_src): @@ -562,14 +833,19 @@ def add_all_mounts(name, project_src, dry_run=False): if git_mount: add_mount(name, *git_mount, dry_run) - # Mount container /tmp/b to host /tmp//b for easy access - tmp_dir = f'/tmp/{name}/b' - os.makedirs(tmp_dir, exist_ok=True) - new_tmpb = not dry_run and not has_mount(name, 'tmpb') - add_mount(name, 'tmpb', tmp_dir, '/tmp/b', dry_run) - if new_tmpb and container_status(name) == 'RUNNING': - tout.notice( - f'Added /tmp/b mount; activate with: uman cc -R {name}') + # Mount /tmp/b if requested, or remove if -O + if output: + tmp_dir = '/tmp/b' + os.makedirs(tmp_dir, exist_ok=True) + new_tmpb = not dry_run and not has_mount(name, 'tmpb') + add_mount(name, 'tmpb', tmp_dir, '/tmp/b', dry_run) + if new_tmpb and container_status(name) == 'RUNNING': + tout.notice( + f'Added /tmp/b mount; activate with: uman cc -R {name}') + elif no_output: + if not dry_run and has_mount(name, 'tmpb'): + remove_mount(name, 'tmpb') + tout.notice('Removed /tmp/b mount') pbuilder = '/var/cache/pbuilder' if os.path.isdir(pbuilder): @@ -579,6 +855,9 @@ def add_all_mounts(name, project_src, dry_run=False): for mname, source, dest in get_config_mounts(): add_mount(name, mname, source, dest, dry_run) + for mname, source, dest in get_cli_mounts(mount_args): + add_mount(name, mname, source, dest, dry_run) + def ensure_running(name, existed, dry_run=False): """Start the container if it is not already running @@ -609,10 +888,54 @@ def show_containers(): tout.notice('No uman containers found') else: home = os.path.expanduser('~') - for cname, status, project in containers: + for cname, status, project, priv in containers: if project.startswith(home): project = '~' + project[len(home):] - tout.notice(f'{cname} {status:8s} {project}') + flags = ' [privileged]' if priv else '' + tout.notice(f'{cname} {status:8s} {project}{flags}') + return 0 + + +def show_mounts(name): + """List mounts for a container + + Args: + name (str): Container name + + Returns: + int: Exit code + """ + result = exec_cmd(['lxc', 'config', 'device', 'show', name], + dry_run=False) + if not result or result.return_code: + tout.error(f'Container not found: {name}') + return 1 + + home = os.path.expanduser('~') + mounts = [] + cur_name = None + source = path = None + for line in result.stdout.splitlines(): + if not line.startswith(' '): + if cur_name and source and path: + mounts.append((cur_name, source, path)) + cur_name = line.rstrip(':') + source = path = None + elif 'source:' in line: + source = line.split(':', 1)[1].strip() + elif 'path:' in line: + path = line.split(':', 1)[1].strip() + if cur_name and source and path: + mounts.append((cur_name, source, path)) + + if not mounts: + tout.notice(f'No mounts for {name}') + return 0 + + for mname, source, path in mounts: + if source.startswith(home): + source = '~' + source[len(home):] + print(f' {mname:14s} {source} -> {path}') return 0 @@ -631,6 +954,27 @@ def run(args): # pylint: disable=too-many-locals,too-many-branches,too-many-sta if args.list_containers: return show_containers() + if args.mounts: + name = args.name or os.path.basename(os.path.realpath(os.getcwd())) + return show_mounts(name) + + if args.mount and not args.shell: + name = args.name or os.path.basename(os.path.realpath(os.getcwd())) + if not args.dry_run and not container_exists(name): + tout.error(f'Container not found: {name}') + return 1 + for mname, source, dest in get_cli_mounts(args.mount): + add_mount(name, mname, source, dest, args.dry_run) + tout.notice(f'Mounted {source} -> {dest} ({mname})') + return 0 + + if args.unmount: + name = args.name or os.path.basename(os.path.realpath(os.getcwd())) + if not args.dry_run and not container_exists(name): + tout.error(f'Container not found: {name}') + return 1 + return 0 if remove_mount(name, args.unmount, args.dry_run) else 1 + if args.delete: if not args.name: tout.error('Container name required for --delete') @@ -684,11 +1028,13 @@ def run(args): # pylint: disable=too-many-locals,too-many-branches,too-many-sta else: tout.notice(f'Container: {name}') + sock_path = os.path.join(project_src, EDITOR_SOCK) try: if not existed: create_container(name, base, dry_run) - add_all_mounts(name, project_src, dry_run) + add_all_mounts(name, project_src, args.mount, args.output, + args.no_output, dry_run) if args.restart and existed: status = container_status(name) @@ -697,8 +1043,71 @@ def run(args): # pylint: disable=too-many-locals,too-many-branches,too-many-sta lxc('stop', name) existed = False + if args.privileged: + lxc('config', 'set', '-q', name, 'security.privileged=true', + dry_run=dry_run) + lxc('config', 'set', '-q', name, 'raw.idmap=', + dry_run=dry_run) + raw_lxc = ('lxc.apparmor.profile=unconfined\n' + 'lxc.seccomp.profile=') + lxc('config', 'set', '-q', name, 'raw.lxc', raw_lxc, + dry_run=dry_run) + lxc('config', 'set', '-q', name, + 'security.nesting=true', dry_run=dry_run) + tout.notice('Enabled privileged mode') + if existed and not dry_run: + status = container_status(name) + if status == 'RUNNING': + tout.notice( + f'Restart needed: um cc -R {name}') + return 0 + elif args.no_privileged: + uid = str(os.getuid()) + gid = str(os.getgid()) + idmap = f'uid {uid} 1000\ngid {gid} 1000' + lxc('config', 'set', '-q', name, + 'security.privileged=false', dry_run=dry_run) + lxc('config', 'set', '-q', name, 'raw.lxc=', + dry_run=dry_run) + lxc('config', 'set', '-q', name, + 'security.nesting=false', dry_run=dry_run) + if not dry_run: + subprocess.run( + ['lxc', 'config', 'set', '-q', name, 'raw.idmap', '-'], + input=idmap.encode(), check=False, capture_output=True) + else: + tout.notice( + f'printf {idmap!r} | lxc config set -q {name} raw.idmap -') + tout.notice('Disabled privileged mode') + if existed and not dry_run: + status = container_status(name) + if status == 'RUNNING': + tout.notice('Restarting container') + lxc('stop', name) + existed = False + elif existed and not dry_run: + if is_privileged(name): + tout.notice( + 'Running in privileged mode (device-mapper enabled)') + ensure_running(name, existed, dry_run) + # In privileged mode, uid namespacing is disabled, so the + # container's ubuntu user (uid 1000) won't match the host uid. + # Fix this by changing ubuntu's uid/gid to match the host. + if args.privileged: + uid = os.getuid() + gid = os.getgid() + lxc_exec(name, + f'usermod -u {uid} ubuntu; groupmod -g {gid} ubuntu;' + f' chown -R {uid}:{gid} /home/ubuntu', + dry_run=dry_run) + elif args.no_privileged: + lxc_exec(name, + 'usermod -u 1000 ubuntu; groupmod -g 1000 ubuntu;' + ' chown -R 1000:1000 /home/ubuntu', + dry_run=dry_run) + # Wait for user and set up (idempotent operations) wait_for_user(name, dry_run) setup_container(name, dry_run) @@ -706,6 +1115,17 @@ def run(args): # pylint: disable=too-many-locals,too-many-branches,too-many-sta install_claude(name, dry_run) setup_uman(name, uboot_tools, dry_run) + # Check X11 access for clipboard (image paste) + if not dry_run and os.path.isdir('/tmp/.X11-unix'): + result = exec_cmd(['xhost'], dry_run=False) + if result and 'LOCAL:' not in result.stdout: + tout.notice( + 'For clipboard access (image paste): ' + 'xhost +local:') + + # Start editor proxy so Ctrl-G opens the host editor + sock_path = start_editor_proxy(project_src, dry_run) + # Launch log_file = get_log_path(name) tout.notice(f'Logging to {log_file}') @@ -716,6 +1136,10 @@ def run(args): # pylint: disable=too-many-locals,too-many-branches,too-many-sta launch_claude(name, args.cont, dry_run, log_file) finally: + # Clean up editor socket + if os.path.exists(sock_path): + os.unlink(sock_path) + # Only delete ephemeral containers that we created if not keep and not existed: delete_container(name, dry_run) diff --git a/uman_pkg/cmddocker.py b/uman_pkg/cmddocker.py new file mode 100644 index 0000000..952681f --- /dev/null +++ b/uman_pkg/cmddocker.py @@ -0,0 +1,245 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2025 Canonical Ltd +# Written by Simon Glass + +"""Docker command for running U-Boot tests in a CI Docker container + +This module handles the 'docker' subcommand which runs U-Boot pytest +tests inside the same Docker image used by CI. +""" + +import os + +import yaml + +# pylint: disable=import-error +from u_boot_pylib import command +from u_boot_pylib import tout + +from uman_pkg import util + + +# Template key in .gitlab-ci.yml +CI_TEMPLATE = '.buildman_and_testpy_template' + +# CI variable substitutions (variable name -> default value) +CI_SUBS = { + 'CI_PROJECT_DIR': '/source', + 'OVERRIDE': '', + 'BUILD_ENV': '', + 'TEST_PY_ID': '', + 'TEST_PY_EXTRA': '', +} + + +def load_ci_yaml(uboot_dir): + """Load and parse .gitlab-ci.yml from a U-Boot tree + + Args: + uboot_dir (str): Path to U-Boot source directory + + Returns: + dict: Parsed YAML data, or None if not found + """ + ci_path = os.path.join(uboot_dir, '.gitlab-ci.yml') + if not os.path.exists(ci_path): + tout.error(f'Cannot find {ci_path}') + return None + + with open(ci_path, 'r', encoding='utf-8') as inf: + return yaml.safe_load(inf) + + +def get_ci_image(data): + """Extract the CI Docker image from parsed .gitlab-ci.yml data + + Looks for the default image and expands ${MIRROR_DOCKER} to + docker.io. + + Args: + data (dict): Parsed YAML data + + Returns: + str: Full Docker image path, or None if not found + """ + image = data.get('image') + if not image: + default = data.get('default', {}) + image = default.get('image') if isinstance(default, dict) else None + if not image: + tout.error('Cannot find image in .gitlab-ci.yml') + return None + + return image.replace('${MIRROR_DOCKER}', 'docker.io') + + +def get_ci_script(data): + """Extract before_script and script from the CI test template + + Args: + data (dict): Parsed YAML data + + Returns: + tuple: (before_script, script) as lists of shell command strings, + or (None, None) if not found + """ + template = data.get(CI_TEMPLATE) + if not template: + tout.error(f'Cannot find {CI_TEMPLATE} in .gitlab-ci.yml') + return None, None + + before = template.get('before_script', []) + script = template.get('script', []) + return before, script + + +def build_script(data, board, test_spec, adjust_cfg=None, + pytest_args=None, gdb=False): + """Generate the shell script to run inside the Docker container + + Parses the before_script and script sections from .gitlab-ci.yml + and applies variable substitutions for the given board and test spec. + + Args: + data (dict): Parsed YAML data + board (str): Board name (e.g. 'sandbox') + test_spec (str or None): pytest -k filter spec + adjust_cfg (list or None): Kconfig adjustments for buildman -a + pytest_args (list or None): Extra pytest flags (e.g. ['-x', '-s']) + gdb (bool): Install gdbserver in the container + + Returns: + str: Shell script to pass to bash -c, or None on error + """ + before, script = get_ci_script(data) + if before is None: + return None + + # Build substitution map; put -a flags into OVERRIDE so they + # are appended to the buildman command line + override = '' + if adjust_cfg: + override = ' '.join(f'-a {cfg}' for cfg in adjust_cfg) + subs = dict(CI_SUBS) + subs['OVERRIDE'] = override + subs['TEST_PY_BD'] = board + subs['TEST_PY_EXTRA'] = ' '.join(pytest_args) if pytest_args else '' + + # Apply substitutions to each command; set test spec as shell + # variables so bash handles ${VAR:+...} expansions natively + spec = test_spec or '' + commands = ['set -e'] + if gdb: + commands.append( + 'which gdbserver >/dev/null 2>&1 ||' + ' (apt-get update -qq && apt-get install -y -qq gdbserver)') + commands.extend(['mkdir -p test/hooks/bin test/hooks/py', + f'export TEST_PY_TEST_SPEC="{spec}"', + f'export TEST_SPEC="{spec}"']) + # Insert gdbserver wrapper just before test.py (the last command + # in script). This wraps the main u-boot binary, not SPL, so gdb + # doesn't need to follow the SPL-to-u-boot exec. + gdb_wrapper = [] + if gdb: + gdb_wrapper = [ + 'bd=$UBOOT_TRAVIS_BUILD_DIR', + 'mv $bd/u-boot $bd/u-boot.real', + 'printf \'#!/bin/bash\\n' + 'exec gdbserver :1234 ' + '"$(dirname "$0")/u-boot.real" "$@"\\n\'' + ' > $bd/u-boot', + 'chmod +x $bd/u-boot', + ] + + all_cmds = before + script + for i, cmd in enumerate(all_cmds): + for var, val in subs.items(): + cmd = cmd.replace(f'${{{var}}}', val) + if gdb_wrapper and i == len(all_cmds) - 1: + commands.extend(gdb_wrapper) + commands.append(cmd) + + return '\n'.join(commands) + + +def run(args): + """Run the docker command + + Args: + args (argparse.Namespace): Parsed arguments + + Returns: + int: Exit code (0 for success, non-zero for failure) + """ + uboot_dir = util.get_uboot_dir() + if not uboot_dir: + tout.error('Not in a U-Boot tree and $USRC not set') + return 1 + + # Load CI config + data = load_ci_yaml(uboot_dir) + if not data: + return 1 + + # Determine Docker image + image = args.image + if not image: + image = get_ci_image(data) + if not image: + return 1 + + board = args.board + + uid_gid = command.output_one_line('id', '-u') + ':' + \ + command.output_one_line('id', '-g') + + docker_cmd = ['docker', 'run', '--rm', + '-e', 'HOME=/tmp', + '-v', '/etc/passwd:/etc/passwd:ro', + '-v', f'{uboot_dir}:/source', + '-w', '/source'] + + # Run as root when gdbserver is needed so we can install it; + # otherwise run as current user for correct file ownership + if args.gdb_phase: + docker_cmd.extend(['--user', '0:0', + '--cap-add=SYS_PTRACE', + '-p', '1234:1234']) + tout.notice('When tests stall, connect from another terminal:') + tout.notice(f' um py -G -B {board}') + tout.notice('Type c to continue; tests then proceed') + else: + docker_cmd.extend(['--user', uid_gid]) + + docker_cmd.append(image) + + if args.interactive: + docker_cmd.append('bash') + tout.notice(f'Starting interactive shell in {image}...') + else: + tout.notice(f'Building and testing {board}...') + spec = ' '.join(args.test_spec) if args.test_spec else None + extra = [] + if args.exitfirst: + extra.append('-x') + if args.show_output: + extra.append('-s') + + # For SPL debugging, pass --gdbserver to test.py so it wraps + # the initial binary (SPL); for u-boot debugging, use a wrapper + # script so gdbserver starts after SPL exec's the main binary + gdb = bool(args.gdb_phase) + if args.gdb_phase and args.gdb_phase != 'u-boot': + extra.extend(['--gdbserver', 'localhost:1234']) + gdb = False + script = build_script(data, board, spec, args.adjust_cfg, + extra or None, gdb=gdb) + if not script: + return 1 + docker_cmd.extend(['bash', '-c', script]) + + result = util.exec_cmd(docker_cmd, args.dry_run, capture=False) + if result and result.return_code: + return result.return_code + + return 0 diff --git a/uman_pkg/cmdgit.py b/uman_pkg/cmdgit.py index 2c92ae7..ffeac27 100644 --- a/uman_pkg/cmdgit.py +++ b/uman_pkg/cmdgit.py @@ -97,6 +97,8 @@ def show_rebase_status(output, return_code=0): label = 'empty commit' tout.notice(f'Rebasing{pos_str} {label} {match.group(1)}... ' f'{match.group(2)}') + else: + tout.error(output.strip()) def show_rb_status(): @@ -133,9 +135,16 @@ def seq_edit_env(action, line=1): """ env = os.environ.copy() if action == 'break': - env['GIT_SEQUENCE_EDITOR'] = f'sed -i "{line}i break"' + script = (f"import sys; p=sys.argv[1]; " + f"lines=open(p).readlines(); " + f"lines.insert({line - 1},'break\\n'); " + f"open(p,'w').writelines(lines)") else: # edit - env['GIT_SEQUENCE_EDITOR'] = f'sed -i "{line}s/^pick/edit/"' + script = (f"import sys,re; p=sys.argv[1]; " + f"lines=open(p).readlines(); " + f"lines[{line - 1}]=re.sub(r'^pick','edit',lines[{line - 1}]); " + f"open(p,'w').writelines(lines)") + env['GIT_SEQUENCE_EDITOR'] = f'python3 -c "{script}"' return env @@ -1354,7 +1363,7 @@ def do_ust(_args): GitAction('cms', 'commit-signoff', 'Commit with signoff', do_cms), GitAction('co', 'checkout', 'Checkout (switch branches/restore)', do_co), GitAction('db', 'diff-branch', 'Diff commit files against upstream', do_db), - GitAction('dh', 'diff-head', 'Show diff of top commit', do_dh), + GitAction('di', 'diff-head', 'Show diff of top commit', do_dh), GitAction('eg', 'errno-grep', 'Search errno.h for error codes', do_eg), GitAction('et', 'edit-todo', 'Edit rebase todo list', do_et), GitAction('g', 'status', 'Show short status', do_g), @@ -1407,7 +1416,7 @@ def do_ust(_args): 'cms': 'git commit -s', 'co': 'git checkout', 'cs': 'git show', - 'dh': 'git difftool HEAD~', + 'di': 'git difftool HEAD~', 'g': 'git status -sb', 'gb': 'git branch', 'gba': 'git branch -a', diff --git a/uman_pkg/cmdline.py b/uman_pkg/cmdline.py index 23ae61d..19511a8 100644 --- a/uman_pkg/cmdline.py +++ b/uman_pkg/cmdline.py @@ -39,6 +39,7 @@ def get_git_action_names(): ALIASES = { 'claude-code': ['cc'], 'config': ['cfg'], + 'docker': ['d'], 'git': ['g'], 'selftest': ['st'], 'pytest': ['py'], @@ -91,12 +92,43 @@ def add_claude_code_subparser(subparsers): cc.add_argument('-l', '--list', action='store_true', dest='list_containers', help='List existing uman containers with project paths') + cc.add_argument('-m', '--mount', action='append', metavar='PATH', + help='Mount a host directory (PATH or HOST:DEST)') + cc.add_argument('-M', '--mounts', action='store_true', + help='List mounts for the container') + cc.add_argument('-u', '--unmount', metavar='NAME', + help='Remove a mount by device name (see -M)') + cc.add_argument('-o', '--output', action='store_true', + help='Mount /tmp/b into the container') + cc.add_argument('-O', '--no-output', action='store_true', + help='Remove /tmp/b mount from the container') + cc.add_argument('-p', '--privileged', action='store_true', + help='Enable privileged mode (e.g. LUKS tests)') + cc.add_argument('-P', '--no-privileged', action='store_true', + help='Disable privileged mode') cc.add_argument('-s', '--shell', nargs='?', const=True, default=False, help='Open shell or run a command in container') return cc +def add_docker_subparser(subparsers): + """Add the 'docker' subparser""" + dtest = subparsers.add_parser( + 'docker', aliases=ALIASES['docker'], + help='Run U-Boot tests in CI Docker container') + add_test_opts(dtest, board_help='Board name (default: sandbox)', + board_default='sandbox') + dtest.add_argument('-a', '--adjust-cfg', action='append', + metavar='CFG', dest='adjust_cfg', + help='Adjust Kconfig (can use multiple times)') + dtest.add_argument('-i', '--image', metavar='IMAGE', + help='Override Docker image') + dtest.add_argument('-I', '--interactive', action='store_true', + help='Drop to shell in container') + return dtest + + def add_ci_subparser(subparsers): """Add the 'ci' subparser""" ci = subparsers.add_parser('ci', help='Push current branch to CI') @@ -145,34 +177,67 @@ def add_selftest_subparser(subparsers): return stest +def add_test_opts(parser, board_help=None, board_default=None): + """Add common test options to a parser + + Args: + parser: Argument parser to add options to + board_help (str or None): Help text for -B flag + board_default (str or None): Default board value + """ + parser.add_argument( + 'test_spec', type=str, nargs='*', + help="Test specification (e.g. 'test_dm', 'not sleep')") + parser.add_argument( + '-B', '--board', metavar='BOARD', default=board_default, + help=board_help or 'Board name to test') + parser.add_argument( + '-g', action='store_true', default=False, dest='gdbserver_flag', + help='Debug with gdbserver (u-boot phase)') + parser.add_argument( + '--gdb-phase', default=None, dest='gdb_phase', + choices=['spl', 'tpl', 'vpl'], + help='Debug a specific phase with gdbserver (implies -g)') + parser.add_argument( + '-s', '--show-output', action='store_true', + help='Show all test output in real-time (pytest -s)') + parser.add_argument( + '-x', '--exitfirst', action='store_true', + help='Stop on first test failure') + parser.add_argument( + '--malloc-dump', metavar='FILE', dest='malloc_dump', + help='Write malloc dump to FILE on exit (use %%d for sequence number)') + + def add_build_opts(parser): """Add common build options to a parser Args: parser: Argument parser to add options to """ - parser.add_argument( + group = parser.add_argument_group('build options') + group.add_argument( '-a', '--adjust-cfg', action='append', metavar='CFG', dest='adjust_cfg', help='Adjust Kconfig setting (use with -b; can use multiple times)') - parser.add_argument( + group.add_argument( '-f', '--force-reconfig', action='store_true', help='Force reconfiguration (use with -b)') - parser.add_argument( + group.add_argument( '-F', '--fresh', action='store_true', help='Delete build dir before building (use with -b)') - parser.add_argument( + group.add_argument( '-j', '--jobs', type=int, metavar='JOBS', help='Number of parallel jobs for build (use with -b)') - parser.add_argument( + group.add_argument( '-L', '--lto', action='store_true', help='Enable LTO when building (use with -b)') - parser.add_argument( + group.add_argument( '-o', '--output-dir', metavar='DIR', dest='output_dir', help='Override build directory (use with -b)') - parser.add_argument( + group.add_argument( '-T', '--trace', action='store_true', help='Enable function tracing (use with -b)') - parser.add_argument( + group.add_argument( '--no-trace-early', action='store_true', dest='no_trace_early', help='Disable TRACE_EARLY when using -T (use with -b)') @@ -182,15 +247,11 @@ def add_pytest_subparser(subparsers): pyt = subparsers.add_parser( 'pytest', aliases=ALIASES['pytest'], help='Run pytest tests for U-Boot') - pyt.add_argument( - 'test_spec', type=str, nargs='*', - help="Test specification (e.g. 'test_dm', 'not sleep')") + add_test_opts(pyt, + board_help='Board name to test (required; use -l to list QEMU boards)') pyt.add_argument( '-b', '--build', action='store_true', help='Build U-Boot before running tests') - pyt.add_argument( - '-B', '--board', metavar='BOARD', - help='Board name to test (required; use -l to list QEMU boards)') pyt.add_argument( '-c', '--show-cmd', action='store_true', help='Show QEMU command line without running tests') @@ -203,9 +264,6 @@ def add_pytest_subparser(subparsers): pyt.add_argument( '--find', metavar='PATTERN', help='Find tests matching PATTERN and show full IDs') - pyt.add_argument( - '-g', action='store_const', const='localhost:1234', dest='gdbserver', - help='Start gdbserver (wait for gdb client to connect)') pyt.add_argument( '-G', '--gdb', action='store_true', help='Launch gdb client (connect to existing gdbserver from -g)') @@ -218,9 +276,6 @@ def add_pytest_subparser(subparsers): pyt.add_argument( '-q', '--quiet', action='store_true', help='Quiet mode: only show build output, progress, and result') - pyt.add_argument( - '-s', '--show-output', action='store_true', - help='Show all test output in real-time (pytest -s)') pyt.add_argument( '-S', '--setup-only', action='store_true', help='Run only fixture setup (create test images) without tests') @@ -231,9 +286,6 @@ def add_pytest_subparser(subparsers): pyt.add_argument( '--no-timeout', action='store_true', help='Disable test timeout') - pyt.add_argument( - '-x', '--exitfirst', action='store_true', - help='Stop on first test failure') pyt.add_argument( '--pollute', metavar='TEST', help='Find which test pollutes TEST (causes it to fail)') @@ -300,7 +352,10 @@ def add_setup_subparser(subparsers): 'setup', help='Build firmware blobs needed for testing') setup.add_argument( 'component', type=str, nargs='?', default=None, - help="Component to build (e.g. 'opensbi'), or omit to build all") + help="Component to set up (e.g. 'opensbi', 'remote'), or omit for all") + setup.add_argument( + 'host', type=str, nargs='?', default=None, + help="Hostname for 'remote' component (e.g. user@host)") setup.add_argument( '-d', '--alias-dir', metavar='DIR', help='Directory for aliases symlinks (default: ~/bin)') @@ -345,6 +400,9 @@ def add_test_subparser(subparsers): test.add_argument( '-s', '--suites', action='store_true', dest='list_suites', help='List available test suites') + test.add_argument( + '--malloc-dump', metavar='FILE', dest='malloc_dump', + help='Write malloc dump to FILE on exit (use %%d for sequence number)') test.add_argument( '-V', '--test-verbose', action='store_true', dest='test_verbose', help='Enable verbose test output') @@ -437,6 +495,7 @@ def setup_parser(): add_claude_code_subparser(subparsers) add_ci_subparser(subparsers) add_config_subparser(subparsers) + add_docker_subparser(subparsers) add_git_subparser(subparsers) add_selftest_subparser(subparsers) add_pytest_subparser(subparsers) @@ -505,6 +564,14 @@ def parse_args(argv=None, prog_name=None): if hasattr(args, 'extra_args'): args.extra_args = extra_args + # Reconcile -g and --gdb-phase into gdb_phase + if hasattr(args, 'gdbserver_flag'): + if args.gdb_phase: + pass # --gdb-phase already set + elif args.gdbserver_flag: + args.gdb_phase = 'u-boot' + del args.gdbserver_flag + # Resolve aliases for full, aliases in ALIASES.items(): if args.cmd in aliases: diff --git a/uman_pkg/cmdpy.py b/uman_pkg/cmdpy.py index bcbfa3e..e09673c 100644 --- a/uman_pkg/cmdpy.py +++ b/uman_pkg/cmdpy.py @@ -461,12 +461,16 @@ def build_pytest_cmd(args): cmd.append('--setup-only') if args.persist: cmd.append('--persist') - if args.gdbserver: - cmd.extend(['--gdbserver', args.gdbserver]) + gdb_channel = args.gdbserver or ( + 'localhost:1234' if args.gdb_phase else None) + if gdb_channel: + cmd.extend(['--gdbserver', gdb_channel]) if args.exitfirst: cmd.append('-x') if not args.flattree_too and has_no_full(): cmd.append('--no-full') + if args.malloc_dump: + cmd.extend(['--malloc-dump', args.malloc_dump]) # Add extra pytest arguments (after --) if args.extra_args: @@ -954,7 +958,7 @@ def run_with_gdb(args): return 1 # Get gdbserver channel - channel = args.gdbserver or 'localhost:1234' + channel = getattr(args, 'gdbserver', None) or 'localhost:1234' # Build gdb command gdb_cmd = [ @@ -964,6 +968,7 @@ def run_with_gdb(args): '-iex', 'set auto-load safe-path /', # Auto-load .gdbinit '-iex', 'set debuginfod enabled off', # Disable debuginfod prompts '-iex', 'set sysroot', # Suppress remote file transfer warnings + '-iex', 'handle SIGUSR2 nostop noprint pass', # Used by sandbox coroutines '-ex', f'target remote {channel}', ] @@ -971,9 +976,8 @@ def run_with_gdb(args): print(' '.join(gdb_cmd)) return 0 - # Run gdb interactively - result = exec_cmd(gdb_cmd, dry_run=False, capture=False) - return result.return_code + # Replace this process with gdb so Ctrl-C is handled by gdb + os.execvp(gdb_cmd[0], gdb_cmd) def collect_tests(args): @@ -1323,9 +1327,9 @@ def do_pytest(args): # pylint: disable=too-many-return-statements,too-many-bran tout.notice('Try: uman setup qemu') return 1 - # Handle -G: set gdbserver if not already set - if args.gdb and not args.gdbserver: - args.gdbserver = 'localhost:1234' + # Handle -G: set gdb_phase if not already set + if args.gdb and not args.gdb_phase: + args.gdb_phase = 'u-boot' # Build with um if requested, rather than letting pytest do it if args.build: @@ -1347,7 +1351,7 @@ def do_pytest(args): # pylint: disable=too-many-return-statements,too-many-bran args.build = False # Don't build again in pytest # Show -G command hint when using -g (not in dry-run mode) - if args.gdbserver and not args.gdb and not args.dry_run: + if args.gdb_phase and not args.gdb and not args.dry_run: tout.notice(f'In another terminal: um py -G -B {args.board}') pytest_vars = pytest_env(args.board) diff --git a/uman_pkg/cmdtest.py b/uman_pkg/cmdtest.py index c71f276..dc3e838 100644 --- a/uman_pkg/cmdtest.py +++ b/uman_pkg/cmdtest.py @@ -9,10 +9,12 @@ """ from collections import namedtuple +import fnmatch import os import re import shlex import struct +import sys import time # pylint: disable=import-error @@ -38,6 +40,8 @@ # Patterns for parsing test output RE_TEST_NAME = re.compile(r'Test:\s*(\S+)') RE_RESULT = re.compile(r'Result:\s*(PASS|FAIL|SKIP):?\s+(\S+)') +RE_SUMMARY = re.compile(r'Tests run:\s*(\d+),.*failures:\s*(\d+)') +RE_TEST_FAILED = re.compile(r"Test '.+' failed \d+ times") # Unit test flags from include/test/test.h UTF_FLAT_TREE = 0x08 @@ -366,7 +370,7 @@ def resolve_specs(sandbox, specs): all_tests = get_tests_from_nm(sandbox) found = False for test_suite, test_name in all_tests: - if test_name.endswith(pattern): + if fnmatch.fnmatch(test_name, f'*{pattern}'): resolved.append((test_suite, pattern)) found = True break # Only add first match @@ -400,7 +404,7 @@ def validate_specs(sandbox, specs): if pattern is None: found = True break - if test_name.endswith(pattern): + if fnmatch.fnmatch(test_name, f'*{pattern}'): found = True break if not found: @@ -409,8 +413,40 @@ def validate_specs(sandbox, specs): return unmatched +def has_no_flat(): + """Check whether the U-Boot tree supports the -F sandbox flag + + Looks for 'noflat' in arch/sandbox/cpu/start.c in the current directory. + + Returns: + bool: True if -F is supported + """ + start_c = os.path.join('arch', 'sandbox', 'cpu', 'start.c') + try: + with open(start_c, encoding='utf-8') as inf: + return 'noflat' in inf.read() + except OSError: + return False + + +def has_emit_result(): + """Check whether the U-Boot tree supports the -E ut flag + + Looks for 'emit_result' in test/cmd_ut.c in the current directory. + + Returns: + bool: True if -E is supported + """ + cmd_ut = os.path.join('test', 'cmd_ut.c') + try: + with open(cmd_ut, encoding='utf-8') as inf: + return 'emit_result' in inf.read() + except OSError: + return False + + def build_ut_cmd(sandbox, specs, full=False, verbose=False, legacy=False, - manual=False): + manual=False, malloc_dump=None): """Build the sandbox command line for running tests Args: @@ -420,14 +456,18 @@ def build_ut_cmd(sandbox, specs, full=False, verbose=False, legacy=False, verbose (bool): Enable verbose test output legacy (bool): Legacy mode (don't use -E flag for older U-Boot) manual (bool): Force manual tests to run + malloc_dump (str or None): File to write malloc dump to on exit Returns: list: Command and arguments """ cmd = [sandbox, '-T'] + if malloc_dump: + cmd.extend(['--malloc_dump', malloc_dump.replace('%d', '0')]) + # Add -F to skip flat-tree tests (live-tree only) unless full mode - if not full: + if not full and has_no_flat(): cmd.append('-F') # Add -v to sandbox to show test output @@ -437,7 +477,7 @@ def build_ut_cmd(sandbox, specs, full=False, verbose=False, legacy=False, # Build ut commands from specs; use -E to emit Result: lines # Flags must come before suite name flags = '' - if not legacy: + if not legacy and has_emit_result(): flags += '-E ' if manual: flags += '-m ' @@ -489,7 +529,9 @@ def parse_legacy_results(output, show_results=False, col=None): for line in output.splitlines(): name_match = RE_TEST_NAME.search(line) - name = name_match.group(1) if name_match else None + if not name_match: + continue + name = name_match.group(1) lower = line.lower() if '... ok' in lower: @@ -511,6 +553,26 @@ def parse_legacy_results(output, show_results=False, col=None): return TestCounts(passed, failed, skipped) +def parse_summary(output): + """Parse 'Tests run:' summary line from test output + + Handles the format: Tests run: N, Xms, average: Xms, failures: N + + Args: + output (str): Test output from sandbox + + Returns: + TestCounts or None: Counts of passed/failed/skipped, or None if none + """ + for line in output.splitlines(): + match = RE_SUMMARY.match(line) + if match: + total = int(match.group(1)) + failed = int(match.group(2)) + return TestCounts(total - failed, failed, 0) + return None + + def parse_results(output, show_results=False, col=None): """Parse test output to extract results from Result: lines @@ -544,6 +606,110 @@ def parse_results(output, show_results=False, col=None): return TestCounts(passed, failed, skipped) +def count_tests(sandbox, specs): + """Count expected tests for the given specs + + Args: + sandbox (str): Path to sandbox executable + specs (list): List of (suite, pattern) tuples + + Returns: + int: Number of matching tests, or 0 if unknown + """ + total = 0 + for suite, pattern in specs: + if suite == 'all': + return len(get_tests_from_nm(sandbox)) + tests = get_tests_from_nm(sandbox, suite) + if pattern: + total += sum(1 for _, name in tests if name.endswith(pattern)) + else: + total += len(tests) + return total + + +class Progress: + """Show live test progress on stderr + + Parses sandbox output as it arrives and displays a running count of + passed/failed/skipped tests, updating in place with carriage return. + + With -E (emit_result=True): counts Result: PASS/FAIL/SKIP lines. + Without -E: counts Test: lines as passes, detects failure lines like + "Test '' failed N times" to adjust the count. + """ + + def __init__(self, emit_result, total=0): + self.emit = emit_result + self.total = total + self.passed = 0 + self.failed = 0 + self.skipped = 0 + self.buf = '' + self.pending = False # A Test: line seen, not yet resolved + + def _show(self): + """Print the progress line, overwriting the previous one""" + col = terminal.Color() + grn = col.start(terminal.Color.GREEN) + red = col.start(terminal.Color.RED) + yel = col.start(terminal.Color.YELLOW) + rst = col.stop() + done = self.passed + self.failed + self.skipped + if self.total: + hdr = f'{done}/{self.total}:' + else: + hdr = f'{done}:' + sys.stderr.write( + f'\r {hdr} {grn}{self.passed} passed{rst}, ' + f'{red}{self.failed} failed{rst}, ' + f'{yel}{self.skipped} skipped{rst}') + sys.stderr.flush() + + def _process_line(self, line): + """Process one complete line of output""" + if self.emit: + match = RE_RESULT.match(line) + if match: + status = match.group(1) + if status == 'PASS': + self.passed += 1 + elif status == 'FAIL': + self.failed += 1 + elif status == 'SKIP': + self.skipped += 1 + self._show() + else: + if RE_TEST_FAILED.search(line): + self.failed += 1 + self.pending = False + self._show() + elif RE_TEST_NAME.match(line): + if self.pending: + self.passed += 1 + self.pending = True + self._show() + + def update(self, _stream, data): # pylint: disable=W9016,W9019 + """output_func callback for command.run_pipe()""" + if isinstance(data, (bytes, bytearray)): + data = data.decode('utf-8', errors='replace') + self.buf += data + while '\n' in self.buf: + line, self.buf = self.buf.split('\n', 1) + self._process_line(line) + + def finish(self): + """Close out the progress line""" + if not self.emit and self.pending: + self.passed += 1 + self.pending = False + if self.passed or self.failed or self.skipped: + self._show() + sys.stderr.write('\n') + sys.stderr.flush() + + def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 """Run sandbox tests @@ -562,7 +728,8 @@ def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 cmd = build_ut_cmd(sandbox, specs, full=args.flattree_too, verbose=args.test_verbose, legacy=args.legacy, - manual=args.manual) + manual=args.manual, + malloc_dump=args.malloc_dump) if args.dry_run: tout.notice(shlex.join(cmd)) @@ -576,9 +743,19 @@ def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 env = os.environ.copy() env['U_BOOT_PERSISTENT_DATA_DIR'] = persist_dir + # Show live progress if stderr is a terminal + emit = has_emit_result() + if sys.stderr.isatty(): + total = count_tests(sandbox, specs) + progress = Progress(emit, total) + else: + progress = None + output_func = progress.update if progress else None + start_time = time.time() try: - result = command.run_one(*cmd, capture=True, env=env) + result = command.run_one(*cmd, capture=True, env=env, + output_func=output_func) except command.CommandExc as exc: # Tests may fail but still produce parseable output result = exc.result @@ -587,6 +764,9 @@ def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 if not result: tout.error(f'Command failed: {exc}') return 1 + finally: + if progress: + progress.finish() elapsed = time.time() - start_time # Detect old U-Boot that doesn't understand -E or -F flags @@ -598,10 +778,13 @@ def run_tests(sandbox, specs, args, col): # pylint: disable=R0914 return 1 # Parse results first to check for failures + legacy = args.legacy or not has_emit_result() res = parse_results(result.stdout, show_results=args.results, col=col) - if not res and args.legacy: + if not res and legacy: res = parse_legacy_results(result.stdout, show_results=args.results, col=col) + if not res: + res = parse_summary(result.stdout) # Print output in verbose mode, if there are failures, or no results if result.stdout and not args.results: diff --git a/uman_pkg/control.py b/uman_pkg/control.py index 2341f71..2936d15 100644 --- a/uman_pkg/control.py +++ b/uman_pkg/control.py @@ -8,6 +8,7 @@ the features of uman. """ +import os import sys # pylint: disable=import-error @@ -328,6 +329,10 @@ def run_command(args): # pylint: disable=R0911 from uman_pkg import cmdconfig return cmdconfig.run(args) + if args.cmd == 'docker': + from uman_pkg import cmddocker + return cmddocker.run(args) + if args.cmd == 'git': from uman_pkg import cmdgit return cmdgit.run(args) @@ -419,6 +424,11 @@ def do_merge_request(args): # pylint: disable=too-many-locals """ # pylint: disable=import-outside-toplevel import gitlab + + uboot_tools = os.path.expanduser( + os.environ.get('UBOOT_TOOLS', '~/u/tools')) + if uboot_tools not in sys.path: + sys.path.insert(0, uboot_tools) from pickman import gitlab_api from u_boot_pylib import gitutil diff --git a/uman_pkg/ftest.py b/uman_pkg/ftest.py index 0aa360b..9935044 100644 --- a/uman_pkg/ftest.py +++ b/uman_pkg/ftest.py @@ -21,8 +21,8 @@ from u_boot_pylib import tout import gitlab -from uman_pkg import (build, cc, cmdconfig, cmdgit, cmdline, cmdpy, cmdtest, - control, gitlab_parser, settings, setup, util) +from uman_pkg import (build, cc, cmdconfig, cmddocker, cmdgit, cmdline, cmdpy, + cmdtest, control, gitlab_parser, settings, setup, util) # Capture stdout and stderr for silent command execution CAPTURE = {'capture': True, 'capture_stderr': True} @@ -133,10 +133,12 @@ def make_args(**kwargs): 'flattree_too': False, 'fresh': False, 'gdb': False, + 'gdb_phase': None, 'gdbserver': None, 'jobs': None, 'list_boards': False, 'lto': False, + 'malloc_dump': None, 'merge': False, 'no_timeout': False, 'null': False, @@ -1152,20 +1154,39 @@ def mock_git_output(*_args): self.assertEqual('', pos) def test_seq_edit_env_break(self): - """Test seq_edit_env creates break command""" + """Test seq_edit_env inserts a break""" env = cmdgit.seq_edit_env('break') self.assertIn('GIT_SEQUENCE_EDITOR', env) - self.assertEqual('sed -i "1i break"', env['GIT_SEQUENCE_EDITOR']) + todo = os.path.join(self.test_dir, 'todo') + with open(todo, 'w', encoding='utf-8') as fil: + fil.write('pick abc First\npick def Second\n') + os.system(f'{env["GIT_SEQUENCE_EDITOR"]} {todo}') + with open(todo, encoding='utf-8') as fil: + self.assertEqual('break\npick abc First\npick def Second\n', + fil.read()) def test_seq_edit_env_edit(self): - """Test seq_edit_env creates edit command""" + """Test seq_edit_env changes pick to edit""" env = cmdgit.seq_edit_env('edit') - self.assertEqual('sed -i "1s/^pick/edit/"', env['GIT_SEQUENCE_EDITOR']) + todo = os.path.join(self.test_dir, 'todo') + with open(todo, 'w', encoding='utf-8') as fil: + fil.write('pick abc First\npick def Second\n') + os.system(f'{env["GIT_SEQUENCE_EDITOR"]} {todo}') + with open(todo, encoding='utf-8') as fil: + self.assertEqual('edit abc First\npick def Second\n', + fil.read()) def test_seq_edit_env_edit_line(self): """Test seq_edit_env with specific line number""" - env = cmdgit.seq_edit_env('edit', 3) - self.assertEqual('sed -i "3s/^pick/edit/"', env['GIT_SEQUENCE_EDITOR']) + env = cmdgit.seq_edit_env('edit', 2) + todo = os.path.join(self.test_dir, 'todo') + with open(todo, 'w', encoding='utf-8') as fil: + fil.write('pick abc First\npick def Second\npick ghi Third\n') + os.system(f'{env["GIT_SEQUENCE_EDITOR"]} {todo}') + with open(todo, encoding='utf-8') as fil: + self.assertEqual( + 'pick abc First\nedit def Second\npick ghi Third\n', + fil.read()) def test_show_rebase_status_success(self): """Test show_rebase_status parses success message""" @@ -1745,7 +1766,7 @@ def test_do_db_with_branch(self): def test_do_dh(self): """Test do_dh runs git difftool HEAD~""" - args = cmdline.parse_args(['git', 'dh']) + args = cmdline.parse_args(['git', 'di']) with mock.patch('u_boot_pylib.command.run_one') as mock_run: mock_run.return_value = mock.Mock(return_code=0) result = cmdgit.do_dh(args) @@ -1755,7 +1776,7 @@ def test_do_dh(self): def test_do_dh_with_count(self): """Test do_dh N runs git difftool HEAD~N""" - args = cmdline.parse_args(['git', 'dh', '3']) + args = cmdline.parse_args(['git', 'di', '3']) with mock.patch('u_boot_pylib.command.run_one') as mock_run: mock_run.return_value = mock.Mock(return_code=0) result = cmdgit.do_dh(args) @@ -1765,7 +1786,7 @@ def test_do_dh_with_count(self): def test_do_dh_with_file(self): """Test do_dh with file path passes it to difftool""" - args = cmdline.parse_args(['git', 'dh', 'some/file.c']) + args = cmdline.parse_args(['git', 'di', 'some/file.c']) with mock.patch('u_boot_pylib.command.run_one') as mock_run: mock_run.return_value = mock.Mock(return_code=0) result = cmdgit.do_dh(args) @@ -1776,7 +1797,7 @@ def test_do_dh_with_file(self): def test_do_dh_with_count_and_file(self): """Test do_dh N file passes both to difftool""" - args = cmdline.parse_args(['git', 'dh', '2', 'some/file.c']) + args = cmdline.parse_args(['git', 'di', '2', 'some/file.c']) with mock.patch('u_boot_pylib.command.run_one') as mock_run: mock_run.return_value = mock.Mock(return_code=0) result = cmdgit.do_dh(args) @@ -2059,7 +2080,7 @@ def mock_git(*args, env=None, dry_run=None): self.assertEqual(('rebase', '-i', 'origin/main'), cap[0]) self.assertIn('GIT_SEQUENCE_EDITOR', cap_env[0]) # rb inserts a break before first commit to stop at upstream - self.assertIn("1i break", cap_env[0]['GIT_SEQUENCE_EDITOR']) + self.assertIn("insert(0,'break", cap_env[0]['GIT_SEQUENCE_EDITOR']) def test_do_rf_no_arg(self): """Test do_rf without arg uses upstream""" @@ -2081,7 +2102,8 @@ def mock_git(*args, env=None, dry_run=None): result = cmdgit.do_rf(args) self.assertEqual(0, result.return_code) self.assertEqual(('rebase', '-i', 'origin/main'), cap[0]) - self.assertIn("1s/^pick/edit/", cap_env[0]['GIT_SEQUENCE_EDITOR']) + self.assertIn("re.sub(r'^pick','edit',lines[0])", + cap_env[0]['GIT_SEQUENCE_EDITOR']) def test_do_rf_with_count(self): """Test do_rf with commit count""" @@ -2168,7 +2190,7 @@ def test_rb_stops_at_upstream(self): self.assertEqual(('rebase', '-i', 'upstream'), call_args[0]) # Check that GIT_SEQUENCE_EDITOR inserts break before first commit env = call_args[1]['env'] - self.assertIn("1i break", env['GIT_SEQUENCE_EDITOR']) + self.assertIn("insert(0,'break", env['GIT_SEQUENCE_EDITOR']) def test_rf_with_count(self): """Test rf N rebases last N commits, stopping at first""" @@ -2181,7 +2203,8 @@ def test_rf_with_count(self): call_args = mock_git.call_args self.assertEqual(('rebase', '-i', 'HEAD~3'), call_args[0]) env = call_args[1]['env'] - self.assertIn("1s/^pick/edit/", env['GIT_SEQUENCE_EDITOR']) + self.assertIn("re.sub(r'^pick','edit',lines[0])", + env['GIT_SEQUENCE_EDITOR']) def test_rp_stops_at_patch_n(self): """Test rp N stops at patch N""" @@ -2194,7 +2217,8 @@ def test_rp_stops_at_patch_n(self): call_args = mock_git.call_args self.assertEqual(('rebase', '-i', 'upstream'), call_args[0]) env = call_args[1]['env'] - self.assertIn("2s/^pick/edit/", env['GIT_SEQUENCE_EDITOR']) + self.assertIn("re.sub(r'^pick','edit',lines[1])", + env['GIT_SEQUENCE_EDITOR']) def test_real_rf_and_rc(self): """Test a real rf followed by rc to complete rebase""" @@ -3424,7 +3448,7 @@ def test_pytest_missing_qemu_binary(self): def test_pytest_gdb_dry_run(self): """Test pytest -G with dry-run prints gdb command""" args = make_args(cmd='pytest', board='sandbox', - gdbserver='localhost:1234', + gdb_phase='u-boot', gdb=True, dry_run=True) # Mock subprocess.run to capture what would be run @@ -3436,6 +3460,7 @@ def test_pytest_gdb_dry_run(self): self.assertIn('gdb-multiarch', output) self.assertIn('/tmp/b/sandbox/u-boot', output) self.assertIn('target remote localhost:1234', output) + self.assertIn('handle SIGUSR2 nostop noprint pass', output) def test_get_uboot_dir_current(self): """Test get_uboot_dir finds U-Boot in current directory""" @@ -4019,6 +4044,134 @@ def test_get_config_mounts_parses(self): finally: os.path.expanduser = orig_expanduser + def test_get_cli_mounts_none(self): + """Test get_cli_mounts with no arguments""" + self.assertEqual([], cc.get_cli_mounts(None)) + + def test_get_cli_mounts_host_only(self): + """Test get_cli_mounts with host path only""" + mounts = cc.get_cli_mounts(['/opt/data']) + self.assertEqual(1, len(mounts)) + self.assertEqual('data', mounts[0][0]) + self.assertEqual('/opt/data', mounts[0][1]) + self.assertEqual('/opt/data', mounts[0][2]) + + def test_get_cli_mounts_host_dest(self): + """Test get_cli_mounts with host:dest format""" + mounts = cc.get_cli_mounts(['/opt/data:/mnt/data']) + self.assertEqual(1, len(mounts)) + self.assertEqual('data', mounts[0][0]) + self.assertEqual('/opt/data', mounts[0][1]) + self.assertEqual('/mnt/data', mounts[0][2]) + + def test_get_cli_mounts_expands_tilde(self): + """Test get_cli_mounts maps ~ to container home""" + mounts = cc.get_cli_mounts(['~/dev/u-boot']) + home = os.path.expanduser('~') + self.assertEqual(f'{home}/dev/u-boot', mounts[0][1]) + self.assertEqual('/home/ubuntu/dev/u-boot', mounts[0][2]) + + def test_get_cli_mounts_dest_tilde(self): + """Test get_cli_mounts maps ~ in dest to container home""" + mounts = cc.get_cli_mounts(['/opt/data:~/data']) + self.assertEqual('/opt/data', mounts[0][1]) + self.assertEqual('/home/ubuntu/data', mounts[0][2]) + + def test_get_cli_mounts_multiple(self): + """Test get_cli_mounts with multiple mounts""" + mounts = cc.get_cli_mounts(['/opt/a', '/opt/b:/mnt/b']) + self.assertEqual(2, len(mounts)) + self.assertEqual('a', mounts[0][0]) + self.assertEqual('/opt/a', mounts[0][1]) + self.assertEqual('b', mounts[1][0]) + self.assertEqual('/mnt/b', mounts[1][2]) + + def test_get_cli_mounts_duplicate_leaf(self): + """Test get_cli_mounts adds suffix for duplicate leaf names""" + mounts = cc.get_cli_mounts(['/opt/data', '/mnt/data']) + self.assertEqual(2, len(mounts)) + self.assertEqual('data', mounts[0][0]) + self.assertEqual('data2', mounts[1][0]) + + def test_show_mounts(self): + """Test show_mounts parses lxc device output""" + yaml = ('datadir:\n' + ' path: /home/ubuntu/project\n' + ' source: /home/sglass/dev/myproj\n' + ' type: disk\n' + 'hostbin:\n' + ' path: /home/ubuntu/bin\n' + ' source: /home/sglass/bin\n' + ' type: disk\n') + result = command.CommandResult(return_code=0, stdout=yaml) + with mock.patch.object(cc, 'exec_cmd', return_value=result): + with terminal.capture() as (out, err): + ret = cc.show_mounts('mybox') + self.assertEqual(0, ret) + self.assertIn('datadir', out.getvalue()) + self.assertIn('/home/ubuntu/project', out.getvalue()) + self.assertFalse(err.getvalue()) + + def test_show_mounts_not_found(self): + """Test show_mounts with missing container""" + result = command.CommandResult(return_code=1, stdout='') + with mock.patch.object(cc, 'exec_cmd', return_value=result): + with terminal.capture() as (out, err): + ret = cc.show_mounts('nobox') + self.assertEqual(1, ret) + self.assertIn('not found', err.getvalue()) + self.assertFalse(out.getvalue()) + + def test_mount_only(self): + """Test -m without -s just adds the mount and exits""" + args = cmdline.parse_args(['cc', '-m', '/opt/data', 'mybox']) + with mock.patch.object(cc, 'container_exists', return_value=True): + with mock.patch.object(cc, 'add_mount') as mock_add: + with terminal.capture() as (out, err): + ret = cc.run(args) + self.assertEqual(0, ret) + mock_add.assert_called_once_with( + 'mybox', 'data', '/opt/data', '/opt/data', False) + self.assertIn('/opt/data -> /opt/data (data)', out.getvalue()) + + def test_mount_only_no_container(self): + """Test -m fails if the container does not exist""" + args = cmdline.parse_args(['cc', '-m', '/opt/data', 'mybox']) + with mock.patch.object(cc, 'container_exists', return_value=False): + with terminal.capture() as (out, err): + ret = cc.run(args) + self.assertEqual(1, ret) + self.assertIn('not found', err.getvalue()) + + def test_unmount(self): + """Test -u removes a mount device""" + args = cmdline.parse_args(['-n', 'cc', '-u', 'linux', 'mybox']) + with terminal.capture() as (out, err): + ret = cc.run(args) + self.assertEqual(0, ret) + self.assertIn('lxc config device remove', out.getvalue()) + self.assertIn('linux', out.getvalue()) + + def test_unmount_missing(self): + """Test -u fails if the device does not exist""" + args = cmdline.parse_args(['cc', '-u', 'nosuch', 'mybox']) + result = command.CommandResult(return_code=1, stdout='', stderr='') + with mock.patch.object(cc, 'container_exists', return_value=True): + with mock.patch.object(cc, 'has_mount', return_value=False): + with terminal.capture() as (out, err): + ret = cc.run(args) + self.assertEqual(1, ret) + self.assertIn('nosuch', err.getvalue()) + + def test_unmount_no_container(self): + """Test -u fails if the container does not exist""" + args = cmdline.parse_args(['cc', '-u', 'linux', 'mybox']) + with mock.patch.object(cc, 'container_exists', return_value=False): + with terminal.capture() as (out, err): + ret = cc.run(args) + self.assertEqual(1, ret) + self.assertIn('not found', err.getvalue()) + def test_get_git_symlink_mount_no_symlink(self): """Test get_git_symlink_mount when .git is not a symlink""" # Create a regular .git directory @@ -4059,9 +4212,8 @@ def test_dry_run(self): self.assertNotIn('lxc delete', output) self.assertIn('test-cc', output) self.assertNotIn('--continue', output) - # /tmp/b mount - self.assertIn('tmpb', output) - self.assertIn('path=/tmp/b', output) + # /tmp/b mount not present without -o + self.assertNotIn('tmpb', output) # uman env written and sourced self.assertIn('.uman_env', output) self.assertIn('.bashrc', output) @@ -4123,6 +4275,88 @@ def test_dry_run_restart(self): finally: os.path.expanduser = orig_expanduser + def test_editor_proxy(self): + """Test editor proxy translates paths and opens host editor""" + import socket as socket_mod + + project_src = self.test_dir + opened = [] + orig_run = subprocess.run + subprocess.run = lambda cmd, **kw: opened.append(cmd) + + sock_path = cc.start_editor_proxy(project_src) + try: + self.assertTrue(os.path.exists(sock_path)) + + # Connect and send a container path + sock = socket_mod.socket(socket_mod.AF_UNIX, + socket_mod.SOCK_STREAM) + sock.connect(sock_path) + sock.sendall(f'{cc.PROJECT_DEST}/foo.c\n'.encode()) + resp = sock.recv(4096).decode().strip() + sock.close() + + self.assertEqual('done', resp) + self.assertEqual(1, len(opened)) + self.assertEqual(os.path.join(project_src, 'foo.c'), + opened[0][1]) + finally: + subprocess.run = orig_run + if os.path.exists(sock_path): + os.unlink(sock_path) + + def test_editor_proxy_content_transfer(self): + """Test editor proxy handles non-project paths via content""" + import json + import socket as socket_mod + + project_src = self.test_dir + opened = [] + orig_run = subprocess.run + subprocess.run = lambda cmd, **kw: opened.append(cmd) + + sock_path = cc.start_editor_proxy(project_src) + try: + sock = socket_mod.socket(socket_mod.AF_UNIX, + socket_mod.SOCK_STREAM) + sock.connect(sock_path) + msg = json.dumps({'content': 'hello', 'ext': '.txt'}) + '\n' + sock.sendall(msg.encode()) + resp_raw = b'' + while True: + chunk = sock.recv(65536) + if not chunk: + break + resp_raw += chunk + sock.close() + + resp = json.loads(resp_raw.decode()) + self.assertIn('content', resp) + self.assertEqual(1, len(opened)) + finally: + subprocess.run = orig_run + if os.path.exists(sock_path): + os.unlink(sock_path) + + def test_editor_proxy_outside_project(self): + """Test editor proxy rejects paths outside the project""" + import socket as socket_mod + + project_src = self.test_dir + sock_path = cc.start_editor_proxy(project_src) + try: + sock = socket_mod.socket(socket_mod.AF_UNIX, + socket_mod.SOCK_STREAM) + sock.connect(sock_path) + sock.sendall(b'/etc/passwd\n') + resp = sock.recv(4096).decode().strip() + sock.close() + + self.assertIn('error', resp) + finally: + if os.path.exists(sock_path): + os.unlink(sock_path) + def test_run_command_cc(self): """Test run_command dispatches to cc correctly""" args = cmdline.parse_args(['-n', 'cc', 'test-cc']) @@ -4244,6 +4478,8 @@ def test_list_containers(self): 'devbox': '/home/user/dev/linux', } + priv = {'mybox': 'true', 'devbox': ''} + def mock_lxc(pipe_list, **_kwargs): cmd = pipe_list[0] if 'device' in cmd: @@ -4254,14 +4490,19 @@ def mock_lxc(pipe_list, **_kwargs): return_code=0, stdout=proj + '\n', stderr='') return command.CommandResult( return_code=1, stdout='', stderr='not found') + if 'security.privileged' in cmd: + name = cmd[cmd.index('get') + 1] + return command.CommandResult( + return_code=0, stdout=priv.get(name, '') + '\n', + stderr='') return command.CommandResult( return_code=0, stdout=csv, stderr='') command.TEST_RESULT = mock_lxc result = cc.list_containers() self.assertEqual( - [('mybox', 'RUNNING', '/home/user/dev/uboot'), - ('devbox', 'STOPPED', '/home/user/dev/linux')], + [('mybox', 'RUNNING', '/home/user/dev/uboot', True), + ('devbox', 'STOPPED', '/home/user/dev/linux', False)], result) def test_list_containers_empty(self): @@ -4688,6 +4929,7 @@ def test_setup_components_dict(self): self.assertIn('gcc', setup.SETUP_COMPONENTS) self.assertIn('qemu', setup.SETUP_COMPONENTS) self.assertIn('opensbi', setup.SETUP_COMPONENTS) + self.assertIn('remote', setup.SETUP_COMPONENTS) self.assertIn('tfa', setup.SETUP_COMPONENTS) self.assertIn('xtensa', setup.SETUP_COMPONENTS) @@ -4771,6 +5013,39 @@ def test_setup_aliases_quiet(self): self.assertEqual(0, res) self.assertFalse(out.getvalue()) + def test_setup_remote_parsing(self): + """Test that setup remote subcommand is parsed correctly""" + args = cmdline.parse_args(['setup', 'remote', 'myhost']) + self.assertEqual('setup', args.cmd) + self.assertEqual('remote', args.component) + self.assertEqual('myhost', args.host) + + def test_setup_remote_no_host(self): + """Test setup remote with no hostname""" + args = argparse.Namespace( + cmd='setup', component='remote', host=None, + list_components=False, force=False, dry_run=False, + verbose=False, debug=False, alias_dir=None) + with terminal.capture() as (_, err): + res = setup.do_setup(args) + self.assertEqual(1, res) + self.assertIn('Hostname required', err.getvalue()) + + def test_setup_remote_dry_run(self): + """Test setup remote in dry-run mode""" + args = argparse.Namespace( + cmd='setup', component='remote', host='testhost', + list_components=False, force=False, dry_run=True, + verbose=False, debug=False, alias_dir=None) + with terminal.capture() as (out, _): + res = setup.do_setup(args) + self.assertEqual(0, res) + output = out.getvalue() + self.assertIn('rsync', output) + self.assertIn('testhost', output) + self.assertIn('ln -sf', output) + self.assertIn('setup aliases', output) + class TestMain(TestBase): """Tests for __main__.py""" @@ -4780,32 +5055,16 @@ def setUpClass(cls): """Get the uman package directory""" cls.uman_dir = os.path.dirname(os.path.dirname(__file__)) - def test_uboot_tools_env_var(self): - """Test UBOOT_TOOLS env var overrides default path""" - # With invalid UBOOT_TOOLS, uman should fail to import u_boot_pylib - # Clear PYTHONPATH to ensure only UBOOT_TOOLS is used + def test_embedded_pylib(self): + """Test uman works without UBOOT_TOOLS using embedded u_boot_pylib""" env = os.environ.copy() env['UBOOT_TOOLS'] = '/nonexistent/path' env.pop('PYTHONPATH', None) + env.pop('UMAN_EXTERNAL_PYLIB', None) result = subprocess.run( ['python3', '-m', 'uman_pkg', '--help'], capture_output=True, env=env, cwd=self.uman_dir, check=False) - # Should fail with import error - self.assertNotEqual(0, result.returncode) - self.assertIn(b'ModuleNotFoundError', result.stderr) - - def test_uboot_tools_tilde_expansion(self): - """Test UBOOT_TOOLS expands ~ in path""" - # Verify tilde is expanded (not passed literally) - # Clear PYTHONPATH to ensure only UBOOT_TOOLS is used - env = os.environ.copy() - env['UBOOT_TOOLS'] = '~/nonexistent' - env.pop('PYTHONPATH', None) - result = subprocess.run( - ['python3', '-m', 'uman_pkg', '--help'], - capture_output=True, env=env, cwd=self.uman_dir, check=False) - # Error should show expanded path, not literal ~ - self.assertNotIn(b"'~/nonexistent'", result.stderr) + self.assertEqual(0, result.returncode) class TestUtil(TestBase): @@ -5016,52 +5275,90 @@ def test_do_test_list_tests_by_suite(self): self.assertIn('dm.test_gpio', stdout) self.assertNotIn('env.test_env_basic', stdout) - def test_build_ut_cmd_no_tests(self): + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_no_tests(self, *_): """Test build_ut_cmd with all specs""" cmd = cmdtest.build_ut_cmd('/sb', [('all', None)]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E all'], cmd) - def test_build_ut_cmd_full(self): + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + def test_build_ut_cmd_full(self, _): """Test build_ut_cmd with full flag (both tree types)""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], full=True) self.assertEqual(['/sb', '-T', '-c', 'ut -E dm'], cmd) - def test_build_ut_cmd_verbose(self): + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_verbose(self, *_): """Test build_ut_cmd with verbose flag""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], verbose=True) self.assertEqual(['/sb', '-T', '-F', '-v', '-c', 'ut -E dm'], cmd) - def test_build_ut_cmd_all_flags(self): + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + def test_build_ut_cmd_all_flags(self, _): """Test build_ut_cmd with all flags""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], full=True, verbose=True) self.assertEqual(['/sb', '-T', '-v', '-c', 'ut -E dm'], cmd) - def test_build_ut_cmd_suite(self): + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_suite(self, *_): """Test build_ut_cmd with suite name""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E dm'], cmd) - def test_build_ut_cmd_specific_test(self): + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_specific_test(self, *_): """Test build_ut_cmd with specific test (suite.test)""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', 'test_one')]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E dm test_one'], cmd) - def test_build_ut_cmd_multiple_tests(self): + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_multiple_tests(self, *_): """Test build_ut_cmd with multiple test specifications""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None), ('env', None)]) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E dm; ut -E env'], cmd) - def test_build_ut_cmd_legacy(self): + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_legacy(self, _): """Test build_ut_cmd with legacy flag omits -E""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], legacy=True) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut dm'], cmd) - def test_build_ut_cmd_manual(self): + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_manual(self, *_): """Test build_ut_cmd with manual flag""" cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], manual=True) self.assertEqual(['/sb', '-T', '-F', '-c', 'ut -E -m dm'], cmd) + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + @mock.patch.object(cmdtest, 'has_no_flat', return_value=False) + def test_build_ut_cmd_no_flat_unsupported(self, *_): + """Test build_ut_cmd omits -F when source tree lacks support""" + cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)]) + self.assertEqual(['/sb', '-T', '-c', 'ut -E dm'], cmd) + + @mock.patch.object(cmdtest, 'has_emit_result', return_value=False) + def test_build_ut_cmd_no_emit_unsupported(self, _): + """Test build_ut_cmd omits -E when source tree lacks support""" + cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)]) + self.assertEqual(['/sb', '-T', '-c', 'ut dm'], cmd) + + @mock.patch.object(cmdtest, 'has_emit_result', return_value=True) + @mock.patch.object(cmdtest, 'has_no_flat', return_value=True) + def test_build_ut_cmd_malloc_dump(self, *_): + """Test build_ut_cmd expands %d in malloc_dump filename""" + cmd = cmdtest.build_ut_cmd('/sb', [('dm', None)], + malloc_dump='/tmp/dump-%d.txt') + self.assertEqual( + ['/sb', '-T', '--malloc_dump', '/tmp/dump-0.txt', + '-F', '-c', 'ut -E dm'], cmd) + def test_run_tests_basic(self): """Test run_tests executes sandbox correctly""" cap = [] @@ -5073,11 +5370,14 @@ def mock_run(*cmd_args, **_kwargs): args = cmdline.parse_args(['test', 'dm']) col = terminal.Color() - with mock.patch.object(command, 'run_one', mock_run): - with mock.patch.object(cmdtest, 'ensure_dm_init_files', - return_value=True): - with terminal.capture(): - result = cmdtest.run_tests('/sb', [('dm', None)], args, col) + with mock.patch.object(cmdtest, 'has_emit_result', return_value=True): + with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): + with mock.patch.object(command, 'run_one', mock_run): + with mock.patch.object(cmdtest, 'ensure_dm_init_files', + return_value=True): + with terminal.capture(): + result = cmdtest.run_tests( + '/sb', [('dm', None)], args, col) self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-F', '-c', 'ut -E dm'), cap[0]) @@ -5092,11 +5392,13 @@ def mock_run(*cmd_args, **_kwargs): args = cmdline.parse_args(['test', '--flattree-too', 'dm']) col = terminal.Color() - with mock.patch.object(command, 'run_one', mock_run): - with mock.patch.object(cmdtest, 'ensure_dm_init_files', - return_value=True): - with terminal.capture(): - result = cmdtest.run_tests('/sb', [('dm', None)], args, col) + with mock.patch.object(cmdtest, 'has_emit_result', return_value=True): + with mock.patch.object(command, 'run_one', mock_run): + with mock.patch.object(cmdtest, 'ensure_dm_init_files', + return_value=True): + with terminal.capture(): + result = cmdtest.run_tests( + '/sb', [('dm', None)], args, col) self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-c', 'ut -E dm'), cap[0]) @@ -5111,11 +5413,14 @@ def mock_run(*cmd_args, **_kwargs): args = cmdline.parse_args(['test', '-V', 'dm']) col = terminal.Color() - with mock.patch.object(command, 'run_one', mock_run): - with mock.patch.object(cmdtest, 'ensure_dm_init_files', - return_value=True): - with terminal.capture(): - result = cmdtest.run_tests('/sb', [('dm', None)], args, col) + with mock.patch.object(cmdtest, 'has_emit_result', return_value=True): + with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): + with mock.patch.object(command, 'run_one', mock_run): + with mock.patch.object(cmdtest, 'ensure_dm_init_files', + return_value=True): + with terminal.capture(): + result = cmdtest.run_tests( + '/sb', [('dm', None)], args, col) self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-F', '-v', '-c', 'ut -E dm'), cap[0]) @@ -5172,6 +5477,25 @@ def test_parse_results_only_result_lines(self): self.assertEqual(0, res.failed) self.assertEqual(1, res.skipped) + def test_parse_summary_basic(self): + """Test parse_summary with standard output""" + output = 'Tests run: 540, 11595 ms, average: 21 ms, failures: 7' + res = cmdtest.parse_summary(output) + self.assertEqual(533, res.passed) + self.assertEqual(7, res.failed) + self.assertEqual(0, res.skipped) + + def test_parse_summary_no_failures(self): + """Test parse_summary with no failures""" + output = 'Tests run: 1, 4 ms, average: 4 ms, failures: 0' + res = cmdtest.parse_summary(output) + self.assertEqual(1, res.passed) + self.assertEqual(0, res.failed) + + def test_parse_summary_empty(self): + """Test parse_summary with no summary line""" + self.assertIsNone(cmdtest.parse_summary('')) + def test_parse_results_empty(self): """Test parse_results with empty output returns None""" self.assertIsNone(cmdtest.parse_results('')) @@ -5246,7 +5570,6 @@ def test_run_tests_shows_output_when_no_results(self): output = ''' U-Boot banner here Missing required argument 'fs_image' for test 'pxe_test_sysboot' -Tests run: 1, failures: 1 ''' def mock_run(*_args, **_kwargs): @@ -5265,6 +5588,26 @@ def mock_run(*_args, **_kwargs): # Error message should be shown in output self.assertIn('Missing required argument', out.getvalue()) + def test_run_tests_parses_summary(self): + """Test run_tests uses summary line when -E is unavailable""" + output = 'Tests run: 10, 100 ms, average: 10 ms, failures: 2' + + def mock_run(*_args, **_kwargs): + return command.CommandResult(return_code=1, stdout=output) + + args = cmdline.parse_args(['test', 'dm']) + col = terminal.Color() + with mock.patch.object(cmdtest, 'has_emit_result', return_value=False): + with mock.patch.object(command, 'run_one', mock_run): + with mock.patch.object(cmdtest, 'ensure_dm_init_files', + return_value=True): + with terminal.capture() as (out, err): + result = cmdtest.run_tests( + '/sb', [('dm', None)], args, col) + self.assertEqual(1, result) + self.assertIn('8 passed', out.getvalue()) + self.assertIn('2 failed', out.getvalue()) + def test_run_tests_detects_segfault(self): """Test run_tests detects segfault (SIGSEGV) and resets terminal""" col = terminal.Color() @@ -5322,18 +5665,105 @@ def mock_resolve(_sandbox, specs): args = cmdline.parse_args(['test', 'dm']) args.col = terminal.Color() - with mock.patch.object(cmdtest, 'get_sandbox_path', return_value='/sb'): - with mock.patch.object(cmdtest, 'resolve_specs', mock_resolve): - with mock.patch.object(cmdtest, 'validate_specs', - return_value=[]): - with mock.patch.object(cmdtest, 'ensure_dm_init_files', - return_value=True): - with mock.patch.object(command, 'run_one', mock_run): - with terminal.capture(): - result = cmdtest.do_test(args) + with mock.patch.object(cmdtest, 'has_emit_result', return_value=True): + with mock.patch.object(cmdtest, 'has_no_flat', return_value=True): + with mock.patch.object(cmdtest, 'get_sandbox_path', + return_value='/sb'): + with mock.patch.object(cmdtest, 'resolve_specs', + mock_resolve): + with mock.patch.object(cmdtest, 'validate_specs', + return_value=[]): + with mock.patch.object( + cmdtest, 'ensure_dm_init_files', + return_value=True): + with mock.patch.object(command, 'run_one', + mock_run): + with terminal.capture(): + result = cmdtest.do_test(args) self.assertEqual(0, result) self.assertEqual(('/sb', '-T', '-F', '-c', 'ut -E dm'), cap[0]) + def test_count_tests_suite(self): + """Test count_tests counts tests in a suite""" + tests = [('dm', 'test_a'), ('dm', 'test_b')] + with mock.patch.object(cmdtest, 'get_tests_from_nm', + return_value=tests) as mock_nm: + result = cmdtest.count_tests('/sb', [('dm', None)]) + self.assertEqual(2, result) + mock_nm.assert_called_once_with('/sb', 'dm') + + def test_count_tests_all(self): + """Test count_tests counts all tests""" + tests = [('dm', 'a'), ('dm', 'b'), ('env', 'c')] + with mock.patch.object(cmdtest, 'get_tests_from_nm', + return_value=tests): + result = cmdtest.count_tests('/sb', [('all', None)]) + self.assertEqual(3, result) + + def test_count_tests_pattern(self): + """Test count_tests counts tests matching a pattern""" + tests = [('dm', 'gpio'), ('dm', 'gpio_irq'), ('dm', 'i2c')] + with mock.patch.object(cmdtest, 'get_tests_from_nm', + return_value=tests): + result = cmdtest.count_tests('/sb', [('dm', 'gpio')]) + self.assertEqual(1, result) + + def test_progress_with_total(self): + """Test Progress shows total in output""" + prog = cmdtest.Progress(emit_result=True, total=10) + self.assertEqual(10, prog.total) + + def test_progress_with_emit(self): + """Test Progress counts Result: lines with -E""" + prog = cmdtest.Progress(emit_result=True) + prog.update(None, b'Result: PASS test1\n') + prog.update(None, b'Result: FAIL test2\n') + prog.update(None, b'Result: SKIP test3\n') + prog.update(None, b'Result: PASS test4\n') + prog.finish() + self.assertEqual(2, prog.passed) + self.assertEqual(1, prog.failed) + self.assertEqual(1, prog.skipped) + + def test_progress_without_emit(self): + """Test Progress counts Test: and failure lines without -E""" + prog = cmdtest.Progress(emit_result=False) + prog.update(None, b'Test: acpi: acpi.c\n') + prog.update(None, b'Test: gpio: gpio.c\n') + prog.update(None, + b"Test 'gpio' failed 1 times\nTest: host: host.c\n") + prog.finish() + self.assertEqual(2, prog.passed) + self.assertEqual(1, prog.failed) + self.assertEqual(0, prog.skipped) + + def test_progress_partial_lines(self): + """Test Progress handles data split across chunks""" + prog = cmdtest.Progress(emit_result=True) + prog.update(None, b'Result: PA') + prog.update(None, b'SS test1\nResult:') + prog.update(None, b' FAIL test2\n') + prog.finish() + self.assertEqual(1, prog.passed) + self.assertEqual(1, prog.failed) + + def test_progress_no_output(self): + """Test Progress with no test output""" + prog = cmdtest.Progress(emit_result=True) + prog.finish() + self.assertEqual(0, prog.passed) + self.assertEqual(0, prog.failed) + self.assertEqual(0, prog.skipped) + + def test_progress_last_test_passes(self): + """Test Progress counts the last test as passed at finish""" + prog = cmdtest.Progress(emit_result=False) + prog.update(None, b'Test: foo: foo.c\n') + self.assertTrue(prog.pending) + prog.finish() + self.assertEqual(1, prog.passed) + self.assertFalse(prog.pending) + def test_parse_one_test_suite(self): """Test parse_one_test with suite name""" self.assertEqual(('dm', None), cmdtest.parse_one_test('dm')) @@ -6149,3 +6579,232 @@ def test_get_board_config_fallback(self, mock_socket, mock_settings): self.assertEqual('qemu', result['console_impl']) self.assertEqual('qemu-system-arm', result['qemu_binary']) + + +class TestDockerTest(TestBase): + """Test the docker (d) subcommand""" + + GITLAB_CI = '''default: + image: ${MIRROR_DOCKER}/sjg20/u-boot-ci-runner:jammy-20250404-abc123 + +.buildman_and_testpy_template: + before_script: + - git config --global --add safe.directory "${CI_PROJECT_DIR}" + - ln -s travis-ci test/hooks/bin/`hostname` + - ln -s travis-ci test/hooks/py/`hostname` + - python3 -m venv /tmp/venv; + . /tmp/venv/bin/activate; + pip install -r test/py/requirements.txt + script: + - export UBOOT_TRAVIS_BUILD_DIR=/tmp/${TEST_PY_BD} + - tools/buildman/buildman -o ${UBOOT_TRAVIS_BUILD_DIR} -w -E -W -e + --board ${TEST_PY_BD} ${OVERRIDE} + - cp /opt/grub/grub_x86.efi $UBOOT_TRAVIS_BUILD_DIR/ + - if [[ -n "${TEST_SPEC}" ]]; then + SPEC="${TEST_SPEC}"; + else + SPEC="${TEST_PY_TEST_SPEC}"; + fi + - export PATH=/opt/qemu/bin:test/hooks/bin:${PATH}; + ./test/py/test.py -ra --bd ${TEST_PY_BD} + ${SPEC:+"-k ${SPEC}"} + --build-dir "$UBOOT_TRAVIS_BUILD_DIR" + ${TEST_PY_EXTRA} + +stages: + - build +''' + + def setUp(self): + super().setUp() + tout.init(tout.NOTICE) + self.orig_cwd = os.getcwd() + self.orig_usrc = os.environ.get('USRC') + if 'USRC' in os.environ: + del os.environ['USRC'] + + # Create a fake U-Boot tree + os.chdir(self.test_dir) + os.makedirs('test/py') + tools.write_file('test/py/test.py', b'# test') + tools.write_file('.gitlab-ci.yml', + self.GITLAB_CI.encode(), binary=True) + + def tearDown(self): + os.chdir(self.orig_cwd) + if self.orig_usrc is not None: + os.environ['USRC'] = self.orig_usrc + elif 'USRC' in os.environ: + del os.environ['USRC'] + super().tearDown() + + def test_load_ci_yaml(self): + """Test loading .gitlab-ci.yml""" + data = cmddocker.load_ci_yaml(self.test_dir) + self.assertIsNotNone(data) + self.assertIn('default', data) + + def test_load_ci_yaml_missing(self): + """Test load_ci_yaml with missing file""" + with terminal.capture() as (out, err): + data = cmddocker.load_ci_yaml(self.orig_cwd) + self.assertIsNone(data) + + def test_get_ci_image(self): + """Test parsing Docker image from parsed YAML""" + data = cmddocker.load_ci_yaml(self.test_dir) + image = cmddocker.get_ci_image(data) + self.assertEqual( + 'docker.io/sjg20/u-boot-ci-runner:jammy-20250404-abc123', + image) + + def test_get_ci_image_no_match(self): + """Test get_ci_image when image is not found""" + with terminal.capture() as (out, err): + image = cmddocker.get_ci_image({'stages': ['build']}) + self.assertIsNone(image) + self.assertEqual('Cannot find image in .gitlab-ci.yml\n', + err.getvalue()) + + def test_get_ci_script(self): + """Test extracting script from parsed YAML""" + data = cmddocker.load_ci_yaml(self.test_dir) + before, script = cmddocker.get_ci_script(data) + self.assertTrue(len(before) > 0) + self.assertTrue(len(script) > 0) + + def test_build_script(self): + """Test script generation without test spec""" + data = cmddocker.load_ci_yaml(self.test_dir) + script = cmddocker.build_script(data, 'sandbox', None) + self.assertIn('--board sandbox', script) + self.assertIn('UBOOT_TRAVIS_BUILD_DIR=/tmp/sandbox', script) + self.assertIn('cp /opt/grub/grub_x86.efi', script) + + def test_build_script_test_spec(self): + """Test script generation with test spec""" + data = cmddocker.load_ci_yaml(self.test_dir) + script = cmddocker.build_script(data, 'sandbox', + 'test_ofplatdata') + self.assertIn('test_ofplatdata', script) + + def test_build_script_gdb(self): + """Test script generation with gdbserver wrapper""" + data = cmddocker.load_ci_yaml(self.test_dir) + script = cmddocker.build_script(data, 'sandbox', None, gdb=True) + self.assertIn('apt-get', script) + self.assertIn('mv $bd/u-boot $bd/u-boot.real', script) + self.assertIn('gdbserver :1234', script) + self.assertIn('chmod +x $bd/u-boot', script) + + def test_build_script_board(self): + """Test script generation with a different board""" + data = cmddocker.load_ci_yaml(self.test_dir) + script = cmddocker.build_script(data, 'sandbox_flattree', None) + self.assertIn('--board sandbox_flattree', script) + self.assertIn('UBOOT_TRAVIS_BUILD_DIR=/tmp/sandbox_flattree', + script) + + def test_run_dry_run(self): + """Test dry-run shows docker command""" + args = cmdline.parse_args(['-n', 'docker']) + with terminal.capture() as (out, err): + res = cmddocker.run(args) + self.assertEqual(0, res) + output = out.getvalue() + self.assertIn('docker run --rm', output) + self.assertIn('--user', output) + self.assertIn('HOME=/tmp', output) + self.assertIn('/etc/passwd:/etc/passwd:ro', output) + self.assertIn('docker.io/sjg20/u-boot-ci-runner', output) + self.assertFalse(err.getvalue()) + + def test_run_dry_run_with_spec(self): + """Test dry-run with test spec""" + args = cmdline.parse_args(['-n', 'd', 'test_ofplatdata']) + with terminal.capture() as (out, err): + res = cmddocker.run(args) + self.assertEqual(0, res) + output = out.getvalue() + self.assertIn('test_ofplatdata', output) + self.assertFalse(err.getvalue()) + + def test_run_dry_run_interactive(self): + """Test dry-run in interactive mode""" + args = cmdline.parse_args(['-n', 'docker', '-I']) + with terminal.capture() as (out, err): + res = cmddocker.run(args) + self.assertEqual(0, res) + output = out.getvalue() + self.assertIn('docker run --rm', output) + # Interactive mode should just run bash, not bash -c