From 258340c70e0b621c87debb05d36309cbcde1a89a Mon Sep 17 00:00:00 2001 From: Lloyd Smart Date: Wed, 5 Aug 2026 11:58:24 +0100 Subject: [PATCH] chore: harden release packaging and CLI (#34) --- .github/workflows/release.yml | 33 +++++++++------ .github/workflows/shellcheck.yml | 4 +- .github/workflows/tests.yml | 2 + Makefile | 18 ++++++-- README.md | 57 ++++++++++++++++++++----- chdtool.sh | 50 +++++++++++++++++++--- scripts/package-release.sh | 73 ++++++++++++++++++++++++++++++++ tests/test_cli.sh | 36 ++++++++++++++++ tests/test_package.sh | 23 ++++++++++ 9 files changed, 260 insertions(+), 36 deletions(-) create mode 100644 scripts/package-release.sh create mode 100644 tests/test_cli.sh create mode 100644 tests/test_package.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 19233ab..d1b9fad 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,11 +15,21 @@ jobs: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" steps: - - name: Check out repo + - name: Check out repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - fetch-tags: true + fetch-tags: true + + - name: Install test and packaging dependencies + run: | + sudo apt-get update + sudo apt-get install -y make shellcheck unzip zip + + - name: Run tests and ShellCheck + run: | + make test + shellcheck chdtool.sh scripts/*.sh tests/*.sh tests/bin/* # Generate release notes for this tag (used as Release body) - name: Generate release notes with git-cliff @@ -33,23 +43,20 @@ jobs: OUTPUT: CHANGELOG_RELEASE.md GITHUB_REPO: ${{ github.repository }} - - name: Prepare artifacts - run: | - set -euo pipefail - chmod +x chdtool.sh || true - cp chdtool.sh chdtool - mkdir -p dist - tar -czf "dist/chdtool-${GITHUB_REF_NAME}.tar.gz" chdtool README.md LICENSE CHANGELOG.md || true - zip -j "dist/chdtool-${GITHUB_REF_NAME}.zip" chdtool README.md LICENSE CHANGELOG.md || true + - name: Prepare artifacts + run: | + set -euo pipefail + release_version="${GITHUB_REF_NAME#v}" + SOURCE_DATE_EPOCH="$(git log -1 --format=%ct)" \ + bash scripts/package-release.sh "$release_version" - name: Create GitHub Release and upload assets uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 with: tag_name: ${{ github.ref_name }} body_path: ${{ steps.cliff_notes.outputs.changelog }} # file generated above - files: | - dist/* - chdtool.sh + files: | + dist/* draft: false prerelease: ${{ contains(github.ref_name, '-') }} env: diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 04fee40..55b271f 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -7,12 +7,14 @@ on: - "chdtool.sh" - "tests/**/*.sh" - "tests/bin/*" + - "scripts/**/*.sh" - ".github/workflows/shellcheck.yml" pull_request: paths: - "chdtool.sh" - "tests/**/*.sh" - "tests/bin/*" + - "scripts/**/*.sh" - ".github/workflows/shellcheck.yml" concurrency: @@ -32,4 +34,4 @@ jobs: - name: Install ShellCheck run: sudo apt-get update && sudo apt-get install -y shellcheck - name: Run ShellCheck - run: shellcheck chdtool.sh tests/*.sh tests/bin/* + run: shellcheck chdtool.sh scripts/*.sh tests/*.sh tests/bin/* diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 17c618d..0601dcc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,12 +7,14 @@ on: - "chdtool.sh" - "Makefile" - "tests/**" + - "scripts/**" - ".github/workflows/tests.yml" pull_request: paths: - "chdtool.sh" - "Makefile" - "tests/**" + - "scripts/**" - ".github/workflows/tests.yml" concurrency: diff --git a/Makefile b/Makefile index 2f15298..0e7f378 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,9 @@ SHELL := /usr/bin/env bash # Path to your script; override with: make test SCRIPT=./path/to/script.sh SCRIPT ?= ./chdtool.sh -.PHONY: test test-m3u test-m3u-single test-partial-resume test-transactional-source-deletion test-temporary-chd-cleanup test-exit-status test-logging test-security-validation test-resource-handling test-regression-gaps test-setup clean changelog changelog-tag +.PHONY: test test-m3u test-m3u-single test-partial-resume test-transactional-source-deletion test-temporary-chd-cleanup test-exit-status test-logging test-security-validation test-resource-handling test-regression-gaps test-cli test-package test-setup clean changelog changelog-tag package -test: test-setup test-m3u test-m3u-single test-partial-resume test-transactional-source-deletion test-temporary-chd-cleanup test-exit-status test-logging test-security-validation test-resource-handling test-regression-gaps +test: test-setup test-m3u test-m3u-single test-partial-resume test-transactional-source-deletion test-temporary-chd-cleanup test-exit-status test-logging test-security-validation test-resource-handling test-regression-gaps test-cli test-package test-setup: chmod +x tests/bin/chdman @@ -18,6 +18,7 @@ test-setup: chmod +x tests/test_security_validation.sh chmod +x tests/test_resource_handling.sh chmod +x tests/test_regression_gaps.sh + chmod +x tests/test_cli.sh tests/test_package.sh scripts/package-release.sh @if [ -f tests/test_logging.sh ]; then chmod +x tests/test_logging.sh; fi test-m3u: @@ -54,11 +55,20 @@ test-resource-handling: test-regression-gaps: SCRIPT=$(SCRIPT) bash tests/test_regression_gaps.sh +test-cli: + SCRIPT=$(SCRIPT) bash tests/test_cli.sh + +test-package: + bash tests/test_package.sh + clean: @echo "Nothing to clean; tests use mktemp dirs." changelog: - git cliff --config .cliff.toml -o CHANGELOG.md + git cliff --config cliff.toml -o CHANGELOG.md changelog-tag: - git cliff --config .cliff.toml --tag $${TAG} --strip header > CHANGELOG_RELEASE.md + git cliff --config cliff.toml --tag $${TAG} --strip header > CHANGELOG_RELEASE.md + +package: + bash scripts/package-release.sh "$${VERSION:-$$(sed -n 's/^CHDTOOL_VERSION="\([^"]*\)"/\1/p' chdtool.sh)}" diff --git a/README.md b/README.md index cd5defe..acbb701 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ๐ŸŽฎ chdtool -[![License](https://img.shields.io/github/license/lloydsmart/chdtool)](https://github.com/lloydsmart/chdtool/blob/main/LICENSE) +[![License](https://img.shields.io/github/license/lloydsmart/chdtool)](https://github.com/lloydsmart/chdtool/blob/main/LICENSE.md) [![Release](https://img.shields.io/github/v/release/lloydsmart/chdtool)](https://github.com/lloydsmart/chdtool/releases) [![ShellCheck](https://img.shields.io/github/actions/workflow/status/lloydsmart/chdtool/shellcheck.yml?branch=main&label=shellcheck)](https://github.com/lloydsmart/chdtool/actions/workflows/shellcheck.yml) @@ -31,7 +31,7 @@ Convert everything in a directory: - ๐Ÿ“‰ Space savings reporting - ๐Ÿงพ Automatic M3U generation for multi-disc sets - ๐Ÿงน Safe temp directory handling with cleanup traps -- ๐Ÿงช Dry-run mode (no changes made) +- ๐Ÿงช Dry-run mode (no source or conversion changes) - ๐Ÿชต Structured logging system: - Console / file / syslog / journald - Configurable verbosity @@ -51,9 +51,18 @@ Convert everything in a directory: | ------ | ----------- | | `-k`, `--keep-originals` | Do not delete source files after conversion | | `-r`, `--recursive` | Process subdirectories | -| `-n`, `--dry-run` | Show what would happen without making changes | -| `-F`, `--file-tee` | Force logging to file | -| `-N`, `--no-file-tee` | Disable logging to file | +| `-n`, `--dry-run` | Show what would happen without making changes | +| `-a`, `--allow-unverified-cue-audio` | Permit lossy or unverified CUE audio tracks (not preservation-safe) | +| `-F`, `--file-tee` | Force logging to file | +| `-N`, `--no-file-tee` | Disable logging to file | +| `-h`, `--help` | Show usage information | +| `-V`, `--version` | Show the packaged version | + +Use `--` to end option parsing when an input directory starts with `-`: + +```bash +./chdtool.sh -- -roms +``` --- @@ -133,15 +142,21 @@ Configurable via environment variables: LOG_DEST=auto|console|file|syslog|journald LOG_LEVEL_THRESHOLD=DEBUG|INFO|WARN|ERROR LOG_TEE_CONSOLE=auto|1|0 -LOG_TEE_FILE=1|0 -LOG_TAG=chdtool +LOG_TEE_FILE=1|0 +LOG_TAG=chdtool +LOGFILE=/custom/path/chdtool.log ``` Default log file: ```text logs/chd_conversion_.log -``` +``` + +`--file-tee` and `--no-file-tee` control the optional file mirror for console, +syslog, and journald backends. When `LOG_DEST=file`, the file is the primary +destination and is therefore still written. A caller-supplied `LOGFILE` path is +used verbatim. --- @@ -174,8 +189,9 @@ The following tools must be installed: - `unzip` - `unrar` - `7z` -- `stat` -- `awk` +- `stat` +- `awk` +- `stdbuf` Optional (enhancements): @@ -194,9 +210,14 @@ Use `--dry-run` to preview actions: ./chdtool.sh -n /roms ``` -- No files are created, moved, or deleted +- No source files, CHDs, playlists, or temporary conversion workspaces are + created, moved, or deleted - All intended operations are logged +Logging remains active during a dry run and may create the configured log file. +Use `--no-file-tee` with the console, syslog, or journald backend to prevent a +file mirror. + ### Resource selection By default, CHD Tool lets `chdman` choose its format-appropriate hunk size. It @@ -222,6 +243,20 @@ result at the available CPU count with a minimum of one. - Archive members are rejected before conversion when paths escape the extraction directory, links are present, or selected entries collide after name sanitisation - Temporary files are cleaned automatically, even on interruption +- `--allow-unverified-cue-audio` relaxes preservation checks for lossy or + otherwise unverified CUE audio tracks; use it only when accepting that risk + +--- + +## ๐Ÿ“ฆ Release assets + +Each release publishes `.tar.gz` and `.zip` archives containing `chdtool`, +`README.md`, `LICENSE.md`, and `CHANGELOG.md`, plus a standalone `chdtool` +script. Verify downloads with the published checksum file: + +```bash +sha256sum --check SHA256SUMS +``` --- diff --git a/chdtool.sh b/chdtool.sh index 5cd957e..9fbddf6 100755 --- a/chdtool.sh +++ b/chdtool.sh @@ -6,7 +6,25 @@ script_start_time=$(date +%s) shopt -s nullglob shopt -s extglob -USAGE="Usage: $0 [--keep-originals|-k] [--recursive|-r] [--dry-run|-n] [--allow-unverified-cue-audio|-a] [--file-tee|-F|--no-file-tee|-N] " +CHDTOOL_VERSION="0.2.4" +PROGRAM_NAME="$(basename -- "$0")" +USAGE="Usage: $PROGRAM_NAME [options] [--] " + +print_usage() { + cat <&2 + echo "$USAGE" >&2; exit 1 + fi + INPUT_DIR="${1%/}" + shift + break ;; -*) echo "โŒ Unknown option: $1" >&2 echo "$USAGE" >&2; exit 1 ;; @@ -51,6 +82,7 @@ done if [[ -z "$INPUT_DIR" ]]; then echo "$USAGE" >&2; exit 1 fi +[[ "$INPUT_DIR" == -* ]] && INPUT_DIR="./$INPUT_DIR" if [[ ! -d "$INPUT_DIR" ]]; then echo "โŒ Input directory does not exist or is not a directory: $INPUT_DIR" >&2 exit 1 @@ -59,7 +91,7 @@ fi # Use a disk-based temp directory to avoid filling up RAM TMP_ROOT="${TMPDIR:-/var/tmp/chdtool}" TMPDIR="$TMP_ROOT/$RUN_ID" -mkdir -p "$TMPDIR" +[[ "$DRY_RUN" == true ]] || mkdir -p "$TMPDIR" LOGFILE="${LOGFILE:-logs/chd_conversion_$(date +%Y%m%d_%H%M%S).log}" @@ -113,7 +145,7 @@ if [[ "$LOG_BACKEND" == journald || "$LOG_BACKEND" == syslog ]]; then fi # ensure the logfile directory exists when we might write to it -if [[ "$LOG_BACKEND" == file || "$LOG_BACKEND" == console ]] || __should_mirror_file; then +if [[ "$LOG_BACKEND" == file ]] || __should_mirror_file; then mkdir -p -- "$(dirname -- "$LOGFILE")" fi @@ -182,9 +214,13 @@ _emit_log() { __should_mirror_console && __console_print "$ts" "$lvl" "$msg" ;; console|*) - while IFS= read -r line; do - printf '[%s] %s: %s\n' "$ts" "$lvl" "$line" | tee -a "$LOGFILE" - done <<< "$msg" + if __should_mirror_file; then + while IFS= read -r line; do + printf '[%s] %s: %s\n' "$ts" "$lvl" "$line" | tee -a "$LOGFILE" + done <<< "$msg" + else + __console_print "$ts" "$lvl" "$msg" + fi ;; esac @@ -503,7 +539,7 @@ validate_extracted_tree() { done < <(find "$root" -type l -print0) } -check_temp_storage "$TMPDIR" +[[ "$DRY_RUN" == true ]] || check_temp_storage "$TMPDIR" # ---------- chdman progress handling ---------- # Config: PROGRESS_STYLE=auto|bar|line|none ; default: auto (TTY -> bar, non-TTY -> none) diff --git a/scripts/package-release.sh b/scripts/package-release.sh new file mode 100644 index 0000000..df569c3 --- /dev/null +++ b/scripts/package-release.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +version="${1:-}" +[[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$ ]] || { + echo "Usage: $0 " >&2 + exit 1 +} + +script_version="$(sed -n 's/^CHDTOOL_VERSION="\([^"]*\)"/\1/p' chdtool.sh)" +[[ "$script_version" == "$version" ]] || { + echo "Release version $version does not match chdtool.sh version $script_version" >&2 + exit 1 +} + +for command in tar gzip zip unzip sha256sum; do + command -v "$command" >/dev/null 2>&1 || { echo "Missing packaging command: $command" >&2; exit 1; } +done + +archive_base="chdtool-v$version" +dist_dir="${DIST_DIR:-$REPO_ROOT/dist}" +[[ "$dist_dir" == /* ]] || dist_dir="$REPO_ROOT/$dist_dir" +[[ "$dist_dir" != "/" && "$dist_dir" != "$REPO_ROOT" ]] || { + echo "Refusing unsafe distribution directory: $dist_dir" >&2 + exit 1 +} +stage_root="$(mktemp -d)" +trap 'rm -rf "$stage_root"' EXIT +stage_dir="$stage_root/$archive_base" +mkdir -p "$stage_dir" + +install -m 0755 chdtool.sh "$stage_dir/chdtool" +install -m 0644 README.md LICENSE.md CHANGELOG.md "$stage_dir/" + +# Use the tagged commit timestamp when available, with a stable fallback for +# source archives or local packaging outside a Git checkout. +source_date_epoch="${SOURCE_DATE_EPOCH:-$(git log -1 --format=%ct 2>/dev/null || printf '0')}" +find "$stage_dir" -exec touch -d "@$source_date_epoch" {} + + +rm -rf -- "$dist_dir" +mkdir -p "$dist_dir" + +tar --sort=name --mtime="@$source_date_epoch" --owner=0 --group=0 --numeric-owner \ + -C "$stage_root" -cf - "$archive_base" | gzip -n > "$dist_dir/$archive_base.tar.gz" + +zip_path="$dist_dir/$archive_base.zip" +( + cd "$stage_root" + find "$archive_base" -type f -print | LC_ALL=C sort | zip -X -q "$zip_path" -@ +) + +install -m 0755 chdtool.sh "$dist_dir/chdtool" + +expected="$(printf '%s\n' \ + "$archive_base/CHANGELOG.md" \ + "$archive_base/LICENSE.md" \ + "$archive_base/README.md" \ + "$archive_base/chdtool")" +tar_contents="$(tar -tzf "$dist_dir/$archive_base.tar.gz" | sed '/\/$/d' | LC_ALL=C sort)" +zip_contents="$(unzip -Z1 "$dist_dir/$archive_base.zip" | sed '/\/$/d' | LC_ALL=C sort)" +[[ "$tar_contents" == "$expected" ]] || { echo "Unexpected tar archive contents" >&2; exit 1; } +[[ "$zip_contents" == "$expected" ]] || { echo "Unexpected zip archive contents" >&2; exit 1; } + +( + cd "$dist_dir" + sha256sum chdtool "$archive_base.tar.gz" "$archive_base.zip" > SHA256SUMS + sha256sum --check SHA256SUMS +) + +printf 'Created validated release assets in %s\n' "$dist_dir" diff --git a/tests/test_cli.sh b/tests/test_cli.sh new file mode 100644 index 0000000..816169c --- /dev/null +++ b/tests/test_cli.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SCRIPT="${SCRIPT:-$REPO_ROOT/chdtool.sh}" +if [[ "$SCRIPT" != /* ]]; then + SCRIPT="$(cd "$(dirname "$SCRIPT")" && pwd)/$(basename "$SCRIPT")" +fi +FIX="$(mktemp -d)" +trap 'rm -rf "$FIX"' EXIT +export PATH="$REPO_ROOT/tests/bin:$PATH" + +help_output="$(bash "$SCRIPT" --help)" +grep -q '^Usage:' <<< "$help_output" || { echo "FAIL: --help omitted usage" >&2; exit 1; } +grep -q -- '--allow-unverified-cue-audio' <<< "$help_output" || { echo "FAIL: --help omitted audio warning option" >&2; exit 1; } + +version_output="$(bash "$SCRIPT" --version)" +[[ "$version_output" =~ ^chdtool\.sh\ [0-9]+\.[0-9]+\.[0-9]+ ]] || { echo "FAIL: unexpected --version output: $version_output" >&2; exit 1; } + +mkdir "$FIX/-input" +( + cd "$FIX" + cli_output="$(LOG_DEST=console LOG_LEVEL_THRESHOLD=ERROR bash "$SCRIPT" --dry-run --no-file-tee -- -input 2>&1)" + [[ "$cli_output" != *'unknown predicate'* ]] || { echo "FAIL: leading-dash path reached find as an option" >&2; exit 1; } +) +[[ ! -e "$FIX/logs" ]] || { echo "FAIL: --no-file-tee created a console-backend logfile" >&2; exit 1; } + +if TMPDIR="$FIX/dry-temp" LOG_DEST=console LOG_LEVEL_THRESHOLD=ERROR \ + bash "$SCRIPT" --dry-run --no-file-tee "$FIX/-input"; then + [[ ! -e "$FIX/dry-temp" ]] || { echo "FAIL: dry-run created a temporary workspace" >&2; exit 1; } +else + echo "FAIL: dry-run CLI invocation failed" >&2 + exit 1 +fi + +echo "PASS: help, version, option terminator, logging, and dry-run CLI behaviour" diff --git a/tests/test_package.sh b/tests/test_package.sh new file mode 100644 index 0000000..115917a --- /dev/null +++ b/tests/test_package.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +version="$(sed -n 's/^CHDTOOL_VERSION="\([^"]*\)"/\1/p' "$REPO_ROOT/chdtool.sh")" +FIX="$(mktemp -d)" +trap 'rm -rf "$FIX"' EXIT +dist_dir="$FIX/dist" + +(cd "$REPO_ROOT" && DIST_DIR="$dist_dir" SOURCE_DATE_EPOCH=0 bash scripts/package-release.sh "$version") + +archive_base="chdtool-v$version" +[[ -s "$dist_dir/$archive_base.tar.gz" ]] || { echo "FAIL: tar archive missing" >&2; exit 1; } +[[ -s "$dist_dir/$archive_base.zip" ]] || { echo "FAIL: zip archive missing" >&2; exit 1; } +[[ -s "$dist_dir/chdtool" && -s "$dist_dir/SHA256SUMS" ]] || { echo "FAIL: standalone script or checksums missing" >&2; exit 1; } +(cd "$dist_dir" && sha256sum --check SHA256SUMS >/dev/null) +first_hashes="$(cd "$dist_dir" && sha256sum chdtool "$archive_base.tar.gz" "$archive_base.zip")" + +(cd "$REPO_ROOT" && DIST_DIR="$dist_dir" SOURCE_DATE_EPOCH=0 bash scripts/package-release.sh "$version" >/dev/null) +second_hashes="$(cd "$dist_dir" && sha256sum chdtool "$archive_base.tar.gz" "$archive_base.zip")" +[[ "$first_hashes" == "$second_hashes" ]] || { echo "FAIL: repeated packaging was not reproducible" >&2; exit 1; } + +echo "PASS: release assets are complete, validated, and checksummed"