diff --git a/.dockerignore b/.dockerignore index b14bf8191..9c651ed80 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,4 +12,4 @@ docs/ .cov/ .venv Dockerfile -.env \ No newline at end of file +.env diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 37b443204..7af33f2e4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,8 @@ updates: allow: - dependency-name: "*" dependency-type: "direct" + cooldown: + default-days: 14 groups: all-dependencies: patterns: @@ -21,18 +23,39 @@ updates: allow: - dependency-name: "*" dependency-type: "direct" + cooldown: + default-days: 14 groups: all-dependencies: patterns: - "*" - - package-ecosystem: docker-compose + - package-ecosystem: "docker" + directories: + - "/" + - "/tests" + schedule: + interval: "monthly" + cooldown: + default-days: 14 + ignore: + - dependency-name: "*" + update-types: ["version-update:semver-major"] + groups: + all-dependencies: + patterns: + - "*" + group-by: dependency-name + + - package-ecosystem: "docker-compose" directory: "/" schedule: interval: "monthly" + cooldown: + default-days: 14 ignore: - - dependency-name: "*" - update-types: ["version-update:semver-major"] + - dependency-name: "*" + update-types: ["version-update:semver-major"] groups: all-dependencies: patterns: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index cccae3471..a22252447 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -38,6 +38,34 @@ jobs: - name: Run typecheck run: | make typecheck + guardrail-check: + name: "Guardrail checks" + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + enable-cache: true + cache-dependency-glob: | + **/pyproject.toml + **/uv.lock + - name: Restore cache + id: cache-venv + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: .venv + key: venv-${{ runner.os }}-${{ hashFiles('**/pyproject.toml', '**/uv.lock') }} + restore-keys: | + venv-${{ runner.os }}- + - name: Install dependencies + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + uv sync --frozen --no-install-project + - name: Run guardrail checks + run: | + make test_guardrail_check unit-test: name: "Unit tests" runs-on: ubuntu-latest @@ -45,8 +73,15 @@ jobs: strategy: fail-fast: true matrix: - db: [postgres, mysql] - test_target: ["tests/app/", "tests/batch/"] + include: + - db: postgres + test_target: "tests/app/" + - db: mysql + test_target: "tests/app/" + - db: postgres + test_target: "tests/batch/" + - db: mysql + test_target: "tests/batch/" timeout-minutes: 60 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/Dockerfile b/Dockerfile index 856b0c0be..4577901a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -FROM ubuntu:24.04 AS builder +FROM ghcr.io/astral-sh/uv:0.9.16@sha256:ae9ff79d095a61faf534a882ad6378e8159d2ce322691153d68d2afac7422840 AS uv + +FROM ubuntu:24.04@sha256:c4a8d5503dfb2a3eb8ab5f807da5bc69a85730fb49b5cfca2330194ebcc41c7b AS builder ENV PYTHON_VERSION=3.14.2 -ENV UV_VERSION=0.9.16 ENV UV_COMPILE_BYTECODE=1 ENV UV_LINK_MODE=copy -ENV UV_INSTALL_DIR="/usr/local/bin" ENV UV_PROJECT_ENVIRONMENT="/home/apl/.venv" # make application directory @@ -34,8 +34,7 @@ RUN apt-get update -q \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # install uv -ADD https://astral.sh/uv/$UV_VERSION/install.sh /uv-installer.sh -RUN INSTALLER_NO_MODIFY_PATH=1 sh /uv-installer.sh && rm /uv-installer.sh +COPY --from=uv /uv /uvx /usr/local/bin/ # install Python RUN uv python install $PYTHON_VERSION @@ -66,7 +65,7 @@ RUN cd /app/ibet-Wallet-API \ && rm -f /app/ibet-Wallet-API/uv.lock \ && rm -rf /app/ibet-Wallet-API/tests/ -FROM ubuntu:24.04 AS runner +FROM ubuntu:24.04@sha256:c4a8d5503dfb2a3eb8ab5f807da5bc69a85730fb49b5cfca2330194ebcc41c7b AS runner # make application directory RUN mkdir -p /app diff --git a/Makefile b/Makefile index 039c164e6..db63498d7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: format lint typecheck doc test test_migrations run +.PHONY: format lint typecheck doc test test_guardrail_check test_guardrail test_migrations run install: uv sync --frozen --no-install-project --all-extras @@ -22,6 +22,11 @@ doc: test: uv run pytest tests/ ${ARG} +test_guardrail_check: + uv run pytest --override-ini addopts='' --noconftest -v -m "guardrail_check" tests/test_guardrail_check.py + +test_guardrail: test_guardrail_check + test_migrations: uv run pytest -vv --test-alembic -m "alembic" diff --git a/README.md b/README.md index e0806a33f..2abe09526 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,12 @@ Install python packages with: $ uv sync --frozen --no-install-project --no-dev ``` +### Dependency update policy + +- Dependabot manages `uv`, GitHub Actions, Dockerfile, and Docker Compose updates in this repository. +- Regular version updates are delayed by a 14-day cooldown to reduce supply-chain risk. Security updates are still handled without that delay. +- Docker base images and external Compose images are pinned by digest, and Dockerfiles must not rely on remote `ADD` or pipe-to-shell installers. + ### Setting environment variables See the following documentation for environment variables that can be set in this system. diff --git a/README_JA.md b/README_JA.md index 306f39ad8..f9e8ddbfd 100644 --- a/README_JA.md +++ b/README_JA.md @@ -60,6 +60,12 @@ $ uv venv $ uv sync --frozen --no-install-project --no-dev ``` +### 依存関係の更新ポリシー + +- このリポジトリでは `uv`、GitHub Actions、Dockerfile、Docker Compose の更新を Dependabot で管理します。 +- 通常のバージョン更新には 14 日の cooldown を設定し、サプライチェーンリスクを下げます。セキュリティ更新はこの遅延の対象外です。 +- Docker の base image と Compose で参照する外部 image は digest で固定し、Dockerfile ではリモート `ADD` や pipe-to-shell installer を使いません。 + ### 環境変数の設定 設定可能な環境変数については以下のドキュメントを確認してください。 diff --git a/docker-compose.yml b/docker-compose.yml index a8bee0c46..fccea68d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: postgres: - image: postgres:17 + image: postgres:17.10@sha256:0027bef26712baaee437a4ea48fdf3d2d2e2bc5f0d81615374408ca320f3c7e3 ports: - "5432:5432" environment: @@ -10,7 +10,7 @@ services: tmpfs: - /var/lib/postgresql/data mysql: - image: mysql:8.4 + image: mysql:8.4.9@sha256:c36050afdca850f23cef85703f84c7531a5ae155a11b5ee1c60acb09937c4084 environment: - MYSQL_USER=apluser - MYSQL_PASSWORD=apluserpass @@ -74,4 +74,4 @@ services: - mysql - anvil-network volumes: - - ./cov:/app/ibet-Wallet-API/cov \ No newline at end of file + - ./cov:/app/ibet-Wallet-API/cov diff --git a/pyproject.toml b/pyproject.toml index 004d8a49e..e7852037a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,8 +101,11 @@ exclude = ["migrations/*", ".venv/*", "cov/*"] combine-as-imports = true [tool.pytest.ini_options] -addopts = "-m 'not alembic'" -markers = ["alembic: tests for alembic"] +addopts = "-m 'not alembic and not guardrail_check'" +markers = [ + "alembic: tests for alembic", + "guardrail_check: tests for repository guardrails", +] asyncio_default_fixture_loop_scope = "session" [tool.coverage.run] diff --git a/tests/Dockerfile_anvil b/tests/Dockerfile_anvil index c13a96332..30fb93f75 100644 --- a/tests/Dockerfile_anvil +++ b/tests/Dockerfile_anvil @@ -1,7 +1,7 @@ -FROM ghcr.io/foundry-rs/foundry:latest +FROM ghcr.io/foundry-rs/foundry:latest@sha256:8347b728d5d393dac1c018691b36f506d23b9dcd78341d40ea0fcb11c3a19cdd WORKDIR /app/ibet-Wallet-API COPY tests/run_anvil.sh /app/ibet-Wallet-API/tests/run_anvil.sh -ENTRYPOINT ["bash", "/app/ibet-Wallet-API/tests/run_anvil.sh"] \ No newline at end of file +ENTRYPOINT ["bash", "/app/ibet-Wallet-API/tests/run_anvil.sh"] diff --git a/tests/Dockerfile_unittest b/tests/Dockerfile_unittest index 00e83be0c..9aba702c4 100644 --- a/tests/Dockerfile_unittest +++ b/tests/Dockerfile_unittest @@ -1,10 +1,10 @@ -FROM ubuntu:24.04 AS builder +FROM ghcr.io/astral-sh/uv:0.9.16@sha256:ae9ff79d095a61faf534a882ad6378e8159d2ce322691153d68d2afac7422840 AS uv + +FROM ubuntu:24.04@sha256:c4a8d5503dfb2a3eb8ab5f807da5bc69a85730fb49b5cfca2330194ebcc41c7b AS builder ENV PYTHON_VERSION=3.14.2 -ENV UV_VERSION=0.9.16 ENV UV_COMPILE_BYTECODE=1 ENV UV_LINK_MODE=copy -ENV UV_INSTALL_DIR="/usr/local/bin" ENV UV_PROJECT_ENVIRONMENT="/home/apl/.venv" # make application directory @@ -34,8 +34,7 @@ RUN apt-get update -q \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # install uv -ADD https://astral.sh/uv/$UV_VERSION/install.sh /uv-installer.sh -RUN INSTALLER_NO_MODIFY_PATH=1 sh /uv-installer.sh && rm /uv-installer.sh +COPY --from=uv /uv /uvx /usr/local/bin/ # install Python RUN uv python install $PYTHON_VERSION @@ -63,7 +62,7 @@ RUN cd /app/ibet-Wallet-API \ && rm -rf "$PYTHON_ROOT"/lib/python*/site-packages/pip "$PYTHON_ROOT"/lib/python*/site-packages/pip-*.dist-info \ && rm -rf /home/apl/.cache/uv -FROM ubuntu:24.04 AS runner +FROM ubuntu:24.04@sha256:c4a8d5503dfb2a3eb8ab5f807da5bc69a85730fb49b5cfca2330194ebcc41c7b AS runner # make application directory RUN mkdir -p /app diff --git a/tests/test_guardrail_check.py b/tests/test_guardrail_check.py new file mode 100644 index 000000000..a2eb2e9c1 --- /dev/null +++ b/tests/test_guardrail_check.py @@ -0,0 +1,280 @@ +""" +Copyright BOOSTRY Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +""" + +import os +import re +import shlex +from pathlib import Path +from typing import Any, cast + +import pytest +from ruamel.yaml import YAML + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEPENDABOT_CONFIG_PATH = REPO_ROOT / ".github" / "dependabot.yml" +REQUIRED_DEPENDABOT_ECOSYSTEMS = ("uv", "github-actions", "docker", "docker-compose") +REQUIRED_COOLDOWN_DAYS = 14 +COMPOSE_FILE_NAMES = ( + "compose.yml", + "compose.yaml", + "docker-compose.yml", + "docker-compose.yaml", +) +IGNORED_DIRECTORY_NAMES = { + ".git", + ".pytest_cache", + ".ruff_cache", + ".venv", + "__pycache__", + "cov", + "htmlcov", + "node_modules", +} + +_REMOTE_ADD_PATTERN = re.compile(r"^\s*ADD\s+https?://", re.IGNORECASE) +_PIPE_TO_SHELL_PATTERN = re.compile( + r"\b(?:curl|wget)\b[^\n]*\|\s*(?:sh|bash)\b", + re.IGNORECASE, +) + +# These tests scan repository files, so keep them out of the normal test run. +pytestmark = pytest.mark.guardrail_check + + +# Walk repository files that the guardrail checks should inspect. +def _walk_repo_files(repo_root: Path): + # Only source-controlled inputs should affect these repository guardrails. + for current_root, dirnames, filenames in os.walk(repo_root): + dirnames[:] = [ + dirname for dirname in dirnames if dirname not in IGNORED_DIRECTORY_NAMES + ] + current_path = Path(current_root) + for filename in filenames: + yield current_path / filename + + +# Find Dockerfiles that may pull base images or remote installers. +def _collect_dockerfiles(repo_root: Path) -> list[Path]: + return sorted( + ( + path + for path in _walk_repo_files(repo_root) + if path.name.startswith("Dockerfile") + ), + key=str, + ) + + +# Find Compose files whose external service images must be pinned. +def _collect_compose_files(repo_root: Path) -> list[Path]: + return sorted( + ( + path + for path in _walk_repo_files(repo_root) + if path.name in COMPOSE_FILE_NAMES + ), + key=str, + ) + + +# Check Docker FROM images and ignore local build-stage references. +def _collect_docker_base_image_violations(file_path: Path) -> list[str]: + stage_names: set[str] = set() + violations: list[str] = [] + relative_path = file_path.relative_to(REPO_ROOT) + + for lineno, line in enumerate( + file_path.read_text(encoding="utf-8").splitlines(), + 1, + ): + stripped = line.strip() + if not stripped or not stripped.upper().startswith("FROM "): + continue + + # Parse FROM like Docker does so flags and quoted values do not hide the image. + tokens = shlex.split(stripped, comments=False, posix=True) + if not tokens or tokens[0].upper() != "FROM": + continue + + # Skip FROM options such as --platform; the next token is the base image. + token_index = 1 + while token_index < len(tokens) and tokens[token_index].startswith("--"): + token_index += 1 + if token_index >= len(tokens): + continue + + base_image = tokens[token_index] + known_stage_names = set(stage_names) + + # Named stages are local build outputs, not images pulled from a registry. + if token_index + 2 < len(tokens) and tokens[token_index + 1].upper() == "AS": + stage_names.add(tokens[token_index + 2]) + + # scratch is also local and has no digest to pin. + if base_image == "scratch" or base_image in known_stage_names: + continue + if "@sha256:" not in base_image: + violations.append( + f"{relative_path}:{lineno} Docker base images must be pinned by digest" + ) + + return violations + + +# Check Dockerfile lines that fetch and run remote content directly. +def _collect_docker_remote_installer_violations(file_path: Path) -> list[str]: + violations: list[str] = [] + relative_path = file_path.relative_to(REPO_ROOT) + + for lineno, line in enumerate( + file_path.read_text(encoding="utf-8").splitlines(), + 1, + ): + # Remote content must not bypass repository review. + if _REMOTE_ADD_PATTERN.search(line): + violations.append(f"{relative_path}:{lineno} Remote URL ADD is not allowed") + if _PIPE_TO_SHELL_PATTERN.search(line): + violations.append( + f"{relative_path}:{lineno} Pipe-to-shell installers are not allowed" + ) + + return violations + + +# Check Compose services that pull external images without digest pins. +def _collect_compose_image_digest_violations(file_path: Path) -> list[str]: + relative_path = file_path.relative_to(REPO_ROOT) + yaml = cast(Any, YAML(typ="safe")) + loaded = cast(object, yaml.load(file_path.read_text(encoding="utf-8"))) + config = cast(dict[str, Any], loaded if isinstance(loaded, dict) else {}) + services = config.get("services") + violations: list[str] = [] + + if not isinstance(services, dict): + return violations + + services_dict = cast(dict[object, object], services) + for service_name_obj, service_obj_obj in services_dict.items(): + if not isinstance(service_name_obj, str) or not isinstance( + service_obj_obj, dict + ): + continue + service_obj = cast(dict[str, Any], service_obj_obj) + # build means the image comes from a local Dockerfile checked above. + if "build" in service_obj: + continue + image = service_obj.get("image") + if not isinstance(image, str): + continue + if "@sha256:" not in image: + violations.append( + f"{relative_path}:{service_name_obj} External Compose images must be pinned by digest" + ) + + return violations + + +# Check that Dependabot keeps the required update policy in place. +def _collect_dependabot_policy_violations(config_path: Path) -> list[str]: + relative_path = config_path.relative_to(REPO_ROOT) + yaml = cast(Any, YAML(typ="safe")) + loaded = cast(object, yaml.load(config_path.read_text(encoding="utf-8"))) + config = cast(dict[str, Any], loaded if isinstance(loaded, dict) else {}) + violations: list[str] = [] + updates = config.get("updates") + + if config.get("version") != 2: + violations.append(f"{relative_path} version must be set to 2") + if not isinstance(updates, list): + violations.append(f"{relative_path} updates must be a list") + return violations + + updates_list = cast(list[object], updates) + updates_by_ecosystem: dict[str, dict[str, Any]] = {} + for raw_update_obj in updates_list: + if not isinstance(raw_update_obj, dict): + continue + raw_update = cast(dict[str, Any], raw_update_obj) + ecosystem = raw_update.get("package-ecosystem") + if isinstance(ecosystem, str): + updates_by_ecosystem[ecosystem] = raw_update + + for ecosystem in REQUIRED_DEPENDABOT_ECOSYSTEMS: + update = updates_by_ecosystem.get(ecosystem) + if update is None: + violations.append( + f"{relative_path} must define an update block for {ecosystem!r}" + ) + continue + + # Use the same cooldown everywhere so dependency updates arrive predictably. + cooldown_obj = update.get("cooldown") + if not isinstance(cooldown_obj, dict): + violations.append(f"{relative_path} must define cooldown for {ecosystem!r}") + continue + cooldown = cast(dict[str, Any], cooldown_obj) + if cooldown.get("default-days") != REQUIRED_COOLDOWN_DAYS: + violations.append( + f"{relative_path} cooldown.default-days for {ecosystem!r} must be {REQUIRED_COOLDOWN_DAYS}" + ) + + # A limit of 0 disables version updates and can hide stale dependencies. + if update.get("open-pull-requests-limit") == 0: + violations.append( + f"{relative_path} must not disable version updates for {ecosystem!r}" + ) + + return violations + + +# This checks that Dockerfiles use fixed base images and avoid remote installers. +def test_dockerfiles_pin_base_images_and_avoid_remote_installers(): + violations: list[str] = [] + # Report all Dockerfile issues together so one run gives a full fix list. + for file_path in _collect_dockerfiles(REPO_ROOT): + violations.extend(_collect_docker_base_image_violations(file_path)) + violations.extend(_collect_docker_remote_installer_violations(file_path)) + + assert not violations, ( + "Supply-chain violations were found in Dockerfiles:\n" + "\n".join(violations) + ) + + +# This checks that Compose services do not use floating external image tags. +def test_compose_files_pin_images_by_digest(): + violations: list[str] = [] + # Report all Compose issues together so one run gives a full fix list. + for file_path in _collect_compose_files(REPO_ROOT): + violations.extend(_collect_compose_image_digest_violations(file_path)) + + assert not violations, ( + "Supply-chain violations were found in Compose files:\n" + "\n".join(violations) + ) + + +# This checks that Dependabot keeps required updates enabled with a fixed cooldown. +def test_dependabot_config_enforces_cooldown_policy(): + assert DEPENDABOT_CONFIG_PATH.exists(), ( + f"Dependabot configuration is missing: {DEPENDABOT_CONFIG_PATH.relative_to(REPO_ROOT)}" + ) + + violations = _collect_dependabot_policy_violations(DEPENDABOT_CONFIG_PATH) + assert not violations, "Dependabot policy violations were found:\n" + "\n".join( + violations + )