diff --git a/.automation/build.py b/.automation/build.py index cc684c85919..639319e41be 100644 --- a/.automation/build.py +++ b/.automation/build.py @@ -143,6 +143,17 @@ "vscode": {"label": "Visual Studio Code", "url": "https://code.visualstudio.com/"}, } +# Vulnerability database settings of the trivy linters, added to the generated +# MegaLinter configuration schema from their descriptor variables +TRIVY_DB_SCHEMA_LINTERS = ["REPOSITORY_TRIVY", "REPOSITORY_TRIVY_SBOM"] +TRIVY_DB_SCHEMA_VARIABLE_TITLES = { + "DB_REPOSITORIES": "Vulnerability database repositories", + "JAVA_DB_REPOSITORIES": "Java vulnerability database repositories", + "DB_RETRY_ATTEMPTS": "Vulnerability database download attempts", + "DB_RETRY_INITIAL_DELAY": "Vulnerability database first retry delay", + "DB_RETRY_MAX_DELAY": "Vulnerability database maximum retry delay", +} + DESCRIPTORS_FOR_BUILD_CACHE = None MAIN_DOCKERFILE = f"{REPO_HOME}/Dockerfile" @@ -1728,6 +1739,14 @@ def process_type(linters_by_type, type1, type_label, linters_tables_md): ], ] ) + # Trivy linters expose vulnerability database mirrors and download + # retry settings consumed by TrivyLinter. Same narrow approach as + # betterleaks above: their descriptor variables are converted into + # configuration schema entries from their known name suffixes. + if linter.name in TRIVY_DB_SCHEMA_LINTERS: + add_in_config_schema_file( + build_trivy_db_config_schema_variables(linter, title_prefix) + ) linter_doc_md += [ f"| {linter.name}_ARGUMENTS | User custom arguments to add in linter CLI call
" f'Ex: `-s --foo "bar"` | |' @@ -3331,6 +3350,32 @@ def validate_config_schema_root_x_metadata() -> None: raise Exception("Config schema root properties missing x-keys") +def build_trivy_db_config_schema_variables(linter, title_prefix): + schema_variables = [] + for variable in linter.variables: + suffix = variable["name"].replace(f"{linter.name}_", "", 1) + if suffix not in TRIVY_DB_SCHEMA_VARIABLE_TITLES: + continue + default_value = variable["default_value"] + variable_schema = { + "$id": f"#/properties/{variable['name']}", + "description": f"{linter.name}: {variable['description']}", + "title": ( + f"{title_prefix}{linter.name}: " + f"{TRIVY_DB_SCHEMA_VARIABLE_TITLES[suffix]}" + ), + } + if suffix.endswith("_REPOSITORIES"): + variable_schema["type"] = ["array", "string"] + variable_schema["items"] = {"type": "string"} + variable_schema["default"] = default_value.split(",") + else: + variable_schema["type"] = "integer" + variable_schema["default"] = int(default_value) + schema_variables += [[variable["name"], variable_schema]] + return schema_variables + + def add_in_config_schema_file(variables): with open(CONFIG_JSON_SCHEMA, "r", encoding="utf-8") as json_file: json_schema = json.load(json_file) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57a0f68a5a0..fe39304dc72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,11 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l - MegaLinter does not crash anymore with **`This module only works with the 'fork' start method`** right after `Processing linters on [N] parallel cores`, which made every v10.0.0 run fail unless `PARALLEL: false` was set ([#8808](https://github.com/oxsecurity/megalinter/issues/8808)) - The **`PARALLEL: false`** workaround is not needed anymore, on the main image as well as on custom flavors - Messages logged by linters **running in parallel** are back in the console and in `megalinter.log`, including the extra output of `LOG_LEVEL: DEBUG` + - **REPOSITORY_TRIVY** and **REPOSITORY_TRIVY_SBOM** do not end the run with `--skip-db-update cannot be specified on the first run` anymore when a registry rate-limits the download of the vulnerability database ([#8807](https://github.com/oxsecurity/megalinter/issues/8807)) + - trivy is now pointed at **all official database mirrors** (`mirror.gcr.io`, `ghcr.io` and `public.ecr.aws`) and uses the first one that answers + - Download retries are **spaced with increasing waits** (10s, 20s, 40s, 60s), so they no longer all land within the same rate limit minute + - The final attempt against an already downloaded database now runs only when there is one, and an explicit message tells you what to do when there is not + - New variables to tune it: `REPOSITORY_TRIVY_DB_REPOSITORIES`, `REPOSITORY_TRIVY_JAVA_DB_REPOSITORIES`, `REPOSITORY_TRIVY_DB_RETRY_ATTEMPTS`, `REPOSITORY_TRIVY_DB_RETRY_INITIAL_DELAY`, `REPOSITORY_TRIVY_DB_RETRY_MAX_DELAY`, and their `REPOSITORY_TRIVY_SBOM_` counterparts - Fixed random **`Segmentation fault`** crashes of MegaLinter itself, which stopped the whole run with no error message ([#8733](https://github.com/oxsecurity/megalinter/issues/8733)). MegaLinter threads now get a full-size stack instead of the 128 KiB default of the Alpine images - Fixed leaked **`git` processes** when **APPLY_FIXES** is active: one was left behind by every fixer linter, which could exhaust the available file descriptors on long runs - Fixed **random crashes of project-mode linters** (`REPOSITORY_TRIVY`, `REPOSITORY_GRYPE`, `REPOSITORY_SYFT`…) caused by MegaLinter writing temporary ignore files inside the analyzed sources: a file appearing then disappearing while another linter walked the repository aborted its scan (`walk dir error: ... no such file or directory`). **MegaLinter now writes only in REPORT_OUTPUT_FOLDER**, never in your sources diff --git a/megalinter/descriptors/repository.megalinter-descriptor.yml b/megalinter/descriptors/repository.megalinter-descriptor.yml index d61051936d5..b1986989766 100644 --- a/megalinter/descriptors/repository.megalinter-descriptor.yml +++ b/megalinter/descriptors/repository.megalinter-descriptor.yml @@ -946,6 +946,14 @@ linters: **Note**: You can ignore specific findings by defining a [.trivyignore file](https://aquasecurity.github.io/trivy/latest/docs/configuration/filtering/#by-finding-ids) at your repository root. + **Vulnerability database download**: the registries hosting the trivy database apply rate limits, and a busy CI can be throttled with a `TOOMANYREQUESTS` error. MegaLinter mitigates it out of the box: + + - trivy is pointed at all official database mirrors (`mirror.gcr.io`, `ghcr.io` and `public.ecr.aws`), and uses the first one that answers + - a download failure is retried with increasing waits, so the retries span more than one rate limit window + - if every attempt fails, trivy runs against the database downloaded by a previous run or shipped within the MegaLinter docker image, when there is one + + You can tune this behaviour with `REPOSITORY_TRIVY_DB_REPOSITORIES`, `REPOSITORY_TRIVY_JAVA_DB_REPOSITORIES`, `REPOSITORY_TRIVY_DB_RETRY_ATTEMPTS`, `REPOSITORY_TRIVY_DB_RETRY_INITIAL_DELAY` and `REPOSITORY_TRIVY_DB_RETRY_MAX_DELAY`. Setting `TRIVY_DB_REPOSITORY` or `TRIVY_JAVA_DB_REPOSITORY`, or defining `db.repository` in your trivy configuration file, disables the MegaLinter mirrors and uses yours instead. Caching the trivy database folder (`TRIVY_CACHE_DIR`) between CI runs avoids the download altogether. + **Tip**: if you see errors related to files that do not exist, you can bypass them using "--skip-dirs" Example: @@ -991,28 +999,38 @@ linters: - identifier: REPOSITORY_TRIVY_ERROR_TOOMANYREQUESTS regex: "TOOMANYREQUESTS" message: |- - trivy was rate-limited by GitHub Container Registry (ghcr.io) while downloading the vulnerability database. + trivy was rate-limited by a container registry while downloading its vulnerability database. This is a registry rate limit, not a vulnerability finding. + MegaLinter already tries every official mirror (mirror.gcr.io, ghcr.io, public.ecr.aws) and retries with increasing waits before giving up. Workarounds: - - Retry the run later (the limit is per-namespace and resets quickly). - - Authenticate pulls so requests count against your own quota. MegaLinter strips token/password env vars by default, so whitelist them for this linter in your .mega-linter.yml: - REPOSITORY_TRIVY_UNSECURED_ENV_VARIABLES: - - GITHUB_TOKEN - - TRIVY_USERNAME - - TRIVY_PASSWORD - - Mirror the trivy-db to your own registry and point trivy at it via `TRIVY_DB_REPOSITORY`. - - Cache `~/.cache/trivy` across CI runs. + - Persist the trivy cache folder between CI runs and point `TRIVY_CACHE_DIR` at it: no download, no rate limit. + - Give the retries more time in your .mega-linter.yml: + REPOSITORY_TRIVY_DB_RETRY_ATTEMPTS: 8 + REPOSITORY_TRIVY_DB_RETRY_MAX_DELAY: 120 + - Mirror the trivy database to a registry you control and list it first: + REPOSITORY_TRIVY_DB_REPOSITORIES: + - my.registry.example.com/trivy-db:2 + - mirror.gcr.io/aquasec/trivy-db:2 + - ghcr.io/aquasecurity/trivy-db:2 - Temporarily mark the linter as non-blocking by adding to your .mega-linter.yml: DISABLE_ERRORS_LINTERS: - REPOSITORY_TRIVY - identifier: REPOSITORY_TRIVY_ERROR_DB_DOWNLOAD_FAILED regex: "(failed to download (vulnerability )?(DB|database)|database download error|could not pull image)" message: |- - trivy could not download or refresh its vulnerability database from the configured registry. + trivy could not download or refresh its vulnerability database from any of the configured registries. Workarounds: - Retry the run; this is often transient. - - Set `TRIVY_DB_REPOSITORY` to an alternative mirror (e.g. an internal registry or AWS ECR Public Gallery). - - Pre-populate `~/.cache/trivy` in CI. + - Persist the trivy cache folder between CI runs and point `TRIVY_CACHE_DIR` at it. + - List a mirror you control first in `REPOSITORY_TRIVY_DB_REPOSITORIES` (the value replaces the default mirrors, so keep the official ones after yours). + - Check that your runner can reach mirror.gcr.io, ghcr.io and public.ecr.aws; a corporate proxy intercepting TLS makes every mirror fail with a certificate error. + - identifier: REPOSITORY_TRIVY_ERROR_FIRST_RUN_NO_DB + regex: "--skip-db-update cannot be specified on the first run" + message: |- + trivy was asked to skip the database update while no database has ever been downloaded, which it refuses to do. + Resolutions: + - Remove `--skip-db-update` from `REPOSITORY_TRIVY_ARGUMENTS` and from `db.skip-update` in your trivy configuration file, or run trivy once with network access to populate its cache. + - When running in an air-gapped environment, download the database elsewhere and mount it into the folder pointed at by `TRIVY_CACHE_DIR`. - identifier: REPOSITORY_TRIVY_ERROR_REGISTRY_UNAUTHORIZED regex: "(UNAUTHORIZED: authentication required|unexpected status (code )?401|status 401 Unauthorized)" message: |- @@ -1044,6 +1062,22 @@ linters: vscode: - name: VSCode Trivy url: https://marketplace.visualstudio.com/items?itemName=AquaSecurityOfficial.trivy-vulnerability-scanner + variables: + - name: REPOSITORY_TRIVY_DB_REPOSITORIES + description: OCI repositories where the vulnerability database is downloaded from, tried in order. Set to an empty value to let trivy use its own defaults + default_value: "mirror.gcr.io/aquasec/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2" + - name: REPOSITORY_TRIVY_JAVA_DB_REPOSITORIES + description: OCI repositories where the Java vulnerability database is downloaded from, tried in order. Set to an empty value to let trivy use its own defaults + default_value: "mirror.gcr.io/aquasec/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1" + - name: REPOSITORY_TRIVY_DB_RETRY_ATTEMPTS + description: Number of times trivy is run again when the vulnerability database can not be downloaded + default_value: "5" + - name: REPOSITORY_TRIVY_DB_RETRY_INITIAL_DELAY + description: Number of seconds waited before the first database download retry, doubled at each new attempt + default_value: "10" + - name: REPOSITORY_TRIVY_DB_RETRY_MAX_DELAY + description: Maximum number of seconds waited between two database download retries + default_value: "60" # TRIVY SBOM - class: TrivySbomLinter @@ -1100,27 +1134,30 @@ linters: - identifier: REPOSITORY_TRIVY_SBOM_ERROR_TOOMANYREQUESTS regex: "TOOMANYREQUESTS" message: |- - trivy-sbom was rate-limited by GitHub Container Registry (ghcr.io) while downloading the trivy DB. + trivy-sbom was rate-limited by a container registry while downloading the trivy database. This is a registry rate limit, not an SBOM error. + MegaLinter already tries every official mirror (mirror.gcr.io, ghcr.io, public.ecr.aws) and retries with increasing waits before giving up. Workarounds: - - Retry the run later. - - Authenticate pulls. MegaLinter strips token/password env vars by default, so whitelist them for this linter in your .mega-linter.yml: - REPOSITORY_TRIVY_SBOM_UNSECURED_ENV_VARIABLES: - - GITHUB_TOKEN - - TRIVY_USERNAME - - TRIVY_PASSWORD - - Mirror trivy-db to your own registry via `TRIVY_DB_REPOSITORY`. + - Persist the trivy cache folder between CI runs and point `TRIVY_CACHE_DIR` at it: no download, no rate limit. + - Give the retries more time in your .mega-linter.yml: + REPOSITORY_TRIVY_SBOM_DB_RETRY_ATTEMPTS: 8 + REPOSITORY_TRIVY_SBOM_DB_RETRY_MAX_DELAY: 120 + - Mirror the trivy database to a registry you control and list it first: + REPOSITORY_TRIVY_SBOM_DB_REPOSITORIES: + - my.registry.example.com/trivy-db:2 + - mirror.gcr.io/aquasec/trivy-db:2 + - ghcr.io/aquasecurity/trivy-db:2 - Temporarily mark the linter as non-blocking by adding to your .mega-linter.yml: DISABLE_ERRORS_LINTERS: - REPOSITORY_TRIVY_SBOM - identifier: REPOSITORY_TRIVY_SBOM_ERROR_DB_DOWNLOAD_FAILED regex: "(failed to download (vulnerability )?(DB|database)|database download error)" message: |- - trivy-sbom could not download or refresh its vulnerability database. + trivy-sbom could not download or refresh its vulnerability database from any of the configured registries. Workarounds: - Retry the run; this is often transient. - - Set `TRIVY_DB_REPOSITORY` to an alternative mirror. - - Pre-populate `~/.cache/trivy` in CI. + - Persist the trivy cache folder between CI runs and point `TRIVY_CACHE_DIR` at it. + - List a mirror you control first in `REPOSITORY_TRIVY_SBOM_DB_REPOSITORIES` (the value replaces the default mirrors, so keep the official ones after yours). test_folder: trivy examples: - "trivy fs --format cyclonedx ." @@ -1141,6 +1178,22 @@ linters: vscode: - name: VSCode Trivy url: https://marketplace.visualstudio.com/items?itemName=AquaSecurityOfficial.trivy-vulnerability-scanner + variables: + - name: REPOSITORY_TRIVY_SBOM_DB_REPOSITORIES + description: OCI repositories where the vulnerability database is downloaded from, tried in order. Set to an empty value to let trivy use its own defaults + default_value: "mirror.gcr.io/aquasec/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2" + - name: REPOSITORY_TRIVY_SBOM_JAVA_DB_REPOSITORIES + description: OCI repositories where the Java vulnerability database is downloaded from, tried in order. Set to an empty value to let trivy use its own defaults + default_value: "mirror.gcr.io/aquasec/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1" + - name: REPOSITORY_TRIVY_SBOM_DB_RETRY_ATTEMPTS + description: Number of times trivy is run again when the vulnerability database can not be downloaded + default_value: "5" + - name: REPOSITORY_TRIVY_SBOM_DB_RETRY_INITIAL_DELAY + description: Number of seconds waited before the first database download retry, doubled at each new attempt + default_value: "10" + - name: REPOSITORY_TRIVY_SBOM_DB_RETRY_MAX_DELAY + description: Maximum number of seconds waited between two database download retries + default_value: "60" # TRUFFLEHOG - class: TruffleHogLinter diff --git a/megalinter/linters/TrivyLinter.py b/megalinter/linters/TrivyLinter.py index daf658f22bc..792fe5395a8 100644 --- a/megalinter/linters/TrivyLinter.py +++ b/megalinter/linters/TrivyLinter.py @@ -4,44 +4,239 @@ """ import logging +import os import time +import yaml from megalinter import Linter, config +# Errors returned by the registries hosting the trivy databases when they are +# rate-limiting us or temporarily unavailable. They are transient: retrying, +# or trying another mirror, can fix them +DB_DOWNLOAD_ERRORS = [ + "TOOMANYREQUESTS", + "failed to download Java DB", + "BLOB_UNKNOWN", + "failed to download vulnerability DB", +] + +# Only an aborted run is worth retrying: a registry error while fetching the +# optional misconfiguration checks bundle just makes trivy fall back to its +# embedded checks, and the scan results are complete +FATAL_ERROR_MARKER = "FATAL" + +# Trivy tries these repositories in order and stops at the first one answering. +# Its own defaults are mirror.gcr.io + ghcr.io: public.ecr.aws is the third +# official mirror, added here because ghcr.io rate-limits per organization and +# a flag value REPLACES the trivy defaults instead of extending them +DEFAULT_DB_REPOSITORIES = [ + "mirror.gcr.io/aquasec/trivy-db:2", + "ghcr.io/aquasecurity/trivy-db:2", + "public.ecr.aws/aquasecurity/trivy-db:2", +] +DEFAULT_JAVA_DB_REPOSITORIES = [ + "mirror.gcr.io/aquasec/trivy-java-db:1", + "ghcr.io/aquasecurity/trivy-java-db:1", + "public.ecr.aws/aquasecurity/trivy-java-db:1", +] + +DB_REPOSITORY_ARGS = [ + ( + "--db-repository", + "DB_REPOSITORIES", + "TRIVY_DB_REPOSITORY", + "db.repository", + DEFAULT_DB_REPOSITORIES, + ), + ( + "--java-db-repository", + "JAVA_DB_REPOSITORIES", + "TRIVY_JAVA_DB_REPOSITORY", + "db.java-repository", + DEFAULT_JAVA_DB_REPOSITORIES, + ), +] + +# Trivy refuses --skip-db-update when either of these files is missing from +# its cache directory: "the first run cannot skip downloading DB" +DB_REQUIRED_FILES = [ + os.path.join("db", "trivy.db"), + os.path.join("db", "metadata.json"), +] + +# Cache directory used to download the database while building the MegaLinter +# docker image (HOME is /root at build time). The runtime HOME can be +# different (GitHub Actions forces HOME=/github/home in container actions), +# which makes trivy ignore the database shipped within the image +IMAGE_CACHE_DIR = "/root/.cache/trivy" + +DEFAULT_RETRY_ATTEMPTS = 5 +DEFAULT_RETRY_INITIAL_DELAY = 10 +DEFAULT_RETRY_MAX_DELAY = 60 + class TrivyLinter(Linter): - def execute_lint_command(self, command): - max_retries = 5 - for attempt in range(max_retries): - return_code, return_output = super().execute_lint_command(command) - if not ( - ("TOOMANYREQUESTS" in return_output) - or ("failed to download Java DB" in return_output) - or ("BLOB_UNKNOWN" in return_output) + def build_lint_command(self, file=None) -> list: + cmd = super().build_lint_command(file) + return self.add_db_repository_arguments(cmd) + + # Send trivy to all known database mirrors, unless the user configured + # their own repositories through arguments, environment or config file + def add_db_repository_arguments(self, cmd): + for ( + arg_name, + config_key, + trivy_env_var, + config_file_key, + default_repositories, + ) in DB_REPOSITORY_ARGS: + if arg_name in cmd: + continue + if config.get(self.request_id, trivy_env_var, "") != "": + continue + if self.get_config_file_value(config_file_key) is not None: + continue + repositories = config.get_list( + self.request_id, f"{self.name}_{config_key}", default_repositories + ) + if len(repositories) == 0: + continue + cmd += [arg_name, ",".join(repositories)] + return cmd + + # Value of a dotted key path in the trivy configuration file, or None + def get_config_file_value(self, config_file_key): + if self.final_config_file is None or not os.path.isfile(self.final_config_file): + return None + with open(self.final_config_file, encoding="utf-8") as config_file: + node = yaml.safe_load(config_file) or {} + for key in config_file_key.split("."): + if not isinstance(node, dict) or key not in node: + return None + node = node[key] + return node + + # Cache directories where a previously downloaded database can be found, + # the first one being the directory trivy uses for this run + def get_trivy_cache_dirs(self, command): + candidates = [] + if isinstance(command, list) and "--cache-dir" in command: + candidates.append(command[command.index("--cache-dir") + 1]) + candidates.append(self.get_config_file_value("cache.dir")) + candidates.append(config.get(self.request_id, "TRIVY_CACHE_DIR", "")) + xdg_cache_home = config.get(self.request_id, "XDG_CACHE_HOME", "") + if xdg_cache_home != "": + candidates.append(os.path.join(xdg_cache_home, "trivy")) + home_dir = config.get(self.request_id, "HOME", "") + if home_dir != "": + candidates.append(os.path.join(home_dir, ".cache", "trivy")) + # Database shipped within the MegaLinter docker image + candidates.append(IMAGE_CACHE_DIR) + cache_dirs = [] + for candidate in candidates: + if candidate and candidate not in cache_dirs: + cache_dirs.append(candidate) + return cache_dirs + + # Cache directory containing a database usable with --skip-db-update, or None + def find_cached_db_dir(self, command): + for cache_dir in self.get_trivy_cache_dirs(command): + if all( + os.path.isfile(os.path.join(cache_dir, db_file)) + for db_file in DB_REQUIRED_FILES ): - return return_code, return_output - if attempt < max_retries - 1: - time.sleep(3.0) - logging.info( - f"[Trivy] Hit TOOMANYREQUESTS: try again (attempt {attempt + 2}/{max_retries})" + return cache_dir + return None + + # Registry rate limits are counted per minute: space the retries so they + # do not all land within the same rate limit window + def get_retry_delay(self, attempt): + initial_delay = float( + config.get( + self.request_id, + f"{self.name}_DB_RETRY_INITIAL_DELAY", + DEFAULT_RETRY_INITIAL_DELAY, + ) + ) + max_delay = float( + config.get( + self.request_id, + f"{self.name}_DB_RETRY_MAX_DELAY", + DEFAULT_RETRY_MAX_DELAY, + ) + ) + return min(initial_delay * (2**attempt), max_delay) + + # Command running trivy against an already downloaded database + def build_offline_command(self, command, cached_db_dir): + offline_args = ["--skip-db-update", "--skip-check-update"] + if cached_db_dir != self.get_trivy_cache_dirs(command)[0]: + offline_args += ["--cache-dir", cached_db_dir] + if isinstance(command, str): + return command + " " + " ".join(offline_args) + return command + offline_args + + def is_db_download_error(self, return_output): + return_output = return_output or "" + if FATAL_ERROR_MARKER not in return_output: + return False + return any(error in return_output for error in DB_DOWNLOAD_ERRORS) + + # Run trivy without letting the base class report the common linter errors: + # their resolution guidance is only relevant once every attempt is over + def execute_trivy_command(self, command): + common_linter_errors = self.common_linter_errors + self.common_linter_errors = [] + try: + return super().execute_lint_command(command) + finally: + self.common_linter_errors = common_linter_errors + + def execute_lint_command(self, command): + max_retries = int( + config.get( + self.request_id, + f"{self.name}_DB_RETRY_ATTEMPTS", + DEFAULT_RETRY_ATTEMPTS, + ) + ) + return_code, return_output = self.execute_trivy_command(command) + attempt = 0 + while attempt < max_retries - 1 and self.is_db_download_error(return_output): + delay = self.get_retry_delay(attempt) + logging.info( + f"[{self.linter_name}] Vulnerability database download failed " + "(registry rate limit or outage): waiting " + f"{delay:.0f}s before attempt {attempt + 2}/{max_retries}" + ) + time.sleep(delay) + return_code, return_output = self.execute_trivy_command(command) + attempt += 1 + if self.is_db_download_error(return_output): + # Last chance: run against a database downloaded by a previous run + # or shipped within the MegaLinter docker image + cached_db_dir = self.find_cached_db_dir(command) + if cached_db_dir is None: + logging.error( + f"[{self.linter_name}] Unable to download the vulnerability " + f"database after {max_retries} attempts, and no previously " + "downloaded database is available to fall back on: trivy can not " + f"run offline on its first run. Raise {self.name}_DB_RETRY_ATTEMPTS," + f" set {self.name}_DB_REPOSITORIES to a mirror you control, or " + "persist the trivy cache directory (TRIVY_CACHE_DIR) between your " + "CI runs." ) else: logging.warning( - "[Trivy] Hit TOOMANYREQUESTS 5 times: Run trivy " - + "with --skip-db-update and --skip-check-update" + f"[{self.linter_name}] Unable to download the vulnerability " + f"database after {max_retries} attempts: running against the " + f"database cached in {cached_db_dir}, whose content may be outdated" + ) + return_code, return_output = self.execute_trivy_command( + self.build_offline_command(command, cached_db_dir) ) - if isinstance(command, str): - command_without_db = ( - command + " --skip-db-update --skip-check-update" - ) - else: - command_without_db = command + [ - "--skip-db-update", - "--skip-check-update", - ] - return super().execute_lint_command(command_without_db) - return return_code, return_output + return self.apply_common_linter_errors(return_code, return_output) def pre_test(self, test_name): if test_name.endswith(("file_lint_mode", "list_of_files_lint_mode")): diff --git a/megalinter/linters/TrivySbomLinter.py b/megalinter/linters/TrivySbomLinter.py index 0a53ce76753..bbd452cb9d4 100644 --- a/megalinter/linters/TrivySbomLinter.py +++ b/megalinter/linters/TrivySbomLinter.py @@ -6,15 +6,18 @@ import json import os -from megalinter import Linter, config, utils +from megalinter import config, utils from megalinter.constants import ( DEFAULT_SARIF_SCHEMA_URI, DEFAULT_SARIF_VERSION, ML_DOC_URL_DESCRIPTORS_ROOT, ) +from megalinter.linters.TrivyLinter import TrivyLinter -class TrivySbomLinter(Linter): +# Inherits from TrivyLinter to get the vulnerability database mirrors and +# download retries: trivy-sbom queries the very same registries +class TrivySbomLinter(TrivyLinter): # Provide additional details in text reporter logs # Add SBOM output file # noinspection PyMethodMayBeStatic diff --git a/megalinter/tests/test_megalinter/trivy_linter_test.py b/megalinter/tests/test_megalinter/trivy_linter_test.py new file mode 100644 index 00000000000..57819361acf --- /dev/null +++ b/megalinter/tests/test_megalinter/trivy_linter_test.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +""" +Unit tests for TrivyLinter vulnerability database download resilience: +mirror repositories, spaced retries, and offline fallback only when a +database has already been downloaded. +""" + +import os +import tempfile +import unittest +import uuid +from unittest import mock + +from megalinter import Linter, config +from megalinter.linters.TrivyLinter import ( + DEFAULT_DB_REPOSITORIES, + DEFAULT_JAVA_DB_REPOSITORIES, + DEFAULT_RETRY_ATTEMPTS, + TrivyLinter, +) + +RATE_LIMITED_OUTPUT = ( + "FATAL Fatal error init error: DB error: failed to download vulnerability DB: " + "OCI repository error: 1 error occurred: failed to fetch the layer: " + "GET https://ghcr.io/v2/aquasecurity/trivy-db/manifests/2: TOOMANYREQUESTS" +) +SUCCESS_OUTPUT = "Total: 0 (UNKNOWN: 0, LOW: 0, MEDIUM: 0, HIGH: 0, CRITICAL: 0)" +# The misconfiguration checks bundle is optional: trivy logs the registry error +# then falls back to its embedded checks and completes the scan +CHECKS_BUNDLE_ERROR_OUTPUT = ( + "ERROR [misconf] Falling back to embedded checks err=failed to download " + "checks bundle: TOOMANYREQUESTS\n" + "Total: 1 (UNKNOWN: 0, LOW: 1, MEDIUM: 0, HIGH: 0, CRITICAL: 0)" +) + + +def make_linter(request_id, name="REPOSITORY_TRIVY", config_file=None): + linter = TrivyLinter.__new__(TrivyLinter) + linter.request_id = request_id + linter.name = name + linter.linter_name = "trivy" + linter.final_config_file = config_file + linter.common_linter_errors = [] + return linter + + +def cache_dir_with_db(root): + db_dir = os.path.join(root, "db") + os.makedirs(db_dir, exist_ok=True) + for db_file in ["trivy.db", "metadata.json"]: + with open(os.path.join(db_dir, db_file), "w", encoding="utf-8") as file_handler: + file_handler.write("{}") + return root + + +class TrivyLinterDbRepositoriesTest(unittest.TestCase): + def setUp(self): + self.request_id = str(uuid.uuid1()) + config.set_config(self.request_id, {}) + + def tearDown(self): + config.delete(self.request_id) + + def test_mirrors_added_by_default(self): + linter = make_linter(self.request_id) + cmd = linter.add_db_repository_arguments(["trivy", "fs", "."]) + self.assertEqual( + cmd[cmd.index("--db-repository") + 1], ",".join(DEFAULT_DB_REPOSITORIES) + ) + self.assertEqual( + cmd[cmd.index("--java-db-repository") + 1], + ",".join(DEFAULT_JAVA_DB_REPOSITORIES), + ) + # Setting the flag replaces the trivy defaults: they must be kept + self.assertIn("ghcr.io/aquasecurity/trivy-db:2", DEFAULT_DB_REPOSITORIES) + self.assertIn("mirror.gcr.io/aquasec/trivy-db:2", DEFAULT_DB_REPOSITORIES) + + def test_mirrors_not_added_when_user_defined_in_arguments(self): + linter = make_linter(self.request_id) + cmd = linter.add_db_repository_arguments( + ["trivy", "fs", "--db-repository", "my.registry/trivy-db:2", "."] + ) + self.assertEqual(cmd.count("--db-repository"), 1) + self.assertEqual( + cmd[cmd.index("--db-repository") + 1], "my.registry/trivy-db:2" + ) + + def test_mirrors_not_added_when_trivy_env_var_is_set(self): + config.set_value( + self.request_id, "TRIVY_DB_REPOSITORY", "my.registry/trivy-db:2" + ) + linter = make_linter(self.request_id) + cmd = linter.add_db_repository_arguments(["trivy", "fs", "."]) + self.assertNotIn("--db-repository", cmd) + # The java database repository is still defaulted to the mirrors + self.assertIn("--java-db-repository", cmd) + + def test_mirrors_not_added_when_defined_in_trivy_config_file(self): + with tempfile.TemporaryDirectory() as workspace: + config_file = os.path.join(workspace, "trivy.yaml") + with open(config_file, "w", encoding="utf-8") as file_handler: + file_handler.write("db:\n repository:\n - my.registry/trivy-db:2\n") + linter = make_linter(self.request_id, config_file=config_file) + cmd = linter.add_db_repository_arguments(["trivy", "fs", "."]) + self.assertNotIn("--db-repository", cmd) + self.assertIn("--java-db-repository", cmd) + + def test_mirrors_can_be_disabled(self): + config.set_value(self.request_id, "REPOSITORY_TRIVY_DB_REPOSITORIES", "") + config.set_value(self.request_id, "REPOSITORY_TRIVY_JAVA_DB_REPOSITORIES", "") + linter = make_linter(self.request_id) + cmd = linter.add_db_repository_arguments(["trivy", "fs", "."]) + self.assertEqual(cmd, ["trivy", "fs", "."]) + + def test_mirrors_can_be_overridden(self): + config.set_value( + self.request_id, + "REPOSITORY_TRIVY_DB_REPOSITORIES", + ["registry1/trivy-db:2", "registry2/trivy-db:2"], + ) + linter = make_linter(self.request_id) + cmd = linter.add_db_repository_arguments(["trivy", "fs", "."]) + self.assertEqual( + cmd[cmd.index("--db-repository") + 1], + "registry1/trivy-db:2,registry2/trivy-db:2", + ) + + def test_sbom_linter_uses_its_own_configuration_keys(self): + config.set_value( + self.request_id, "REPOSITORY_TRIVY_SBOM_DB_REPOSITORIES", "sbom/trivy-db:2" + ) + linter = make_linter(self.request_id, name="REPOSITORY_TRIVY_SBOM") + cmd = linter.add_db_repository_arguments(["trivy", "fs", "."]) + self.assertEqual(cmd[cmd.index("--db-repository") + 1], "sbom/trivy-db:2") + + +class TrivyLinterCacheTest(unittest.TestCase): + def setUp(self): + self.request_id = str(uuid.uuid1()) + config.set_config(self.request_id, {}) + + def tearDown(self): + config.delete(self.request_id) + + def test_cached_db_found_in_trivy_cache_dir(self): + with tempfile.TemporaryDirectory() as cache_dir: + cache_dir_with_db(cache_dir) + config.set_value(self.request_id, "TRIVY_CACHE_DIR", cache_dir) + linter = make_linter(self.request_id) + self.assertEqual(linter.find_cached_db_dir(["trivy", "fs", "."]), cache_dir) + + def test_cached_db_found_from_cache_dir_argument(self): + with tempfile.TemporaryDirectory() as cache_dir: + cache_dir_with_db(cache_dir) + linter = make_linter(self.request_id) + command = ["trivy", "fs", "--cache-dir", cache_dir, "."] + self.assertEqual(linter.find_cached_db_dir(command), cache_dir) + + def test_cached_db_found_in_home_cache_dir(self): + with tempfile.TemporaryDirectory() as home_dir: + cache_dir_with_db(os.path.join(home_dir, ".cache", "trivy")) + config.set_value(self.request_id, "HOME", home_dir) + linter = make_linter(self.request_id) + self.assertEqual( + linter.find_cached_db_dir(["trivy", "fs", "."]), + os.path.join(home_dir, ".cache", "trivy"), + ) + + def test_no_cached_db_when_metadata_is_missing(self): + with tempfile.TemporaryDirectory() as cache_dir: + cache_dir_with_db(cache_dir) + os.remove(os.path.join(cache_dir, "db", "metadata.json")) + config.set_value(self.request_id, "TRIVY_CACHE_DIR", cache_dir) + config.set_value(self.request_id, "HOME", cache_dir) + config.set_value(self.request_id, "XDG_CACHE_HOME", cache_dir) + linter = make_linter(self.request_id) + # /root/.cache/trivy is the last candidate and does not exist here + self.assertIsNone(linter.find_cached_db_dir(["trivy", "fs", "."])) + + def test_offline_command_adds_skip_arguments(self): + with tempfile.TemporaryDirectory() as cache_dir: + cache_dir_with_db(cache_dir) + config.set_value(self.request_id, "TRIVY_CACHE_DIR", cache_dir) + linter = make_linter(self.request_id) + cmd = linter.build_offline_command(["trivy", "fs", "."], cache_dir) + self.assertEqual( + cmd, ["trivy", "fs", ".", "--skip-db-update", "--skip-check-update"] + ) + + def test_offline_command_points_to_the_cache_dir_of_the_found_database(self): + with tempfile.TemporaryDirectory() as workspace: + image_cache_dir = cache_dir_with_db(os.path.join(workspace, "image-cache")) + config.set_value( + self.request_id, "TRIVY_CACHE_DIR", os.path.join(workspace, "empty") + ) + linter = make_linter(self.request_id) + cmd = linter.build_offline_command(["trivy", "fs", "."], image_cache_dir) + self.assertEqual(cmd[-2:], ["--cache-dir", image_cache_dir]) + + +class TrivyLinterRetryTest(unittest.TestCase): + def setUp(self): + self.request_id = str(uuid.uuid1()) + config.set_config(self.request_id, {}) + + def tearDown(self): + config.delete(self.request_id) + + # Replace Linter.execute_lint_command, which spawns a real process, by a + # stub returning the given outputs, and neutralize the retry waits + def run_lint(self, outputs, command=None, common_linter_errors=None): + if command is None: + command = ["trivy", "fs", "."] + results = list(outputs) + executed_commands = [] + + def fake_execute(_self, cmd): + executed_commands.append(cmd) + return 1, results.pop(0) if len(results) > 1 else results[0] + + linter = make_linter(self.request_id) + linter.common_linter_errors = common_linter_errors or [] + with ( + mock.patch.object(Linter, "execute_lint_command", fake_execute), + mock.patch("megalinter.linters.TrivyLinter.time.sleep") as sleep_mock, + ): + return_code, return_output = linter.execute_lint_command(command) + delays = [call.args[0] for call in sleep_mock.call_args_list] + return executed_commands, delays, return_code, return_output + + def test_no_retry_and_no_wait_when_command_succeeds(self): + executed_commands, delays, _, _ = self.run_lint([SUCCESS_OUTPUT]) + self.assertEqual(len(executed_commands), 1) + self.assertEqual(delays, []) + + def test_no_retry_on_a_regular_vulnerability_finding(self): + executed_commands, delays, _, _ = self.run_lint( + ["Total: 3 (UNKNOWN: 0, LOW: 1, MEDIUM: 2, HIGH: 0, CRITICAL: 0)"] + ) + self.assertEqual(len(executed_commands), 1) + self.assertEqual(delays, []) + + def test_no_retry_when_only_the_checks_bundle_download_failed(self): + # The scan completed: retrying would waste minutes for nothing + executed_commands, delays, _, _ = self.run_lint([CHECKS_BUNDLE_ERROR_OUTPUT]) + self.assertEqual(len(executed_commands), 1) + self.assertEqual(delays, []) + + def test_retries_then_succeeds(self): + executed_commands, delays, _, return_output = self.run_lint( + [RATE_LIMITED_OUTPUT, SUCCESS_OUTPUT] + ) + self.assertEqual(len(executed_commands), 2) + self.assertEqual(delays, [10.0]) + self.assertNotIn("TOOMANYREQUESTS", return_output) + + def test_retries_are_spaced_over_more_than_one_rate_limit_window(self): + with tempfile.TemporaryDirectory() as empty_dir: + config.set_value(self.request_id, "TRIVY_CACHE_DIR", empty_dir) + config.set_value(self.request_id, "HOME", empty_dir) + _, delays, _, _ = self.run_lint([RATE_LIMITED_OUTPUT]) + self.assertEqual(delays, [10.0, 20.0, 40.0, 60.0]) + self.assertGreater(sum(delays), 60) + + def test_retry_delays_are_configurable(self): + config.set_value(self.request_id, "REPOSITORY_TRIVY_DB_RETRY_ATTEMPTS", "3") + config.set_value( + self.request_id, "REPOSITORY_TRIVY_DB_RETRY_INITIAL_DELAY", "2" + ) + config.set_value(self.request_id, "REPOSITORY_TRIVY_DB_RETRY_MAX_DELAY", "3") + with tempfile.TemporaryDirectory() as empty_dir: + config.set_value(self.request_id, "TRIVY_CACHE_DIR", empty_dir) + config.set_value(self.request_id, "HOME", empty_dir) + executed_commands, delays, _, _ = self.run_lint([RATE_LIMITED_OUTPUT]) + self.assertEqual(len(executed_commands), 3) + self.assertEqual(delays, [2.0, 3.0]) + + def test_offline_fallback_when_a_database_is_cached(self): + with tempfile.TemporaryDirectory() as cache_dir: + cache_dir_with_db(cache_dir) + config.set_value(self.request_id, "TRIVY_CACHE_DIR", cache_dir) + executed_commands, _, _, _ = self.run_lint([RATE_LIMITED_OUTPUT]) + self.assertEqual(len(executed_commands), DEFAULT_RETRY_ATTEMPTS + 1) + self.assertIn("--skip-db-update", executed_commands[-1]) + self.assertIn("--skip-check-update", executed_commands[-1]) + + def test_resolution_guidance_is_reported_only_once(self): + common_linter_errors = [ + { + "identifier": "REPOSITORY_TRIVY_ERROR_TOOMANYREQUESTS", + "regex": "TOOMANYREQUESTS", + "message": "Rate limited by the registry", + } + ] + with tempfile.TemporaryDirectory() as empty_dir: + config.set_value(self.request_id, "TRIVY_CACHE_DIR", empty_dir) + config.set_value(self.request_id, "HOME", empty_dir) + _, _, _, return_output = self.run_lint( + [RATE_LIMITED_OUTPUT], common_linter_errors=common_linter_errors + ) + self.assertEqual(return_output.count("Rate limited by the registry"), 1) + + def test_no_offline_fallback_on_a_first_run(self): + with tempfile.TemporaryDirectory() as empty_dir: + config.set_value(self.request_id, "TRIVY_CACHE_DIR", empty_dir) + config.set_value(self.request_id, "HOME", empty_dir) + config.set_value(self.request_id, "XDG_CACHE_HOME", empty_dir) + executed_commands, _, _, return_output = self.run_lint( + [RATE_LIMITED_OUTPUT] + ) + # An extra --skip-db-update run would be fatal without a database + self.assertEqual(len(executed_commands), DEFAULT_RETRY_ATTEMPTS) + for command in executed_commands: + self.assertNotIn("--skip-db-update", command) + self.assertIn("TOOMANYREQUESTS", return_output) + + +if __name__ == "__main__": + unittest.main()