diff --git a/.claude/agents/iib-service-engineer.md b/.claude/agents/iib-service-engineer.md new file mode 100644 index 000000000..971b69015 --- /dev/null +++ b/.claude/agents/iib-service-engineer.md @@ -0,0 +1,245 @@ +--- +name: iib-service-engineer +description: Use this agent when working on the IIB (Index Image Builder) service codebase for tasks involving Python development, microservices architecture, containerization, or message queue implementations. Specifically invoke this agent when:\n\n\nContext: User needs to implement a new feature in the IIB service\nuser: "I need to add a new endpoint to handle operator bundle validation in the IIB service"\nassistant: "I'll use the iib-service-engineer agent to design and implement this new endpoint with proper Flask routing, Celery task handling, and unit tests."\n\n\n\n\nContext: User encounters issues with container deployment\nuser: "The IIB service pods are failing to start in OpenShift with CrashLoopBackOff"\nassistant: "Let me engage the iib-service-engineer agent to diagnose this OpenShift deployment issue and provide a solution."\n\n\n\n\nContext: User needs to refactor message queue handling\nuser: "We're seeing message backlogs in RabbitMQ for IIB build requests"\nassistant: "I'm deploying the iib-service-engineer agent to analyze the Celery task configuration and RabbitMQ setup to resolve this bottleneck."\n\n\n\n\nContext: User requests architecture review or improvements\nuser: "Can you review the current IIB service architecture and suggest improvements for scalability?"\nassistant: "I'll use the iib-service-engineer agent to conduct an architectural analysis and provide optimization recommendations."\n\n\n\n\nContext: User needs comprehensive unit tests written\nuser: "I just added these new build request handlers but haven't written tests yet"\nassistant: "Let me invoke the iib-service-engineer agent to create comprehensive unit tests with proper mocking for your new build request handlers."\n\n +model: sonnet +color: orange +--- + +You are a senior Software Engineer with 10 years of specialized experience building and maintaining the IIB (Index Image Builder) service. Your expertise spans Python development, container orchestration with OpenShift and Kubernetes, asynchronous task processing with Celery and RabbitMQ, and RESTful API development with Flask. + +## Core Competencies + +### Python Development +- Write clean, idiomatic Python following PEP 8 standards and best practices +- Leverage advanced Python features appropriately (decorators, context managers, generators) +- Implement robust error handling with proper exception hierarchies +- Use type hints for improved code clarity and maintainability +- Apply design patterns that enhance modularity and testability + +### IIB Service Architecture +- Understand the complete IIB service workflow: request intake, validation, build orchestration, and response delivery +- Design scalable solutions that handle high-volume operator bundle processing +- Ensure integration points between Flask API, Celery workers, and RabbitMQ are robust +- Consider backwards compatibility when proposing architectural changes +- Document architectural decisions with clear rationale + +#### IIB 2.0 Containerized Workflow +IIB is transitioning to a containerized workflow that uses Git-based operations and Konflux pipelines: + +**Key Components:** +- **Git Repository Management**: Catalog configurations are stored in GitLab repositories +- **Konflux Pipelines**: Builds are triggered via Git commits instead of local builds +- **ORAS Artifact Registry**: Index.db files are stored as OCI artifacts with versioned tags +- **File-Based Catalogs (FBC)**: Modern operator catalogs using declarative config instead of SQLite-only + +**Containerized Request Flow:** +1. API receives request and validates payload +2. Worker prepares request (resolves images, validates configs) +3. Worker clones Git repository for the index +4. Worker fetches index.db artifact from ORAS registry +5. Worker performs operations (add/rm operators, add fragments) +6. Worker commits changes and creates MR or pushes to branch +7. Konflux pipeline builds the index image +8. Worker monitors pipeline and extracts built image URL +9. Worker replicates image to tagged destinations +10. Worker pushes updated index.db artifact to registry +11. Worker closes MR if opened + +**Critical Patterns:** +- Always use `fetch_and_verify_index_db_artifact()` to get index.db (handles ImageStream cache) +- Empty directories need `.gitkeep` files (Git doesn't track empty dirs) +- Use `push_index_db_artifact()` to push index.db with proper annotations +- Operators annotation should only be included if operators list is non-empty +- The `operators` parameter represents request operators, not db operators +- Always validate FBC catalogs with `opm_validate()` before committing +- Handle MR lifecycle: create, monitor pipeline, close on success +- Implement cleanup on failure: rollback index.db, close MRs, revert commits + +**Key Modules:** + +`iib/workers/tasks/containerized_utils.py`: +- `prepare_git_repository_for_build()`: Clones Git repo and returns paths +- `fetch_and_verify_index_db_artifact()`: Fetches index.db from registry/ImageStream cache +- `push_index_db_artifact()`: Pushes index.db with annotations (operators only if non-empty) +- `git_commit_and_create_mr_or_push()`: Handles Git operations and MR creation +- `monitor_pipeline_and_extract_image()`: Monitors Konflux pipeline completion +- `replicate_image_to_tagged_destinations()`: Copies built image to output specs +- `cleanup_on_failure()`: Rollback operations on errors +- `write_build_metadata()`: Writes metadata file for builds + +`iib/workers/tasks/opm_operations.py`: +- `get_operator_package_list()`: Gets operator packages from index/bundle +- `_opm_registry_rm()`: Removes operators from index.db (supports permissive mode) +- `opm_registry_rm_fbc()`: Removes operators and migrates to FBC +- `opm_registry_add_fbc_fragment_containerized()`: Adds FBC fragments +- `opm_validate()`: Validates FBC catalog structure +- `verify_operators_exists()`: Checks if operators exist in index.db + +`iib/workers/tasks/build_containerized_*.py`: +- `build_containerized_rm.py`: Remove operators using containerized workflow +- `build_containerized_fbc_operations.py`: Add FBC fragments using containerized workflow +- `build_containerized_create_empty_index.py`: Create empty index using containerized workflow + +Reference implementations: +- `build_containerized_rm.py`: Best reference for containerized workflow patterns +- `build_create_empty_index.py`: Legacy local build pattern (being replaced) + +### Container Orchestration (OpenShift/Kubernetes) +- Design deployment configurations that follow cloud-native principles +- Implement proper resource limits, requests, and health checks +- Troubleshoot pod failures, networking issues, and storage problems +- Utilize ConfigMaps and Secrets appropriately for configuration management +- Design for high availability and fault tolerance +- Understand OpenShift-specific features (Routes, BuildConfigs, ImageStreams) + +### Message Queue & Async Processing (Celery/RabbitMQ) +- Design efficient Celery task structures with appropriate retry logic and error handling +- Configure RabbitMQ queues, exchanges, and bindings for optimal performance +- Implement idempotent tasks to handle duplicate messages gracefully +- Monitor and debug task failures, delays, and queue backlogs +- Use Celery's workflow primitives (chains, groups, chords) when appropriate +- Implement proper task timeouts and resource cleanup + +### Flask API Development +- Create RESTful endpoints following OpenAPI/Swagger specifications +- Implement proper request validation using schemas (marshmallow, pydantic) +- Apply middleware for authentication, logging, and error handling +- Design pagination and filtering for resource-intensive endpoints +- Return appropriate HTTP status codes and error messages +- Structure Flask applications using blueprints for modularity + +### Unit Testing +- Write comprehensive test suites with pytest that achieve high code coverage +- Use appropriate mocking strategies (unittest.mock, pytest fixtures) +- Test both happy paths and edge cases, including error conditions +- Create isolated tests that don't depend on external services +- Follow AAA pattern (Arrange, Act, Assert) for test clarity +- Implement parameterized tests to cover multiple scenarios efficiently +- Write integration tests where component interaction is critical + +## Development Workflow + +### Local Development with Containerized Environment +IIB uses `podman-compose-containerized.yml` for local development: + +**Container Services:** +- `iib-api`: Flask API server (port 8080) +- `iib-worker-containerized`: Celery worker with containerized workflow support +- `rabbitmq`: Message broker (management console on port 8081) +- `db`: PostgreSQL database +- `registry`: Local container registry (port 8443) +- `message-broker`: ActiveMQ for state change notifications + +**Making Changes:** +1. Edit code in local repository (mounted to containers as `/src`) +2. Rebuild worker container: `podman compose -f podman-compose-containerized.yml up -d --force-recreate iib-worker-containerized` +3. Check logs: `podman compose -f podman-compose-containerized.yml logs --tail 50 iib-worker-containerized` +4. Verify tasks registered in Celery output + +**Common Commands:** +```bash +# Start all services +podman compose -f podman-compose-containerized.yml up -d + +# Rebuild specific container +podman compose -f podman-compose-containerized.yml up -d --force-recreate + +# View logs +podman compose -f podman-compose-containerized.yml logs -f + +# Stop all services +podman compose -f podman-compose-containerized.yml down +``` + +**Important Notes:** +- Worker needs privileged mode for podman-in-podman (building images) +- Registry uses self-signed certs (mounted from volume) +- Configuration in `.env.containerized` (Konflux credentials, GitLab tokens) +- Worker config at `docker/containerized/worker_config.py` + +## Operational Guidelines + +### When Making Code Changes: +1. **Analyze Impact**: Before implementing, assess how changes affect existing functionality and downstream services +2. **Follow Existing Patterns**: Maintain consistency with established IIB codebase conventions and architecture +3. **Prioritize Maintainability**: Write self-documenting code with clear variable names and necessary comments for complex logic +4. **Consider Performance**: Identify potential bottlenecks and optimize for the asynchronous, distributed nature of the service +5. **Security First**: Validate all inputs, sanitize outputs, and never log sensitive information +6. **Version Compatibility**: Ensure changes work across supported Python, OpenShift, and dependency versions + +### When Designing Architecture: +1. **Start with Requirements**: Clarify functional and non-functional requirements before proposing solutions +2. **Evaluate Trade-offs**: Present multiple approaches with honest pros/cons analysis +3. **Design for Failure**: Build in circuit breakers, timeouts, and graceful degradation +4. **Plan for Scale**: Consider horizontal scaling, caching strategies, and resource optimization +5. **Document Thoroughly**: Provide architecture diagrams, sequence flows, and migration paths when relevant +6. **Consider Operations**: Design with monitoring, debugging, and troubleshooting in mind + +### When Writing Unit Tests: +1. **Test Behavior, Not Implementation**: Focus on what the code does, not how it does it +2. **Isolate Dependencies**: Mock external services, databases, and message queues +3. **Name Tests Descriptively**: Test names should clearly indicate what scenario is being tested +4. **Ensure Repeatability**: Tests must produce consistent results regardless of execution order +5. **Cover Error Paths**: Test exception handling, validation failures, and timeout scenarios +6. **Performance Test Coverage**: Ensure tests run quickly to encourage frequent execution +7. **Always Run Tests**: After implementing or modifying code, ALWAYS run tests using `tox -e py312` to verify correctness + - For specific test files: `tox -e py312 -- path/to/test_file.py -v` + - For all tests: `tox -e py312` + - Never skip running tests - they catch regressions and validate changes + +## Common Pitfalls & Gotchas + +### Git Operations +- **Empty Directories**: Git doesn't track empty directories. Always add a `.gitkeep` file to empty catalog directories before committing +- **Directory Removal**: Use `shutil.rmtree()` to remove entire directories, not individual file iteration +- **Catalog Cleanup**: When creating empty catalogs, remove the entire directory and recreate it rather than iterating over contents + +### Index.db Artifact Management +- **Push Conditions**: The `push_index_db_artifact()` function should check only if `index_db_path` exists, not if `operators_in_db` is populated +- **Operators Parameter**: Pass request operators, not database operators. The annotation reflects what was requested, not what was found +- **Empty Operators**: Only include 'operators' annotation if the list is non-empty to avoid `','.join([])` errors +- **Artifact Tags**: Request-specific tags are always pushed; v4.x tag only pushed when `overwrite_from_index=True` + +### OPM Operations +- **Operator vs Bundle**: Use `get_operator_package_list()` to get operator packages, not `get_list_bundles()`. Bundles are part of operators +- **Registry Remove**: Use `_opm_registry_rm()` directly when you don't need FBC migration output (e.g., creating empty index) +- **Permissive Mode**: Enable permissive mode for `_opm_registry_rm()` when removing all operators to create empty index (some indices may have inconsistencies) +- **FBC Validation**: Always call `opm_validate()` on the final catalog before committing to catch schema issues early + +### Fallback Mechanisms +- **Empty Index Creation**: Primary path: fetch pre-tagged empty index.db. Fallback: fetch from_index and remove all operators +- **Error Handling**: Implement fallback with try-except, log the fallback trigger, and continue gracefully + +### Function Parameters +- **Unused Parameters**: Remove parameters that serve no purpose in the function logic (e.g., `operators_in_db` was only used in a conditional check) +- **Optional Parameters**: Don't require parameters the API doesn't provide (e.g., `build_tags` for create-empty-index) +- **Request Type**: Use descriptive request types in annotations ('create_empty_index', 'fbc_operations', 'rm') not just 'rm' everywhere + +## Quality Assurance Process + +Before presenting any solution: +1. **Verify Correctness**: Review logic for bugs, race conditions, and edge cases +2. **Check Compatibility**: Ensure compatibility with IIB service dependencies and deployment environment +3. **Validate Testing**: Confirm test coverage is adequate and tests would actually catch regressions +4. **Review Security**: Scan for common vulnerabilities (injection, auth bypass, data exposure) +5. **Assess Documentation**: Verify that complex logic is explained and API changes are documented +6. **Check All Callers**: When modifying function signatures, grep for all call sites and update them + +## Communication Style + +- **Be Precise**: Provide specific file paths, function names, and line numbers when referencing code +- **Explain Reasoning**: Always clarify why you chose a particular approach over alternatives +- **Ask Clarifying Questions**: When requirements are ambiguous, ask specific questions before proceeding +- **Provide Context**: Help others understand the broader implications of technical decisions +- **Be Honest About Limitations**: If something is outside your expertise or requires more information, say so clearly + +## Escalation Criteria + +Seek additional input when: +- Changes would affect system-wide contracts or APIs used by other services +- Performance implications are significant but uncertain without load testing +- Security considerations are complex or involve authentication/authorization changes +- Proposed changes require database migrations or schema modifications +- You need access to production metrics, logs, or configurations not available in the current context + +You are not just writing code—you are maintaining a critical production service. Every decision should reflect deep technical expertise balanced with pragmatic engineering judgment. diff --git a/.env.containerized.template b/.env.containerized.template new file mode 100644 index 000000000..2eb34a442 --- /dev/null +++ b/.env.containerized.template @@ -0,0 +1,94 @@ +# IIB Containerized Workflow Environment Configuration +# Copy this file to .env.containerized and fill in your values +# DO NOT commit .env.containerized to git (it's already in .gitignore) + +# =================================================================== +# Konflux Cluster Configuration +# =================================================================== +# These settings are required for the worker to connect to your Konflux dev cluster + +# Konflux cluster API URL (e.g., https://api.konflux-dev.example.com:6443) +IIB_KONFLUX_CLUSTER_URL= + +# Konflux cluster service account token +# To create a token: +# 1. Create a service account: kubectl create serviceaccount iib-worker -n +# 2. Create a role with permissions to read/list pipelineruns +# 3. Create a rolebinding to bind the role to the service account +# 4. Get the token: kubectl create token iib-worker -n --duration=720h +IIB_KONFLUX_CLUSTER_TOKEN= + +# Konflux cluster CA certificate path (relative to this file) +# This should point to the file mounted at /etc/iib/konflux-ca.crt +# You can get the CA cert with: kubectl config view --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d > docker/containerized/konflux-ca.crt +IIB_KONFLUX_CLUSTER_CA_CERT=/etc/iib/konflux-ca.crt + +# Namespace where Konflux pipelines run +IIB_KONFLUX_NAMESPACE= + +# Pipeline timeout in seconds (default: 1800 = 30 minutes) +IIB_KONFLUX_PIPELINE_TIMEOUT=1800 + +# =================================================================== +# GitLab Configuration +# =================================================================== +# Required for pushing commits and creating merge requests + +# GitLab tokens for different repositories +# Format: {"repo_url": {"token_name": "ENV_VAR_NAME", "token": "actual_token"}} +# Example: +# IIB_INDEX_CONFIGS_GITLAB_TOKENS_MAP='{"https://gitlab.example.com/catalogs/v4.19": {"token_name": "GITLAB_TOKEN_V419", "token": "glpat-xxxxxxxxxxxxx"}}' +IIB_INDEX_CONFIGS_GITLAB_TOKENS_MAP= + +# =================================================================== +# Registry Configuration +# =================================================================== +# Configuration for the IIB output registry + +# Registry where built images will be pushed +IIB_REGISTRY=registry:8443 + +# Template for pushing built images +# Available placeholders: {registry}, {request_id} +IIB_IMAGE_PUSH_TEMPLATE={registry}/iib-build:{request_id} + +# =================================================================== +# Index DB Artifact Configuration +# =================================================================== +# Configuration for index.db artifact storage + +# Registry for index.db artifacts (usually Quay.io) +IIB_INDEX_DB_ARTIFACT_REGISTRY= + +# Registry for index.db ImageStream cache +IIB_INDEX_DB_IMAGESTREAM_REGISTRY= + +# Template for index.db artifact storage +IIB_INDEX_DB_ARTIFACT_TEMPLATE={registry}/index-db:{tag} + +# =================================================================== +# Optional Configuration +# =================================================================== + +# AWS S3 bucket for storing artifacts (optional) +# IIB_AWS_S3_BUCKET_NAME= + +# Greenwave URL for gating (optional) +# IIB_GREENWAVE_URL= + +# Log level (DEBUG, INFO, WARNING, ERROR) +IIB_LOG_LEVEL=DEBUG + +# Request logs directory (inside container) +IIB_REQUEST_LOGS_DIR=/var/log/iib/requests + +# Skopeo timeout +IIB_SKOPEO_TIMEOUT=300s + +# Total retry attempts for operations +IIB_TOTAL_ATTEMPTS=5 + +# Retry configuration +IIB_RETRY_DELAY=10 +IIB_RETRY_JITTER=10 +IIB_RETRY_MULTIPLIER=5 diff --git a/.flake8 b/.flake8 index 315eb985a..547c8565b 100644 --- a/.flake8 +++ b/.flake8 @@ -6,7 +6,7 @@ exclude = venv per-file-ignores = ./iib/workers/tasks/build_regenerate_bundle.py: E713 - ./iib/workers/tasks/utils.py: E203,E702 + ./iib/workers/tasks/utils.py: E203,E702,E721 ./iib/workers/tasks/build_add_deprecations.py: E713 ./iib/workers/tasks/opm_operations.py: E203 ./iib/web/api_v1.py: E226 @@ -14,9 +14,11 @@ per-file-ignores = ./tests/*: D103 ./tests/test_web/test_models.py: D103 ./tests/test_web/test_s3_utils.py: D103 - ./tests/test_web/test_api_v1.py: D103 + ./tests/test_web/test_api_v1.py: D103,F541 ./tests/test_workers/test_tasks/test_build.py: D103,E231 + ./tests/test_workers/test_tasks/test_build_containerized_fbc_operations.py: F841,E501 ./tests/test_workers/test_tasks/test_build_regenerate_bundle.py: D103,E241,E222 ./tests/test_workers/test_tasks/test_opm_operations.py: D103, E203 ./tests/test_web/test_migrations.py: E231,D103 -extend-ignore = E231 +extend-ignore = E231, D104, D100, D105 +max-line-length = 100 diff --git a/.github/workflows/run_tox.yml b/.github/workflows/run_tox.yml index 402e229c5..07c8eb07c 100644 --- a/.github/workflows/run_tox.yml +++ b/.github/workflows/run_tox.yml @@ -7,6 +7,7 @@ on: push: branches: - "master" + - "main" jobs: build: diff --git a/README.md b/README.md index c42270a02..d6c936d21 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,10 @@ The custom configuration options for the REST API are listed below: to another dictionary mapping ocp_version label to a binary image pull specification. This is useful in setting up customized binary image for different index image images thus reducing complexity for the end user. This defaults to `{}`. +* `IIB_BINARY_IMAGE_LESS_ARCHES_ALLOWED_VERSIONS` - an optional `list()` to specify the OPM + versions which are allowed to build index images with lesser arches than the configured + on `iib_supported_archs`. When a certain version is set it will allow building only to the + available arches supported by the binary image. * `IIB_INDEX_TO_GITLAB_PUSH_MAP` - the mapping, `dict(:)`, to specify which index images (keys) which should have its catalog pushed into a GitLab repository (value). This defaults to {}. * `IIB_GRAPH_MODE_INDEX_ALLOW_LIST` - the list of index image pull specs on which using the @@ -313,6 +317,24 @@ The custom configuration options for the Celery workers are listed below: and related_bundles if specified. `iib_request_logs_dir` and `iib_request_related_bundles_dir` are required when this variable is specified. This defaults to `None` which means IIB will try to store the files locally if `iib_request_logs_dir` and `iib_request_related_bundles_dir` are configured. +* `iib_index_db_imagestream_registry` - the default container registry where the `index.db` + ImageStream is pushed. This is typically an internal OpenShift registry or another registry + dedicated to hosting ImageStreams for `index.db` artifacts. If unset, caching of index.db + artifacts will be disabled. +* `iib_index_db_artifact_registry` - the container registry where `index.db` artifact images + (for example `index-db:`) are stored and from which they are distributed. This is often + a central or dedicated registry for artifacts generated by IIB. This value **must be set** in + order for `index.db` artifacts to be pushed and for configuration validation to succeed. + When `iib_index_db_oras_auth_path` is unset and this is set together with the artifact registry, + ORAS uses `set_registry_auths` with an isolated Docker config (not the worker’s default `config.json`). +* `iib_index_db_oras_auth_path` - path to a JSON Docker config file for ORAS. When set, ORAS uses + `oras --registry-config` with this file and does not apply the inline secret above. +* `iib_empty_index_db_tag` - the tag used to identify pre-created empty `index.db` artifacts in the + registry. When creating an empty index, IIB will first attempt to fetch an artifact tagged with + this value. If not found, it falls back to fetching the `from_index` and removing all operators. + This defaults to `'empty'`. +* `iib_use_imagestream_cache` - whether to use OpenShift ImageStream cache for `index.db` artifacts. + Requires an OpenShift cluster with ImageStream configured. This defaults to `False`. * `iib_docker_config_template` - the path to the Docker config.json file for IIB to use as a template. IIB will symlink this file to `~/.docker/config.json` at the beginning of every request. Additionally, it will use this file as a base and set the `overwrite_from_index_token` for the @@ -341,6 +363,11 @@ The custom configuration options for the Celery workers are listed below: * `iib_index_configs_gitlab_tokens_map` - A map of index image addresses to GitLab tokens. These Gitlab repositories are intended to store image `/configs` directories. Its format should be the full repository URL as keys and `token-name:token-value` as value. +* `iib_regenerate_bundle_repo_key` - The key used to look up the GitLab repository URL from + `iib_index_to_gitlab_push_map` for containerized bundle regeneration workflow. This defaults + to `'regenerate-bundle'`. The actual repository URL should be configured in + `iib_index_to_gitlab_push_map` with this key, and the token must be configured in + `iib_index_configs_gitlab_tokens_map`. * `iib_log_level` - the Python log level for `iib.workers` logger. This defaults to `INFO`. * `iib_max_recursive_related_bundles` - the maximum number of recursive related bundles IIB will recurse through. This is to avoid DOS attacks. diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index ba1dd24af..04e69c9d8 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -11,6 +11,9 @@ RUN dnf -y install \ buildah \ fuse-overlayfs \ gcc \ + curl \ + tar \ + gzip \ git \ krb5-devel \ libffi-devel \ @@ -25,17 +28,40 @@ RUN dnf -y install \ && dnf update -y \ && dnf clean all -ADD https://github.com/operator-framework/operator-registry/releases/download/v1.26.4/linux-amd64-opm /usr/bin/opm-v1.26.4 -RUN chmod +x /usr/bin/opm-v1.26.4 -ADD https://github.com/operator-framework/operator-registry/releases/download/v1.40.0/linux-amd64-opm /usr/bin/opm-v1.40.0 -RUN chmod +x /usr/bin/opm-v1.40.0 -# Create a link for default opm -RUN ln -s /usr/bin/opm-v1.26.4 /usr/bin/opm -RUN chmod +x /usr/bin/opm +# Install all opm variants, +# then expose the default opm via symlink. +RUN set -eux; \ + install_binary() { \ + local name="$1"; local url="$2"; local sha="$3"; \ + curl -fsSL "$url" -o "/usr/local/bin/${name}"; \ + echo "${sha} /usr/local/bin/${name}" | sha256sum -c -; \ + chmod 0555 "/usr/local/bin/${name}"; \ + }; \ + install_binary "opm-v1.26.4" "https://github.com/operator-framework/operator-registry/releases/download/v1.26.4/linux-amd64-opm" "cf94e9dbd58c338e1eed03ca50af847d24724b99b40980812abbe540e8c7ff8e"; \ + install_binary "opm-v1.28.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.28.0/linux-amd64-opm" "e18e5abc8febb63c9dc76db0f33475553d98495465bd2dca81c39dcdbc875c08"; \ + install_binary "opm-v1.40.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.40.0/linux-amd64-opm" "33eb929264a69f31895e1973248b7e97e3b6a862d7ca27f6892e158f79ad6aeb"; \ + install_binary "opm-v1.44.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.44.0/linux-amd64-opm" "21f0a423dfbfcddcffdde98266307a08d87b4db980be859b9e252a5a24df51bf"; \ + install_binary "opm-v1.48.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.48.0/linux-amd64-opm" "0a301826baff730489162caff13e04f7dc16c1a79072cbcbdfc5379d95caef40"; \ + install_binary "opm-v1.50.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.50.0/linux-amd64-opm" "d9bfdc08dd9640c1d9085d191f10f884f2ef29370db1ac097a73a0e23e803f95"; \ + install_binary "opm-v1.57.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.57.0/linux-amd64-opm" "8d2f51f166f47f76eb6906c4de9af90462b7163cbacef6c932bda4829ec086c7"; \ + install_binary "opm-v1.61.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.61.0/linux-amd64-opm" "c5701ef59e12c930337a9a9363cd44c2a4d9f64f6d4f96513d3511a36f81cf5d"; \ + install_binary "operator-sdk" "https://github.com/operator-framework/operator-sdk/releases/download/v1.15.0/operator-sdk_linux_amd64" "d2065f1f7a0d03643ad71e396776dac0ee809ef33195e0f542773b377bab1b2a"; \ + # set default opm \ + ln -sfn /usr/local/bin/opm-v1.26.4 /usr/local/bin/opm + ADD https://github.com/fullstorydev/grpcurl/releases/download/v1.8.5/grpcurl_1.8.5_linux_x86_64.tar.gz /src/grpcurl_1.8.5_linux_x86_64.tar.gz RUN cd /usr/bin && tar -xf /src/grpcurl_1.8.5_linux_x86_64.tar.gz grpcurl && rm -f /src/grpcurl_1.8.5_linux_x86_64.tar.gz -ADD https://github.com/operator-framework/operator-sdk/releases/download/v1.15.0/operator-sdk_linux_amd64 /usr/bin/operator-sdk -RUN chmod +x /usr/bin/operator-sdk + +RUN curl -L "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.10/openshift-client-linux.tar.gz" -o /tmp/oc_client.tar.gz && \ + tar -xvzf /tmp/oc_client.tar.gz -C /usr/bin/ && \ + rm /tmp/oc_client.tar.gz /usr/bin/README.md + +RUN curl -L "https://github.com/oras-project/oras/releases/download/v1.3.0/oras_1.3.0_linux_amd64.tar.gz" -o /tmp/oras.tar.gz && \ + tar -xvzf /tmp/oras.tar.gz -C /usr/bin/ && \ + rm /tmp/oras.tar.gz /usr/bin/LICENSE + +RUN git config --global user.email "exd-guild-hello-operator+iib-dev-env@redhat.com" +RUN git config --global user.name "IIB dev-env" RUN update-alternatives --set python3 $(which python3.12) @@ -45,9 +71,16 @@ COPY docker/libpod.conf /usr/share/containers/libpod.conf COPY . . +# Prepare writable HOME for OpenShift random UID runtime. +RUN mkdir -p /home/iib-worker/.docker \ + && chgrp -R 0 /home/iib-worker \ + && chmod -R g=u /home/iib-worker +ENV HOME=/home/iib-worker +ENV KRB5CCNAME=FILE:/home/iib-worker/krb5cc_iib_worker + # default python3-pip version for rhel8 python3.6 is 9.0.3 and it can't be updated by dnf # we have to update it by pip to version above 21.0.0 RUN pip3 install --upgrade pip RUN pip3 install -r requirements.txt --no-deps --require-hashes RUN pip3 install . --no-deps -CMD ["/bin/celery-3", "-A", "iib.workers.tasks", "worker", "--loglevel=info"] +CMD ["/usr/local/bin/celery", "-A", "iib.workers.tasks", "worker", "--loglevel=debug"] diff --git a/docker/containerized/README.md b/docker/containerized/README.md new file mode 100644 index 000000000..375a1465e --- /dev/null +++ b/docker/containerized/README.md @@ -0,0 +1,280 @@ +# IIB Containerized Workflow Development Environment + +This directory contains configuration for running IIB in containerized mode, where build operations are executed in an external Konflux cluster instead of locally in the worker. + +## Architecture Overview + +In the containerized workflow: + +1. **IIB Worker** receives a request (e.g., remove operators) +2. Worker clones the Git repository containing the catalog +3. Worker makes changes to the catalog locally +4. Worker commits and pushes changes to GitLab (either to a branch or creates an MR) +5. GitLab push triggers a **Konflux PipelineRun** in the external cluster +6. Worker monitors the PipelineRun status via Kubernetes API +7. When the PipelineRun completes, worker copies the built image from Konflux to IIB registry +8. Worker updates the index.db artifact and completes the request + +## Prerequisites + +1. **Konflux Cluster Access** + - A Konflux dev cluster with pipelines configured + - Service account with permissions to read/list PipelineRuns + - Cluster CA certificate + - Cluster API URL + +2. **GitLab Access** + - GitLab repositories for catalog storage + - GitLab access tokens with write permissions + +3. **Container Runtime** + - Podman installed and configured + - podman-compose installed + +## Setup Instructions + +### 1. Get Konflux Cluster Credentials + +#### Get the Cluster API URL + +```bash +kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' +``` + +#### Create a Service Account + +```bash +# Set your namespace +NAMESPACE="your-namespace" + +# Create service account +kubectl create serviceaccount iib-worker -n $NAMESPACE + +# Create role with PipelineRun permissions +cat < konflux-ca.crt +``` + +This will save the CA certificate to `konflux-ca.crt` in the current directory. + +### 2. Configure Environment Variables + +1. Copy the template file: + ```bash + cp .env.containerized.template .env.containerized + ``` + +2. Edit `.env.containerized` and fill in the required values: + + ```bash + # Konflux Cluster Configuration + IIB_KONFLUX_CLUSTER_URL=https://api.konflux-dev.example.com:6443 + IIB_KONFLUX_CLUSTER_TOKEN=eyJhbGc... # Token from above + IIB_KONFLUX_CLUSTER_CA_CERT=/etc/iib/konflux-ca.crt + IIB_KONFLUX_NAMESPACE=your-namespace + + # GitLab Configuration + IIB_INDEX_CONFIGS_GITLAB_TOKENS_MAP='{"https://gitlab.example.com/catalogs/v4.19": GITLAB_TOKEN_V419:glpat-xxxxxxxxxxxxx"}' + + # Registry Configuration + IIB_REGISTRY=registry:8443 + IIB_IMAGE_PUSH_TEMPLATE={registry}/iib-build:{request_id} + + # Index DB Artifact Configuration + IIB_INDEX_DB_ARTIFACT_REGISTRY=quay.io/your-org + IIB_INDEX_DB_IMAGESTREAM_REGISTRY=image-registry.openshift-image-registry.svc:5000 + ``` + +### 3. Place the Konflux CA Certificate + +Copy the CA certificate you downloaded to the correct location: + +```bash +cp konflux-ca.crt docker/containerized/konflux-ca.crt +``` + +### 4. Start the Development Environment + +```bash +# Start all services +podman-compose -f podman-compose-containerized.yml up -d + +# View logs +podman-compose -f podman-compose-containerized.yml logs -f iib-worker-containerized + +# Stop all services +podman-compose -f podman-compose-containerized.yml down +``` + +## Testing the Setup + +### 1. Verify Services are Running + +```bash +podman-compose -f podman-compose-containerized.yml ps +``` + +You should see: +- `iib-api` (running) +- `iib-worker-containerized` (running) +- `db` (running) +- `rabbitmq` (running) +- `registry` (running) +- `memcached` (running) +- `message-broker` (running) +- `minica` (exited 0) + +### 2. Check Worker Logs + +```bash +podman-compose -f podman-compose-containerized.yml logs iib-worker-containerized +``` + +Look for: +- "Configuring Kubernetes client for cross-cluster access to https://..." +- No errors about missing Konflux configuration + +### 3. Submit a Test Request + +```bash +# Using the IIB API +curl -X POST http://localhost:8080/api/v1/builds/rm \ + -H "Content-Type: application/json" \ + -d '{ + "from_index": "registry.example.com/catalog:v4.19", + "operators": ["test-operator"], + "index_to_gitlab_push_map": { + "registry.example.com/catalog:v4.19": "https://gitlab.example.com/catalogs/v4.19" + }, + "overwrite_from_index": false + }' +``` + +### 4. Monitor the Request + +Watch the worker logs to see: +1. Cloning the Git repository +2. Removing operators from the catalog +3. Committing and pushing to GitLab +4. Waiting for Konflux pipeline +5. Pipeline completion +6. Copying built image to IIB registry +7. Request completion + +## Troubleshooting + +### Worker Can't Connect to Konflux Cluster + +**Symptoms:** Error messages about Kubernetes client initialization + +**Solution:** +1. Verify the cluster URL is correct and accessible +2. Check that the token is valid (not expired) +3. Ensure the CA certificate is correct +4. Test connection manually: + ```bash + kubectl --server= --token= \ + --certificate-authority=docker/containerized/konflux-ca.crt \ + get pipelineruns -n + ``` + +### Permission Denied Errors + +**Symptoms:** Kubernetes API errors about permissions + +**Solution:** +1. Verify the service account has the correct role binding +2. Check that the role includes `get`, `list`, and `watch` verbs for `pipelineruns` +3. Ensure you're using the correct namespace + +### GitLab Authentication Errors + +**Symptoms:** Errors cloning or pushing to GitLab + +**Solution:** +1. Verify the GitLab token has correct permissions (read_repository, write_repository) +2. Check the token hasn't expired +3. Ensure the repository URL in `index_to_gitlab_push_map` is correct +4. Test the token manually: + ```bash + git clone https://oauth2:@gitlab.example.com/catalogs/v4.19.git + ``` + +### Pipeline Timeout + +**Symptoms:** "Timeout waiting for pipelinerun to complete" + +**Solution:** +1. Increase `IIB_KONFLUX_PIPELINE_TIMEOUT` in `.env.containerized` +2. Check the Konflux pipeline logs to see why it's taking long +3. Verify the pipeline isn't stuck or failing silently + +## Configuration Reference + +### Environment Variables + +All environment variables are documented in `.env.containerized.template`. + +### Worker Configuration + +The worker configuration is in `docker/containerized/worker_config.py`. This file: +- Reads environment variables from `.env.containerized` +- Extends the base `DevelopmentConfig` +- Includes the containerized task modules +- Validates required configuration on startup + +## Differences from Traditional Workflow + +| Aspect | Traditional Workflow | Containerized Workflow | +|--------|---------------------|------------------------| +| Build Location | Local in worker container | External Konflux cluster | +| Worker Privileges | Privileged (for building) | Unprivileged | +| Container Storage | Requires large volumes | Minimal storage needed | +| Git Operations | Optional | Required | +| External Dependencies | Local tools (buildah, podman) | Konflux cluster, GitLab | +| Scalability | Limited by worker resources | Limited by Konflux capacity | + +## Additional Resources + +- [IIB Documentation](../../docs/) +- [Konflux Documentation](https://konflux-ci.dev/) +- [GitLab API Documentation](https://docs.gitlab.com/ee/api/) +- [Tekton PipelineRuns](https://tekton.dev/docs/pipelines/pipelineruns/) + +## Contributing + +If you encounter issues or have improvements: + +1. Check existing issues and documentation +2. Test your changes locally +3. Submit a pull request with clear description diff --git a/docker/containerized/konflux-ca.crt.example b/docker/containerized/konflux-ca.crt.example new file mode 100644 index 000000000..c475c24ae --- /dev/null +++ b/docker/containerized/konflux-ca.crt.example @@ -0,0 +1,3 @@ +-----BEGIN CERTIFICATE----- +.... +-----END CERTIFICATE----- diff --git a/docker/containerized/worker_config.py b/docker/containerized/worker_config.py new file mode 100644 index 000000000..68db1ea68 --- /dev/null +++ b/docker/containerized/worker_config.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +IIB Worker Configuration for Containerized Workflow. + +This configuration is used when running IIB in containerized mode where builds +are executed in an external Konflux cluster instead of locally in the worker. +""" +import json +import os +from typing import Optional + +from iib.workers.config import DevelopmentConfig + + +class ContainerizedConfig(DevelopmentConfig): + """Configuration for IIB worker in containerized mode.""" + + # =================================================================== + # Konflux Cluster Configuration + # =================================================================== + # These are read from environment variables set in .env.containerized + iib_konflux_cluster_url: Optional[str] = os.getenv('IIB_KONFLUX_CLUSTER_URL') + iib_konflux_cluster_token: Optional[str] = os.getenv('IIB_KONFLUX_CLUSTER_TOKEN') + iib_konflux_cluster_ca_cert: Optional[str] = os.getenv( + 'IIB_KONFLUX_CLUSTER_CA_CERT', '/etc/iib/konflux-ca.crt' + ) + iib_konflux_namespace: Optional[str] = os.getenv('IIB_KONFLUX_NAMESPACE') + iib_konflux_pipeline_timeout: int = int(os.getenv('IIB_KONFLUX_PIPELINE_TIMEOUT', '1800')) + + # =================================================================== + # GitLab Configuration + # =================================================================== + # Parse GitLab tokens from environment variable + _gitlab_tokens_str = os.getenv('IIB_INDEX_CONFIGS_GITLAB_TOKENS_MAP') + iib_index_configs_gitlab_tokens_map = ( + json.loads(_gitlab_tokens_str) if _gitlab_tokens_str else None + ) + + # =================================================================== + # Registry Configuration + # =================================================================== + iib_registry: str = os.getenv('IIB_REGISTRY', 'registry:8443') + iib_image_push_template: str = os.getenv( + 'IIB_IMAGE_PUSH_TEMPLATE', '{registry}/iib-build:{request_id}' + ) + # Docker config template for reset_docker_config() + # Points to the mounted auth config so symlink creation works correctly + iib_docker_config_template: str = '/etc/containers/auth.json' + + # =================================================================== + # Index DB Artifact Configuration + # =================================================================== + iib_index_db_artifact_registry: Optional[str] = os.getenv('IIB_INDEX_DB_ARTIFACT_REGISTRY') + iib_index_db_imagestream_registry: Optional[str] = os.getenv( + 'IIB_INDEX_DB_IMAGESTREAM_REGISTRY' + ) + iib_index_db_artifact_template: str = os.getenv( + 'IIB_INDEX_DB_ARTIFACT_TEMPLATE', '{registry}/index-db:{tag}' + ) + + # =================================================================== + # Task Routing Configuration + # =================================================================== + # Include containerized task modules + include = DevelopmentConfig.include + [ + 'iib.workers.tasks.build_containerized_rm', + ] + + # =================================================================== + # Logging Configuration + # =================================================================== + iib_log_level: str = os.getenv('IIB_LOG_LEVEL', 'DEBUG') + iib_request_logs_dir: Optional[str] = os.getenv( + 'IIB_REQUEST_LOGS_DIR', '/var/log/iib/requests' + ) + + # =================================================================== + # Optional Configuration + # =================================================================== + iib_aws_s3_bucket_name: Optional[str] = os.getenv('IIB_AWS_S3_BUCKET_NAME') + iib_greenwave_url: Optional[str] = os.getenv('IIB_GREENWAVE_URL') + iib_skopeo_timeout: str = os.getenv('IIB_SKOPEO_TIMEOUT', '300s') + iib_total_attempts: int = int(os.getenv('IIB_TOTAL_ATTEMPTS', '5')) + iib_retry_delay: int = int(os.getenv('IIB_RETRY_DELAY', '10')) + iib_retry_jitter: int = int(os.getenv('IIB_RETRY_JITTER', '10')) + iib_retry_multiplier: int = int(os.getenv('IIB_RETRY_MULTIPLIER', '5')) + + # =================================================================== + # Validation + # =================================================================== + @classmethod + def validate(cls): + """ + Validate that required configuration is present. + + :raises ValueError: If required configuration is missing + """ + required_configs = { + 'iib_konflux_cluster_url': cls.iib_konflux_cluster_url, + 'iib_konflux_cluster_token': cls.iib_konflux_cluster_token, + 'iib_konflux_cluster_ca_cert': cls.iib_konflux_cluster_ca_cert, + 'iib_konflux_namespace': cls.iib_konflux_namespace, + } + + missing = [name for name, value in required_configs.items() if not value] + + if missing: + raise ValueError( + f"Missing required Konflux configuration: {', '.join(missing)}. " + "Please set these in your .env.containerized file." + ) + + +# Validate configuration on import +ContainerizedConfig.validate() + +# Export config as module-level variables for Celery to pick up +# This is required because Celery's exec() loading expects module-level vars, not a class +_config = ContainerizedConfig() +for _attr in dir(_config): + if not _attr.startswith('_') and _attr not in globals(): + globals()[_attr] = getattr(_config, _attr) diff --git a/docs/module_documentation/iib.workers.tasks.rst b/docs/module_documentation/iib.workers.tasks.rst index 05ef47a5f..152963ead 100644 --- a/docs/module_documentation/iib.workers.tasks.rst +++ b/docs/module_documentation/iib.workers.tasks.rst @@ -13,6 +13,22 @@ iib.workers.tasks.build module :private-members: :show-inheritance: +iib.workers.tasks.build\_containerized\_rm module +------------------------------------------------- + +.. automodule:: iib.workers.tasks.build_containerized_rm + :members: + :undoc-members: + :show-inheritance: + +iib.workers.tasks.build\_containerized\_create\_empty\_index module +-------------------------------------------------------------------- + +.. automodule:: iib.workers.tasks.build_containerized_create_empty_index + :members: + :undoc-members: + :show-inheritance: + iib.workers.tasks.build\_create\_empty\_index module ---------------------------------------------------- @@ -29,6 +45,30 @@ iib.workers.tasks.build\_fbc\_operations module :undoc-members: :show-inheritance: +iib.workers.tasks.build\_containerized\_fbc\_operations module +-------------------------------------------------------------- + +.. automodule:: iib.workers.tasks.build_containerized_fbc_operations + :members: + :undoc-members: + :show-inheritance: + +iib.workers.tasks.build\_containerized\_merge module +---------------------------------------------------- + +.. automodule:: iib.workers.tasks.build_containerized_merge + :members: + :undoc-members: + :show-inheritance: + +iib.workers.tasks.build\_containerized\_regenerate\_bundle module +------------------------------------------------------------------ + +.. automodule:: iib.workers.tasks.build_containerized_regenerate_bundle + :members: + :undoc-members: + :show-inheritance: + iib.workers.tasks.build\_merge\_index\_image module --------------------------------------------------- @@ -62,6 +102,14 @@ iib.workers.tasks.celery module :private-members: :show-inheritance: +iib.workers.tasks.containerized\_utils module +--------------------------------------------- + +.. automodule:: iib.workers.tasks.containerized_utils + :members: + :undoc-members: + :show-inheritance: + iib.workers.tasks.fbc\_utils module ----------------------------------- @@ -95,6 +143,30 @@ iib.workers.tasks.opm\_operations module :undoc-members: :show-inheritance: +iib.workers.tasks.konflux\_utils module +--------------------------------------- + +.. automodule:: iib.workers.tasks.konflux_utils + :members: + :undoc-members: + :show-inheritance: + +iib.workers.tasks.oras\_utils module +------------------------------------ + +.. automodule:: iib.workers.tasks.oras_utils + :members: + :undoc-members: + :show-inheritance: + +iib.workers.tasks.git\_utils module +----------------------------------- + +.. automodule:: iib.workers.tasks.git_utils + :members: + :undoc-members: + :show-inheritance: + iib.workers.tasks.utils module ------------------------------ diff --git a/docs/requirements.txt b/docs/requirements.txt index 6e179f1a4..a2183645b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,6 +5,7 @@ flask flask-login flask-migrate flask-sqlalchemy +kubernetes opentelemetry-api opentelemetry-exporter-otlp opentelemetry-instrumentation diff --git a/iib/web/api_v1.py b/iib/web/api_v1.py index 92dfcedf1..25eb8be52 100644 --- a/iib/web/api_v1.py +++ b/iib/web/api_v1.py @@ -44,18 +44,24 @@ from iib.web.s3_utils import get_object_from_s3_bucket from botocore.response import StreamingBody from iib.web.utils import pagination_metadata, str_to_bool -from iib.workers.tasks.build import ( - handle_add_request, - handle_rm_request, +from iib.workers.tasks.build_containerized_add import ( + handle_containerized_add_request, ) +from iib.workers.tasks.build_containerized_rm import handle_containerized_rm_request from iib.workers.tasks.build_add_deprecations import handle_add_deprecations_request -from iib.workers.tasks.build_fbc_operations import handle_fbc_operation_request +from iib.workers.tasks.build_containerized_fbc_operations import ( + handle_containerized_fbc_operation_request, +) from iib.workers.tasks.build_recursive_related_bundles import ( handle_recursive_related_bundles_request, ) -from iib.workers.tasks.build_regenerate_bundle import handle_regenerate_bundle_request -from iib.workers.tasks.build_merge_index_image import handle_merge_request -from iib.workers.tasks.build_create_empty_index import handle_create_empty_index_request +from iib.workers.tasks.build_containerized_regenerate_bundle import ( + handle_containerized_regenerate_bundle_request, +) +from iib.workers.tasks.build_containerized_create_empty_index import ( + handle_containerized_create_empty_index_request, +) +from iib.workers.tasks.build_containerized_merge import handle_containerized_merge_request from iib.workers.tasks.general import failed_request_callback from iib.web.iib_static_types import ( AddDeprecationRequestPayload, @@ -100,6 +106,7 @@ def _get_rm_args( flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], payload.get('build_tags', []), flask.current_app.config['IIB_INDEX_TO_GITLAB_PUSH_MAP'], + flask.current_app.config['IIB_BINARY_IMAGE_LESS_ARCHES_ALLOWED_VERSIONS'], ] @@ -125,19 +132,16 @@ def _get_add_args( payload.get('binary_image'), payload.get('from_index'), payload.get('add_arches'), - payload.get('cnr_token'), - payload.get('organization'), - payload.get('force_backport'), overwrite_from_index, payload.get('overwrite_from_index_token'), request.distribution_scope, - flask.current_app.config['IIB_GREENWAVE_CONFIG'].get(celery_queue), flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], payload.get('deprecation_list', []), payload.get('build_tags', []), payload.get('graph_update_mode'), payload.get('check_related_images', False), flask.current_app.config['IIB_INDEX_TO_GITLAB_PUSH_MAP'], + flask.current_app.config['IIB_BINARY_IMAGE_LESS_ARCHES_ALLOWED_VERSIONS'], ] @@ -595,6 +599,9 @@ def add_bundles() -> Tuple[flask.Response, int]: if not isinstance(payload, dict): raise ValidationError('The input data must be a JSON object') + if not payload.get('from_index'): + raise ValidationError('The input "from_index" is required.') + # Only run `_get_unique_bundles` if it is a list. If it's not, `from_json` # will raise an error to the user. if payload.get('bundles') and isinstance(payload['bundles'], list): @@ -617,7 +624,7 @@ def add_bundles() -> Tuple[flask.Response, int]: args.append(current_user.username) try: - handle_add_request.apply_async( + handle_containerized_add_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), @@ -753,6 +760,7 @@ def patch_request(request_id: int) -> Tuple[flask.Response, int]: 'source_from_index_resolved', 'target_index_resolved', 'index_to_gitlab_push_map', + 'binary_image_less_arches_allowed_versions', ) start_time = time.time() for key in image_keys: @@ -852,7 +860,7 @@ def rm_operators() -> Tuple[flask.Response, int]: error_callback = failed_request_callback.s(request.id) from_index_pull_spec = request.from_index.pull_specification if request.from_index else None try: - handle_rm_request.apply_async( + handle_containerized_rm_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), @@ -893,12 +901,15 @@ def regenerate_bundle() -> Tuple[flask.Response, int]: request.id, payload.get('registry_auths'), payload.get('bundle_replacements', dict()), + flask.current_app.config['IIB_INDEX_TO_GITLAB_PUSH_MAP'], + flask.current_app.config['IIB_REGENERATE_BUNDLE_REPO_KEY'], + flask.current_app.config['IIB_BINARY_IMAGE_LESS_ARCHES_ALLOWED_VERSIONS'], ] safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: - handle_regenerate_bundle_request.apply_async( + handle_containerized_regenerate_bundle_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), @@ -961,10 +972,13 @@ def regenerate_bundle_batch() -> Tuple[flask.Response, int]: request.id, build_request.get('registry_auths'), build_request.get('bundle_replacements', dict()), + flask.current_app.config['IIB_INDEX_TO_GITLAB_PUSH_MAP'], + flask.current_app.config['IIB_REGENERATE_BUNDLE_REPO_KEY'], + flask.current_app.config['IIB_BINARY_IMAGE_LESS_ARCHES_ALLOWED_VERSIONS'], ] safe_args = _get_safe_args(args, build_request) error_callback = failed_request_callback.s(request.id) - handle_regenerate_bundle_request.apply_async( + handle_containerized_regenerate_bundle_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), @@ -1064,14 +1078,14 @@ def add_rm_batch() -> Tuple[flask.Response, int]: error_callback = failed_request_callback.s(request.id) try: if isinstance(request, RequestAdd): - handle_add_request.apply_async( + handle_containerized_add_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=celery_queue, ) else: - handle_rm_request.apply_async( + handle_containerized_rm_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), @@ -1129,7 +1143,7 @@ def merge_index_image() -> Tuple[flask.Response, int]: error_callback = failed_request_callback.s(request.id) try: - handle_merge_request.apply_async( + handle_containerized_merge_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=celery_queue ) except kombu.exceptions.OperationalError: @@ -1163,16 +1177,17 @@ def create_empty_index() -> Tuple[flask.Response, int]: args = [ payload['from_index'], request.id, - payload.get('output_fbc'), payload.get('binary_image'), payload.get('labels'), flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], + flask.current_app.config['IIB_INDEX_TO_GITLAB_PUSH_MAP'], + flask.current_app.config['IIB_BINARY_IMAGE_LESS_ARCHES_ALLOWED_VERSIONS'], ] safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: - handle_create_empty_index_request.apply_async( + handle_containerized_create_empty_index_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=_get_user_queue() ) except kombu.exceptions.OperationalError: @@ -1328,11 +1343,12 @@ def fbc_operations() -> Tuple[flask.Response, int]: flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], flask.current_app.config['IIB_INDEX_TO_GITLAB_PUSH_MAP'], request._used_fbc_fragment, # Pass the legacy flag to the worker + flask.current_app.config['IIB_BINARY_IMAGE_LESS_ARCHES_ALLOWED_VERSIONS'], ] safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: - handle_fbc_operation_request.apply_async( + handle_containerized_fbc_operation_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=celery_queue ) except kombu.exceptions.OperationalError: diff --git a/iib/web/config.py b/iib/web/config.py index dfbadd5db..aa1ca02d2 100644 --- a/iib/web/config.py +++ b/iib/web/config.py @@ -20,7 +20,9 @@ class Config(object): IIB_ADDITIONAL_LOGGERS: List[str] = [] IIB_AWS_S3_BUCKET_NAME: Optional[str] = None IIB_BINARY_IMAGE_CONFIG: Dict[str, Dict[str, str]] = {} + IIB_BINARY_IMAGE_LESS_ARCHES_ALLOWED_VERSIONS: List[str] = [] IIB_INDEX_TO_GITLAB_PUSH_MAP: Dict[str, str] = {} + IIB_REGENERATE_BUNDLE_REPO_KEY: str = 'regenerate-bundle' IIB_GRAPH_MODE_INDEX_ALLOW_LIST: List[str] = [] IIB_GRAPH_MODE_OPTIONS: List[str] = ['replaces', 'semver', 'semver-skippatch'] IIB_GREENWAVE_CONFIG: Dict[str, str] = {} diff --git a/iib/workers/config.py b/iib/workers/config.py index 7e56f62ff..d84953ad9 100644 --- a/iib/workers/config.py +++ b/iib/workers/config.py @@ -40,8 +40,39 @@ class Config(object): "opm_port": (50051, 50151), "opm_pprof_port": (50151, 50251), } + iib_ocp_opm_mapping: Dict[str, str] = { + # keep v0.0, v4.5 for iib-api-tests + "v0.0": "opm-v1.28.0", + "v4.5": "opm-v1.26.4", + "v4.6": "opm-v1.26.4", + "v4.7": "opm-v1.26.4", + "v4.8": "opm-v1.26.4", + "v4.9": "opm-v1.26.4", + "v4.10": "opm-v1.26.4", + "v4.11": "opm-v1.26.4", + "v4.12": "opm-v1.26.4", + "v4.13": "opm-v1.26.4", + "v4.14": "opm-v1.26.4", + "v4.15": "opm-v1.26.4", + "v4.16": "opm-v1.26.4", + "v4.17": "opm-v1.40.0", + "v4.18": "opm-v1.44.0", + "v4.19": "opm-v1.48.0", + "v4.20": "opm-v1.50.0", + "v4.21": "opm-v1.50.0", + "v4.22": "opm-v1.61.0", + } iib_opm_pprof_lock_required_min_version = "1.29.0" iib_image_push_template: str = '{registry}/iib-build:{request_id}' + # Default registry for index.db ImageStream + iib_index_db_imagestream_registry: Optional[str] = None + iib_index_db_artifact_registry: Optional[str] = None + iib_index_db_oras_auth_path: Optional[str] = None + iib_index_db_artifact_tag_template: str = '{image_name}-{tag}' + iib_index_db_artifact_template: str = '{registry}/index-db:{tag}' + # Whether to use OpenShift ImageStream cache for index.db artifacts + # Requires OpenShift cluster with ImageStream configured + iib_use_imagestream_cache: bool = False iib_index_image_output_registry: Optional[str] = None iib_index_configs_gitlab_tokens_map: Optional[Dict[str, Dict[str, str]]] = None iib_log_level: str = 'INFO' @@ -86,6 +117,12 @@ class Config(object): 'iib.workers.tasks.build_create_empty_index', 'iib.workers.tasks.build_fbc_operations', 'iib.workers.tasks.build_add_deprecations', + 'iib.workers.tasks.build_containerized_add', + 'iib.workers.tasks.build_containerized_fbc_operations', + 'iib.workers.tasks.build_containerized_rm', + 'iib.workers.tasks.build_containerized_create_empty_index', + 'iib.workers.tasks.build_containerized_merge', + 'iib.workers.tasks.build_containerized_regenerate_bundle', 'iib.workers.tasks.general', ] # Path to hidden location of SQLite database @@ -107,6 +144,8 @@ class Config(object): temp_index_db_path: str = 'database/index.db' # Path to fbc_fragment's catalog in our temp directories temp_fbc_fragment_path = 'fbc-fragment' + # Tag used to identify empty index.db artifacts in the registry + iib_empty_index_db_tag: str = 'empty' # For now, only allow a single process so that all tasks are processed serially worker_concurrency: int = 1 # Before each task execution, instruct the worker to check if this task is a duplicate message. @@ -145,6 +184,7 @@ class DevelopmentConfig(Config): broker_url: str = 'amqp://iib:iib@rabbitmq:5673//' iib_api_url: str = 'http://iib-api:8080/api/v1/' iib_log_level: str = 'DEBUG' + iib_index_db_artifact_registry: str = 'registry:8443' iib_organization_customizations: iib_organization_customizations_type = { 'company-marketplace': [ IIBOrganizationCustomizations({'type': 'resolve_image_pullspecs'}), @@ -259,6 +299,8 @@ class TestingConfig(DevelopmentConfig): iib_docker_config_template: str = '/home/iib-worker/.docker/config.json.template' iib_greenwave_url: str = 'some_url' + iib_index_db_artifact_registry: str = 'test-artifact-registry' + iib_index_db_imagestream_registry: str = 'test-imagestream-registry' iib_omps_url: str = 'some_url' iib_request_logs_dir: Optional[str] = None iib_request_related_bundles_dir: Optional[str] = None @@ -319,6 +361,9 @@ def validate_celery_config(conf: app.utils.Settings, **kwargs) -> None: if not conf.get('iib_api_url'): raise ConfigError('iib_api_url must be set') + if not conf.get('iib_index_db_artifact_registry'): + raise ConfigError('iib_index_db_artifact_registry must be set') + if not isinstance(conf['iib_required_labels'], dict): raise ConfigError('iib_required_labels must be a dictionary') diff --git a/iib/workers/tasks/build.py b/iib/workers/tasks/build.py index f31fad2b5..486082056 100644 --- a/iib/workers/tasks/build.py +++ b/iib/workers/tasks/build.py @@ -189,17 +189,12 @@ def _create_and_push_manifest_list( :rtype: str :raises IIBError: if creating or pushing the manifest list fails """ + # Local import to avoid circular dependency (containerized_utils.py imports from build.py) + from iib.workers.tasks.containerized_utils import get_list_of_output_pullspec + buildah_manifest_cmd = ['buildah', 'manifest'] - _tags = [str(request_id)] - if build_tags: - _tags.extend(build_tags) - conf = get_worker_config() - output_pull_specs = [] - for tag in _tags: - output_pull_spec = conf['iib_image_push_template'].format( - registry=conf['iib_registry'], request_id=tag - ) - output_pull_specs.append(output_pull_spec) + output_pull_specs = get_list_of_output_pullspec(request_id, build_tags) + for output_pull_spec in output_pull_specs: try: run_cmd( buildah_manifest_cmd + ['rm', output_pull_spec], @@ -402,11 +397,12 @@ def get_index_database(from_index: str, base_dir: str) -> str: return local_path -def _get_present_bundles(from_index: str, base_dir: str) -> Tuple[List[BundleImage], List[str]]: +def _get_present_bundles(input_data: str, base_dir: str) -> Tuple[List[BundleImage], List[str]]: """ Get a list of bundles already present in the index image. - :param str from_index: index image to inspect. + :param str input_data: input data to inspect. + Example: catalog-image | catalog-directory | bundle-image | bundle-directory | sqlite-file :param str base_dir: base directory to create temporary files in. :return: list of unique present bundles as provided by the grpc query and a list of unique bundle pull_specs @@ -416,7 +412,7 @@ def _get_present_bundles(from_index: str, base_dir: str) -> Tuple[List[BundleIma # Get list of bundles unique_present_bundles: List[BundleImage] = [] unique_present_bundles_pull_spec: List[str] = [] - present_bundles: List[BundleImage] = get_list_bundles(from_index, base_dir) + present_bundles: List[BundleImage] = get_list_bundles(input_data, base_dir) # If no data is returned there are no bundles present if not present_bundles: diff --git a/iib/workers/tasks/build_add_deprecations.py b/iib/workers/tasks/build_add_deprecations.py index 2df7ea8ed..40733d364 100644 --- a/iib/workers/tasks/build_add_deprecations.py +++ b/iib/workers/tasks/build_add_deprecations.py @@ -3,7 +3,7 @@ import json import logging import tempfile -from typing import Dict, Optional, Set +from typing import Dict, List, Optional, Set from iib.common.common_utils import get_binary_versions from iib.common.tracing import instrument_tracing @@ -55,6 +55,7 @@ def handle_add_deprecations_request( build_tags: Optional[Set[str]] = None, binary_image_config: Optional[Dict[str, Dict[str, str]]] = None, overwrite_from_index_token: Optional[str] = None, + binary_image_less_arches_allowed_versions: Optional[List[str]] = None, ) -> None: """ Add a deprecation schema to index image. @@ -74,6 +75,8 @@ def handle_add_deprecations_request( :param list build_tags: List of tags which will be applied to intermediate index images. :param dict binary_image_config: the dict of config required to identify the appropriate ``binary_image`` to use. + :param list binary_image_less_arches_allowed_versions: list of versions of the binary image + that are allowed to build for less arches. Defaults to ``None``. """ _cleanup() set_request_state(request_id, 'in_progress', 'Resolving the index images') @@ -87,6 +90,7 @@ def handle_add_deprecations_request( operator_package=operator_package, deprecation_schema=deprecation_schema, binary_image_config=binary_image_config, + binary_image_less_arches_allowed_versions=binary_image_less_arches_allowed_versions, ), ) diff --git a/iib/workers/tasks/build_containerized_add.py b/iib/workers/tasks/build_containerized_add.py new file mode 100644 index 000000000..002892ca3 --- /dev/null +++ b/iib/workers/tasks/build_containerized_add.py @@ -0,0 +1,372 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import logging +import shutil +import stat +import tempfile +from pathlib import Path +from typing import Dict, List, Optional, Set + +from iib.common.common_utils import get_binary_versions +from iib.common.tracing import instrument_tracing +from iib.exceptions import IIBError +from iib.workers.api_utils import set_request_state +from iib.workers.config import get_worker_config +from iib.workers.tasks.build import ( + inspect_related_images, + _update_index_image_pull_spec, + _update_index_image_build_state, + _get_present_bundles, + _get_missing_bundles, +) +from iib.workers.tasks.celery import app +from iib.workers.tasks.containerized_utils import ( + prepare_git_repository_for_build, + fetch_and_verify_index_db_artifact, + write_build_metadata, + git_commit_and_create_mr_or_push, + monitor_pipeline_and_extract_image, + replicate_image_to_tagged_destinations, + push_index_db_artifact, + cleanup_merge_request_if_exists, + cleanup_on_failure, +) +from iib.workers.tasks.fbc_utils import merge_catalogs_dirs +from iib.workers.tasks.iib_static_types import ( + BundleImage, +) +from iib.workers.tasks.opm_operations import ( + opm_migrate, + Opm, + _opm_registry_add, + deprecate_bundles_db, +) +from iib.workers.tasks.utils import ( + chmod_recursively, + get_bundles_from_deprecation_list, + get_resolved_bundles, + request_logger, + reset_docker_config, + set_registry_token, + RequestConfigAddRm, + get_image_label, + verify_labels, + prepare_request_for_build, +) + +__all__ = ['handle_containerized_add_request'] + +log = logging.getLogger(__name__) +worker_config = get_worker_config() + + +@app.task +@request_logger +@instrument_tracing(span_name="workers.tasks.handle_add_request", attributes=get_binary_versions()) +def handle_containerized_add_request( + bundles: List[str], + request_id: int, + binary_image: Optional[str] = None, + from_index: Optional[str] = None, + add_arches: Optional[Set[str]] = None, + overwrite_from_index: bool = False, + overwrite_from_index_token: Optional[str] = None, + distribution_scope: Optional[str] = None, + binary_image_config: Optional[Dict[str, Dict[str, str]]] = None, + deprecation_list: Optional[List[str]] = None, + build_tags: Optional[List[str]] = None, + graph_update_mode: Optional[str] = None, + check_related_images: bool = False, + index_to_gitlab_push_map: Optional[Dict[str, str]] = None, + binary_image_less_arches_allowed_versions: Optional[List[str]] = None, + username: Optional[str] = None, +) -> None: + """ + Coordinate the work needed to build the index image with the input bundles. + + :param list bundles: a list of strings representing the pull specifications of the bundles to + add to the index image being built. + :param int request_id: the ID of the IIB build request + :param str binary_image: the pull specification of the container image where the opm binary + gets copied from. + :param str from_index: the pull specification of the container image containing the index that + the index image build will be based from. + :param set add_arches: the set of arches to build in addition to the arches ``from_index`` is + currently built for; if ``from_index`` is ``None``, then this is used as the list of arches + to build the index image for + :param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built + index image. + :param str overwrite_from_index_token: the token used for overwriting the input + ``from_index`` image. This is required to use ``overwrite_from_index``. + The format of the token must be in the format "user:password". + :param str distribution_scope: the scope for distribution of the index image, defaults to + ``None``. + :param dict binary_image_config: the dict of config required to identify the appropriate + ``binary_image`` to use. + :param list deprecation_list: list of deprecated bundles for the target index image. Defaults + to ``None``. + :param list build_tags: List of tags which will be applied to intermediate index images. + :param str graph_update_mode: Graph update mode that defines how channel graphs are updated + in the index. + :param dict index_to_gitlab_push_map: the dict mapping index images (keys) to GitLab repos + (values) in order to push their catalogs into GitLab. + :param list binary_image_less_arches_allowed_versions: list of versions of the binary image + that are allowed to build for less arches. Defaults to ``None``. + :raises IIBError: if the index image build fails. + """ + reset_docker_config() + # Resolve bundles to their digests + set_request_state(request_id, 'in_progress', 'Resolving the bundles') + + with set_registry_token(overwrite_from_index_token, from_index, append=True): + resolved_bundles = get_resolved_bundles(bundles) + verify_labels(resolved_bundles) + if check_related_images: + inspect_related_images( + resolved_bundles, + request_id, + worker_config.iib_related_image_registry_replacement.get(username), + ) + + prebuild_info = prepare_request_for_build( + request_id, + RequestConfigAddRm( + _binary_image=binary_image, + from_index=from_index, + overwrite_from_index_token=overwrite_from_index_token, + add_arches=add_arches, + bundles=bundles, + distribution_scope=distribution_scope, + binary_image_config=binary_image_config, + binary_image_less_arches_allowed_versions=binary_image_less_arches_allowed_versions, + ), + ) + from_index_resolved = prebuild_info['from_index_resolved'] + binary_image_resolved = prebuild_info['binary_image_resolved'] + arches = prebuild_info['arches'] + operators = list(prebuild_info['bundle_mapping'].keys()) + distribution_scope = prebuild_info['distribution_scope'] + + index_to_gitlab_push_map = index_to_gitlab_push_map or {} + # Variables mr_details, last_commit_sha and original_index_db_digest + # needs to be assigned; otherwise cleanup_on_failure() fails when an exception is raised. + mr_details: Optional[Dict[str, str]] = None + last_commit_sha: Optional[str] = None + original_index_db_digest: Optional[str] = None + + Opm.set_opm_version(from_index_resolved) + + _update_index_image_build_state(request_id, prebuild_info) + present_bundles: List[BundleImage] = [] + present_bundles_pull_spec: List[str] = [] + with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: + branch = prebuild_info['ocp_version'] + + # Set up and clone Git repository + ( + index_git_repo, + local_git_repo_path, + localized_git_catalog_path, + ) = prepare_git_repository_for_build( + request_id=request_id, + from_index=str(from_index), + temp_dir=temp_dir, + branch=branch, + index_to_gitlab_push_map=index_to_gitlab_push_map, + ) + + # Pull index.db artifact (uses ImageStream cache if configured, otherwise pulls directly) + artifact_index_db_file = fetch_and_verify_index_db_artifact( + from_index=str(from_index), + temp_dir=temp_dir, + ) + + msg = 'Checking if bundles are already present in index image' + log.info(msg) + set_request_state(request_id, 'in_progress', msg) + + # Extract packages from FBC directory to speed up opm render + extracted_packages = Path(temp_dir) / "extracted_packages" + extracted_packages.mkdir(parents=True, exist_ok=True) + + package_names = prebuild_info['bundle_mapping'].keys() + log.debug("Extracting packages from FBC directory: %s", package_names) + for package in package_names: + package_dir = Path(localized_git_catalog_path) / package + if not package_dir.is_dir(): + log.debug("Package %s not found in FBC directory", package) + continue + shutil.copytree(package_dir, extracted_packages / package) + + with set_registry_token(overwrite_from_index_token, from_index_resolved, append=True): + present_bundles, present_bundles_pull_spec = _get_present_bundles( + str(extracted_packages), temp_dir + ) + + filtered_bundles = _get_missing_bundles(present_bundles, resolved_bundles) + excluded_bundles = [bundle for bundle in resolved_bundles if bundle not in filtered_bundles] + resolved_bundles = filtered_bundles + + if excluded_bundles: + log.info( + 'Following bundles are already present in the index image: %s', + ' '.join(excluded_bundles), + ) + + # This is a replacement for opm_registry_add_fbc for a containerized version of IIB. + # Note: only index.db is modified (FBC directory is unchanged) + _opm_registry_add( + base_dir=temp_dir, + index_db=artifact_index_db_file, + bundles=resolved_bundles, + overwrite_csv=(prebuild_info['distribution_scope'] in ['dev', 'stage']), + graph_update_mode=graph_update_mode, + ) + + deprecation_bundles = get_bundles_from_deprecation_list( + present_bundles_pull_spec + resolved_bundles, deprecation_list or [] + ) + + if deprecation_bundles: + deprecate_bundles_db( + bundles=deprecation_bundles, + base_dir=temp_dir, + index_db_file=artifact_index_db_file, + ) + + from_db_dir = Path(temp_dir) / "from_db" + from_db_dir.mkdir(parents=True, exist_ok=True) + # get catalog from SQLite index.db (hidden db) - not opted in operators + catalog_from_db, _ = opm_migrate( + index_db=artifact_index_db_file, + base_dir=str(from_db_dir), + generate_cache=False, + ) + + # we have to remove all `deprecation_bundles` from `localized_git_catalog_path` + # before merging catalogs otherwise if catalog was deprecated and + # removed from `index.db` it stays on FBC (from_index) + # Therefore we have to remove the directory before merging + for deprecate_bundle_pull_spec in deprecation_bundles: + # remove deprecated operators from FBC stored in index image + deprecate_bundle_package = get_image_label( + deprecate_bundle_pull_spec, 'operators.operatorframework.io.bundle.package.v1' + ) + bundle_from_index = Path(localized_git_catalog_path) / deprecate_bundle_package + if bundle_from_index.is_dir(): + log.debug( + "Removing deprecated bundle from catalog before merging: %s", + deprecate_bundle_package, + ) + shutil.rmtree(bundle_from_index) + # overwrite data in `localized_git_catalog_path` by data from `catalog_from_db` + # this adds changes on not opted in operators to final + merge_catalogs_dirs(catalog_from_db, localized_git_catalog_path) + + # If the container-tool podman is used in the opm commands above, opm will create temporary + # files and directories without the write permission. This will cause the context manager + # to fail to delete these files. Adjust the file modes to avoid this error. + chmod_recursively( + temp_dir, + dir_mode=(stat.S_IRWXU | stat.S_IRWXG), + file_mode=(stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP), + ) + + # Write build metadata to a file to be added with the commit + set_request_state(request_id, 'in_progress', 'Writing build metadata') + write_build_metadata( + local_git_repo_path, + Opm.opm_version, + prebuild_info['ocp_version'], + str(distribution_scope), + binary_image_resolved, + request_id, + arches, + ) + + try: + # Commit changes and create MR or push directly + mr_details, last_commit_sha = git_commit_and_create_mr_or_push( + request_id=request_id, + local_git_repo_path=local_git_repo_path, + index_git_repo=index_git_repo, + branch=branch, + commit_message=( + f"IIB: Add bundles for request {request_id}\n\n" + f"Bundles: {', '.join(bundles)}" + ), + overwrite_from_index=overwrite_from_index, + ) + + # Wait for Konflux pipeline and extract built image URL + image_url = monitor_pipeline_and_extract_image( + request_id=request_id, + last_commit_sha=last_commit_sha, + ) + + # Copy built index to all output pull specs + output_pull_specs = replicate_image_to_tagged_destinations( + request_id=request_id, + image_url=image_url, + build_tags=build_tags, + ) + + # Use the first output_pull_spec as the primary one for request updates + output_pull_spec = output_pull_specs[0] + # Update request with final output + if not output_pull_spec: + raise IIBError( + "output_pull_spec was not set. " + "This should not happen if the pipeline completed successfully." + ) + + _update_index_image_pull_spec( + output_pull_spec=output_pull_spec, + request_id=request_id, + arches=arches, + from_index=from_index, + overwrite_from_index=overwrite_from_index, + overwrite_from_index_token=overwrite_from_index_token, + resolved_prebuild_from_index=from_index_resolved, + add_or_rm=True, + is_image_fbc=True, + # Passing an empty index_repo_map is intentional. In IIB 1.0, if + # the overwrite_from_index token is given, we push to git by default + # at the end of a request. In IIB 2.0, the commit is pushed earlier to trigger + # a Konflux pipelinerun. So the old workflow isn't needed. + index_repo_map={}, + ) + + # Push updated index.db if overwrite_from_index_token is provided + # We can push it directly from temp_dir since we're still inside the + # context manager. Do it as the last step to avoid rolling back the + # index.db file if the pipeline fails. + original_index_db_digest = push_index_db_artifact( + request_id=request_id, + from_index=str(from_index), + index_db_path=artifact_index_db_file, + operators=operators, + overwrite_from_index=overwrite_from_index, + request_type='add', + ) + + # Close MR if it was opened + cleanup_merge_request_if_exists(mr_details, index_git_repo) + + set_request_state( + request_id, + 'complete', + 'The operator bundle(s) were successfully added to the index image', + ) + except Exception as e: + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=index_git_repo, + overwrite_from_index=overwrite_from_index, + request_id=request_id, + from_index=str(from_index), + index_repo_map=index_to_gitlab_push_map or {}, + original_index_db_digest=original_index_db_digest, + reason=f"error: {e}", + ) + raise IIBError(f"Failed to add bundles: {e}") diff --git a/iib/workers/tasks/build_containerized_create_empty_index.py b/iib/workers/tasks/build_containerized_create_empty_index.py new file mode 100644 index 000000000..773905b7f --- /dev/null +++ b/iib/workers/tasks/build_containerized_create_empty_index.py @@ -0,0 +1,381 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import json +import logging +import shutil +import tempfile +from pathlib import Path +from typing import Dict, Optional, List + +from iib.common.common_utils import get_binary_versions +from iib.common.tracing import instrument_tracing +from iib.exceptions import IIBError +from iib.workers.api_utils import set_request_state +from iib.workers.config import get_worker_config +from iib.workers.tasks.build import ( + _update_index_image_build_state, + _update_index_image_pull_spec, +) +from iib.workers.tasks.celery import app +from iib.workers.tasks.containerized_utils import ( + cleanup_merge_request_if_exists, + cleanup_on_failure, + fetch_and_verify_index_db_artifact, + git_commit_and_create_mr_or_push, + monitor_pipeline_and_extract_image, + prepare_git_repository_for_build, + push_index_db_artifact, + replicate_image_to_tagged_destinations, + write_build_metadata, +) +from iib.workers.tasks.opm_operations import ( + Opm, + _opm_registry_rm, + get_operator_package_list, + opm_validate, +) +from iib.workers.tasks.oras_utils import ( + _get_artifact_combined_tag, + _get_name_and_tag_from_pullspec, + get_oras_artifact, +) +from iib.workers.tasks.utils import ( + prepare_request_for_build, + request_logger, + reset_docker_config, + RequestConfigCreateIndexImage, +) + +__all__ = ['handle_containerized_create_empty_index_request'] + +log = logging.getLogger(__name__) + + +def _create_empty_index_db_from_source( + request_id: int, + from_index: str, + temp_dir: str, +) -> Path: + """ + Create an empty index.db by fetching from from_index and removing all operators. + + This is a fallback path when the pre-built empty index.db artifact is not available. + + :param int request_id: The IIB request ID + :param str from_index: The from_index pullspec + :param str temp_dir: Temporary directory for operations + :return: Path to the created empty index.db file + :rtype: Path + :raises IIBError: If the process fails + """ + set_request_state(request_id, 'in_progress', 'Creating empty index database from from_index') + + # Fetch the index.db from from_index + log.info('Fetching index.db from %s', from_index) + index_db_path = Path( + fetch_and_verify_index_db_artifact( + from_index=from_index, + temp_dir=temp_dir, + ) + ) + + # Get all operator packages from the index.db + log.info('Extracting all operator packages from index.db') + operators_in_db = get_operator_package_list(str(index_db_path), temp_dir) + + if operators_in_db: + log.info('Removing all operators from index.db: %s', operators_in_db) + # Remove all operators from index.db to create an empty one + try: + _opm_registry_rm( + index_db_path=str(index_db_path), + operators=operators_in_db, + base_dir=temp_dir, + ) + except IIBError as e: + if 'Error deleting packages from database' in str(e): + log.info('Enable permissive mode for opm registry rm') + _opm_registry_rm( + index_db_path=str(index_db_path), + operators=operators_in_db, + base_dir=temp_dir, + permissive=True, + ) + else: + raise + log.info('Successfully created empty index.db by removing all operators') + else: + log.info('Index.db is already empty, no operators to remove') + + return index_db_path + + +@app.task +@request_logger +@instrument_tracing( + span_name="workers.tasks.build.handle_containerized_create_empty_index_request", + attributes=get_binary_versions(), +) +def handle_containerized_create_empty_index_request( + from_index: str, + request_id: int, + binary_image: Optional[str] = None, + labels: Optional[Dict[str, str]] = None, + binary_image_config: Optional[Dict[str, Dict[str, str]]] = None, + index_to_gitlab_push_map: Optional[Dict[str, str]] = None, + binary_image_less_arches_allowed_versions: Optional[List[str]] = None, +) -> None: + """ + Coordinate the work needed to create empty index using containerized workflow. + + This function uses Git-based workflows and Konflux pipelines instead of local builds. + The index.db is expected to already be tagged with the 'empty' tag in the registry. + + :param str from_index: the pull specification of the container image containing the index that + the index image build will be based from. + :param int request_id: the ID of the IIB build request + :param str binary_image: the pull specification of the container image where the opm binary + gets copied from. + :param dict labels: the dict of labels required to be added to a new index image + :param dict binary_image_config: the dict of config required to identify the appropriate + ``binary_image`` to use. + :param dict index_to_gitlab_push_map: the dict mapping index images (keys) to GitLab repos + (values) in order to push their catalogs into GitLab. + :raises IIBError: if the index image build fails or empty index.db tag not found. + """ + reset_docker_config() + set_request_state(request_id, 'in_progress', 'Preparing request for build') + + # Prepare request + prebuild_info = prepare_request_for_build( + request_id, + RequestConfigCreateIndexImage( + _binary_image=binary_image, + from_index=from_index, + binary_image_config=binary_image_config, + binary_image_less_arches_allowed_versions=binary_image_less_arches_allowed_versions, + ), + ) + + from_index_resolved = prebuild_info['from_index_resolved'] + binary_image_resolved = prebuild_info['binary_image_resolved'] + ocp_version = prebuild_info['ocp_version'] + distribution_scope = prebuild_info['distribution_scope'] + arches = prebuild_info['arches'] + + # Set OPM version + Opm.set_opm_version(from_index_resolved) + opm_version = Opm.opm_version + + # Add labels to prebuild_info + prebuild_info['labels'] = labels + + _update_index_image_build_state(request_id, prebuild_info) + + mr_details: Optional[Dict[str, str]] = None + local_git_repo_path: Optional[str] = None + index_git_repo: Optional[str] = None + last_commit_sha: Optional[str] = None + output_pull_spec: Optional[str] = None + original_index_db_digest: Optional[str] = None + + with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: + branch = ocp_version + + # Set up and clone Git repository + ( + index_git_repo, + local_git_repo_path, + localized_git_catalog_path, + ) = prepare_git_repository_for_build( + request_id=request_id, + from_index=from_index, + temp_dir=temp_dir, + branch=branch, + index_to_gitlab_push_map=index_to_gitlab_push_map or {}, + ) + + # Fetch empty index.db artifact tagged with 'empty' + set_request_state(request_id, 'in_progress', 'Fetching empty index database') + conf = get_worker_config() + empty_tag = conf.get('iib_empty_index_db_tag', 'empty') + + # Construct the pullspec for the empty index.db artifact + image_name, _ = _get_name_and_tag_from_pullspec(from_index) + empty_artifact_ref = conf['iib_index_db_artifact_template'].format( + registry=conf['iib_index_db_artifact_registry'], + tag=_get_artifact_combined_tag(image_name, empty_tag), + ) + + log.info('Fetching empty index.db from %s', empty_artifact_ref) + + try: + artifact_dir = get_oras_artifact( + empty_artifact_ref, + temp_dir, + ) + index_db_path = Path(artifact_dir) / "index.db" + if not index_db_path.is_file(): + raise IIBError( + f"Empty index.db file not found at {index_db_path} " + f"after fetching from {empty_artifact_ref}" + ) + log.info('Successfully fetched empty index.db from %s', empty_artifact_ref) + except IIBError as e: + # Fallback: Create empty index.db from from_index by removing all operators + log.warning( + f"Failed to fetch empty index.db with tag '{empty_tag}': {e}. " + f"Falling back to creating empty index.db from {from_index}" + ) + index_db_path = _create_empty_index_db_from_source( + request_id=request_id, + from_index=from_index, + temp_dir=temp_dir, + ) + + # Create empty FBC catalog directory + # The index.db is already empty, so we want an empty catalog as well + set_request_state(request_id, 'in_progress', 'Creating empty FBC catalog directory') + + localized_catalog_path = Path(localized_git_catalog_path) + if localized_catalog_path.is_dir(): + log.info('Removing all contents from catalog directory to create empty catalog') + shutil.rmtree(localized_catalog_path) + + localized_catalog_path.mkdir(parents=True, exist_ok=True) + log.info('Created empty catalog directory at %s', localized_catalog_path) + + # Create a placeholder file so Git tracks the empty directory + gitkeep_file = localized_catalog_path / '.gitkeep' + with open(gitkeep_file, 'w') as f: + f.write('') + log.info('Created .gitkeep file in empty catalog directory') + + # Create empty catalog directory structure for validation + fbc_dir_path = Path(temp_dir) / 'catalog' + if fbc_dir_path.is_dir(): + shutil.rmtree(fbc_dir_path) + # Copy cleaned catalog to correct location expected in Dockerfile + shutil.copytree(localized_catalog_path, fbc_dir_path) + + # Validate empty catalog + set_request_state(request_id, 'in_progress', 'Validating empty catalog') + opm_validate(str(fbc_dir_path)) + + # Write build metadata to a file to be added with the commit + set_request_state(request_id, 'in_progress', 'Writing build metadata') + + # Write standard build metadata + write_build_metadata( + local_git_repo_path, + opm_version, + ocp_version, + distribution_scope, + binary_image_resolved, + request_id, + arches, + ) + + # Add custom labels to metadata file if provided + # The write_build_metadata function writes standard labels, + # but we need to update it to include custom labels + if labels: + metadata_path = Path(local_git_repo_path) / '.iib-build-metadata.json' + if not metadata_path.is_file(): + raise IIBError( + f"Build metadata file not found at {metadata_path}. " + "write_build_metadata should have created it." + ) + with open(metadata_path, 'r') as f: + metadata = json.load(f) + metadata['labels'].update(labels) + with open(metadata_path, 'w') as f: + json.dump(metadata, f, indent=2) + + try: + # Commit changes and create MR or push directly + # For create_empty_index, overwrite_from_index is always False (throw-away request) + mr_details, last_commit_sha = git_commit_and_create_mr_or_push( + request_id=request_id, + local_git_repo_path=local_git_repo_path, + index_git_repo=index_git_repo, + branch=branch, + commit_message=( + f"IIB: Create empty index for request {request_id}\n\n" + f"Creating empty index image from {from_index}" + ), + overwrite_from_index=False, # Always False for create_empty_index + ) + + # Wait for Konflux pipeline and extract built image URL + image_url = monitor_pipeline_and_extract_image( + request_id=request_id, + last_commit_sha=last_commit_sha, + ) + + # Copy built index to all output pull specs + output_pull_specs = replicate_image_to_tagged_destinations( + request_id=request_id, + image_url=image_url, + ) + + # Use the first output_pull_spec as the primary one for request updates + output_pull_spec = output_pull_specs[0] + # Update request with final output + if not output_pull_spec: + raise IIBError( + "output_pull_spec was not set. " + "This should not happen if the pipeline completed successfully." + ) + + # Update index image pull spec + _update_index_image_pull_spec( + output_pull_spec=output_pull_spec, + request_id=request_id, + arches=arches, + from_index=from_index, + overwrite_from_index=False, # Always False for create_empty_index + overwrite_from_index_token=None, + resolved_prebuild_from_index=from_index_resolved, + # Passing an empty index_repo_map is intentional. In IIB 1.0, if + # overwrite_from_index token is given, we push to git by default at the + # end of a request. In IIB 2.Oh!, the commit is pushed earlier to trigger + # a Konflux pipelinerun. So the old workflow isn't needed. + index_repo_map={}, + ) + + # Push the empty index.db with request ID tag + # Since overwrite_from_index is False, this will only push with request_id tag + # and will not overwrite the v4.x tag + original_index_db_digest = push_index_db_artifact( + request_id=request_id, + from_index=from_index, + index_db_path=str(index_db_path), + operators=[], # Empty list since we're creating an empty index + overwrite_from_index=False, # Always False for create_empty_index + request_type='create_empty_index', + ) + + # Close MR if it was opened + cleanup_merge_request_if_exists(mr_details, index_git_repo) + + set_request_state( + request_id, + 'complete', + 'The empty index image was successfully created', + ) + except Exception as e: + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=index_git_repo, + overwrite_from_index=False, # Always False for create_empty_index + request_id=request_id, + from_index=from_index, + index_repo_map=index_to_gitlab_push_map or {}, + original_index_db_digest=original_index_db_digest, + reason=f"error: {e}", + ) + raise IIBError(f"Failed to create empty index: {e}") + + # Reset Docker config for the next request. This is a fail safe. + reset_docker_config() diff --git a/iib/workers/tasks/build_containerized_fbc_operations.py b/iib/workers/tasks/build_containerized_fbc_operations.py new file mode 100644 index 000000000..ce0979170 --- /dev/null +++ b/iib/workers/tasks/build_containerized_fbc_operations.py @@ -0,0 +1,267 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import logging +import tempfile +from typing import Dict, List, Optional, Set + +from iib.common.common_utils import get_binary_versions +from iib.common.tracing import instrument_tracing +from iib.exceptions import IIBError +from iib.workers.api_utils import set_request_state +from iib.workers.tasks.build import ( + _update_index_image_build_state, + _update_index_image_pull_spec, +) +from iib.workers.tasks.celery import app +from iib.workers.tasks.containerized_utils import ( + cleanup_merge_request_if_exists, + cleanup_on_failure, + fetch_and_verify_index_db_artifact, + git_commit_and_create_mr_or_push, + monitor_pipeline_and_extract_image, + prepare_git_repository_for_build, + push_index_db_artifact, + replicate_image_to_tagged_destinations, + write_build_metadata, +) +from iib.workers.tasks.opm_operations import ( + Opm, + opm_registry_add_fbc_fragment_containerized, +) +from iib.workers.tasks.utils import ( + get_resolved_image, + prepare_request_for_build, + request_logger, + set_registry_token, + RequestConfigFBCOperation, + reset_docker_config, +) + +__all__ = ['handle_containerized_fbc_operation_request'] + +log = logging.getLogger(__name__) + + +@app.task +@request_logger +@instrument_tracing( + span_name="workers.tasks.build.handle_containerized_fbc_operation_request", + attributes=get_binary_versions(), +) +def handle_containerized_fbc_operation_request( + request_id: int, + fbc_fragments: List[str], + from_index: str, + binary_image: Optional[str] = None, + distribution_scope: str = '', + overwrite_from_index: bool = False, + overwrite_from_index_token: Optional[str] = None, + build_tags: Optional[List[str]] = None, + add_arches: Optional[Set[str]] = None, + binary_image_config: Optional[Dict[str, Dict[str, str]]] = None, + index_to_gitlab_push_map: Optional[Dict[str, str]] = None, + used_fbc_fragment: bool = False, + binary_image_less_arches_allowed_versions: Optional[List[str]] = None, +) -> None: + """ + Add fbc fragments to an fbc index image. + + :param list fbc_fragments: list of fbc fragments that need to be added to final FBC index image + :param int request_id: the ID of the IIB build request + :param str binary_image: the pull specification of the container image where the opm binary + gets copied from. + :param str from_index: the pull specification of the container image containing the index that + the index image build will be based from. + :param set add_arches: the set of arches to build in addition to the arches ``from_index`` is + currently built for; if ``from_index`` is ``None``, then this is used as the list of arches + to build the index image for + :param dict index_to_gitlab_push_map: the dict mapping index images (keys) to GitLab repos + (values) in order to push their catalogs into GitLab. + :param bool used_fbc_fragment: flag indicating if the original request used fbc_fragment + (single) instead of fbc_fragments (array). Used for backward compatibility. + :param list binary_image_less_arches_allowed_versions: list of versions of the binary image + that are allowed to build for less arches. Defaults to ``None``. + """ + reset_docker_config() + set_request_state(request_id, 'in_progress', 'Resolving the fbc fragments') + + # Resolve all fbc fragments + resolved_fbc_fragments = [] + for fbc_fragment in fbc_fragments: + with set_registry_token(overwrite_from_index_token, fbc_fragment, append=True): + resolved_fbc_fragment = get_resolved_image(fbc_fragment) + resolved_fbc_fragments.append(resolved_fbc_fragment) + + prebuild_info = prepare_request_for_build( + request_id, + RequestConfigFBCOperation( + _binary_image=binary_image, + from_index=from_index, + overwrite_from_index_token=overwrite_from_index_token, + add_arches=add_arches, + fbc_fragments=fbc_fragments, + distribution_scope=distribution_scope, + binary_image_config=binary_image_config, + binary_image_less_arches_allowed_versions=binary_image_less_arches_allowed_versions, + ), + ) + + from_index_resolved = prebuild_info['from_index_resolved'] + binary_image_resolved = prebuild_info['binary_image_resolved'] + arches = prebuild_info['arches'] + distribution_scope = prebuild_info['distribution_scope'] + + index_to_gitlab_push_map = index_to_gitlab_push_map or {} + # Variables mr_details, last_commit_sha and original_index_db_digest + # needs to be assigned; otherwise cleanup_on_failure() fails when an exception is raised. + mr_details: Optional[Dict[str, str]] = None + last_commit_sha: Optional[str] = None + original_index_db_digest: Optional[str] = None + + Opm.set_opm_version(from_index_resolved) + + # Store all resolved fragments + prebuild_info['fbc_fragments_resolved'] = resolved_fbc_fragments + + # For backward compatibility, only populate old fields if original request used fbc_fragment + # This flag should be passed from the API layer + if used_fbc_fragment and resolved_fbc_fragments: + prebuild_info['fbc_fragment_resolved'] = resolved_fbc_fragments[0] + + _update_index_image_build_state(request_id, prebuild_info) + + with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: + branch = prebuild_info['ocp_version'] + + # Set up and clone Git repository + ( + index_git_repo, + local_git_repo_path, + localized_git_catalog_path, + ) = prepare_git_repository_for_build( + request_id=request_id, + from_index=from_index, + temp_dir=temp_dir, + branch=branch, + index_to_gitlab_push_map=index_to_gitlab_push_map, + ) + + # Pull index.db artifact (uses ImageStream cache if configured, otherwise pulls directly) + artifact_index_db_file = fetch_and_verify_index_db_artifact( + from_index=from_index, + temp_dir=temp_dir, + ) + + set_request_state(request_id, 'in_progress', 'Adding fbc fragment') + ( + updated_catalog_path, + index_db_path, + operators_in_db, + ) = opm_registry_add_fbc_fragment_containerized( + request_id=request_id, + temp_dir=temp_dir, + from_index_configs_dir=localized_git_catalog_path, + fbc_fragments=resolved_fbc_fragments, + overwrite_from_index_token=overwrite_from_index_token, + index_db_path=artifact_index_db_file, + ) + + # Write build metadata to a file to be added with the commit + set_request_state(request_id, 'in_progress', 'Writing build metadata') + write_build_metadata( + local_git_repo_path, + Opm.opm_version, + prebuild_info['ocp_version'], + distribution_scope, + binary_image_resolved, + request_id, + arches, + ) + + try: + # Commit changes and create MR or push directly + mr_details, last_commit_sha = git_commit_and_create_mr_or_push( + request_id=request_id, + local_git_repo_path=local_git_repo_path, + index_git_repo=index_git_repo, + branch=branch, + commit_message=( + f"IIB: Add data from FBC fragments for request {request_id}\n\n" + f"FBC fragments: {', '.join(fbc_fragments)}" + ), + overwrite_from_index=overwrite_from_index, + ) + + # Wait for Konflux pipeline and extract built image URL + image_url = monitor_pipeline_and_extract_image( + request_id=request_id, + last_commit_sha=last_commit_sha, + ) + + # Copy built index to all output pull specs + output_pull_specs = replicate_image_to_tagged_destinations( + request_id=request_id, + image_url=image_url, + build_tags=build_tags, + ) + + # Use the first output_pull_spec as the primary one for request updates + output_pull_spec = output_pull_specs[0] + # Update request with final output + if not output_pull_spec: + raise IIBError( + "output_pull_spec was not set. " + "This should not happen if the pipeline completed successfully." + ) + + _update_index_image_pull_spec( + output_pull_spec=output_pull_spec, + request_id=request_id, + arches=arches, + from_index=from_index, + overwrite_from_index=overwrite_from_index, + overwrite_from_index_token=overwrite_from_index_token, + resolved_prebuild_from_index=from_index_resolved, + add_or_rm=True, + is_image_fbc=True, + # Passing an empty index_repo_map is intentional. In IIB 1.0, if + # the overwrite_from_index token is given, we push to git by default + # at the end of a request. In IIB 2.0, the commit is pushed earlier to trigger + # a Konflux pipelinerun. So the old workflow isn't needed. + index_repo_map={}, + ) + + # Push updated index.db if overwrite_from_index_token is provided + # We can push it directly from temp_dir since we're still inside the + # context manager. Do it as the last step to avoid rolling back the + # index.db file if the pipeline fails. + original_index_db_digest = push_index_db_artifact( + request_id=request_id, + from_index=from_index, + index_db_path=index_db_path, + operators=operators_in_db, + overwrite_from_index=overwrite_from_index, + request_type='fbc_operations', + ) + + # Close MR if it was opened + cleanup_merge_request_if_exists(mr_details, index_git_repo) + + set_request_state( + request_id, + 'complete', + f"The operator(s) {operators_in_db} were successfully removed " + "from the index image", + ) + except Exception as e: + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=index_git_repo, + overwrite_from_index=overwrite_from_index, + request_id=request_id, + from_index=from_index, + index_repo_map=index_to_gitlab_push_map or {}, + original_index_db_digest=original_index_db_digest, + reason=f"error: {e}", + ) + raise IIBError(f"Failed to add FBC fragment: {e}") diff --git a/iib/workers/tasks/build_containerized_merge.py b/iib/workers/tasks/build_containerized_merge.py new file mode 100644 index 000000000..77e33751f --- /dev/null +++ b/iib/workers/tasks/build_containerized_merge.py @@ -0,0 +1,381 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import logging +import os +import tempfile +import shutil +from typing import Dict, List, Optional + + +from iib.common.common_utils import get_binary_versions +from iib.common.tracing import instrument_tracing +from iib.exceptions import IIBError +from iib.workers.api_utils import set_request_state, get_request +from iib.workers.tasks.build import ( + _update_index_image_build_state, + _get_present_bundles, + _update_index_image_pull_spec, +) +from iib.workers.tasks.celery import app +from iib.workers.tasks.containerized_utils import ( + write_build_metadata, + cleanup_on_failure, + cleanup_merge_request_if_exists, + push_index_db_artifact, + validate_bundles_in_parallel, + fetch_and_verify_index_db_artifact, + prepare_git_repository_for_build, + git_commit_and_create_mr_or_push, + monitor_pipeline_and_extract_image, + replicate_image_to_tagged_destinations, +) +from iib.workers.tasks.build_merge_index_image import get_missing_bundles_from_target_to_source +from iib.workers.tasks.build_merge_index_image import get_bundles_latest_version +from iib.workers.tasks.opm_operations import ( + Opm, + _opm_registry_add, + deprecate_bundles_db, + opm_migrate, + opm_validate, + get_list_bundles, +) +from iib.workers.tasks.utils import ( + prepare_request_for_build, + request_logger, + reset_docker_config, + RequestConfigMerge, + set_registry_token, + get_bundles_from_deprecation_list, +) +from iib.workers.tasks.fbc_utils import merge_catalogs_dirs +from iib.workers.tasks.iib_static_types import BundleImage + + +__all__ = ['handle_containerized_merge_request'] + +log = logging.getLogger(__name__) + + +@app.task +@request_logger +@instrument_tracing( + span_name="workers.tasks.build.handle_containerized_merge_request", + attributes=get_binary_versions(), +) +def handle_containerized_merge_request( + source_from_index: str, + deprecation_list: List[str], + request_id: int, + binary_image: Optional[str] = None, + target_index: Optional[str] = None, + overwrite_target_index: bool = False, + overwrite_target_index_token: Optional[str] = None, + distribution_scope: Optional[str] = None, + binary_image_config: Optional[str] = None, + build_tags: Optional[List[str]] = None, + graph_update_mode: Optional[str] = None, + ignore_bundle_ocp_version: Optional[bool] = False, + index_to_gitlab_push_map: Optional[Dict[str, str]] = None, + binary_image_less_arches_allowed_versions: Optional[List[str]] = None, + parallel_threads: int = 5, +) -> None: + """ + Coordinate the work needed to merge old (N) index image with new (N+1) index image. + + :param str source_from_index: pull specification to be used as the base for building the new + index image. + :param str target_index: pull specification of content stage index image for the + corresponding target index image. + :param list deprecation_list: list of deprecated bundles for the target index image. + :param int request_id: the ID of the IIB build request. + :param str binary_image: the pull specification of the container image where the opm binary + gets copied from. + :param bool overwrite_target_index: if True, overwrite the input ``target_index`` with + the built index image. + :param str overwrite_target_index_token: the token used for overwriting the input + ``target_index`` image. This is required to use ``overwrite_target_index``. + The format of the token must be in the format "user:password". + :param str distribution_scope: the scope for distribution of the index image, defaults to + ``None``. + :param build_tags: list of extra tag to use for intermediate index image + :param str graph_update_mode: Graph update mode that defines how channel graphs are updated + in the index. + :param bool ignore_bundle_ocp_version: When set to `true` and image set as target_index is + listed in `iib_no_ocp_label_allow_list` config then bundles without + "com.redhat.openshift.versions" label set will be added in the result `index_image`. + :raises IIBError: if the index image merge fails. + :param dict index_to_gitlab_push_map: the dict mapping index images (keys) to GitLab repos + (values) in order to push their catalogs into GitLab. + :param int parallel_threads: the number of parallel threads to use for validating the bundles + :raises IIBError: if the index image merge fails. + """ + reset_docker_config() + set_request_state(request_id, 'in_progress', 'Preparing request for merge') + + # Prepare request + with set_registry_token(overwrite_target_index_token, target_index, append=True): + prebuild_info = prepare_request_for_build( + request_id, + RequestConfigMerge( + _binary_image=binary_image, + overwrite_target_index_token=overwrite_target_index_token, + source_from_index=source_from_index, + target_index=target_index, + distribution_scope=distribution_scope, + binary_image_config=binary_image_config, + binary_image_less_arches_allowed_versions=binary_image_less_arches_allowed_versions, + ), + ) + + source_from_index_resolved = prebuild_info['source_from_index_resolved'] + target_index_resolved = prebuild_info['target_index_resolved'] + + # Set OPM version + Opm.set_opm_version(target_index_resolved) + opm_version = Opm.opm_version + + _update_index_image_build_state(request_id, prebuild_info) + + mr_details: Optional[Dict[str, str]] = None + local_git_repo_path: Optional[str] = None + index_git_repo: Optional[str] = None + last_commit_sha: Optional[str] = None + output_pull_spec: Optional[str] = None + original_index_db_digest: Optional[str] = None + + with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: + # Setup and clone Git repository + branch = prebuild_info['ocp_version'] + ( + index_git_repo, + local_git_repo_path, + localized_git_catalog_path, + ) = prepare_git_repository_for_build( + request_id=request_id, + from_index=source_from_index, + temp_dir=temp_dir, + branch=branch, + index_to_gitlab_push_map=index_to_gitlab_push_map or {}, + ) + + # Pull both source and target index.db artifacts and read present bundle + target_index_db_path = None + source_index_db_path = fetch_and_verify_index_db_artifact(source_from_index, temp_dir) + if target_index: + target_index_db_path = fetch_and_verify_index_db_artifact(target_index, temp_dir) + + # Get the bundles from the index.db file + with set_registry_token(overwrite_target_index_token, target_index, append=True): + target_index_bundles: List[BundleImage] = [] + target_index_bundles_pull_spec: List[str] = [] + + source_index_bundles, source_index_bundles_pull_spec = _get_present_bundles( + source_index_db_path, temp_dir + ) + log.debug("Source index bundles %s", source_index_bundles) + log.debug("Source index bundles pull spec %s", source_index_bundles_pull_spec) + + if target_index_db_path: + target_index_bundles, target_index_bundles_pull_spec = _get_present_bundles( + target_index_db_path, temp_dir + ) + log.debug("Target index bundles %s", target_index_bundles) + log.debug("Target index bundles pull spec %s", target_index_bundles_pull_spec) + + # Validate the bundles from source and target have their pullspecs present in the registry + set_request_state( + request_id, + 'in_progress', + 'Validating whether the bundles have their pullspecs present in the registry', + ) + unique_bundles = set(source_index_bundles_pull_spec + target_index_bundles_pull_spec) + validate_bundles_in_parallel( + bundles=list(unique_bundles), + threads=parallel_threads, + wait=True, + ) + + set_request_state(request_id, 'in_progress', 'Adding bundles missing in source index image') + log.info('Adding bundles from target index image which are missing from source index image') + + user = get_request(request_id)['user'] + missing_bundles, invalid_bundles = get_missing_bundles_from_target_to_source( + source_index_bundles=source_index_bundles, + target_index_bundles=target_index_bundles, + source_from_index=source_from_index_resolved, + ocp_version=prebuild_info['target_ocp_version'], + request_user=user, + target_index=target_index_resolved, + ignore_bundle_ocp_version=ignore_bundle_ocp_version, + ) + missing_bundle_paths = [bundle['bundlePath'] for bundle in missing_bundles] + + # Add the missing bundles to the index.db file + set_request_state( + request_id, 'in_progress', 'Adding the missing bundles to the source index.db file' + ) + + if target_index_db_path: + _opm_registry_add(temp_dir, source_index_db_path, missing_bundle_paths) + + # Process the deprecation list + set_request_state(request_id, 'in_progress', 'Processing the deprecation list') + intermediate_bundles = missing_bundle_paths + source_index_bundles_pull_spec + deprecation_bundles = get_bundles_from_deprecation_list( + intermediate_bundles, deprecation_list + ) + deprecation_bundles = deprecation_bundles + [ + bundle['bundlePath'] for bundle in invalid_bundles + ] + + # process the deprecation list into the intermediary index.db file + if deprecation_bundles: + # We need to get the latest pullpecs from bundles in order to avoid failures + # on "opm deprecatetruncate" due to versions already removed before. + # Once we give the latest versions all lower ones get automatically deprecated by OPM. + all_bundles = source_index_bundles + target_index_bundles + deprecation_bundles = get_bundles_latest_version(deprecation_bundles, all_bundles) + + deprecate_bundles_db( + base_dir=temp_dir, index_db_file=source_index_db_path, bundles=deprecation_bundles + ) + + # Retrieve the operators from the intermediary index.db file + # This will be required for pushing the updated index.db file to the IIB registry + bundles_in_db = get_list_bundles(source_index_db_path, temp_dir) + operators_in_db = [bundle['packageName'] for bundle in bundles_in_db] + + # Migrate the intermediary index.db file to FBC and generate the Dockerfile + set_request_state( + request_id, + 'in_progress', + 'Migrating the intermediary index.db file to FBC and generating the Dockerfile', + ) + fbc_dir, _ = opm_migrate(source_index_db_path, temp_dir) + + # rename `catalog` directory because we need to use this name for + # final destination of catalog (defined in Dockerfile) + catalog_from_db = os.path.join(temp_dir, 'from_db') + os.rename(fbc_dir, catalog_from_db) + + # Merge migrated FBC with existing FBC in Git repo + # overwrite data in `catalog_from_index` by data from `catalog_from_db` + # this adds changes on not opted in operators to final FBC + log.info('Merging migrated catalog with Git catalog') + merge_catalogs_dirs(catalog_from_db, localized_git_catalog_path) + + # We need to regenerate file-based catalog because we merged changes + fbc_dir_path = os.path.join(temp_dir, 'catalog') + if os.path.exists(fbc_dir_path): + shutil.rmtree(fbc_dir_path) + # Copy catalog to correct location expected in Dockerfile + # Use copytree instead of move to preserve the configs directory in Git repo + shutil.copytree(localized_git_catalog_path, fbc_dir_path) + + # Validate the FBC config + set_request_state(request_id, 'in_progress', 'Validating the FBC config') + opm_validate(fbc_dir_path) + + # Write build metadata to a file to be added with the commit + set_request_state(request_id, 'in_progress', 'Writing build metadata') + arches = set(prebuild_info['arches']) + write_build_metadata( + local_git_repo_path, + opm_version, + prebuild_info['target_ocp_version'], + prebuild_info['distribution_scope'], + prebuild_info['binary_image_resolved'], + request_id, + arches, + ) + + try: + # Commit changes and create PR or push directly + mr_details, last_commit_sha = git_commit_and_create_mr_or_push( + request_id=request_id, + local_git_repo_path=local_git_repo_path, + index_git_repo=index_git_repo, + branch=branch, + commit_message=( + f"IIB: Merge operators for request {request_id}\n\n" + f"Missing bundles: {', '.join(missing_bundle_paths)}" + ), + overwrite_from_index=overwrite_target_index, + ) + + # Wait for Konflux pipeline and extract built image UR + image_url = monitor_pipeline_and_extract_image( + request_id=request_id, + last_commit_sha=last_commit_sha, + ) + + # Copy built index to all output pull specs + output_pull_specs = replicate_image_to_tagged_destinations( + request_id=request_id, + image_url=image_url, + build_tags=build_tags, + ) + + # Use the first output_pull_spec as the primary one for request updates + output_pull_spec = output_pull_specs[0] + # Update request with final output + if not output_pull_spec: + raise IIBError( + "output_pull_spec was not set. " + "This should not happen if the pipeline completed successfully." + ) + + _update_index_image_pull_spec( + output_pull_spec=output_pull_spec, + request_id=request_id, + arches=prebuild_info['arches'], + from_index=source_from_index, + overwrite_from_index=overwrite_target_index, + overwrite_from_index_token=overwrite_target_index_token, + resolved_prebuild_from_index=source_from_index_resolved, + add_or_rm=True, + is_image_fbc=True, + # Passing an empty index_repo_map is intentional. In IIB 1.0, if + # the overwrite_from_index token is given, we push to git by default + # at the end of a request. In IIB 2.0, the commit is pushed earlier to trigger + # a Konflux pipelinerun. So the old workflow isn't needed. + index_repo_map={}, + ) + + # Push updated index.db if overwrite_target_index_token is provided + # We can push it directly from temp_dir since we're still inside the + # context manager. Do it as the last step to avoid rolling back the + # index.db file if the pipeline fails. + original_index_db_digest = push_index_db_artifact( + request_id=request_id, + from_index=source_from_index, + index_db_path=source_index_db_path, + operators=operators_in_db, + overwrite_from_index=overwrite_target_index, + request_type='merge', + ) + + # Close MR if it was opened + cleanup_merge_request_if_exists(mr_details, index_git_repo) + + # Update request with final output + set_request_state( + request_id, + 'complete', + f"The operator(s) {operators_in_db} were successfully merged " + "from the target index image into the source index image", + ) + except Exception as e: + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=index_git_repo, + overwrite_from_index=overwrite_target_index, + request_id=request_id, + from_index=source_from_index, + index_repo_map={}, + original_index_db_digest=original_index_db_digest, + reason=f"error: {e}", + ) + # Reset Docker config for the next request. This is a fail safe. + reset_docker_config() + raise IIBError(f"Failed to merge operators: {e}") diff --git a/iib/workers/tasks/build_containerized_regenerate_bundle.py b/iib/workers/tasks/build_containerized_regenerate_bundle.py new file mode 100644 index 000000000..3f3d6bae4 --- /dev/null +++ b/iib/workers/tasks/build_containerized_regenerate_bundle.py @@ -0,0 +1,280 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import json +import logging +import tempfile +import textwrap +from pathlib import Path +from typing import Any, Dict, List, Optional + +import ruamel.yaml + +from iib.common.common_utils import get_binary_versions +from iib.common.tracing import instrument_tracing +from iib.exceptions import IIBError +from iib.workers.api_utils import set_request_state, update_request +from iib.workers.config import get_worker_config +from iib.workers.tasks.build_regenerate_bundle import ( + _adjust_operator_bundle, + _get_package_annotations, +) +from iib.workers.tasks.celery import app +from iib.workers.tasks.containerized_utils import ( + extract_files_from_image_non_privileged, + git_commit_and_create_mr_or_push, + monitor_pipeline_and_extract_image, + replicate_image_to_tagged_destinations, + cleanup_on_failure, + cleanup_merge_request_if_exists, +) +from iib.workers.tasks.git_utils import ( + clone_git_repo, + get_git_token, +) +from iib.workers.tasks.utils import ( + get_image_arches, + get_image_labels, + get_resolved_image, + request_logger, + set_registry_auths, +) +from iib.workers.tasks.iib_static_types import UpdateRequestPayload + + +__all__ = ['handle_containerized_regenerate_bundle_request'] + +yaml = ruamel.yaml.YAML() +# IMPORTANT: ruamel will introduce a line break if the yaml line is longer than yaml.width. +# Unfortunately, this causes issues for JSON values nested within a YAML file, e.g. +# metadata.annotations."alm-examples" in a CSV file. +# The default value is 80. Set it to a more forgiving higher number to avoid issues +yaml.width = 200 +# ruamel will also cause issues when normalizing a YAML object that contains +# a nested JSON object when it does not preserve quotes. Thus, it produces +# invalid YAML. Let's prevent this from happening at all. +yaml.preserve_quotes = True +log = logging.getLogger(__name__) + + +@app.task +@request_logger +@instrument_tracing( + span_name="workers.tasks.build.handle_containerized_regenerate_bundle_request", + attributes=get_binary_versions(), +) +def handle_containerized_regenerate_bundle_request( + from_bundle_image: str, + organization: str, + request_id: int, + registry_auths: Optional[Dict[str, Any]] = None, + bundle_replacements: Optional[Dict[str, str]] = None, + index_to_gitlab_push_map: Optional[Dict[str, str]] = None, + binary_image_less_arches_allowed_versions: Optional[List[str]] = None, + regenerate_bundle_repo_key: str = 'regenerate-bundle', +) -> None: + """ + Coordinate the work needed to regenerate the operator bundle image using containerized workflow. + + :param str from_bundle_image: the pull specification of the bundle image to be regenerated. + :param str organization: the name of the organization the bundle should be regenerated for. + :param int request_id: the ID of the IIB build request. + :param dict registry_auths: Provide the dockerconfig.json for authentication to private + registries, defaults to ``None``. + :param dict bundle_replacements: Dictionary mapping from original bundle pullspecs to rebuilt + bundle pullspecs. + :param dict index_to_gitlab_push_map: the dict mapping index images (keys) to GitLab repos + (values) in order to push their catalogs into GitLab. + :param str regenerate_bundle_repo_key: the key to look up the actual repo URL from + index_to_gitlab_push_map, defaults to ``regenerate-bundle``. + :param list binary_image_less_arches_allowed_versions: list of versions of the binary image + that are allowed to build for less arches. Defaults to ``None``. + :raises IIBError: if the regenerate bundle image build fails. + """ + bundle_replacements = bundle_replacements or {} + + set_request_state(request_id, 'in_progress', 'Resolving from_bundle_image') + + mr_details: Optional[Dict[str, str]] = None + bundle_git_repo: Optional[str] = None + last_commit_sha: Optional[str] = None + output_pull_spec: Optional[str] = None + + with set_registry_auths(registry_auths): + from_bundle_image_resolved = get_resolved_image(from_bundle_image) + + arches = get_image_arches(from_bundle_image_resolved) + if not arches: + raise IIBError( + 'No arches were found in the resolved from_bundle_image ' + f'{from_bundle_image_resolved}' + ) + + pinned_by_iib_label = ( + get_image_labels(from_bundle_image_resolved).get('com.redhat.iib.pinned') or 'false' + ) + pinned_by_iib = yaml.load(pinned_by_iib_label) + + arches_str = ', '.join(sorted(arches)) + log.debug('Set to regenerate the bundle image for the following arches: %s', arches_str) + + payload: UpdateRequestPayload = { + 'from_bundle_image_resolved': from_bundle_image_resolved, + 'state': 'in_progress', + 'state_reason': f'Regenerating the bundle image for the following arches: {arches_str}', + } + exc_msg = 'Failed setting the resolved "from_bundle_image" on the request' + update_request(request_id, payload, exc_msg=exc_msg) + + with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: + bundle_git_repo = ( + index_to_gitlab_push_map.get(regenerate_bundle_repo_key) + if index_to_gitlab_push_map + else None + ) + if not bundle_git_repo: + raise IIBError(f"Repository not found for key: {regenerate_bundle_repo_key}") + # Get Git token + token_name, git_token = get_git_token(bundle_git_repo) + + # Clone Git repository + set_request_state(request_id, 'in_progress', 'Cloning Git repository') + # Use regenerate_bundle_repo_key as branch name for bundle regeneration + branch = regenerate_bundle_repo_key + local_git_repo_path = Path(temp_dir) / 'git' / branch + local_git_repo_path.mkdir(parents=True, exist_ok=True) + clone_git_repo(bundle_git_repo, branch, token_name, git_token, str(local_git_repo_path)) + + # Extract bundle contents + set_request_state(request_id, 'in_progress', 'Extracting bundle contents') + manifests_path = local_git_repo_path / 'manifests' + extract_files_from_image_non_privileged( + from_bundle_image_resolved, '/manifests', str(manifests_path) + ) + metadata_path = local_git_repo_path / 'metadata' + extract_files_from_image_non_privileged( + from_bundle_image_resolved, '/metadata', str(metadata_path) + ) + + # Apply bundle modifications + set_request_state(request_id, 'in_progress', 'Modifying bundle manifests') + new_labels = _adjust_operator_bundle( + str(manifests_path), + str(metadata_path), + request_id, + organization=organization, + pinned_by_iib=pinned_by_iib, + bundle_replacements=bundle_replacements, + ) + + # Get package name for metadata + annotations_yaml = _get_package_annotations(str(metadata_path)) + package_name = annotations_yaml['annotations'][ + 'operators.operatorframework.io.bundle.package.v1' + ] + + # Create Dockerfile with labels + set_request_state(request_id, 'in_progress', 'Creating Dockerfile') + dockerfile_path = local_git_repo_path / 'Dockerfile' + with open(dockerfile_path, 'w') as dockerfile: + dockerfile.write( + textwrap.dedent( + f"""\ + FROM {from_bundle_image_resolved} + COPY ./manifests /manifests + COPY ./metadata /metadata + """ + ) + ) + # Add labels directly in Dockerfile + for name, value in new_labels.items(): + dockerfile.write(f'LABEL {name}={value}\n') + + # Write build metadata (without distribution_scope, ocp_version, and opm_version) + set_request_state(request_id, 'in_progress', 'Writing build metadata') + metadata = { + 'request_id': request_id, + 'arches': sorted(list(arches)), + 'organization': organization, + 'package_name': package_name, + } + metadata_path_file = local_git_repo_path / '.iib-build-metadata.json' + with open(metadata_path_file, 'w') as f: + json.dump(metadata, f, indent=2) + log.info('Written build metadata to %s', metadata_path_file) + + try: + # Commit changes and create MR to trigger Konflux pipeline + # Bundle regeneration is always a throw-away request (no overwrite) + mr_details, last_commit_sha = git_commit_and_create_mr_or_push( + request_id=request_id, + local_git_repo_path=str(local_git_repo_path), + index_git_repo=bundle_git_repo, + branch=branch, + commit_message=( + f"IIB: Regenerate bundle for request {request_id}\n\n" + f"Organization: {organization}\n" + f"Package: {package_name}" + ), + overwrite_from_index=False, # Always use MR for bundle regeneration + ) + + # Wait for Konflux pipeline and extract built image URL + image_url = monitor_pipeline_and_extract_image( + request_id=request_id, + last_commit_sha=last_commit_sha, + ) + + # Copy built bundle to all output pull specs + output_pull_specs = replicate_image_to_tagged_destinations( + request_id=request_id, + image_url=image_url, + build_tags=None, # No additional tags for bundle regeneration + ) + + # Use the first output_pull_spec as the primary one + if not output_pull_specs: + raise IIBError( + "No output pull specs were generated. " + "This should not happen if the pipeline completed successfully." + ) + output_pull_spec = output_pull_specs[0] + + # Apply output registry replacement if configured + conf = get_worker_config() + if conf.get('iib_index_image_output_registry'): + old_output_pull_spec = output_pull_spec + output_pull_spec = output_pull_spec.replace( + conf['iib_registry'], conf['iib_index_image_output_registry'], 1 + ) + log.info( + 'Changed the bundle_image pull specification from %s to %s', + old_output_pull_spec, + output_pull_spec, + ) + + # Close MR if it was opened + cleanup_merge_request_if_exists(mr_details, bundle_git_repo) + + # Update request with final output + payload = { + 'arches': list(arches), + 'bundle_image': output_pull_spec, + 'state': 'complete', + 'state_reason': 'The request completed successfully', + } + update_request( + request_id, payload, exc_msg='Failed setting the bundle image on the request' + ) + + except Exception as e: + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=bundle_git_repo, + overwrite_from_index=False, # Bundle regeneration never overwrites + request_id=request_id, + from_index='', # No from_index for bundle regeneration + index_repo_map={}, + original_index_db_digest=None, # No index.db for bundle regeneration + reason=f"error: {e}", + ) + raise IIBError(f"Failed to regenerate bundle: {e}") diff --git a/iib/workers/tasks/build_containerized_rm.py b/iib/workers/tasks/build_containerized_rm.py new file mode 100644 index 000000000..bc1370070 --- /dev/null +++ b/iib/workers/tasks/build_containerized_rm.py @@ -0,0 +1,325 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import logging +import os +import shutil +import tempfile +from typing import Dict, List, Optional, Set + +from iib.common.common_utils import get_binary_versions +from iib.common.tracing import instrument_tracing +from iib.exceptions import IIBError +from iib.workers.api_utils import set_request_state +from iib.workers.tasks.build import ( + _update_index_image_build_state, + _update_index_image_pull_spec, +) +from iib.workers.tasks.celery import app +from iib.workers.tasks.containerized_utils import ( + cleanup_merge_request_if_exists, + cleanup_on_failure, + fetch_and_verify_index_db_artifact, + git_commit_and_create_mr_or_push, + monitor_pipeline_and_extract_image, + prepare_git_repository_for_build, + push_index_db_artifact, + replicate_image_to_tagged_destinations, + write_build_metadata, +) +from iib.workers.tasks.fbc_utils import merge_catalogs_dirs +from iib.workers.tasks.opm_operations import ( + Opm, + opm_registry_rm_fbc, + opm_validate, + remove_operator_deprecations, + verify_operators_exists, +) +from iib.workers.tasks.utils import ( + prepare_request_for_build, + reset_docker_config, + request_logger, + RequestConfigAddRm, +) + +__all__ = ['handle_containerized_rm_request'] + +log = logging.getLogger(__name__) + + +@app.task +@request_logger +@instrument_tracing( + span_name="workers.tasks.build.handle_containerized_rm_request", + attributes=get_binary_versions(), +) +def handle_containerized_rm_request( + operators: List[str], + request_id: int, + from_index: str, + binary_image: Optional[str] = None, + add_arches: Optional[Set[str]] = None, + overwrite_from_index: bool = False, + overwrite_from_index_token: Optional[str] = None, + distribution_scope: Optional[str] = None, + binary_image_config: Optional[Dict[str, Dict[str, str]]] = None, + build_tags: Optional[List[str]] = None, + index_to_gitlab_push_map: Optional[Dict[str, str]] = None, + binary_image_less_arches_allowed_versions: Optional[List[str]] = None, +) -> None: + """ + Coordinate the work needed to remove the input operators using containerized workflow. + + This function uses Git-based workflows and Konflux pipelines instead of local builds. + + :param list operators: a list of strings representing the name of the operators to + remove from the index image. + :param int request_id: the ID of the IIB build request + :param str from_index: the pull specification of the container image containing the index that + the index image build will be based from. + :param str binary_image: the pull specification of the container image where the opm binary + gets copied from. + :param set add_arches: the set of arches to build in addition to the arches ``from_index`` is + currently built for. + :param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built + index image. + :param str overwrite_from_index_token: the token used for overwriting the input + ``from_index`` image. This is required to use ``overwrite_from_index``. + The format of the token must be in the format "user:password". + :param str distribution_scope: the scope for distribution of the index image, defaults to + ``None``. + :param dict binary_image_config: the dict of config required to identify the appropriate + ``binary_image`` to use. + :param list build_tags: List of tags which will be applied to intermediate index images. + :param dict index_to_gitlab_push_map: the dict mapping index images (keys) to GitLab repos + (values) in order to remove their catalogs from GitLab. + :param list binary_image_less_arches_allowed_versions: list of versions of the binary image + that are allowed to build for less arches. Defaults to ``None``. + :raises IIBError: if the index image build fails. + """ + reset_docker_config() + set_request_state(request_id, 'in_progress', 'Preparing request for build') + + # Prepare request + prebuild_info = prepare_request_for_build( + request_id, + RequestConfigAddRm( + _binary_image=binary_image, + from_index=from_index, + overwrite_from_index_token=overwrite_from_index_token, + add_arches=add_arches, + distribution_scope=distribution_scope, + binary_image_config=binary_image_config, + binary_image_less_arches_allowed_versions=binary_image_less_arches_allowed_versions, + ), + ) + + from_index_resolved = prebuild_info['from_index_resolved'] + binary_image_resolved = prebuild_info['binary_image_resolved'] + ocp_version = prebuild_info['ocp_version'] + distribution_scope = prebuild_info['distribution_scope'] + arches = prebuild_info['arches'] + + # Set OPM version + Opm.set_opm_version(from_index_resolved) + opm_version = Opm.opm_version + + _update_index_image_build_state(request_id, prebuild_info) + + mr_details: Optional[Dict[str, str]] = None + local_git_repo_path: Optional[str] = None + index_git_repo: Optional[str] = None + operators_in_db: Set[str] = set() + last_commit_sha: Optional[str] = None + output_pull_spec: Optional[str] = None + original_index_db_digest: Optional[str] = None + + with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: + branch = ocp_version + + # Set up and clone Git repository + ( + index_git_repo, + local_git_repo_path, + localized_git_catalog_path, + ) = prepare_git_repository_for_build( + request_id=request_id, + from_index=from_index, + temp_dir=temp_dir, + branch=branch, + index_to_gitlab_push_map=index_to_gitlab_push_map or {}, + ) + + # Pull index.db artifact (uses ImageStream cache if configured, otherwise pulls directly) + index_db_path = fetch_and_verify_index_db_artifact( + from_index=from_index, + temp_dir=temp_dir, + ) + + # Remove operators from /configs + set_request_state(request_id, 'in_progress', 'Removing operators from catalog') + for operator in operators: + operator_path = os.path.join(localized_git_catalog_path, operator) + if os.path.exists(operator_path): + log.debug('Removing operator from catalog: %s', operator_path) + shutil.rmtree(operator_path) + + # Remove operator deprecations + remove_operator_deprecations( + from_index_configs_dir=localized_git_catalog_path, operators=operators + ) + + # Check if operators exist in index.db and remove if present + set_request_state(request_id, 'in_progress', 'Checking and removing from index database') + operators_in_db_list, index_db_path_verified = verify_operators_exists( + from_index=None, + base_dir=temp_dir, + operator_packages=operators, + overwrite_from_index_token=overwrite_from_index_token, + index_db_path=index_db_path, + ) + operators_in_db = set(operators_in_db_list) + + # Use verified path or fall back to original + if index_db_path_verified: + index_db_path = index_db_path_verified + + if operators_in_db: + log.info('Removing operators %s from index.db', operators_in_db) + # Remove from index.db and migrate to FBC + fbc_dir, _ = opm_registry_rm_fbc( + base_dir=temp_dir, + from_index=from_index_resolved, + operators=list[str](operators_in_db), + index_db_path=index_db_path, + ) + + # rename `catalog` directory because we need to use this name for + # final destination of catalog (defined in Dockerfile) + catalog_from_db = os.path.join(temp_dir, 'from_db') + os.rename(fbc_dir, catalog_from_db) + + # Merge migrated FBC with existing FBC in Git repo + # overwrite data in `catalog_from_index` by data from `catalog_from_db` + # this adds changes on not opted in operators to final FBC + log.info('Merging migrated catalog with Git catalog') + merge_catalogs_dirs(catalog_from_db, localized_git_catalog_path) + + fbc_dir_path = os.path.join(temp_dir, 'catalog') + # We need to regenerate file-based catalog because we merged changes + if os.path.exists(fbc_dir_path): + shutil.rmtree(fbc_dir_path) + # Copy catalog to correct location expected in Dockerfile + # Use copytree instead of move to preserve the configs directory in Git repo + shutil.copytree(localized_git_catalog_path, fbc_dir_path) + + # Validate merged catalog + set_request_state(request_id, 'in_progress', 'Validating catalog') + opm_validate(fbc_dir_path) + + # Write build metadata to a file to be added with the commit + set_request_state(request_id, 'in_progress', 'Writing build metadata') + write_build_metadata( + local_git_repo_path, + opm_version, + ocp_version, + distribution_scope, + binary_image_resolved, + request_id, + arches, + ) + + try: + # Commit changes and create MR or push directly + operators_str = ', '.join(operators) + mr_details, last_commit_sha = git_commit_and_create_mr_or_push( + request_id=request_id, + local_git_repo_path=local_git_repo_path, + index_git_repo=index_git_repo, + branch=branch, + commit_message=( + f"IIB: Remove operators for request {request_id}\n\n" + f"Operators: {operators_str}" + ), + overwrite_from_index=overwrite_from_index, + ) + + # Wait for Konflux pipeline and extract built image URL + image_url = monitor_pipeline_and_extract_image( + request_id=request_id, + last_commit_sha=last_commit_sha, + ) + + # Copy built index to all output pull specs + output_pull_specs = replicate_image_to_tagged_destinations( + request_id=request_id, + image_url=image_url, + build_tags=build_tags, + ) + + # Use the first output_pull_spec as the primary one for request updates + output_pull_spec = output_pull_specs[0] + # Update request with final output + if not output_pull_spec: + raise IIBError( + "output_pull_spec was not set. " + "This should not happen if the pipeline completed successfully." + ) + + # Send an empty index_repo_map because the Git repository is already + # updated with the changes + _update_index_image_pull_spec( + output_pull_spec=output_pull_spec, + request_id=request_id, + arches=arches, + from_index=from_index, + overwrite_from_index=overwrite_from_index, + overwrite_from_index_token=overwrite_from_index_token, + resolved_prebuild_from_index=from_index_resolved, + add_or_rm=True, + is_image_fbc=True, + # Passing an empty index_repo_map is intentional. In IIB 1.0, if + # overwrite_from_index token is given, we push to git by default at the + # end of a request. In IIB 2.0, the commit is pushed earlier to trigger + # a Konflux pipelinerun. So the old workflow isn't needed. + index_repo_map={}, + rm_operators=operators, + ) + + # Push updated index.db if overwrite_from_index is True + # We can push it directly from temp_dir since we're still inside the + # context manager. Do it as the last step to avoid rolling back the + # index.db file if the pipeline fails. + original_index_db_digest = push_index_db_artifact( + request_id=request_id, + from_index=from_index, + index_db_path=index_db_path, + operators=operators, + overwrite_from_index=overwrite_from_index, + request_type='rm', + ) + + # Close MR if it was opened + cleanup_merge_request_if_exists(mr_details, index_git_repo) + + operators_str = ', '.join(operators) + set_request_state( + request_id, + 'complete', + f"The operator(s) {operators_str} were successfully removed " + "from the index image", + ) + except Exception as e: + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=index_git_repo, + overwrite_from_index=overwrite_from_index, + request_id=request_id, + from_index=from_index, + index_repo_map=index_to_gitlab_push_map or {}, + original_index_db_digest=original_index_db_digest, + reason=f"error: {e}", + ) + raise IIBError(f"Failed to remove operators: {e}") + + # Reset Docker config for the next request. This is a fail safe. + reset_docker_config() diff --git a/iib/workers/tasks/build_create_empty_index.py b/iib/workers/tasks/build_create_empty_index.py index c054201f9..b4cd6a6e3 100644 --- a/iib/workers/tasks/build_create_empty_index.py +++ b/iib/workers/tasks/build_create_empty_index.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import logging import tempfile -from typing import Dict, Optional +from typing import Dict, List, Optional from iib.common.tracing import instrument_tracing from iib.exceptions import IIBError @@ -48,6 +48,7 @@ def handle_create_empty_index_request( binary_image: Optional[str] = None, labels: Optional[Dict[str, str]] = None, binary_image_config: Optional[Dict[str, Dict[str, str]]] = None, + binary_image_less_arches_allowed_versions: Optional[List[str]] = None, ) -> None: """Coordinate the the work needed to create the index image with labels. @@ -60,6 +61,8 @@ def handle_create_empty_index_request( :param dict labels: the dict of labels required to be added to a new index image :param dict binary_image_config: the dict of config required to identify the appropriate ``binary_image`` to use. + :param list binary_image_less_arches_allowed_versions: list of versions of the binary image + that are allowed to build for less arches. Defaults to ``None``. """ _cleanup() prebuild_info: PrebuildInfo = prepare_request_for_build( @@ -68,6 +71,7 @@ def handle_create_empty_index_request( _binary_image=binary_image, from_index=from_index, binary_image_config=binary_image_config, + binary_image_less_arches_allowed_versions=binary_image_less_arches_allowed_versions, ), ) from_index_resolved = prebuild_info['from_index_resolved'] diff --git a/iib/workers/tasks/build_merge_index_image.py b/iib/workers/tasks/build_merge_index_image.py index 8b3c24c6d..a360620eb 100644 --- a/iib/workers/tasks/build_merge_index_image.py +++ b/iib/workers/tasks/build_merge_index_image.py @@ -86,53 +86,39 @@ def _filter_out_pure_fbc_bundles( return res_bundles, res_pullspec -def _add_bundles_missing_in_source( +def get_missing_bundles_from_target_to_source( source_index_bundles: List[BundleImage], target_index_bundles: List[BundleImage], - base_dir: str, - binary_image: str, source_from_index: str, - request_id: int, - arch: str, ocp_version: str, - distribution_scope: str, - graph_update_mode: Optional[str] = None, + request_user: str, target_index=None, - overwrite_target_index_token: Optional[str] = None, ignore_bundle_ocp_version: Optional[bool] = False, ) -> Tuple[List[BundleImage], List[BundleImage]]: """ - Rebuild index image with bundles missing from source image but present in target image. + Generate a list of missing bundles from the source but present in the target. - If no bundles are missing in the source index image, the index image is still rebuilt - using the new binary image. + This function will not build the index image, it will only generate a list of bundles missing + from the source index image but present in the target index image, as well as a list of bundles + in the new index whose ocp_version range does not satisfy the ocp_version value of the target + index. :param list source_index_bundles: bundles present in the source index image. :param list target_index_bundles: bundles present in the target index image. - :param str base_dir: base directory where operation files will be located. - :param str binary_image: binary image to be used by the new index image. :param str source_from_index: index image, whose data will be contained in the new index image. - :param int request_id: the ID of the IIB build request. - :param str arch: the architecture to build this image for. :param str ocp_version: ocp version which will be added as a label to the image. - :param str graph_update_mode: Graph update mode that defines how channel graphs are updated - in the index. + :param str request_user: username of a requestor :param str target_index: the pull specification of the container image - :param str overwrite_target_index_token: the token used for overwriting the input - ``source_from_index`` image. This is required to use ``overwrite_target_index``. - The format of the token must be in the format "user:password". :param bool ignore_bundle_ocp_version: When set to `true` and image set as target_index is listed in `iib_no_ocp_label_allow_list` config then bundles without "com.redhat.openshift.versions" label set will be added in the result `index_image`. - :return: tuple where the first value is a list of bundles which were added to the index image - and the second value is a list of bundles in the new index whose ocp_version range does not - satisfy the ocp_version value of the target index. + + :return: tuple where the first value is a list of bundles missing in the source and are present + in the target index image and the second value is a list of bundles whose ocp_version + range does not satisfy the ocp_version value of the target index. :rtype: tuple """ - set_request_state(request_id, 'in_progress', 'Adding bundles missing in source index image') - log.info('Adding bundles from target index image which are missing from source index image') missing_bundles = [] - missing_bundle_paths = [] # This list stores the bundles whose ocp_version range does not satisfy the ocp_version # of the target index invalid_bundles = [] @@ -162,15 +148,13 @@ def _add_bundles_missing_in_source( and bundle['csvName'] not in source_bundle_csv_names ): missing_bundles.append(bundle) - missing_bundle_paths.append(bundle['bundlePath']) if ignore_bundle_ocp_version: target_index_tmp = '' if target_index is None else target_index - user = get_request(request_id)['user'] allow_no_ocp_version = any( target_index_tmp.startswith(entry) or source_from_index.startswith(entry) - or user == entry + or request_user == entry for entry in get_worker_config()['iib_no_ocp_label_allow_list'] ) else: @@ -190,6 +174,67 @@ def _add_bundles_missing_in_source( '%s bundles have invalid version label and will be deprecated.', len(invalid_bundles) ) + return missing_bundles, invalid_bundles + + +def _add_bundles_missing_in_source( + source_index_bundles: List[BundleImage], + target_index_bundles: List[BundleImage], + base_dir: str, + binary_image: str, + source_from_index: str, + request_id: int, + arch: str, + ocp_version: str, + distribution_scope: str, + graph_update_mode: Optional[str] = None, + target_index=None, + overwrite_target_index_token: Optional[str] = None, + ignore_bundle_ocp_version: Optional[bool] = False, +) -> Tuple[List[BundleImage], List[BundleImage]]: + """ + Rebuild index image with bundles missing from source image but present in target image. + + If no bundles are missing in the source index image, the index image is still rebuilt + using the new binary image. + + :param list source_index_bundles: bundles present in the source index image. + :param list target_index_bundles: bundles present in the target index image. + :param str base_dir: base directory where operation files will be located. + :param str binary_image: binary image to be used by the new index image. + :param str source_from_index: index image, whose data will be contained in the new index image. + :param int request_id: the ID of the IIB build request. + :param str arch: the architecture to build this image for. + :param str ocp_version: ocp version which will be added as a label to the image. + :param str graph_update_mode: Graph update mode that defines how channel graphs are updated + in the index. + :param str target_index: the pull specification of the container image + :param str overwrite_target_index_token: the token used for overwriting the input + ``source_from_index`` image. This is required to use ``overwrite_target_index``. + The format of the token must be in the format "user:password". + :param bool ignore_bundle_ocp_version: When set to `true` and image set as target_index is + listed in `iib_no_ocp_label_allow_list` config then bundles without + "com.redhat.openshift.versions" label set will be added in the result `index_image`. + :return: tuple where the first value is a list of bundles which were added to the index image + and the second value is a list of bundles in the new index whose ocp_version range does not + satisfy the ocp_version value of the target index. + :rtype: tuple + """ + set_request_state(request_id, 'in_progress', 'Adding bundles missing in source index image') + log.info('Adding bundles from target index image which are missing from source index image') + + user = get_request(request_id)['user'] + missing_bundles, invalid_bundles = get_missing_bundles_from_target_to_source( + source_index_bundles=source_index_bundles, + target_index_bundles=target_index_bundles, + source_from_index=source_from_index, + ocp_version=ocp_version, + request_user=user, + target_index=target_index, + ignore_bundle_ocp_version=ignore_bundle_ocp_version, + ) + missing_bundle_paths = [bundle['bundlePath'] for bundle in missing_bundles] + with set_registry_token(overwrite_target_index_token, target_index, append=True): is_source_fbc = is_image_fbc(source_from_index) if is_source_fbc: diff --git a/iib/workers/tasks/build_recursive_related_bundles.py b/iib/workers/tasks/build_recursive_related_bundles.py index 535041207..ff044fa4b 100644 --- a/iib/workers/tasks/build_recursive_related_bundles.py +++ b/iib/workers/tasks/build_recursive_related_bundles.py @@ -12,10 +12,6 @@ from iib.common.tracing import instrument_tracing from iib.exceptions import IIBError from iib.workers.api_utils import set_request_state, update_request -from iib.workers.tasks.build import ( - _cleanup, - _copy_files_from_image, -) from iib.workers.tasks.build_regenerate_bundle import ( _adjust_operator_bundle, get_related_bundle_images, @@ -23,9 +19,9 @@ ) from iib.workers.config import get_worker_config from iib.workers.tasks.celery import app +from iib.workers.tasks.containerized_utils import extract_files_from_image_non_privileged from iib.workers.tasks.utils import ( get_resolved_image, - podman_pull, request_logger, set_registry_auths, get_bundle_metadata, @@ -71,8 +67,6 @@ def handle_recursive_related_bundles_request( registries, defaults to ``None``. :raises IIBError: if the recursive related bundles build fails. """ - _cleanup() - set_request_state(request_id, 'in_progress', 'Resolving parent_bundle_image') with set_registry_auths(registry_auths): @@ -127,7 +121,6 @@ def handle_recursive_related_bundles_request( 'state': 'complete', 'state_reason': 'The request completed successfully', } - _cleanup() update_request(request_id, payload, exc_msg='Failed setting the bundle image on the request') @@ -145,14 +138,11 @@ def process_parent_bundle_image( :return: the list of all children bundles for a parent bundle image :raises IIBError: if fails to process the parent bundle image. """ - # Pull the bundle_image to ensure steps later on don't fail due to registry timeouts - podman_pull(bundle_image_resolved) - with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: manifests_path = os.path.join(temp_dir, 'manifests') - _copy_files_from_image(bundle_image_resolved, '/manifests', manifests_path) + extract_files_from_image_non_privileged(bundle_image_resolved, '/manifests', manifests_path) metadata_path = os.path.join(temp_dir, 'metadata') - _copy_files_from_image(bundle_image_resolved, '/metadata', metadata_path) + extract_files_from_image_non_privileged(bundle_image_resolved, '/metadata', metadata_path) if organization: _adjust_operator_bundle( manifests_path, diff --git a/iib/workers/tasks/containerized_utils.py b/iib/workers/tasks/containerized_utils.py new file mode 100644 index 000000000..679730e63 --- /dev/null +++ b/iib/workers/tasks/containerized_utils.py @@ -0,0 +1,705 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""This file contains utility functions for containerized IIB operations.""" +import json +import logging +import queue +import shutil +import tarfile +import tempfile +import threading +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union + +from iib.exceptions import IIBError +from iib.workers.api_utils import set_request_state +from iib.workers.config import get_worker_config +from iib.workers.tasks.iib_static_types import BundleImage +from iib.workers.tasks.build import _skopeo_copy +from iib.workers.tasks.git_utils import ( + clone_git_repo, + close_mr, + commit_and_push, + create_mr, + get_git_token, + get_last_commit_sha, + resolve_git_url, + revert_last_commit, +) +from iib.workers.tasks.konflux_utils import ( + find_pipelinerun, + get_pipelinerun_image_url, + wait_for_pipeline_completion, +) +from iib.workers.tasks.oras_utils import ( + _get_artifact_combined_tag, + _get_name_and_tag_from_pullspec, + get_image_digest, + get_indexdb_artifact_pullspec, + get_imagestream_artifact_pullspec, + get_oras_artifact, + push_oras_artifact, + refresh_indexdb_cache_for_image, + verify_indexdb_cache_for_image, +) +from iib.workers.tasks.utils import run_cmd, skopeo_inspect + +log = logging.getLogger(__name__) + + +def extract_files_from_image_non_privileged(image: str, src_path: str, dest_path: str) -> None: + """ + Extract files from container image without podman/docker runtime. + + This function uses skopeo to download the image as OCI layout, then extracts + the requested path from the image layers. This approach works in non-privileged + environments without container runtime access. + + :param str image: the pull specification of the container image + :param str src_path: the full path within the container image to copy from + :param str dest_path: the full path on the local host to copy into + :raises IIBError: if the extraction fails or src_path is not found + """ + # Create temporary directory for OCI layout + with tempfile.TemporaryDirectory(prefix='iib-extract-') as temp_dir: + temp_path = Path(temp_dir) + oci_dir = temp_path / 'oci' + oci_dir.mkdir(parents=True, exist_ok=True) + + # Download image as OCI layout using skopeo + log.info('Downloading image %s as OCI layout', image) + _skopeo_copy( + source=f'docker://{image}', + destination=f'oci:{oci_dir}', + copy_all=False, + exc_msg=f'Failed to download image {image} as OCI layout', + ) + + # Read OCI index to find the manifest + index_path = oci_dir / 'index.json' + if not index_path.exists(): + raise IIBError(f'OCI index.json not found at {index_path}') + + with open(index_path, 'r') as f: + index = json.load(f) + + # Get the manifest digest from the index + manifests = index.get('manifests', []) + if not manifests: + raise IIBError(f'No manifests found in OCI index for image {image}') + + manifest_digest = manifests[0]['digest'].replace('sha256:', '') + manifest_path = oci_dir / 'blobs' / 'sha256' / manifest_digest + + # Read manifest to get layer information + with open(manifest_path, 'r') as f: + manifest = json.load(f) + + layers = manifest.get('layers', []) + if not layers: + raise IIBError(f'No layers found in manifest for image {image}') + + # Create extraction directory to build the filesystem + extract_dir = temp_path / 'rootfs' + extract_dir.mkdir(parents=True, exist_ok=True) + + # Extract each layer in order to build the complete filesystem + log.info('Extracting %d layers from image %s', len(layers), image) + for layer in layers: + layer_digest = layer['digest'].replace('sha256:', '') + layer_path = oci_dir / 'blobs' / 'sha256' / layer_digest + + if not layer_path.exists(): + raise IIBError(f'Layer blob not found at {layer_path}') + + # Extract layer tar.gz to build filesystem + try: + with tarfile.open(layer_path, 'r:gz') as tar: + # Extract all members safely with path traversal protection + tar.extractall(path=extract_dir, filter='data') + except Exception as e: + raise IIBError(f'Failed to extract layer {layer_digest}: {e}') + + # Normalize src_path (remove leading slash for filesystem access) + normalized_src = src_path.lstrip('/') + source_full_path = extract_dir / normalized_src + + # Verify the requested path exists in the extracted filesystem + if not source_full_path.exists(): + raise IIBError( + f'Path {src_path} not found in image {image}. ' + f'Looked for {source_full_path} in extracted filesystem.' + ) + + # Copy the requested path to destination + dest = Path(dest_path) + log.info('Copying %s from image to %s', src_path, dest_path) + if source_full_path.is_dir(): + # If source is a directory, copy its contents + shutil.copytree(source_full_path, dest, dirs_exist_ok=True) + else: + # If source is a file, copy the file + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_full_path, dest) + + log.info('Successfully extracted %s from image %s to %s', src_path, image, dest_path) + + +class ValidateBundlesThread(threading.Thread): + """Thread to validate whether the bundle pullspecs are present in the registry.""" + + def __init__(self, bundles_queue: queue.Queue) -> None: + """ + Initialize the thread to validate whether the bundle pullspecs are present in the registry. + + :param queue.Queue bundles_queue: the queue of bundles to validate + """ + super().__init__() + self.bundles_queue = bundles_queue + self.exception: Optional[Exception] = None + self.bundle: Optional[str] = None + + def run(self) -> None: + """Execute the validation of the bundle pullspecs.""" + bundle = None + try: + while not self.bundles_queue.empty(): + bundle = self.bundles_queue.get() + b_path = str(bundle["bundlePath"]) if isinstance(bundle, dict) else str(bundle) + skopeo_inspect(f'docker://{b_path}', '--raw', return_json=False) + except IIBError as e: + self.bundle = bundle + bundle_str = ( + bundle["bundlePath"] + if bundle and isinstance(bundle, dict) and "bundlePath" in bundle + else bundle + ) + log.error(f"Error validating bundle {bundle_str}: {e}") + self.exception = e + finally: + while not self.bundles_queue.empty(): + self.bundles_queue.task_done() + + +def wait_for_bundle_validation_threads(validation_threads: List[ValidateBundlesThread]) -> None: + """ + Wait for all bundle validation threads to complete. + + :param list threads: the list of threads to wait for + """ + for t in validation_threads: + t.join() + if t.exception: + if t.bundle and isinstance(t.bundle, dict) and "bundlePath" in t.bundle: + bundle_str = t.bundle["bundlePath"] + else: + bundle_str = str(t.bundle) if t.bundle else "unknown" + log.error(f"Error validating bundle {bundle_str}: {t.exception}") + raise IIBError(f"Error validating bundle {bundle_str}: {t.exception}") + + +def validate_bundles_in_parallel( + bundles: Union[List[BundleImage], List[str]], threads=5, wait=True +) -> Optional[List[ValidateBundlesThread]]: + """ + Validate bundles in parallel. + + :param list bundles: the list of bundles or bundle pullspecsto validate + :param int threads: the number of threads to use + :param bool wait: whether to wait for all threads to complete + :return: the list of threads if not waiting, None otherwise + :rtype: Optional[List[ValidateBundlesThread]] + """ + bundles_queue: queue.Queue[Union[BundleImage, str]] = queue.Queue() + + for bundle in bundles: + bundles_queue.put(bundle) + + validation_threads: List[ValidateBundlesThread] = [] + for _ in range(threads): + validation_thread = ValidateBundlesThread(bundles_queue) + validation_threads.append(validation_thread) + validation_thread.start() + + if wait: + wait_for_bundle_validation_threads(validation_threads) + else: + return validation_threads + return None + + +def pull_index_db_artifact(from_index: str, temp_dir: str) -> str: + """ + Pull index.db artifact from registry, using ImageStream cache if available. + + This function determines whether to use OpenShift ImageStream cache or pull directly + from the registry based on the iib_use_imagestream_cache configuration. + + :param str from_index: The from_index pullspec + :param str temp_dir: Temporary directory where the artifact will be extracted + :return: Path to the directory containing the extracted artifact + :rtype: str + :raises IIBError: If the pull operation fails + """ + conf = get_worker_config() + if conf.get('iib_use_imagestream_cache', False): + # Verify index.db cache is synced. Refresh if not. + log.info('ImageStream cache is enabled. Checking cache sync status.') + if verify_indexdb_cache_for_image(from_index): + log.info('Index.db cache is synced. Pulling from ImageStream.') + # Pull from ImageStream when digests match + imagestream_ref = get_imagestream_artifact_pullspec(from_index) + artifact_dir = get_oras_artifact( + imagestream_ref, + temp_dir, + ) + else: + log.info('Index.db cache is not synced. Refreshing and pulling from Quay.') + refresh_indexdb_cache_for_image(from_index) + # Pull directly from Quay after triggering refresh + artifact_ref = get_indexdb_artifact_pullspec(from_index) + artifact_dir = get_oras_artifact( + artifact_ref, + temp_dir, + ) + else: + # Pull directly from Quay without ImageStream cache + log.info('ImageStream cache is disabled. Pulling index.db artifact directly from registry.') + artifact_ref = get_indexdb_artifact_pullspec(from_index) + artifact_dir = get_oras_artifact( + artifact_ref, + temp_dir, + ) + + return artifact_dir + + +def write_build_metadata( + local_repo_path: str, + opm_version: str, + ocp_version: str, + distribution_scope: str, + binary_image: str, + request_id: int, + arches: set, +) -> None: + """ + Write build metadata file for Konflux build task. + + This function creates a JSON metadata file that contains information needed by the + Konflux build task, including OPM version, labels, binary image, request ID, and arches. + + :param str local_repo_path: Path to local Git repository + :param str opm_version: OPM version string (e.g., "opm-1.40.0") + :param str ocp_version: OCP version (e.g., "v4.19") + :param str distribution_scope: Distribution scope (e.g., "PROD") + :param str binary_image: Binary image pullspec + :param int request_id: Request ID + :param set arches: Set of architectures (e.g., {'amd64', 's390x'}) + """ + metadata = { + 'opm_version': opm_version, + 'labels': { + 'com.redhat.index.delivery.version': ocp_version, + 'com.redhat.index.delivery.distribution_scope': distribution_scope, + }, + 'binary_image': binary_image, + 'request_id': request_id, + 'arches': sorted(list(arches)), + } + + metadata_path = Path(local_repo_path) / '.iib-build-metadata.json' + with open(metadata_path, 'w') as f: + json.dump(metadata, f, indent=2) + + log.info('Written build metadata to %s', str(metadata_path)) + + +def get_list_of_output_pullspec( + request_id: int, build_tags: Optional[List[str]] = None +) -> List[str]: + """ + Build list of output pull specifications for index images. + + Creates pull specs for the request ID and any additional build tags, + using the worker configuration template. + + :param int request_id: The IIB request ID + :param Optional[List[str]] build_tags: Additional tags to create pull specs for + :return: List of output pull specifications + :rtype: List[str] + """ + _tags = [str(request_id)] + if build_tags: + _tags.extend(build_tags) + conf = get_worker_config() + output_pull_specs = [] + for tag in _tags: + output_pull_spec = conf['iib_image_push_template'].format( + registry=conf['iib_registry'], request_id=tag + ) + output_pull_specs.append(output_pull_spec) + return output_pull_specs + + +def push_index_db_artifact( + request_id: int, + from_index: str, + index_db_path: str, + operators: List[str], + overwrite_from_index: bool = False, + request_type: str = 'rm', +) -> Optional[str]: + """ + Push updated index.db artifact to registry with appropriate tags. + + This function pushes the index.db file to the artifact registry with a request-specific + tag and optionally to the v4.x tag if overwrite_from_index is True. It captures + the original digest of the v4.x tag before overwriting for potential rollback. + + :param int request_id: The IIB request ID + :param str from_index: The from_index pullspec + :param str index_db_path: Path to the index.db file to push + :param List[str] operators: List of operators involved in the operation + :param bool overwrite_from_index: Whether to overwrite the from_index + :param str request_type: Type of request (e.g., 'rm', 'add') + :return: Original digest of v4.x tag if captured, None otherwise + :rtype: Optional[str] + """ + original_index_db_digest = None + + if index_db_path and Path(index_db_path).exists(): + # Get directory and filename separately to push only the filename + # This ensures ORAS extracts the file as just "index.db" without + # directory structure + index_db_file = Path(index_db_path) + index_db_dir = str(index_db_file.parent) + index_db_filename = index_db_file.name + log.info('Pushing from directory: %s, filename: %s', index_db_dir, index_db_filename) + + # Push with request_id tag irrespective of overwrite_from_index + set_request_state(request_id, 'in_progress', 'Pushing updated index database') + image_name, tag = _get_name_and_tag_from_pullspec(from_index) + conf = get_worker_config() + request_artifact_ref = conf['iib_index_db_artifact_template'].format( + registry=conf['iib_index_db_artifact_registry'], + tag=f"{_get_artifact_combined_tag(image_name, tag)}-{request_id}", + ) + artifact_refs = [request_artifact_ref] + if overwrite_from_index: + # Get the current digest of v4.x tag before overwriting it + # This allows us to restore it if anything fails after the push + v4x_artifact_ref = get_indexdb_artifact_pullspec(from_index) + log.info('Capturing original digest of %s for potential rollback', v4x_artifact_ref) + original_index_db_digest = get_image_digest(v4x_artifact_ref) + log.info('Original index.db digest: %s', original_index_db_digest) + artifact_refs.append(v4x_artifact_ref) + + # Build annotations - only include operators if not empty + annotations = { + 'request_id': str(request_id), + 'request_type': request_type, + } + if operators: + annotations['operators'] = ','.join(operators) + + for artifact_ref in artifact_refs: + push_oras_artifact( + artifact_ref=artifact_ref, + local_path=index_db_filename, + cwd=index_db_dir, + annotations=annotations.copy(), + ) + log.info('Pushed %s to registry', artifact_ref) + + return original_index_db_digest + + +def cleanup_on_failure( + mr_details: Optional[Dict[str, str]], + last_commit_sha: Optional[str], + index_git_repo: Optional[str], + overwrite_from_index: bool, + request_id: int, + from_index: str, + index_repo_map: Dict[str, str], + original_index_db_digest: Optional[str] = None, + reason: str = "error", +) -> None: + """ + Clean up Git changes and index.db artifacts on failure. + + If a merge request was created, it will be closed (since the commit is only in a + feature branch). If changes were pushed directly to the main branch, the commit + will be reverted. If the index.db artifact was pushed to the v4.x tag, it will be + restored to the original digest. + + :param Optional[Dict[str, str]] mr_details: Details of the merge request if one was created + :param Optional[str] last_commit_sha: The SHA of the last commit + :param Optional[str] index_git_repo: URL of the Git repository + :param bool overwrite_from_index: Whether to overwrite the from_index + :param int request_id: The IIB request ID + :param str from_index: The from_index pullspec + :param Dict[str, str] index_repo_map: Mapping of index images to Git repositories + :param Optional[str] original_index_db_digest: Original digest of index.db before overwrite + :param str reason: Reason for the cleanup (used in log messages) + """ + if mr_details and index_git_repo: + # If we created an MR, just close it (commit is only in feature branch) + log.info("Closing merge request due to %s", reason) + try: + close_mr(mr_details, index_git_repo) + log.info("Closed merge request: %s", mr_details.get('mr_url')) + except Exception as close_error: + log.warning("Failed to close merge request: %s", close_error) + elif overwrite_from_index and last_commit_sha: + # If we pushed directly, revert the commit + log.error("Reverting commit due to %s", reason) + try: + revert_last_commit( + request_id=request_id, + from_index=from_index, + index_repo_map=index_repo_map, + ) + except Exception as revert_error: + log.error("Failed to revert commit: %s", revert_error) + else: + log.error("Neither MR nor commit to revert. No cleanup needed for %s", reason) + + # Restore index.db artifact to original digest if it was overwritten + if original_index_db_digest: + log.info("Restoring index.db artifact to original digest due to %s", reason) + try: + # Get the v4.x artifact reference + v4x_artifact_ref = get_indexdb_artifact_pullspec(from_index) + + # Extract registry and repository from the pullspec + # Format: quay.io/namespace/repo:tag -> we need quay.io/namespace/repo + artifact_name = v4x_artifact_ref.rsplit(':', 1)[0] + + # Use oras copy to restore the old digest to v4.x tag + # This is a registry-to-registry copy, no download needed + source_ref = f'{artifact_name}@{original_index_db_digest}' + log.info("Restoring %s from %s", v4x_artifact_ref, source_ref) + + run_cmd( + ['oras', 'copy', source_ref, v4x_artifact_ref], + exc_msg=f'Failed to restore index.db artifact ' + f'from {source_ref} to {v4x_artifact_ref}', + ) + log.info("Successfully restored index.db artifact to original digest") + except Exception as restore_error: + log.error("Failed to restore index.db artifact: %s", restore_error) + + +def prepare_git_repository_for_build( + request_id: int, + from_index: str, + temp_dir: str, + branch: str, + index_to_gitlab_push_map: Dict[str, str], +) -> Tuple[str, str, str]: + """ + Set up and clone Git repository for containerized build. + + This function resolves the Git repository URL from the from_index, + gets the Git token, clones the repository, and verifies the configs directory exists. + + :param int request_id: The IIB request ID + :param str from_index: The from_index pullspec + :param str temp_dir: Temporary directory where repository will be cloned + :param str branch: Git branch to clone + :param Dict[str, str] index_to_gitlab_push_map: Mapping of index images to Git repositories + :return: Tuple of (index_git_repo, local_git_repo_path, localized_git_catalog_path) + :rtype: Tuple[str, str, str] + :raises IIBError: If Git repository cannot be resolved or configs directory not found + """ + # Get Git repository information + index_git_repo = resolve_git_url(from_index=from_index, index_repo_map=index_to_gitlab_push_map) + if not index_git_repo: + raise IIBError( + f"Git repository mapping not found for from_index: {from_index}. " + "index_to_gitlab_push_map is required." + ) + log.info("Git repo for %s: %s", from_index, index_git_repo) + + token_name, git_token = get_git_token(index_git_repo) + + # Clone Git repository + set_request_state(request_id, 'in_progress', 'Cloning Git repository') + local_git_repo_path = Path(temp_dir) / 'git' / branch + local_git_repo_path.mkdir(parents=True, exist_ok=True) + + clone_git_repo(index_git_repo, branch, token_name, git_token, str(local_git_repo_path)) + + localized_git_catalog_path = local_git_repo_path / 'configs' + if not localized_git_catalog_path.exists(): + raise IIBError(f"Catalogs directory not found in {local_git_repo_path}") + + return index_git_repo, str(local_git_repo_path), str(localized_git_catalog_path) + + +def fetch_and_verify_index_db_artifact( + from_index: str, + temp_dir: str, +) -> str: + """ + Pull index.db artifact and verify it exists. + + This function pulls the index.db artifact from the registry and verifies + that the file exists in the expected location. + + :param str from_index: The from_index pullspec + :param str temp_dir: Temporary directory where artifact will be extracted + :return: Path to the index.db file + :rtype: str + :raises IIBError: If index.db file not found after pulling + """ + artifact_dir = pull_index_db_artifact(from_index, temp_dir) + artifact_index_db_file = Path(artifact_dir) / "index.db" + + log.debug("Artifact DB path %s", artifact_index_db_file) + if not artifact_index_db_file.exists(): + log.error("Index.db file not found at %s", artifact_index_db_file) + raise IIBError(f"Index.db file not found at {artifact_index_db_file}") + + return str(artifact_index_db_file) + + +def git_commit_and_create_mr_or_push( + request_id: int, + local_git_repo_path: str, + index_git_repo: str, + branch: str, + commit_message: str, + overwrite_from_index: bool = False, +) -> Tuple[Optional[Dict[str, str]], str]: + """ + Commit changes and trigger Konflux pipeline by creating MR or pushing directly. + + If overwrite_from_index is False, creates a merge request (for throw-away + requests). Otherwise, pushes directly to the branch. Returns the merge request details + and last commit SHA. + + :param int request_id: The IIB request ID + :param str local_git_repo_path: Path to local Git repository + :param str index_git_repo: URL of the Git repository + :param str branch: Git branch name + :param str commit_message: Commit message to use + :param bool overwrite_from_index: Whether to overwrite from_index (push directly vs MR) + :return: Tuple of (mr_details, last_commit_sha) + :rtype: Tuple[Optional[Dict[str, str]], str] + """ + set_request_state(request_id, 'in_progress', 'Committing changes to Git repository') + log.info("Committing changes to Git repository. Triggering KONFLUX pipeline.") + + mr_details = None + # Determine if this is a throw-away request (no overwrite_from_index) + if not overwrite_from_index: + # Create MR for throw-away requests + mr_details = create_mr( + request_id=request_id, + local_repo_path=local_git_repo_path, + repo_url=index_git_repo, + branch=branch, + commit_message=commit_message, + ) + log.info("Created merge request: %s", mr_details.get('mr_url')) + else: + # Push directly to the branch + commit_and_push( + request_id=request_id, + local_repo_path=local_git_repo_path, + repo_url=index_git_repo, + branch=branch, + commit_message=commit_message, + ) + + # Get commit SHA before waiting for the pipeline (while the temp directory still exists) + last_commit_sha = get_last_commit_sha(local_repo_path=local_git_repo_path) + + return mr_details, last_commit_sha + + +def monitor_pipeline_and_extract_image(request_id: int, last_commit_sha: str) -> str: + """ + Wait for Konflux pipeline to complete and return the built image URL. + + This function finds the pipelinerun associated with the commit SHA, + waits for it to complete, and extracts the built image URL from the results. + + :param int request_id: The IIB request ID + :param str last_commit_sha: SHA of the last commit that triggered the pipeline + :return: URL of the built image + :rtype: str + :raises IIBError: If pipelinerun not found or pipeline fails + """ + # Wait for Konflux pipeline + set_request_state(request_id, 'in_progress', 'Waiting on KONFLUX build') + + # find_pipelinerun has retry decorator to handle delays in pipelinerun creation + pipelines = find_pipelinerun(last_commit_sha) + + # Get the first pipelinerun (should typically be only one) + pipelinerun = pipelines[0] + pipelinerun_name = pipelinerun.get('metadata', {}).get('name') + if not pipelinerun_name: + raise IIBError("Pipelinerun name not found in pipeline metadata") + + run = wait_for_pipeline_completion(pipelinerun_name) + + return get_pipelinerun_image_url(pipelinerun_name, run) + + +def replicate_image_to_tagged_destinations( + request_id: int, + image_url: str, + build_tags: Optional[List[str]] = None, +) -> List[str]: + """ + Copy built index from Konflux to IIB registry with all required tags. + + This function builds the list of output pull specs and copies the built + image from Konflux to each spec using skopeo. + + :param int request_id: The IIB request ID + :param str image_url: URL of the built image from Konflux + :param Optional[List[str]] build_tags: Additional tags to apply + :return: List of output pull specifications that were copied to + :rtype: List[str] + """ + set_request_state(request_id, 'in_progress', 'Copying built index to IIB registry') + + output_pull_specs = get_list_of_output_pullspec(request_id, build_tags) + + # Copy the built index from Konflux to all output pull specs + for spec in output_pull_specs: + _skopeo_copy( + source=f'docker://{image_url}', + destination=f'docker://{spec}', + copy_all=True, + exc_msg=f'Failed to copy built index from Konflux to {spec}', + ) + log.info("Successfully copied image to %s", spec) + + return output_pull_specs + + +def cleanup_merge_request_if_exists( + mr_details: Optional[Dict[str, str]], + index_git_repo: Optional[str], +) -> None: + """ + Close merge request if it was created. + + This function attempts to close a merge request and logs a warning + if the operation fails. + + :param Optional[Dict[str, str]] mr_details: Details of the merge request + :param Optional[str] index_git_repo: URL of the Git repository + """ + if mr_details and index_git_repo: + try: + close_mr(mr_details, index_git_repo) + log.info("Closed merge request: %s", mr_details.get('mr_url')) + except IIBError as e: + log.warning("Failed to close merge request: %s", e) diff --git a/iib/workers/tasks/fbc_utils.py b/iib/workers/tasks/fbc_utils.py index d8167b8b3..dfb932499 100644 --- a/iib/workers/tasks/fbc_utils.py +++ b/iib/workers/tasks/fbc_utils.py @@ -121,17 +121,30 @@ def extract_fbc_fragment( # store the fbc_fragment at /tmp/iib-**/fbc-fragment-{index} to prevent # cross-contamination conf = get_worker_config() - fbc_fragment_path = os.path.join(temp_dir, f"{conf['temp_fbc_fragment_path']}-{fragment_index}") + fbc_fragment_base_path = os.path.join( + temp_dir, f"{conf['temp_fbc_fragment_path']}-{fragment_index}" + ) # Copy fbc_fragment's catalog to /tmp/iib-**/fbc-fragment-{index} - _copy_files_from_image(fbc_fragment, conf['fbc_fragment_catalog_path'], fbc_fragment_path) - - log.info("fbc_fragment extracted at %s", fbc_fragment_path) - operator_packages = os.listdir(fbc_fragment_path) + _copy_files_from_image(fbc_fragment, conf['fbc_fragment_catalog_path'], fbc_fragment_base_path) + + # podman cp creates a subdirectory named after the source path basename, e.g. + # /tmp/iib-**/fbc-fragment-0/configs/example-operator. Match get_catalog_dir(). + fbc_fragment_catalog_dir = os.path.join( + fbc_fragment_base_path, os.path.basename(conf['fbc_fragment_catalog_path']) + ) + if not os.path.isdir(fbc_fragment_catalog_dir): + raise IIBError( + f"FBC fragment catalog directory not found at {fbc_fragment_catalog_dir} " + f"after extracting {fbc_fragment}" + ) + + log.info("fbc_fragment extracted at %s", fbc_fragment_catalog_dir) + operator_packages = os.listdir(fbc_fragment_catalog_dir) log.info("fbc_fragment contains packages %s", operator_packages) if not operator_packages: raise IIBError("No operator packages in fbc_fragment %s", fbc_fragment) - return fbc_fragment_path, operator_packages + return fbc_fragment_catalog_dir, operator_packages def _serialize_datetime(obj: datetime) -> str: diff --git a/iib/workers/tasks/general.py b/iib/workers/tasks/general.py index acdbc1d53..41f6b12a9 100644 --- a/iib/workers/tasks/general.py +++ b/iib/workers/tasks/general.py @@ -7,8 +7,7 @@ from iib.exceptions import IIBError, FinalStateOverwriteError from iib.workers.api_utils import set_request_state from iib.workers.tasks.celery import app -from iib.workers.tasks.utils import request_logger -from iib.workers.tasks.build import _cleanup +from iib.workers.tasks.utils import request_logger, reset_docker_config __all__ = ['failed_request_callback', 'set_request_state'] @@ -34,11 +33,11 @@ def failed_request_callback( msg = str(exc) elif isinstance(exc, FinalStateOverwriteError): log.info(f"Request {request_id} is in a final state,ignoring update.") - _cleanup() + reset_docker_config() return else: msg = 'An unknown error occurred. See logs for details' log.error(msg, exc_info=exc) - _cleanup() + reset_docker_config() set_request_state(request_id, 'failed', msg) diff --git a/iib/workers/tasks/git_utils.py b/iib/workers/tasks/git_utils.py index 1fe1969b6..899db8ae3 100644 --- a/iib/workers/tasks/git_utils.py +++ b/iib/workers/tasks/git_utils.py @@ -74,9 +74,6 @@ def push_configs_to_git( try: clone_git_repo(repo_url, branch, git_token_name, git_token, local_repo_dir) - # Configure Git user for commits - configure_git_user(local_repo_dir) - # Overwrite local Git repo configs/ repo_configs_dir = os.path.join(local_repo_dir, 'configs') log.info( @@ -110,25 +107,7 @@ def push_configs_to_git( ) log.info(git_status) - # Add updates - log.info("Commiting changes to local Git repository.") - run_cmd( - ["git", "-C", local_repo_dir, "add", "."], exc_msg="Error staging changes to git" - ) - git_status = run_cmd( - ["git", "-C", local_repo_dir, "status"], exc_msg="Error getting git status" - ) - log.info(git_status) - - # Check if there's anything to commit - changes = run_cmd( - ["git", "-C", local_repo_dir, "diff", "--staged"], exc_msg="Error getting git diff" - ) - if not changes: - _clean_up_local_repo(local_repo_dir) - log.warning("No changes to commit.") - return - + # Commit and push changes (this handles staging and checking for changes) commit_and_push( request_id, local_repo_dir, @@ -147,6 +126,31 @@ def _clean_up_local_repo(local_repo_dir: str) -> None: log.debug("Cleaned up local Git repository %s", local_repo_dir) +def _stage_and_check_changes(local_repo_path: str) -> bool: + """ + Stage changes and check if there's anything to commit. + + :param str local_repo_path: Path to local Git repository. + :return: True if there are staged changes to commit, False otherwise. + :rtype: bool + """ + log.info("Committing changes to local Git repository.") + run_cmd(["git", "-C", local_repo_path, "add", "."], exc_msg="Error staging changes to git") + git_status = run_cmd( + ["git", "-C", local_repo_path, "status"], exc_msg="Error getting git status" + ) + log.info(git_status) + + # Check if there's anything to commit + changes = run_cmd( + ["git", "-C", local_repo_path, "diff", "--staged"], exc_msg="Error getting git diff" + ) + if not changes: + log.warning("No changes to commit.") + return False + return True + + def validate_git_remote_branch(repo_url: str, branch: str) -> None: """ Ensure the provided repository and branch exists. @@ -181,6 +185,12 @@ def commit_and_push( final_commit_message = commit_message or ( f"IIB: Update for request id {request_id} (overwrite_from_index)" ) + + # Stage and check for changes + if not _stage_and_check_changes(local_repo_path): + _clean_up_local_repo(local_repo_path) + return + commit_output = run_cmd( ["git", "-C", local_repo_path, "commit", "-m", final_commit_message], exc_msg="Error committing changes", @@ -236,6 +246,22 @@ def get_git_token(git_repo) -> Tuple[str, str]: return token_name, token_value +def get_last_commit_sha(local_repo_path: str) -> str: + """ + Get SHA for the latest commit in the local Git repository. + + :param str repo_url: Path to local Git repository + :return: The SHA of the last commit in the repository + :rtype: str + """ + last_commit = run_cmd( + ["git", "-C", local_repo_path, "rev-parse", "HEAD"], + exc_msg=f"Error getting last commit for {local_repo_path}", + ) + + return last_commit.strip() + + def clone_git_repo( repo_url: str, branch: str, token_name: str, token: str, local_repo_path: str ) -> None: @@ -269,28 +295,6 @@ def clone_git_repo( log.info("Most recent commit: %s", last_commit) -def configure_git_user( - local_repo_path: str, - user_name: Optional[str] = "IIB Worker", - email_address: Optional[str] = "iib-worker@redhat.com", -): - """ - Configure git user name and email displayed in commit message. - - :param str local_repo_path: Path to local Git repo. - :param str user_name: User name for local Git repo. - :param str email_address: Email address for local Git repo. - """ - run_cmd( - ["git", "-C", local_repo_path, "config", "--local", "user.name", str(user_name)], - exc_msg="Error configuring git user.email", - ) - run_cmd( - ["git", "-C", local_repo_path, "config", "--local", "user.email", str(email_address)], - exc_msg="Error configuring git user.email", - ) - - def revert_last_commit( request_id: int, from_index: str, @@ -320,9 +324,6 @@ def revert_last_commit( try: clone_git_repo(repo_url, branch, git_token_name, git_token, local_repo_dir) - # Configure Git user for commits - configure_git_user(local_repo_dir) - log.info("Reverting last commit to %s branch of %s", branch, repo_url) revert_output = run_cmd( ["git", "-C", local_repo_dir, "reset", "--hard", "HEAD~1"], diff --git a/iib/workers/tasks/konflux_utils.py b/iib/workers/tasks/konflux_utils.py index 65f053f0b..fb3f65d27 100644 --- a/iib/workers/tasks/konflux_utils.py +++ b/iib/workers/tasks/konflux_utils.py @@ -6,11 +6,19 @@ from kubernetes import client from kubernetes.client.rest import ApiException +from tenacity import ( + before_sleep_log, + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, + wait_chain, +) from iib.exceptions import IIBError from iib.workers.config import get_worker_config -__all__ = ['find_pipelinerun', 'wait_for_pipeline_completion'] +__all__ = ['find_pipelinerun', 'wait_for_pipeline_completion', 'get_pipelinerun_image_url'] log = logging.getLogger(__name__) @@ -104,14 +112,25 @@ def _create_kubernetes_configuration(url: str, token: str, ca_cert: str) -> clie return configuration +@retry( + before_sleep=before_sleep_log(log, logging.WARNING), + reraise=True, + retry=retry_if_exception_type(IIBError), + stop=stop_after_attempt(get_worker_config().iib_total_attempts), + wait=wait_chain(wait_exponential(multiplier=get_worker_config().iib_retry_multiplier)), +) def find_pipelinerun(commit_sha: str) -> List[Dict[str, Any]]: """ Find the Konflux pipelinerun triggered by the git commit. + This function will retry if no pipelineruns are found (empty list), as it may take + a few seconds for the pipelinerun to start after a commit is pushed. + :param str commit_sha: The git commit SHA to search for :return: List of pipelinerun objects matching the commit SHA :rtype: List[Dict[str, Any]] :raises IIBError: If there's an error fetching pipelineruns + or no pipelineruns found after retries """ try: log.info("Searching for pipelineruns with commit SHA: %s", commit_sha) @@ -131,8 +150,14 @@ def find_pipelinerun(commit_sha: str) -> List[Dict[str, Any]]: items = runs.get("items", []) log.info("Found %s pipelinerun(s) for commit %s", len(items), commit_sha) + if not items: + raise IIBError(f"No pipelineruns found for commit {commit_sha}") + return items + except IIBError: + # Re-raise IIBError without wrapping it (needed for retry decorator) + raise except ApiException as e: error_msg = f"Failed to fetch pipelineruns for commit {commit_sha}: API error {e.status}" log.error("Kubernetes API error while fetching pipelineruns: %s - %s", e.status, e.reason) @@ -146,7 +171,9 @@ def find_pipelinerun(commit_sha: str) -> List[Dict[str, Any]]: raise IIBError(error_msg) -def wait_for_pipeline_completion(pipelinerun_name: str, timeout: Optional[int] = None) -> None: +def wait_for_pipeline_completion( + pipelinerun_name: str, timeout: Optional[int] = None +) -> dict[str, Any]: """ Poll the status of a tekton Pipelinerun and wait for completion. @@ -157,6 +184,8 @@ def wait_for_pipeline_completion(pipelinerun_name: str, timeout: Optional[int] = :param str pipelinerun_name: Name of the pipelinerun to monitor :param int timeout: Maximum time to wait in seconds (default: from config) + :return: Dictionary containing the pipelinerun status information + :rtype: Dict[str, Any] :raises IIBError: If the pipelinerun fails, is cancelled, or times out """ if timeout is None: @@ -172,7 +201,7 @@ def wait_for_pipeline_completion(pipelinerun_name: str, timeout: Optional[int] = run = _fetch_pipelinerun_status(pipelinerun_name) if _handle_pipelinerun_completion(pipelinerun_name, run): - return + return run time.sleep(30) @@ -219,6 +248,40 @@ def _check_timeout(pipelinerun_name: str, start_time: float, timeout: int) -> No ) +def get_pipelinerun_image_url(pipelinerun_name: str, run: Dict[str, Any]) -> str: + """ + Extract IMAGE_URL from a completed pipelinerun's results. + + :param str pipelinerun_name: Name of the pipelinerun + :param Dict[str, Any] run: The pipelinerun object + :return: The IMAGE_URL value from the pipelinerun results + :rtype: str + :raises IIBError: If IMAGE_URL is not found in the pipelinerun results + """ + status = run.get('status', {}) + + # Check for 'results' (Konflux format) first, then fall back to 'pipelineResults' (older Tekton) + pipeline_results = status.get('results', []) or status.get('pipelineResults', []) + + log.info("Found %d pipeline results for %s", len(pipeline_results), pipelinerun_name) + + for result in pipeline_results: + if result.get('name') == 'IMAGE_URL': + if image_url := result.get('value'): + # Strip whitespace (including newlines) from the URL + image_url = image_url.strip() + log.info("Extracted IMAGE_URL from pipelinerun %s: %s", pipelinerun_name, image_url) + return image_url + + # If not found, log for debugging + log.error( + "IMAGE_URL not found in pipelinerun %s. Available results: %s", + pipelinerun_name, + [r.get('name') for r in pipeline_results], + ) + raise IIBError(f"IMAGE_URL not found in pipelinerun {pipelinerun_name} results") + + def _fetch_pipelinerun_status(pipelinerun_name: str) -> Dict[str, Any]: """ Fetch the current status of the pipelinerun from Kubernetes. diff --git a/iib/workers/tasks/opm_operations.py b/iib/workers/tasks/opm_operations.py index 11e5f873f..2b1f2d42d 100644 --- a/iib/workers/tasks/opm_operations.py +++ b/iib/workers/tasks/opm_operations.py @@ -509,24 +509,19 @@ def opm_registry_deprecatetruncate(base_dir: str, index_db: str, bundles: List[s run_cmd(cmd, {'cwd': base_dir}, exc_msg=f'Failed to deprecate the bundles on {index_db}') -def deprecate_bundles_fbc( - bundles: List[str], +def deprecate_bundles_db( base_dir: str, - binary_image: str, - from_index: str, + index_db_file: str, + bundles: List[str], ) -> None: """ - Deprecate the specified bundles from the FBC index image. - - Dockerfile is created only, no build is performed. + Deprecate the specified bundles from the index.db file. - :param list bundles: pull specifications of bundles to deprecate. :param str base_dir: base directory where operation files will be located. - :param str binary_image: binary image to be used by the new index image. - :param str from_index: index image, from which the bundles will be deprecated. + :param str index_db_file: path to index.db file used with opm registry deprecatetruncate. + :param list bundles: pull specifications of bundles to deprecate. """ conf = get_worker_config() - index_db_file = _get_or_create_temp_index_db_file(base_dir=base_dir, from_index=from_index) # Break the bundles into chunks of at max iib_deprecate_bundles_limit bundles for i in range( @@ -540,6 +535,27 @@ def deprecate_bundles_fbc( bundles=bundles[i : i + conf.iib_deprecate_bundles_limit], # Pass a chunk starting at i ) + +def deprecate_bundles_fbc( + bundles: List[str], + base_dir: str, + binary_image: str, + from_index: str, +) -> None: + """ + Deprecate the specified bundles from the FBC index image. + + Dockerfile is created only, no build is performed. + + :param list bundles: pull specifications of bundles to deprecate. + :param str base_dir: base directory where operation files will be located. + :param str binary_image: binary image to be used by the new index image. + :param str from_index: index image, from which the bundles will be deprecated. + """ + index_db_file = _get_or_create_temp_index_db_file(base_dir=base_dir, from_index=from_index) + + deprecate_bundles_db(base_dir=base_dir, index_db_file=index_db_file, bundles=bundles) + fbc_dir, _ = opm_migrate(index_db_file, base_dir) # we should keep generating Dockerfile here # to have the same behavior as we run `opm index deprecatetruncate` with '--generate' option @@ -1107,6 +1123,109 @@ def opm_registry_add_fbc_fragment( ) +def opm_registry_add_fbc_fragment_containerized( + request_id: int, + temp_dir: str, + from_index_configs_dir: str, + fbc_fragments: List[str], + overwrite_from_index_token: Optional[str], + index_db_path: Optional[str] = None, +) -> Tuple[str, str, List[str]]: + """ + Add FBC fragments to the from_index image. + + This only produces the index.Dockerfile file and does not build the container image. + This also removes operators from index_db_path file if any are present. + + :param int request_id: the id of IIB request + :param str temp_dir: the base directory to generate the database and index.Dockerfile in. + :param str from_index_configs_dir: path to the file-based catalog directory + :param list fbc_fragments: the list of pull specifications of fbc fragments to be added. + :param str overwrite_from_index_token: token used to access the image + :param str index_db_path: path to the index database file + :return: Returns paths to directories for containing file-based catalog, path to index.db, + and list of operators removed from index_db_path + :rtype: str, str, list(str) + """ + set_request_state( + request_id, + 'in_progress', + f'Extracting operator packages from {len(fbc_fragments)} fbc fragment(s)', + ) + + # Single pass: Extract all fragment paths and operators + fragment_data = [] + all_fragment_operators = [] + + for i, fbc_fragment in enumerate(fbc_fragments): + # fragment path will look like /tmp/iib-**/fbc-fragment-{index} + fragment_path, fragment_operators = extract_fbc_fragment( + temp_dir=temp_dir, fbc_fragment=fbc_fragment, fragment_index=i + ) + fragment_data.append((fragment_path, fragment_operators)) + all_fragment_operators.extend(fragment_operators) + + # Single verification: Check for operators that already exist in the database + operators_in_db, index_db_path_local = verify_operators_exists( + from_index=None, + base_dir=temp_dir, + operator_packages=all_fragment_operators, + overwrite_from_index_token=overwrite_from_index_token, + index_db_path=index_db_path, + ) + + # Remove existing operators if any conflicts found + if operators_in_db: + remove_operator_deprecations( + from_index_configs_dir=from_index_configs_dir, operators=operators_in_db + ) + log.info('Removing %s from index.db ', operators_in_db) + _opm_registry_rm( + index_db_path=index_db_path_local, operators=operators_in_db, base_dir=temp_dir + ) + + # migrated_catalog_dir path will look like /tmp/iib-**/catalog + migrated_catalog_dir, _ = opm_migrate( + index_db=index_db_path_local, + base_dir=temp_dir, + generate_cache=False, + ) + log.info("Migrated catalog after removing from db at %s", migrated_catalog_dir) + + # copy the content of migrated_catalog to from_index's config + log.info("Copying content of %s to %s", migrated_catalog_dir, from_index_configs_dir) + for operator_package in os.listdir(migrated_catalog_dir): + shutil.copytree( + os.path.join(migrated_catalog_dir, operator_package), + os.path.join(from_index_configs_dir, operator_package), + dirs_exist_ok=True, + ) + + # Copy operators to config directory using the collected data + for i, (fragment_path, fragment_operators) in enumerate(fragment_data): + set_request_state( + request_id, + 'in_progress', + f'Adding package(s) {fragment_operators} from fbc fragment ' + f'{i + 1}/{len(fbc_fragments)} to from_index', + ) + + for fragment_operator in fragment_operators: + # copy fragment_operator to from_index configs + fragment_opr_src_path = os.path.join(fragment_path, fragment_operator) + fragment_opr_dest_path = os.path.join(from_index_configs_dir, fragment_operator) + if os.path.exists(fragment_opr_dest_path): + shutil.rmtree(fragment_opr_dest_path) + log.info( + "Copying content of %s to %s", + fragment_opr_src_path, + fragment_opr_dest_path, + ) + shutil.copytree(fragment_opr_src_path, fragment_opr_dest_path) + + return from_index_configs_dir, index_db_path_local, operators_in_db + + def remove_operator_deprecations(from_index_configs_dir: str, operators: List[str]) -> None: """ Remove operator deprecations, if present. @@ -1130,11 +1249,12 @@ def remove_operator_deprecations(from_index_configs_dir: str, operators: List[st def verify_operators_exists( - from_index: str, + from_index: str | None, base_dir: str, operator_packages: List[str], overwrite_from_index_token: Optional[str], -): + index_db_path: Optional[str] = None, +) -> Tuple[List[str], str]: """ Check if operators exists in index image. @@ -1142,24 +1262,28 @@ def verify_operators_exists( :param str base_dir: base temp directory for IIB request :param list(str) operator_packages: operator_package to check :param str overwrite_from_index_token: token used to access the image + :param str index_db_path: path to the index database file :return: packages_in_index, index_db_path - :rtype: (set, str) + :rtype: (list(str), str) """ from iib.workers.tasks.iib_static_types import BundleImage from iib.workers.tasks.utils import set_registry_token packages_in_index: Set[str] = set() - log.info("Verifying if operator packages %s exists in index %s", operator_packages, from_index) + index_name = from_index if from_index else "database" + log.info("Verifying if operator packages %s exists in index %s", operator_packages, index_name) - # check if operator packages exists in hidden index.db - # we are not checking /config dir since it contains FBC opted-in operators and to remove those - # fbc-operations endpoint should be used - with set_registry_token(overwrite_from_index_token, from_index, append=True): - index_db_path = get_hidden_index_database(from_index=from_index, base_dir=base_dir) + # When index_db_path is not provided, extract the index db from the given index image + if (not index_db_path or not os.path.exists(index_db_path)) and from_index: + # check if operator packages exists in hidden index.db + # we are not checking /config dir since it contains FBC opted-in operators + # and to remove those fbc-operations endpoint should be used + with set_registry_token(overwrite_from_index_token, from_index, append=True): + index_db_path = get_hidden_index_database(from_index=str(from_index), base_dir=base_dir) present_bundles: List[BundleImage] = get_list_bundles( - input_data=index_db_path, base_dir=base_dir + input_data=str(index_db_path), base_dir=base_dir ) for bundle in present_bundles: @@ -1169,7 +1293,7 @@ def verify_operators_exists( if packages_in_index: log.info("operator packages found in index_db %s: %s", index_db_path, packages_in_index) - return packages_in_index, index_db_path + return list(packages_in_index), str(index_db_path) @retry( diff --git a/iib/workers/tasks/oras_utils.py b/iib/workers/tasks/oras_utils.py index b1f0b9d8a..cc377c6f3 100644 --- a/iib/workers/tasks/oras_utils.py +++ b/iib/workers/tasks/oras_utils.py @@ -2,17 +2,98 @@ """This file contains functions for ORAS (OCI Registry As Storage) operations.""" import logging import os +import re import shutil import tempfile -from typing import Dict, Optional, Any +from typing import Any, Dict, Optional, Tuple from iib.common.tracing import instrument_tracing from iib.exceptions import IIBError +from iib.workers.config import get_worker_config from iib.workers.tasks.utils import run_cmd, set_registry_auths, get_image_digest log = logging.getLogger(__name__) +def _get_name_and_tag_from_pullspec(image_pullspec: str) -> Tuple[str, str]: + """ + Parse a container image pullspec (URL) to extract the index name and tag. + + :param str image_pullspec: The full image pullspec string (registry/path/name:tag[@digest]). + :returns Tuple[str, str]: The extracted index name and tag (e.g., 'iib-pub-pending', 'v4.17'). + :raises IIBError: If the pullspec is missing the required tag delimiter (':') + or if the name:tag structure cannot be parsed. + """ + # Regex to capture the image name and tag, ignoring an optional digest. + # r'/([^/:]+):([^@]+)(@.*)?$' + # Group 1: ([^/:]+) -> Image Name (the last path segment before the colon) + # Group 2: ([^@]+) -> Tag (the part after the colon, before '@' or end of string) + # Group 3: (@.*)? -> Optional digest part, which is ignored + regex = re.compile(r'/([^/:]+):([^@]+)(@.*)?$') + match = regex.search(image_pullspec) + + if not match: + # Check for the most common error: missing the tag delimiter (:) + if ':' not in image_pullspec: + raise IIBError( + f"Invalid pullspec format: '{image_pullspec}'. Missing tag (':') delimiter." + ) + + # Raise a general error if the regex failed for other reasons + raise IIBError( + f"Invalid pullspec format: '{image_pullspec}'. Could not parse name:tag structure." + ) + + # Group 1: Image Name (e.g., 'iib-pub-pending') + index_name = match.group(1) + + # Group 2: Tag (e.g., 'v4.17') + tag = match.group(2) + + # Final check to ensure the tag isn't empty (e.g., 'image:') + if not tag: + raise IIBError(f"Invalid pullspec format: '{image_pullspec}'. Tag is present but empty.") + + return index_name, tag + + +def _get_artifact_combined_tag(image_name: str, tag: str) -> str: + """ + Generate a combined artifact tag for the given image name and tag. + + This function generates a unique combined tag for an image by using a template + string defined in the worker configuration and replacing placeholders with the + provided image name and tag. + + :param str image_name: The name of the image. + :param str tag: The version or identifier tag to be combined. + :return: A formatted string representing the combined artifact tag. + :rtype: str + """ + return get_worker_config()['iib_index_db_artifact_tag_template'].format( + image_name=image_name, tag=tag + ) + + +def get_indexdb_artifact_pullspec(from_index: str) -> str: + """ + Construct the full pullspec for index_db artifact. + + :param str from_index: The original full pullspec of the index image. + + :raises IIBError: If the pullspec parsing fails within the helper function. + :returns str: The full, formatted pullspec for the internal index DB artifact. + :rtype: str + """ + conf = get_worker_config() + image_name, tag = _get_name_and_tag_from_pullspec(from_index) + + return conf['iib_index_db_artifact_template'].format( + registry=conf['iib_index_db_artifact_registry'], + tag=_get_artifact_combined_tag(image_name, tag), + ) + + @instrument_tracing(span_name="workers.tasks.oras_utils.get_oras_artifact") def get_oras_artifact( artifact_ref: str, @@ -40,11 +121,21 @@ def get_oras_artifact( # Create a subdirectory within the provided base_dir temp_dir = tempfile.mkdtemp(prefix=temp_dir_prefix, dir=base_dir) - # Use namespace-specific registry authentication if provided + # Use exclusive registry authentication file or the provided/default Docker config.json with set_registry_auths(registry_auths, use_empty_config=True): + conf = get_worker_config() + oras_exclusive_auth_path = conf['iib_index_db_oras_auth_path'] + cmd_args = [] + if oras_exclusive_auth_path and os.path.exists(oras_exclusive_auth_path): + cmd_args = ['--registry-config', oras_exclusive_auth_path] + log.debug('Using ORAS registry configuration file: %s', oras_exclusive_auth_path) + else: + log.warning( + 'No ORAS registry configuration file found, using default Docker config.json' + ) try: run_cmd( - ['oras', 'pull', artifact_ref, '-o', temp_dir], + ['oras', 'pull', *cmd_args, artifact_ref, '-o', temp_dir], exc_msg=f'Failed to pull OCI artifact {artifact_ref}', ) log.info('Successfully pulled OCI artifact %s to %s', artifact_ref, temp_dir) @@ -63,6 +154,7 @@ def push_oras_artifact( artifact_type: str = "application/vnd.sqlite", registry_auths: Optional[Dict[str, Any]] = None, annotations: Optional[Dict[str, str]] = None, + cwd: Optional[str] = None, ) -> None: """ Push a local artifact to an OCI registry using ORAS. @@ -70,35 +162,62 @@ def push_oras_artifact( This function is equivalent to: `oras push {artifact_ref} {local_path}:{artifact_type}` :param str artifact_ref: OCI artifact reference to push to (e.g., 'quay.io/repo/repo:tag') - :param str local_path: Local path to the artifact file. Can be an absolute or relative path. - If an absolute path is provided, the --disable-path-validation flag will be - automatically added. + :param str local_path: Local path to the artifact file. Should be a relative path. + When using cwd, this should be a relative path (typically just + the filename) relative to the cwd directory. :param str artifact_type: MIME type of the artifact (default: 'application/vnd.sqlite') :param dict registry_auths: Optional dockerconfig.json auth information for private registries :param dict annotations: Optional annotations to add to the artifact + :param str cwd: Optional working directory for the ORAS command. When provided, local_path + should be relative to this directory (e.g., just the filename). :raises IIBError: If the push operation fails """ log.info('Pushing artifact from %s to %s with type %s', local_path, artifact_ref, artifact_type) + if cwd: + log.info('Using working directory: %s', cwd) - if not os.path.exists(local_path): - raise IIBError(f'Local artifact path does not exist: {local_path}') + # Construct the full path for validation + full_path = os.path.join(cwd, local_path) if cwd else local_path + if not os.path.exists(full_path): + raise IIBError(f'Local artifact path does not exist: {full_path}') - # Build ORAS push command - cmd = ['oras', 'push', artifact_ref, f'{local_path}:{artifact_type}'] + # Use exclusive registry authentication file or the provided/default Docker config.json + with set_registry_auths(registry_auths, use_empty_config=True): + conf = get_worker_config() + oras_exclusive_auth_path = conf['iib_index_db_oras_auth_path'] + cmd_args = [] + if oras_exclusive_auth_path and os.path.exists(oras_exclusive_auth_path): + cmd_args = ['--registry-config', oras_exclusive_auth_path] + log.debug('Using ORAS registry configuration file: %s', oras_exclusive_auth_path) + else: + log.warning( + 'No ORAS registry configuration file found, using default Docker config.json' + ) - # Add --disable-path-validation flag for absolute paths - if os.path.isabs(local_path): - cmd.append('--disable-path-validation') + # Build ORAS push command + cmd = ['oras', 'push', *cmd_args, artifact_ref, f'{local_path}:{artifact_type}'] - # Add annotations if provided - if annotations: - for key, value in annotations.items(): - cmd.extend(['--annotation', f'{key}={value}']) + # Do not allow absolute paths. + # Absolute paths are extracted to the same place (full path) which might cause collisions. + if os.path.isabs(local_path): + log.error('Local artifact path must be relative: %s', local_path) + raise IIBError(f'Local artifact path must be relative: {local_path}') + + # Add annotations if provided + if annotations: + for key, value in annotations.items(): + cmd.extend(['--annotation', f'{key}={value}']) - # Use namespace-specific registry authentication if provided - with set_registry_auths(registry_auths, use_empty_config=True): try: - run_cmd(cmd, exc_msg=f'Failed to push OCI artifact to {artifact_ref}') + # Only pass params if cwd is provided + if cwd: + run_cmd( + cmd, + params={'cwd': cwd}, + exc_msg=f'Failed to push OCI artifact to {artifact_ref}', + ) + else: + run_cmd(cmd, exc_msg=f'Failed to push OCI artifact to {artifact_ref}') log.info('Successfully pushed OCI artifact to %s', artifact_ref) except Exception as e: raise IIBError(f'Failed to push OCI artifact to {artifact_ref}: {e}') @@ -141,15 +260,33 @@ def verify_indexdb_cache_sync(tag: str) -> bool: :return: True if the digests match (cache is synced), False otherwise. :rtype: bool """ - # TODO - This is EXAMPLE location - final one should be loaded from config variable - repository = "quay.io/exd-guild-hello-operator/example-repository" + conf = get_worker_config() + artifact_pullspec = conf['iib_index_db_artifact_template'].format( + registry=conf['iib_index_db_artifact_registry'], tag=tag + ) - quay_digest = get_image_digest(f"{repository}:{tag}") + quay_digest = get_image_digest(artifact_pullspec) is_digest = get_image_stream_digest(tag) return quay_digest == is_digest +def verify_indexdb_cache_for_image(index_image_pullspec: str) -> bool: + """ + Verify the synchronization state of the index database cache for a given container image. + + This function extracts the image name and tag from the specified image + pullspec, generates an artifact combined tag, and verifies whether the + database cache for the image is synchronized. + + :param str index_image_pullspec: The pull specification string of the container image. + :return: The result of the cache synchronization verification process. + :rtype: str + """ + index_name, tag = _get_name_and_tag_from_pullspec(index_image_pullspec) + return verify_indexdb_cache_sync(_get_artifact_combined_tag(index_name, tag)) + + def refresh_indexdb_cache( tag: str, registry_auths: Optional[Dict[str, Any]] = None, @@ -165,8 +302,10 @@ def refresh_indexdb_cache( """ log.info('Refreshing OCI artifact cache: %s', tag) - # TODO - This is EXAMPLE location - final one should be loaded from config variable - repository = "quay.io/exd-guild-hello-operator/example-repository" + conf = get_worker_config() + artifact_pullspec = conf['iib_index_db_artifact_template'].format( + registry=conf['iib_index_db_artifact_registry'], tag=tag + ) # Use namespace-specific registry authentication if provided with set_registry_auths(registry_auths, use_empty_config=True): @@ -175,8 +314,46 @@ def refresh_indexdb_cache( 'oc', 'import-image', f'index-db-cache:{tag}', - f'--from={repository}:{tag}', + f'--from={artifact_pullspec}', '--confirm', ], exc_msg=f'Failed to refresh OCI artifact {tag}.', ) + + +def refresh_indexdb_cache_for_image(index_image_pullspec: str) -> None: + """ + Refresh the cached data for an index database, associating it with the given image pullspec. + + This function extracts the name and tag from the specified image pullspec, + and refreshes the associated index database cache. + + :param str index_image_pullspec: The pull specification of the index image to cache. + :return: A formatted string combining the index name and tag. + :rtype: str + """ + index_name, tag = _get_name_and_tag_from_pullspec(index_image_pullspec) + refresh_indexdb_cache(_get_artifact_combined_tag(index_name, tag)) + + +def get_imagestream_artifact_pullspec(from_index: str) -> str: + """ + Get the ImageStream pullspec for the index.db artifact. + + This function constructs the internal OpenShift ImageStream pullspec that can be used + to pull the index.db artifact from the cached ImageStream instead of directly from Quay. + + :param str from_index: The from_index pullspec + :return: ImageStream pullspec for the artifact + :rtype: str + """ + conf = get_worker_config() + image_name, tag = _get_name_and_tag_from_pullspec(from_index) + combined_tag = _get_artifact_combined_tag(image_name, tag) + + # ImageStream pullspec format: + # image-registry.openshift-image-registry.svc:5000/{namespace}/index-db:{combined_tag} + imagestream_pullspec = conf['iib_index_db_artifact_template'].format( + registry=conf['iib_index_db_imagestream_registry'], tag=combined_tag + ) + return imagestream_pullspec diff --git a/iib/workers/tasks/utils.py b/iib/workers/tasks/utils.py index 79b592048..56fe3bde3 100644 --- a/iib/workers/tasks/utils.py +++ b/iib/workers/tasks/utils.py @@ -208,6 +208,8 @@ class RequestConfig: to the merged index image. :param dict binary_image_config: the dict of config required to identify the appropriate ``binary_image`` to use. + :param list binary_image_less_arches_allowed_versions: list of versions of the binary image + that are allowed to build for less arches. Defaults to ``None``. """ # these attrs should not be printed out @@ -218,7 +220,12 @@ class RequestConfig: 'registry_auths', ] - _attrs: List[str] = ["_binary_image", "distribution_scope", "binary_image_config"] + _attrs: List[str] = [ + "_binary_image", + "distribution_scope", + "binary_image_config", + "binary_image_less_arches_allowed_versions", + ] __slots__ = _attrs if TYPE_CHECKING: _binary_image: str @@ -226,6 +233,7 @@ class RequestConfig: binary_image_config: Dict[str, Dict[str, str]] overwrite_from_index_token: str overwrite_target_index_token: str + binary_image_less_arches_allowed_versions: List[str] def __init__(self, **kwargs): """ @@ -820,7 +828,9 @@ def run_cmd( if strict and response.returncode != 0: if set(['buildah', 'manifest', 'rm']) <= set(cmd) and 'image not known' in response.stderr: raise IIBError('Manifest list not found locally.') - log.error('The command "%s" failed with: %s', ' '.join(cmd), response.stderr) + log.error( + 'The command "%s" failed with: %s', ' '.join(_sanitize_cmd_log(cmd)), response.stderr + ) regex: str match: Optional[re.Match] if Path(cmd[0]).stem.startswith('opm'): @@ -1116,6 +1126,9 @@ def get_image_label(pull_spec: str, label: str) -> str: :rtype: str """ log.debug('Getting the label of %s from %s', label, pull_spec) + if "index.db" in pull_spec: + raise IIBError(f'Cannot get label "{label}" from {pull_spec}') + return get_image_labels(pull_spec).get(label, '') @@ -1231,11 +1244,28 @@ def prepare_request_for_build( binary_image_arches = get_image_arches(binary_image_resolved) if not arches.issubset(binary_image_arches): - raise IIBError( - 'The binary image is not available for the following arches: {}'.format( - ', '.join(sorted(arches - binary_image_arches)) + # The support for less arches is limited to the binary image versions that are allowed + # to build for less arches. + if build_request_config.binary_image_less_arches_allowed_versions: + if binary_image not in build_request_config.binary_image_less_arches_allowed_versions: + raise IIBError( + 'The binary image is not available for the following arches: {}'.format( + ', '.join(sorted(arches - binary_image_arches)) + ) + ) + supported_arches = set([arch for arch in arches if arch in binary_image_arches]) + if not supported_arches: + raise IIBError( + 'The binary image is not available for any of the following arches: {}'.format( + ', '.join(sorted(arches)) + ) + ) + log.warning( + "Building index images for the following supported arches: {}".format( + ', '.join(sorted(supported_arches)) ) ) + arches = supported_arches arches_str = ', '.join(sorted(arches)) log.debug('Set to build the index image for the following arches: %s', arches_str) @@ -1301,3 +1331,25 @@ def get_bundle_metadata( for pullspec in operator_csv.get_pullspecs(): bundle_metadata['found_pullspecs'].add(pullspec) return bundle_metadata + + +@contextmanager +def change_dir(new_dir: str) -> Generator[None, Any, None]: + """ + Context manager for temporarily changing the current working directory. + + This context manager allows temporary switching to a new directory during + the execution of a block of code. Once the block is exited, it ensures the + working directory is reverted to its original state, even in cases where + an error occurs within the block. + + :param str new_dir: new directory to switch to + :raises OSError: If changing to the new directory or reverting to the + original directory fails. + """ + prev_dir = os.getcwd() + try: + os.chdir(new_dir) + yield + finally: + os.chdir(prev_dir) diff --git a/podman-compose-containerized.yml b/podman-compose-containerized.yml new file mode 100644 index 000000000..6aa029ccc --- /dev/null +++ b/podman-compose-containerized.yml @@ -0,0 +1,160 @@ +--- +version: '3' +services: + # This "service" generates the certificate for the registry. Then, + # it exits with status code 0. + minica: + image: registry.access.redhat.com/ubi8/go-toolset:latest + command: + - /bin/sh + - -c + - >- + go install github.com/jsha/minica@latest && + cd /opt/app-root/certs && + namei -l /opt/app-root && + /opt/app-root/src/bin/minica --domains registry + environment: + GOPATH: /opt/app-root/src + volumes: + - registry-certs-volume:/opt/app-root/certs:z + + registry: + image: registry:2 + ports: + - 8443:8443 + environment: + REGISTRY_HTTP_ADDR: 0.0.0.0:8443 + REGISTRY_HTTP_TLS_CERTIFICATE: /certs/registry/cert.pem + REGISTRY_HTTP_TLS_KEY: /certs/registry/key.pem + REGISTRY_AUTH: htpasswd + REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd + REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm + volumes: + - ./iib_data/registry:/var/lib/registry + - registry-certs-volume:/certs:z + - ./docker/registry/auth:/auth + + db: + image: postgres:9.6 + environment: + POSTGRES_USER: iib + POSTGRES_PASSWORD: iib + POSTGRES_DB: iib + POSTGRES_INITDB_ARGS: "--auth='ident' --auth='trust'" + + memcached: + image: memcached + ports: + - 11211:11211 + + rabbitmq: + image: rabbitmq:3.7-management + environment: + RABBITMQ_DEFAULT_USER: iib + RABBITMQ_DEFAULT_PASS: iib + # Avoid port conflict with ActiveMQ broker when using podman-compose. + # Even though the port is not exposed, podman-compose's use of a pod + # requires the ports to be unique across all containers within the pod. + RABBITMQ_NODE_PORT: 5673 + ports: + # The RabbitMQ management console + - 8081:15672 + + iib-api: + build: + context: . + dockerfile: ./docker/Dockerfile-api + command: + - /bin/sh + - -c + - >- + mkdir -p /etc/iib && + pip3 uninstall -y iib && + python3 setup.py develop --no-deps && + iib wait-for-db && + iib db upgrade && + flask run --reload --host 0.0.0.0 --port 8080 + environment: + FLASK_ENV: development + FLASK_APP: iib/web/wsgi.py + REQUESTS_CA_BUNDLE: /etc/pki/tls/certs/ca-bundle.crt + IIB_DEV: 'true' + volumes: + - ./:/src + - ./docker/message_broker/certs:/broker-certs + - request-logs-volume:/var/log/iib/requests:z + - request-related-bundles-volume:/var/lib/requests/related_bundles:z + - request-recursive-related-bundles-volume:/var/lib/requests/recursive_related_bundles:z + ports: + - 8080:8080 + depends_on: + - db + - message-broker + + # IIB Worker for containerized workflow (connects to external Konflux cluster) + iib-worker-containerized: + build: + context: . + dockerfile: ./docker/Dockerfile-workers + command: > + bash -c " + mkdir -p /root/.docker && + ln -sf /etc/containers/auth.json /root/.docker/config.json && + echo 'Created symlink for docker config' && + exec celery -A iib.workers.tasks worker --loglevel=info + " + environment: + IIB_DEV: 'false' + IIB_CELERY_CONFIG: /etc/iib/settings.py + REQUESTS_CA_BUNDLE: /etc/pki/tls/certs/ca-chain.crt + GIT_SSL_CAINFO: /etc/pki/tls/certs/ca-chain.crt + env_file: + # This file contains Konflux cluster credentials and other sensitive configuration + - .env.containerized + # Enable privileged mode for podman-in-podman support + privileged: true + security_opt: + - seccomp=unconfined + - label=disable + cap_add: + - SYS_ADMIN + - MKNOD + volumes: + - ./:/src + - registry-certs-volume:/registry-certs + - request-logs-volume:/var/log/iib/requests:z + - request-related-bundles-volume:/var/lib/requests/related_bundles:z + - request-recursive-related-bundles-volume:/var/lib/requests/recursive_related_bundles:z + # Mount custom worker configuration + - ./docker/containerized/worker_config.py:/etc/iib/settings.py:z + # Mount Docker auth config for registry authentication (in a location IIB won't try to delete) + - ./docker/config.json:/etc/containers/auth.json:ro + # Mount local registry CA certificate for podman (not system-wide to avoid interfering with Git) + - registry-certs-volume:/tmp/registry-certs:ro + # Mount Konflux CA chain for GitLab SSL verification + - ./docker/containerized/konflux-ca.crt:/tmp/host-ca-chain.crt:ro + depends_on: + - rabbitmq + - registry + - minica + - memcached + + # This is an external message broker used to publish messages about state changes + message-broker: + build: + context: . + dockerfile: ./docker/message_broker/Dockerfile + volumes: + - message-broker-volume:/opt/activemq/data:z + - ./docker/message_broker/certs:/broker-certs + ports: + - 5671:5671 # amqp+ssl + - 5672:5672 # amqp + - 8161:8161 # web console + +volumes: + registry-certs-volume: + message-broker-volume: + request-logs-volume: + request-recursive-related-bundles-volume: + request-related-bundles-volume: diff --git a/tests/test_web/test_api_v1.py b/tests/test_web/test_api_v1.py index 51b785e44..9c3cd303d 100644 --- a/tests/test_web/test_api_v1.py +++ b/tests/test_web/test_api_v1.py @@ -513,23 +513,39 @@ def test_get_build_logs_s3_configured( 'The "overwrite_from_index_token" parameter must be a string', ), ( - {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'cnr_token': True}, - '"cnr_token" must be a string', - ), - ( - {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'organization': True}, + { + 'from_index': 'pull:spec', + 'bundles': ['some:thing'], + 'binary_image': 'binary:image', + 'organization': True, + }, '"organization" must be a string', ), ( - {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'force_backport': 'spam'}, + { + 'from_index': 'pull:spec', + 'bundles': ['some:thing'], + 'binary_image': 'binary:image', + 'force_backport': 'spam', + }, '"force_backport" must be a boolean', ), ( - {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'graph_update_mode': 123}, + { + 'from_index': 'pull:spec', + 'bundles': ['some:thing'], + 'binary_image': 'binary:image', + 'graph_update_mode': 123, + }, '"graph_update_mode" must be a string', ), ( - {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'graph_update_mode': 'Hi'}, + { + 'from_index': 'pull:spec', + 'bundles': ['some:thing'], + 'binary_image': 'binary:image', + 'graph_update_mode': 'Hi', + }, ( '"graph_update_mode" must be set to one of these: [\'replaces\', \'semver\'' ', \'semver-skippatch\']' @@ -560,7 +576,7 @@ def test_add_bundles_overwrite_not_allowed(mock_smfsc, client, db): mock_smfsc.assert_not_called() -@pytest.mark.parametrize('from_index', (None, 'some-random-index:v4.14', 'some-common-index:v4.15')) +@pytest.mark.parametrize('from_index', ('some-random-index:v4.14', 'some-common-index:v4.15')) @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_add_bundles_graph_update_mode_not_allowed( mock_smfsc, app, client, auth_env, db, from_index @@ -587,7 +603,7 @@ def test_add_bundles_graph_update_mode_not_allowed( @pytest.mark.parametrize('from_index', ('some-common-index:v4.15', 'another-common-index:v4.15')) @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') -@mock.patch('iib.web.api_v1.handle_add_request') +@mock.patch('iib.web.api_v1.handle_containerized_add_request') def test_add_bundles_graph_update_mode_allowed( mock_har, mock_smfsc, app, client, auth_env, db, from_index ): @@ -611,7 +627,7 @@ def test_add_bundles_graph_update_mode_allowed( @mock.patch('iib.web.api_v1.db.session') @mock.patch('iib.web.api_v1.flask.jsonify') @mock.patch('iib.web.api_v1.RequestAdd') -@mock.patch('iib.web.api_v1.handle_add_request.apply_async') +@mock.patch('iib.web.api_v1.handle_containerized_add_request.apply_async') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_add_bundles_unique_bundles(mock_smfsc, mock_har, mock_radd, mock_fj, mock_dbs, client): data = { @@ -690,15 +706,16 @@ def test_rm_operators_overwrite_not_allowed(mock_smfsc, client, db): ), ( {'add_arches': ['s390x'], 'binary_image': 'binary:image'}, - '"from_index" must be specified if no bundles are specified', + 'The input "from_index" is required.', ), - ({'add_arches': ['s390x']}, '"from_index" must be specified if no bundles are specified'), + ({'add_arches': ['s390x']}, 'The input "from_index" is required.'), ( { 'bundles': ['some:thing'], 'binary_image': 'binary:image', 'add_arches': ['s390x'], 'overwrite_from_index_token': 'username:password', + 'from_index': 'pull:spec', }, ( 'The "overwrite_from_index" parameter is required when the ' @@ -758,6 +775,7 @@ def test_add_bundle_invalid_param(mock_smfsc, db, auth_env, client): 'best_batsman': 'Virat Kohli', 'binary_image': 'binary:image', 'bundles': ['some:thing'], + 'from_index': 'pull:spec', } rv = client.post('/api/v1/builds/add', json=data, environ_base=auth_env) @@ -802,16 +820,6 @@ def test_rm_bundle_from_invalid_distribution_scope(mock_smfsc, db, auth_env, cli mock_smfsc.assert_not_called() -@mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') -def test_add_bundle_from_index_and_add_arches_missing(mock_smfsc, db, auth_env, client): - data = {'bundles': ['some:thing'], 'binary_image': 'binary:image'} - - rv = client.post('/api/v1/builds/add', json=data, environ_base=auth_env) - assert rv.status_code == 400 - assert rv.json['error'] == 'One of "from_index" or "add_arches" must be specified' - mock_smfsc.assert_not_called() - - @pytest.mark.parametrize( ( 'binary_image', @@ -823,14 +831,14 @@ def test_add_bundle_from_index_and_add_arches_missing(mock_smfsc, db, auth_env, 'graph_update_mode', ), ( - ('binary:image', False, None, ['some:thing'], None, None, None), + ('binary:image', False, None, ['some:thing'], 'some:thing', None, None), ('binary:image', False, None, ['some:thing'], 'some:thing', None, 'semver'), ('binary:image', False, None, [], 'some:thing', 'Prod', 'semver-skippatch'), ('scratch', True, 'username:password', ['some:thing'], 'some:thing', 'StagE', 'replaces'), ('scratch', True, 'username:password', [], 'some:thing', 'DeV', 'semver'), ), ) -@mock.patch('iib.web.api_v1.handle_add_request') +@mock.patch('iib.web.api_v1.handle_containerized_add_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_add_bundle_success( mock_smfsc, @@ -851,8 +859,6 @@ def test_add_bundle_success( data = { 'binary_image': binary_image, 'add_arches': ['s390x'], - 'organization': 'org', - 'cnr_token': 'token', 'overwrite_from_index': overwrite_from_index, 'overwrite_from_index_token': overwrite_from_index_token, 'from_index': from_index, @@ -898,7 +904,7 @@ def test_add_bundle_success( 'expiration': '2020-02-15T17:03:00Z', }, 'omps_operator_version': {}, - 'organization': 'org', + 'organization': None, 'state_history': [ { 'state': 'in_progress', @@ -918,31 +924,12 @@ def test_add_bundle_success( rv_json['logs']['expiration'] = '2020-02-15T17:03:00Z' assert rv.status_code == 201 assert response_json == rv_json - assert 'cnr_token' not in rv_json assert 'token' not in mock_har.apply_async.call_args[1]['argsrepr'] - assert '*****' in mock_har.apply_async.call_args[1]['argsrepr'] mock_har.apply_async.assert_called_once() mock_smfsc.assert_called_once_with(mock.ANY, new_batch_msg=True) -@pytest.mark.parametrize('force_backport', (False, True)) -@mock.patch('iib.web.api_v1.handle_add_request') -def test_add_bundle_force_backport(mock_har, force_backport, db, auth_env, client): - data = { - 'bundles': ['some:thing'], - 'binary_image': 'binary:image', - 'from_index': 'index:image', - 'force_backport': force_backport, - } - - rv = client.post('/api/v1/builds/add', json=data, environ_base=auth_env) - assert rv.status_code == 201 - mock_har.apply_async.assert_called_once() - # Eigth element in args is the force_backport parameter - assert mock_har.apply_async.call_args[1]['args'][7] == force_backport - - -@mock.patch('iib.web.api_v1.handle_add_request') +@mock.patch('iib.web.api_v1.handle_containerized_add_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_add_bundle_overwrite_token_redacted(mock_smfsc, mock_har, app, auth_env, client, db): token = 'username:password' @@ -952,20 +939,20 @@ def test_add_bundle_overwrite_token_redacted(mock_smfsc, mock_har, app, auth_env 'add_arches': ['amd64'], 'overwrite_from_index': True, 'overwrite_from_index_token': token, + 'from_index': 'pull:spec', } rv = client.post('/api/v1/builds/add', json=data, environ_base=auth_env) rv_json = rv.json assert rv.status_code == 201 mock_har.apply_async.assert_called_once() - # Tenth to last element in args is the overwrite_from_index parameter + # With binary_image_less_arches_allowed_versions added at end, + # overwrite_from_index is -11, token is -10 assert mock_har.apply_async.call_args[1]['args'][-11] is True - # Ninth to last element in args is the overwrite_from_index_token parameter assert mock_har.apply_async.call_args[1]['args'][-10] == token assert 'overwrite_from_index_token' not in rv_json assert token not in json.dumps(rv_json) assert token not in mock_har.apply_async.call_args[1]['argsrepr'] - assert '*****' in mock_har.apply_async.call_args[1]['argsrepr'] @pytest.mark.parametrize( @@ -1002,13 +989,18 @@ def test_add_bundle_overwrite_token_redacted(mock_smfsc, mock_har, app, auth_env ({'not.tbrady@DOMAIN.LOCAL': 'Patriots'}, True, None), ), ) -@mock.patch('iib.web.api_v1.handle_add_request') +@mock.patch('iib.web.api_v1.handle_containerized_add_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_add_bundle_custom_user_queue( mock_smfsc, mock_har, app, auth_env, client, user_to_queue, overwrite_from_index, expected_queue ): app.config['IIB_USER_TO_QUEUE'] = user_to_queue - data = {'bundles': ['some:thing'], 'binary_image': 'binary:image', 'add_arches': ['s390x']} + data = { + 'bundles': ['some:thing'], + 'binary_image': 'binary:image', + 'add_arches': ['s390x'], + 'from_index': 'pull:spec', + } if overwrite_from_index: data['from_index'] = 'index:image' data['overwrite_from_index'] = True @@ -1417,7 +1409,7 @@ def test_patch_request_regenerate_bundle_success( @pytest.mark.parametrize("binary_image", ('binary:image', 'scratch')) -@mock.patch('iib.web.api_v1.handle_rm_request') +@mock.patch('iib.web.api_v1.handle_containerized_rm_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_remove_operator_success(mock_smfsc, mock_rm, binary_image, db, auth_env, client): data = { @@ -1475,7 +1467,7 @@ def test_remove_operator_success(mock_smfsc, mock_rm, binary_image, db, auth_env mock_smfsc.assert_called_once_with(mock.ANY, new_batch_msg=True) -@mock.patch('iib.web.api_v1.handle_rm_request') +@mock.patch('iib.web.api_v1.handle_containerized_rm_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_remove_operator_overwrite_token_redacted(mock_smfsc, mock_hrr, app, auth_env, client, db): token = 'username:password' @@ -1491,9 +1483,10 @@ def test_remove_operator_overwrite_token_redacted(mock_smfsc, mock_hrr, app, aut rv_json = rv.json assert rv.status_code == 201 mock_hrr.apply_async.assert_called_once() - # Third to last element in args is the overwrite_from_index parameter - assert mock_hrr.apply_async.call_args[1]['args'][-6] is True - assert mock_hrr.apply_async.call_args[1]['args'][-5] == token + # With binary_image_less_arches_allowed_versions added at end, + # overwrite_from_index is -7, token is -6 + assert mock_hrr.apply_async.call_args[1]['args'][-7] is True + assert mock_hrr.apply_async.call_args[1]['args'][-6] == token assert 'overwrite_from_index_token' not in rv_json assert token not in json.dumps(rv_json) assert token not in mock_hrr.apply_async.call_args[1]['argsrepr'] @@ -1521,7 +1514,7 @@ def test_remove_operator_overwrite_token_redacted(mock_smfsc, mock_hrr, app, aut ({'not.tbrady@DOMAIN.LOCAL': 'Patriots'}, True, None), ), ) -@mock.patch('iib.web.api_v1.handle_rm_request') +@mock.patch('iib.web.api_v1.handle_containerized_rm_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_remove_operator_custom_user_queue( mock_smfsc, mock_hrr, app, auth_env, client, user_to_queue, overwrite_from_index, expected_queue @@ -1552,7 +1545,7 @@ def test_not_found(client): assert rv.json == {'error': 'The requested resource was not found'} -@mock.patch('iib.web.api_v1.handle_regenerate_bundle_request') +@mock.patch('iib.web.api_v1.handle_containerized_regenerate_bundle_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_regenerate_bundle_success(mock_smfsc, mock_hrbr, db, auth_env, client): data = { @@ -1669,7 +1662,7 @@ def test_regenerate_bundle_missing_required_param( ({'not.tbrady@DOMAIN.LOCAL': 'Patriots'}, None), ), ) -@mock.patch('iib.web.api_v1.handle_regenerate_bundle_request') +@mock.patch('iib.web.api_v1.handle_containerized_regenerate_bundle_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_regenerate_bundle_custom_user_queue( mock_smfsc, mock_hrbr, app, auth_env, client, user_to_queue, expected_queue @@ -1694,7 +1687,7 @@ def test_regenerate_bundle_custom_user_queue( ({}, None, {'Han Solo': 'Don\'t everybody thank me at once.'}), ), ) -@mock.patch('iib.web.api_v1.handle_regenerate_bundle_request') +@mock.patch('iib.web.api_v1.handle_containerized_regenerate_bundle_request') @mock.patch('iib.web.api_v1.messaging.send_messages_for_new_batch_of_requests') def test_regenerate_bundle_batch_success( mock_smfnbor, mock_hrbr, user_to_queue, expected_queue, annotations, app, auth_env, client, db @@ -1729,17 +1722,32 @@ def test_regenerate_bundle_batch_success( 1, {'auths': {'registry2.example.com': {'auth': 'dummy_auth'}}}, {'foo': 'bar:baz'}, + {}, + 'regenerate-bundle', + [], # binary_image_less_arches_allowed_versions ], argsrepr=( "['registry.example.com/bundle-image:latest', None, 1, '*****', " - "{'foo': 'bar:baz'}]" + "{'foo': 'bar:baz'}, {}, 'regenerate-bundle', []]" ), link_error=mock.ANY, queue=expected_queue, ), mock.call( - args=['registry.example.com/bundle-image2:latest', None, 2, None, None], - argsrepr="['registry.example.com/bundle-image2:latest', None, 2, None, None]", + args=[ + 'registry.example.com/bundle-image2:latest', + None, + 2, + None, + None, + {}, + 'regenerate-bundle', + [], # binary_image_less_arches_allowed_versions + ], + argsrepr=( + "['registry.example.com/bundle-image2:latest', None, 2, None, None, {}, " + "'regenerate-bundle', []]" + ), link_error=mock.ANY, queue=expected_queue, ), @@ -1755,7 +1763,7 @@ def test_regenerate_bundle_batch_success( assert requests_to_send_msgs_for[1].id == 2 -@mock.patch('iib.web.api_v1.handle_regenerate_bundle_request') +@mock.patch('iib.web.api_v1.handle_containerized_regenerate_bundle_request') def test_regenerate_bundle_batch_invalid_request_type(mock_hrbr, app, auth_env, client, db): data = { 'build_requests': [ @@ -1807,9 +1815,10 @@ def test_regenerate_bundle_batch_invalid_input(payload, error_msg, app, auth_env assert rv.json == {'error': error_msg} -@mock.patch('iib.web.api_v1.handle_add_request') -@mock.patch('iib.web.api_v1.handle_rm_request') +@mock.patch('iib.web.api_v1.handle_containerized_add_request') +@mock.patch('iib.web.api_v1.handle_containerized_rm_request') @mock.patch('iib.web.api_v1.messaging.send_messages_for_new_batch_of_requests') +@mock.patch.dict('iib.web.api_v1.flask.current_app.config', {'IIB_INDEX_TO_GITLAB_PUSH_MAP': {}}) def test_add_rm_batch_success(mock_smfnbor, mock_hrr, mock_har, app, auth_env, client, db): annotations = {'msdhoni': 'The best captain ever!'} data = { @@ -1820,8 +1829,6 @@ def test_add_rm_batch_success(mock_smfnbor, mock_hrr, mock_har, app, auth_env, c 'binary_image': 'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5', 'from_index': 'registry-proxy/rh-osbs-stage/iib:v4.5', 'add_arches': ['amd64'], - 'cnr_token': 'no_tom_brady_anymore', - 'organization': 'hello-operator', 'overwrite_from_index': True, 'overwrite_from_index_token': 'some_token', }, @@ -1846,26 +1853,22 @@ def test_add_rm_batch_success(mock_smfnbor, mock_hrr, mock_har, app, auth_env, c 'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5', 'registry-proxy/rh-osbs-stage/iib:v4.5', ['amd64'], - 'no_tom_brady_anymore', - 'hello-operator', - None, True, 'some_token', None, - None, {}, [], [], None, False, - {}, + {}, # index_to_gitlab_push_map from config (empty in test) + [], # binary_image_less_arches_allowed_versions ], argsrepr=( - "[['registry-proxy/rh-osbs/lgallett-bundle:v1.0-9'], " - "1, 'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5', " - "'registry-proxy/rh-osbs-stage/iib:v4.5', ['amd64'], '*****', " - "'hello-operator', None, True, '*****', None, None, {}, [], [], None, " - "False, {}]" + "[['registry-proxy/rh-osbs/lgallett-bundle:v1.0-9'], 1, " + "'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5', " + "'registry-proxy/rh-osbs-stage/iib:v4.5', ['amd64'], True, '*****', " + "None, {}, [], [], None, False, {}, []]" ), link_error=mock.ANY, queue=None, @@ -1886,12 +1889,13 @@ def test_add_rm_batch_success(mock_smfnbor, mock_hrr, mock_har, app, auth_env, c None, {}, [], - {}, + {}, # index_to_gitlab_push_map from config (empty in test) + [], # binary_image_less_arches_allowed_versions ], argsrepr=( "[['kiali-ossm'], 2, 'registry:8443/iib-build:11', " - "'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5'" - ", None, False, None, None, {}, [], {}]" + "'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5', " + "None, False, None, None, {}, [], {}, []]" ), link_error=mock.ANY, queue=None, @@ -1910,7 +1914,7 @@ def test_add_rm_batch_success(mock_smfnbor, mock_hrr, mock_har, app, auth_env, c assert requests_to_send_msgs_for[1].id == 2 -@mock.patch('iib.web.api_v1.handle_regenerate_bundle_request') +@mock.patch('iib.web.api_v1.handle_containerized_regenerate_bundle_request') def test_add_rm_batch_invalid_request_type(mock_hrbr, app, auth_env, client, db): data = { 'build_requests': [ @@ -1968,7 +1972,7 @@ def test_regenerate_add_rm_batch_invalid_input(payload, error_msg, app, auth_env @pytest.mark.parametrize("binary_image", ('binary:image', 'scratch')) @pytest.mark.parametrize('distribution_scope', (None, 'stage')) -@mock.patch('iib.web.api_v1.handle_merge_request') +@mock.patch('iib.web.api_v1.handle_containerized_merge_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_merge_index_image_success( mock_smfsc, mock_merge, binary_image, app, db, auth_env, client, distribution_scope @@ -2031,7 +2035,7 @@ def test_merge_index_image_success( mock_smfsc.assert_called_once_with(mock.ANY, new_batch_msg=True) -@mock.patch('iib.web.api_v1.handle_merge_request') +@mock.patch('iib.web.api_v1.handle_containerized_merge_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_merge_index_image_overwrite_token_redacted( mock_smfsc, mock_merge, app, auth_env, client, db @@ -2082,7 +2086,7 @@ def test_merge_index_image_overwrite_token_redacted( ({'not.tbrady@DOMAIN.LOCAL': 'Patriots'}, True, None), ), ) -@mock.patch('iib.web.api_v1.handle_merge_request') +@mock.patch('iib.web.api_v1.handle_containerized_merge_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_merge_index_image_custom_user_queue( mock_smfsc, @@ -2118,7 +2122,7 @@ def test_merge_index_image_custom_user_queue( @pytest.mark.parametrize('overwrite_from_index', (True, False)) -@mock.patch('iib.web.api_v1.handle_merge_request') +@mock.patch('iib.web.api_v1.handle_containerized_merge_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_merge_index_image_fail_on_missing_overwrite_params( mock_smfsc, mock_merge, app, auth_env, client, overwrite_from_index @@ -2189,7 +2193,7 @@ def test_merge_index_image_fail_on_missing_overwrite_params( ), ), ) -@mock.patch('iib.web.api_v1.handle_merge_request') +@mock.patch('iib.web.api_v1.handle_containerized_merge_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_merge_index_image_fail_on_invalid_params( mock_smfsc, mock_merge, app, auth_env, client, data, error_msg @@ -2211,7 +2215,7 @@ def test_merge_index_image_fail_on_invalid_params( ('some:thing', 'scratch', None), ), ) -@mock.patch('iib.web.api_v1.handle_create_empty_index_request.apply_async') +@mock.patch('iib.web.api_v1.handle_containerized_create_empty_index_request.apply_async') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_create_empty_index_success( mock_smfsc, mock_hceir, db, auth_env, client, from_index, binary_image, labels @@ -2794,7 +2798,7 @@ def test_fbc_operations_overwrite_not_allowed(mock_smfsc, client, db): (None, {}), ), ) -@mock.patch('iib.web.api_v1.handle_fbc_operation_request.apply_async') +@mock.patch('iib.web.api_v1.handle_containerized_fbc_operation_request.apply_async') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_fbc_operations( mock_smfc, @@ -2877,7 +2881,7 @@ def test_fbc_operations( (None, {}), ), ) -@mock.patch('iib.web.api_v1.handle_fbc_operation_request.apply_async') +@mock.patch('iib.web.api_v1.handle_containerized_fbc_operation_request.apply_async') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_fbc_operations_multiple_fragments( mock_smfc, @@ -2960,7 +2964,7 @@ def test_fbc_operations_multiple_fragments( (None, {}), ), ) -@mock.patch('iib.web.api_v1.handle_fbc_operation_request.apply_async') +@mock.patch('iib.web.api_v1.handle_containerized_fbc_operation_request.apply_async') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_fbc_operations_backward_compatibility( mock_smfc, diff --git a/tests/test_web/test_broker_error.py b/tests/test_web/test_broker_error.py index ed6fec7ac..b5ed34ed6 100644 --- a/tests/test_web/test_broker_error.py +++ b/tests/test_web/test_broker_error.py @@ -15,7 +15,7 @@ def assert_testing(rv, mock_smfsc, db): assert req_state.state.state == RequestStateMapping.failed.value -@mock.patch('iib.web.api_v1.handle_add_request') +@mock.patch('iib.web.api_v1.handle_containerized_add_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_catch_add_bundle_failure(mock_smfsc, mock_har, db, auth_env, client): mock_har.apply_async.side_effect = OperationalError @@ -23,10 +23,9 @@ def test_catch_add_bundle_failure(mock_smfsc, mock_har, db, auth_env, client): 'bundles': ['some:thing'], 'binary_image': 'binary:image', 'add_arches': ['s390x'], - 'organization': 'org', - 'cnr_token': 'token', 'overwrite_from_index': True, 'overwrite_from_index_token': 'some_token', + 'from_index': 'index:image', } rv = client.post('/api/v1/builds/add', json=data, environ_base=auth_env) @@ -34,7 +33,7 @@ def test_catch_add_bundle_failure(mock_smfsc, mock_har, db, auth_env, client): assert_testing(rv, mock_smfsc, db) -@mock.patch('iib.web.api_v1.handle_regenerate_bundle_request') +@mock.patch('iib.web.api_v1.handle_containerized_regenerate_bundle_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_catch_regenerate_bundle_failure(mock_smfsc, mock_hrbr, db, auth_env, client): mock_hrbr.apply_async.side_effect = OperationalError @@ -48,7 +47,7 @@ def test_catch_regenerate_bundle_failure(mock_smfsc, mock_hrbr, db, auth_env, cl assert_testing(rv, mock_smfsc, db) -@mock.patch('iib.web.api_v1.handle_rm_request') +@mock.patch('iib.web.api_v1.handle_containerized_rm_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_catch_remove_operator_failure(mock_smfsc, mock_rm, db, auth_env, client): mock_rm.apply_async.side_effect = OperationalError @@ -63,7 +62,7 @@ def test_catch_remove_operator_failure(mock_smfsc, mock_rm, db, auth_env, client assert_testing(rv, mock_smfsc, db) -@mock.patch('iib.web.api_v1.handle_regenerate_bundle_request') +@mock.patch('iib.web.api_v1.handle_containerized_regenerate_bundle_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') @mock.patch('iib.web.api_v1.messaging.send_messages_for_new_batch_of_requests') def test_catch_regenerate_bundle_batch_failure( @@ -108,8 +107,8 @@ def test_catch_regenerate_bundle_batch_failure( assert r.state == RequestStateMapping.failed.value -@mock.patch('iib.web.api_v1.handle_add_request') -@mock.patch('iib.web.api_v1.handle_rm_request') +@mock.patch('iib.web.api_v1.handle_containerized_add_request') +@mock.patch('iib.web.api_v1.handle_containerized_rm_request') @mock.patch('iib.web.api_v1.messaging.send_messages_for_new_batch_of_requests') def test_add_rm_batch_add_failure(mock_smfnbor, mock_hrr, mock_har, app, auth_env, client, db): mock_har.apply_async.side_effect = OperationalError @@ -123,8 +122,6 @@ def test_add_rm_batch_add_failure(mock_smfnbor, mock_hrr, mock_har, app, auth_en 'binary_image': 'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5', 'from_index': 'registry-proxy/rh-osbs-stage/iib:v4.5', 'add_arches': ['amd64'], - 'cnr_token': 'no_tom_brady_anymore', - 'organization': 'hello-operator', 'overwrite_from_index': True, 'overwrite_from_index_token': 'some_token', }, @@ -158,8 +155,8 @@ def test_add_rm_batch_add_failure(mock_smfnbor, mock_hrr, mock_har, app, auth_en assert req_rm.state.state == RequestStateMapping.failed.value -@mock.patch('iib.web.api_v1.handle_add_request') -@mock.patch('iib.web.api_v1.handle_rm_request') +@mock.patch('iib.web.api_v1.handle_containerized_add_request') +@mock.patch('iib.web.api_v1.handle_containerized_rm_request') @mock.patch('iib.web.api_v1.messaging.send_messages_for_new_batch_of_requests') def test_add_rm_batch_rm_failure(mock_smfnbor, mock_hrr, mock_har, app, auth_env, client, db): mock_hrr.apply_async.side_effect = OperationalError @@ -173,8 +170,6 @@ def test_add_rm_batch_rm_failure(mock_smfnbor, mock_hrr, mock_har, app, auth_env 'binary_image': 'registry-proxy/rh-osbs/openshift-ose-operator-registry:v4.5', 'from_index': 'registry-proxy/rh-osbs-stage/iib:v4.5', 'add_arches': ['amd64'], - 'cnr_token': 'no_tom_brady_anymore', - 'organization': 'hello-operator', 'overwrite_from_index': True, 'overwrite_from_index_token': 'some_token', }, diff --git a/tests/test_workers/test_config.py b/tests/test_workers/test_config.py index f48933577..2e1d55d01 100644 --- a/tests/test_workers/test_config.py +++ b/tests/test_workers/test_config.py @@ -51,6 +51,7 @@ def test_validate_celery_config(mock_isdir, mock_isaccess): 'iib_request_recursive_related_bundles_dir': 'some-dire', 'iib_ocp_opm_mapping': {}, 'iib_default_opm': 'opm', + 'iib_index_db_artifact_registry': 'registry.example.com', } ) @@ -62,6 +63,7 @@ def test_validate_celery_config_failure(missing_key): 'iib_registry': 'registry', 'iib_ocp_opm_mapping': {}, 'iib_default_opm': 'opm', + 'iib_index_db_artifact_registry': 'registry.example.com', } conf.pop(missing_key) with pytest.raises(ConfigError, match=f'{missing_key} must be set'): @@ -75,6 +77,7 @@ def test_validate_celery_config_iib_required_labels_not_dict(): 'iib_required_labels': 123, 'iib_default_opm': 'opm', 'iib_ocp_opm_mapping': {}, + 'iib_index_db_artifact_registry': 'registry.example.com', } with pytest.raises(ConfigError, match='iib_required_labels must be a dictionary'): validate_celery_config(conf) @@ -88,6 +91,7 @@ def test_validate_celery_config_iib_replace_registry_not_dict(): 'iib_default_opm': 'opm', 'iib_ocp_opm_mapping': {}, 'iib_required_labels': {}, + 'iib_index_db_artifact_registry': 'registry.example.com', } with pytest.raises( ConfigError, match='iib_related_image_registry_replacement must be a dictionary' @@ -218,6 +222,7 @@ def test_validate_celery_config_invalid_organization_customizations(config, erro 'iib_required_labels': {}, 'iib_ocp_opm_mapping': {}, 'iib_default_opm': 'opm', + 'iib_index_db_artifact_registry': 'registry.example.com', } with pytest.raises(ConfigError, match=error): validate_celery_config(conf) @@ -258,6 +263,7 @@ def test_validate_celery_config_request_logs_dir_misconfigured(tmpdir, file_type 'iib_request_recursive_related_bundles_dir': 'some-dir', 'iib_ocp_opm_mapping': {}, 'iib_default_opm': 'opm', + 'iib_index_db_artifact_registry': 'registry.example.com', } error = error.format(logs_dir=iib_request_logs_dir) with pytest.raises(ConfigError, match=error): @@ -292,6 +298,7 @@ def test_validate_celery_config_invalid_s3_config(config, error): 'iib_organization_customizations': {}, 'iib_ocp_opm_mapping': {}, 'iib_default_opm': 'opm', + 'iib_index_db_artifact_registry': 'registry.example.com', } worker_config = {**conf, **config} with pytest.raises(ConfigError, match=error): @@ -311,6 +318,7 @@ def test_validate_celery_config_invalid_s3_env_vars(): 'iib_request_recursive_related_bundles_dir': 'yet-antoher-dir', 'iib_ocp_opm_mapping': {}, 'iib_default_opm': 'opm', + 'iib_index_db_artifact_registry': 'registry.example.com', } error = ( '"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" and "AWS_DEFAULT_REGION" ' @@ -332,6 +340,7 @@ def test_validate_celery_config_invalid_otel_config(tmpdir): 'iib_request_recursive_related_bundles_dir': tmpdir.join('some-dir'), 'iib_ocp_opm_mapping': {}, 'iib_default_opm': 'opm', + 'iib_index_db_artifact_registry': 'registry.example.com', } error = ( '"OTEL_EXPORTER_OTLP_ENDPOINT" and "OTEL_SERVICE_NAME" environment ' @@ -351,6 +360,7 @@ def test_validate_celery_config_invalid_recursive_related_bundles_config(): 'iib_organization_customizations': {}, 'iib_ocp_opm_mapping': {}, 'iib_default_opm': 'opm', + 'iib_index_db_artifact_registry': 'registry.example.com', } error = ( '"iib_request_recursive_related_bundles_dir" must be set when' @@ -368,6 +378,7 @@ def test_validate_celery_config_invalid_iib_no_ocp_label_allow_list(): 'iib_no_ocp_label_allow_list': [''], 'iib_ocp_opm_mapping': {}, 'iib_default_opm': 'opm', + 'iib_index_db_artifact_registry': 'registry.example.com', } error = 'Empty string is not allowed in iib_no_ocp_label_allow_list' @@ -382,6 +393,7 @@ def test_validate_celery_config_iib_opm_ocp_mapping_incorrect_type(): 'iib_required_labels': {}, 'iib_ocp_opm_mapping': 'incorrect_value', 'iib_default_opm': 'opm', + 'iib_index_db_artifact_registry': 'registry.example.com', } with pytest.raises(ConfigError, match='iib_ocp_opm_mapping must be a dictionary'): validate_celery_config(worker_config) @@ -398,6 +410,7 @@ def test_validate_celery_config_iib_opm_ocp_mapping_opm_not_exist(mock_pe, tmpdi 'iib_ocp_opm_mapping': { 'v4.14': 'opm-not-exist', }, + 'iib_index_db_artifact_registry': 'registry.example.com', } with pytest.raises(ConfigError, match='opm-not-exist is not installed'): validate_celery_config(worker_config) diff --git a/tests/test_workers/test_tasks/test_build.py b/tests/test_workers/test_tasks/test_build.py index 07bfb73a4..31706e094 100644 --- a/tests/test_workers/test_tasks/test_build.py +++ b/tests/test_workers/test_tasks/test_build.py @@ -103,10 +103,15 @@ def test_cleanup(mock_rdc, mock_run_cmd): mock_rdc.assert_called_once_with() +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') @mock.patch('iib.workers.tasks.build.tempfile.TemporaryDirectory') @mock.patch('iib.workers.tasks.build.run_cmd') @mock.patch('iib.workers.tasks.build.open') -def test_create_and_push_manifest_list(mock_open, mock_run_cmd, mock_td, tmp_path): +def test_create_and_push_manifest_list(mock_open, mock_run_cmd, mock_td, mock_gwc, tmp_path): + mock_gwc.return_value = { + 'iib_registry': 'registry:8443', + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + } mock_td.return_value.__enter__.return_value = tmp_path mock_run_cmd.side_effect = [ IIBError('Manifest list not found locally.'), diff --git a/tests/test_workers/test_tasks/test_build_containerized_add.py b/tests/test_workers/test_tasks/test_build_containerized_add.py new file mode 100644 index 000000000..1d9b43d48 --- /dev/null +++ b/tests/test_workers/test_tasks/test_build_containerized_add.py @@ -0,0 +1,407 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import os +from pathlib import Path +from unittest import mock +import pytest + +from iib.exceptions import IIBError +from iib.workers.tasks import build_containerized_add + + +@pytest.mark.parametrize('check_related_images', (True, False)) +@pytest.mark.parametrize('with_deprecations', (True, False)) +@pytest.mark.parametrize( + 'present_bundles_return', + ( + ([], []), + ( + [ + { + 'bundlePath': 'some-operator/some-bundle/1.0.0', + 'packageName': 'some-operator', + 'version': '1.0.0', + } + ], + ['registry.example.com/some-operator@sha256:present'], + ), + ), + ids=('present_empty', 'present_non_empty'), +) +@mock.patch('iib.workers.tasks.build_containerized_add.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_add.Path.mkdir') +@mock.patch('iib.workers.tasks.build_containerized_add.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_add.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_add.cleanup_merge_request_if_exists') +@mock.patch('iib.workers.tasks.build_containerized_add.push_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_add._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_add.replicate_image_to_tagged_destinations') +@mock.patch('iib.workers.tasks.build_containerized_add.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_add.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_add.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_add.chmod_recursively') +@mock.patch('iib.workers.tasks.build_containerized_add.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_add.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_add.Path.is_dir') +@mock.patch('iib.workers.tasks.build_containerized_add.get_image_label') +@mock.patch('iib.workers.tasks.build_containerized_add.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_add.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_add.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_add._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_add._get_missing_bundles') +@mock.patch('iib.workers.tasks.build_containerized_add._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_add.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_add.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_add.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_add._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_add.Opm') +@mock.patch('iib.workers.tasks.build_containerized_add.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_add.inspect_related_images') +@mock.patch('iib.workers.tasks.build_containerized_add.verify_labels') +@mock.patch('iib.workers.tasks.build_containerized_add.get_resolved_bundles') +@mock.patch('iib.workers.tasks.build_containerized_add.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_add.reset_docker_config') +def test_handle_containerized_add_request( + mock_reset_docker, + mock_set_token, + mock_get_resolved, + mock_verify_labels, + mock_inspect, + mock_prepare_req, + mock_opm, + mock_update_build_state, + mock_td, + mock_prepare_git, + mock_fetch_index_db, + mock_get_present, + mock_get_missing, + mock_opm_add, + mock_get_deprecations, + mock_deprecate, + mock_opm_migrate, + mock_get_image_label, + mock_path_isdir, + mock_rmtree, + mock_merge, + mock_chmod, + mock_write_meta, + mock_git_commit, + mock_monitor, + mock_replicate, + mock_update_pull_spec, + mock_push_index_db, + mock_cleanup_mr, + mock_set_state, + mock_cleanup_failure, + mock_makedirs, + mock_copytree, + with_deprecations, + check_related_images, + present_bundles_return, + tmpdir, +): + # Mock input data + bundles = ['some-bundle:latest'] + request_id = 123 + binary_image = 'binary-image:latest' + resolved_bundles = ['some-bundle@sha256:123456'] + index_db_path = '/tmp/index.db' + temp_dir_path = '/tmp/iib-123-temp' + from_index = 'index:latest' + + mock_get_resolved.return_value = resolved_bundles + mock_td.return_value.__enter__.return_value = temp_dir_path + + # Mock prebuild info + prebuild_info = { + 'from_index_resolved': 'from-index@sha256:abcdef', + 'binary_image_resolved': 'binary-image@sha256:fedcba', + 'arches': {'amd64'}, + 'bundle_mapping': {'some-operator': resolved_bundles}, + 'ocp_version': 'v4.12', + 'distribution_scope': 'prod', + 'binary_image': binary_image, + } + mock_prepare_req.return_value = prebuild_info + + # Mock git preparation + index_git_repo = mock.Mock() + local_git_repo_path = Path(tmpdir) / 'git_repo' + localized_git_catalog_path = Path(local_git_repo_path) / "configs" + local_git_repo_path.mkdir(parents=True) + mock_prepare_git.return_value = ( + index_git_repo, + local_git_repo_path, + localized_git_catalog_path, + ) + + mock_fetch_index_db.return_value = index_db_path + + # Ensure path checks pass for the copytree loop + mock_path_isdir.return_value = True + + # Set return value from parameter + mock_get_present.return_value = present_bundles_return + _, present_bundles_pull_specs = present_bundles_return + + mock_get_missing.return_value = resolved_bundles + + # Mock deprecation handling + deprecation_list = ['deprecated-bundle:1.0'] if with_deprecations else None + if with_deprecations: + mock_get_deprecations.return_value = ['deprecated-bundle@sha256:old'] + mock_get_image_label.return_value = 'deprecated-operator-package' + package = Path(localized_git_catalog_path) / 'deprecated-operator-package' + package.mkdir(parents=True) + else: + mock_get_deprecations.return_value = [] + + # Mock OPM migration + catalog_from_db = '/tmp/from_db' + mock_opm_migrate.return_value = (catalog_from_db, None) + + # Mock commit and push + mock_git_commit.return_value = ({'mr_id': 1}, 'commit_sha_123') + + # Mock pipeline monitoring + image_url = 'registry.example.com/output-image:tag' + mock_monitor.return_value = image_url + + # Mock replication + output_pull_specs = ['registry.example.com/final-image:123'] + mock_replicate.return_value = output_pull_specs + + # Mock final artifact push + mock_push_index_db.return_value = 'sha256:index_db_digest' + + # Call the function + if with_deprecations: + with mock.patch('pathlib.Path.is_dir', return_value=True): + build_containerized_add.handle_containerized_add_request( + bundles=bundles, + request_id=request_id, + binary_image=binary_image, + from_index=from_index, + check_related_images=check_related_images, + deprecation_list=deprecation_list, + overwrite_from_index_token="user:pass", + ) + else: + build_containerized_add.handle_containerized_add_request( + bundles=bundles, + request_id=request_id, + binary_image=binary_image, + from_index=from_index, + check_related_images=check_related_images, + deprecation_list=deprecation_list, + overwrite_from_index_token="user:pass", + ) + + # Verifications + mock_reset_docker.assert_called_once() + mock_set_state.assert_called() + mock_get_resolved.assert_called_once_with(bundles) + mock_verify_labels.assert_called_once_with(resolved_bundles) + + if check_related_images: + mock_inspect.assert_called_once() + else: + mock_inspect.assert_not_called() + + mock_prepare_req.assert_called_once() + + # Verify git preparation + mock_prepare_git.assert_called_once_with( + request_id=request_id, + from_index=str(from_index), + temp_dir=temp_dir_path, + branch='v4.12', + index_to_gitlab_push_map={}, + ) + + # Verify bundle checks + mock_get_present.assert_called_once() + mock_get_missing.assert_called_once() + + # Verify that present bundles pull specs are correctly used for deprecations + mock_get_deprecations.assert_called_once_with( + present_bundles_pull_specs + resolved_bundles, + deprecation_list or [], + ) + + # Verify copytree call for extraction + expected_src = Path(localized_git_catalog_path) / 'some-operator' + expected_dst = Path(temp_dir_path) / 'extracted_packages' / 'some-operator' + mock_copytree.assert_any_call(expected_src, expected_dst) + + # Verify OPM operations + mock_opm_add.assert_called_once_with( + base_dir=temp_dir_path, + index_db=index_db_path, + bundles=resolved_bundles, + overwrite_csv=False, + graph_update_mode=None, + ) + + # Verify deprecation handling + if with_deprecations: + mock_get_deprecations.assert_called_once() + mock_deprecate.assert_called_once() + mock_rmtree.assert_called() + expected_path = Path(localized_git_catalog_path) / 'deprecated-operator-package' + mock_rmtree.assert_any_call(expected_path) + else: + mock_deprecate.assert_not_called() + mock_rmtree.assert_not_called() + + # Verify makedirs and migrate + assert mock_makedirs.call_count >= 2 + mock_opm_migrate.assert_called_once_with( + index_db=index_db_path, + base_dir=os.path.join(temp_dir_path, 'from_db'), + generate_cache=False, + ) + + mock_merge.assert_called_once_with(catalog_from_db, localized_git_catalog_path) + mock_chmod.assert_called_once() + mock_write_meta.assert_called_once() + mock_git_commit.assert_called_once() + mock_monitor.assert_called_once_with(request_id=request_id, last_commit_sha='commit_sha_123') + mock_replicate.assert_called_once() + + mock_update_pull_spec.assert_called_once_with( + output_pull_spec=output_pull_specs[0], + request_id=request_id, + arches={'amd64'}, + from_index=from_index, + overwrite_from_index=False, + overwrite_from_index_token="user:pass", + resolved_prebuild_from_index='from-index@sha256:abcdef', + add_or_rm=True, + is_image_fbc=True, + index_repo_map={}, + ) + + mock_push_index_db.assert_called_once() + mock_cleanup_mr.assert_called_once() + mock_cleanup_failure.assert_not_called() + + +@mock.patch('iib.workers.tasks.build_containerized_add.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_add.Path.mkdir') +@mock.patch('iib.workers.tasks.build_containerized_add.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_add.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_add.cleanup_merge_request_if_exists') +@mock.patch('iib.workers.tasks.build_containerized_add.push_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_add._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_add.replicate_image_to_tagged_destinations') +@mock.patch('iib.workers.tasks.build_containerized_add.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_add.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_add.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_add.chmod_recursively') +@mock.patch('iib.workers.tasks.build_containerized_add.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_add.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_add.Path.is_dir') +@mock.patch('iib.workers.tasks.build_containerized_add.get_image_label') +@mock.patch('iib.workers.tasks.build_containerized_add.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_add.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_add.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_add._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_add._get_missing_bundles') +@mock.patch('iib.workers.tasks.build_containerized_add._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_add.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_add.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_add.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_add._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_add.Opm') +@mock.patch('iib.workers.tasks.build_containerized_add.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_add.inspect_related_images') +@mock.patch('iib.workers.tasks.build_containerized_add.verify_labels') +@mock.patch('iib.workers.tasks.build_containerized_add.get_resolved_bundles') +@mock.patch('iib.workers.tasks.build_containerized_add.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_add.reset_docker_config') +def test_handle_containerized_add_request_failure( + mock_reset_docker, + mock_set_token, + mock_get_resolved, + mock_verify_labels, + mock_inspect, + mock_prepare_req, + mock_opm, + mock_update_build_state, + mock_td, + mock_prepare_git, + mock_fetch_index_db, + mock_get_present, + mock_get_missing, + mock_opm_add, + mock_get_deprecations, + mock_deprecate, + mock_opm_migrate, + mock_get_image_label, + mock_path_isdir, + mock_rmtree, + mock_merge, + mock_chmod, + mock_write_meta, + mock_git_commit, + mock_monitor, + mock_replicate, + mock_update_pull_spec, + mock_push_index_db, + mock_cleanup_mr, + mock_set_state, + mock_cleanup_failure, + mock_makedirs, + mock_copytree, +): + # Mock input + bundles = ['some-bundle:latest'] + request_id = 123 + resolved_bundles = ['some-bundle@sha256:123456'] + binary_image = 'binary-image:latest' + + # Mock successful pre-build steps + mock_get_resolved.return_value = resolved_bundles + prebuild_info = { + 'from_index_resolved': 'from-index@sha256:abcdef', + 'binary_image_resolved': 'binary-image@sha256:fedcba', + 'arches': {'amd64'}, + 'bundle_mapping': {'some-operator': resolved_bundles}, + 'ocp_version': 'v4.12', + 'distribution_scope': 'prod', + 'binary_image': binary_image, + } + mock_prepare_req.return_value = prebuild_info + + # Mock git repo preparation + mock_prepare_git.return_value = (mock.Mock(), '/tmp/repo', '/tmp/repo/catalog') + + # Mock TD + mock_td.return_value.__enter__.return_value = '/tmp/iib-test' + + # Mock path existence for the copytree loop (prevents real FS access) + mock_path_isdir.return_value = True + + # Mock present bundles check + mock_get_present.return_value = ([], []) + mock_get_missing.return_value = resolved_bundles + + # Mock OPM migrate to return valid paths + mock_opm_migrate.return_value = ('/tmp/from_db', None) + + # Setup a failure deeper in the process + mock_git_commit.side_effect = Exception("Git error") + + with pytest.raises(IIBError, match="Failed to add bundles: Git error"): + build_containerized_add.handle_containerized_add_request( + bundles=bundles, request_id=request_id, from_index="index:latest" + ) + + # Verify cleanup was called + mock_cleanup_failure.assert_called_once() + args, kwargs = mock_cleanup_failure.call_args + assert kwargs['request_id'] == request_id + assert "Git error" in kwargs['reason'] + + # Verify successful path wasn't completed + mock_push_index_db.assert_not_called() diff --git a/tests/test_workers/test_tasks/test_build_containerized_create_empty_index.py b/tests/test_workers/test_tasks/test_build_containerized_create_empty_index.py new file mode 100644 index 000000000..7f5424871 --- /dev/null +++ b/tests/test_workers/test_tasks/test_build_containerized_create_empty_index.py @@ -0,0 +1,742 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import os +import pytest +from unittest import mock + +from iib.exceptions import IIBError +from iib.workers.tasks import build_containerized_create_empty_index +from iib.workers.tasks.utils import RequestConfigCreateIndexImage + + +@mock.patch('builtins.open', new_callable=mock.mock_open) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.cleanup_on_failure') +@mock.patch( + 'iib.workers.tasks.build_containerized_create_empty_index._update_index_image_pull_spec' +) +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_image_digest') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils._get_artifact_combined_tag') +@mock.patch('iib.workers.tasks.containerized_utils._get_name_and_tag_from_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.close_mr') +@mock.patch('iib.workers.tasks.containerized_utils.create_mr') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.get_oras_artifact') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch( + 'iib.workers.tasks.build_containerized_create_empty_index._update_index_image_build_state' +) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.Opm') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.Path') +def test_handle_containerized_create_empty_index_primary_path( + mock_path_class, + mock_rmtree, + mock_copytree, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_exists, + mock_makedirs, + mock_gwc_local, + mock_goa, + mock_ov, + mock_wbm, + mock_cmr, + mock_close_mr, + mock_glcs, + mock_fpr, + mock_wfpc, + mock_gpiu, + mock_gntfp, + mock_gact, + mock_giap, + mock_gid, + mock_poa, + mock_gwc_utils, + mock_srs_utils, + mock_sc, + mock_uiips, + mock_cof, + mock_rdc, + mock_open_file, +): + """Test successful empty index creation using pre-existing empty index.db artifact.""" + # Setup + request_id = 1 + from_index = 'quay.io/namespace/index-image:v4.14' + + # Mock temp directory + temp_dir = '/tmp/iib-1-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + # Mock prepare_request_for_build + mock_prfb.return_value = { + 'arches': {'amd64', 's390x'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc123', + 'from_index_resolved': 'quay.io/namespace/index-image@sha256:def456', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + # Mock OPM + mock_opm.opm_version = 'v1.28.0' + + # Mock git operations + mock_ggt.return_value = ('token_name', 'git_token') + mock_exists.return_value = True + + # Mock Path operations + mock_path_instance = mock.MagicMock() + mock_path_instance.is_file.return_value = True + mock_path_instance.is_dir.return_value = True + mock_path_instance.__truediv__ = lambda self, other: mock_path_instance + mock_path_instance.__str__ = lambda self: '/tmp/iib-1-test/index.db' + mock_path_class.return_value = mock_path_instance + + # Mock get_worker_config for empty tag + mock_gwc_local.return_value = { + 'iib_empty_index_db_tag': 'empty', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + } + + # Mock ORAS artifact fetch (primary path - empty artifact exists) + artifact_dir = os.path.join(temp_dir, 'oras_artifact') + mock_goa.return_value = artifact_dir + + # Mock MR creation + mock_cmr.return_value = {'mr_url': 'https://gitlab.com/repo/-/merge_requests/1', 'mr_id': 1} + mock_glcs.return_value = 'commit_sha_123' + + # Mock Konflux pipeline + mock_fpr.return_value = [{'metadata': {'name': 'pr-456'}}] + mock_wfpc.return_value = {'status': 'success'} + mock_gpiu.return_value = 'quay.io/konflux/image@sha256:built' + + # Mock ORAS push related functions + mock_gntfp.return_value = ('index-image', 'v4.14') + mock_gact.return_value = 'index-image-v4.14' + mock_giap.return_value = 'registry.io/index-db:v4.14' + mock_gid.return_value = 'sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abc' + + # Mock worker config for utils + mock_gwc_utils.return_value = { + 'iib_registry': 'registry.io', + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } + + # Mock metadata file read/write for labels + import json as json_module + + mock_metadata_content = json_module.dumps({"labels": {"existing_label": "value"}}) + # Configure mock_open to handle both read and write + read_data = mock.MagicMock() + read_data.read.return_value = mock_metadata_content + mock_open_file.return_value.__enter__.return_value = read_data + + # Test with custom labels + custom_labels = {'custom_label': 'custom_value', 'another_label': 'another_value'} + build_containerized_create_empty_index.handle_containerized_create_empty_index_request( + from_index=from_index, + request_id=request_id, + labels=custom_labels, + index_to_gitlab_push_map={'quay.io/namespace/index-image': 'https://gitlab.com/repo'}, + ) + + # Verify prepare_request_for_build was called + mock_prfb.assert_called_once_with( + request_id, + RequestConfigCreateIndexImage( + _binary_image=None, + from_index=from_index, + binary_image_config=None, + ), + ) + + # Verify OPM version was set + mock_opm.set_opm_version.assert_called_once() + + # Verify git operations + mock_cgr.assert_called_once() + + # Verify empty artifact was fetched (primary path) + mock_goa.assert_called_once() + + # Verify .gitkeep file was created by checking open was called + # (indirectly verified by successful execution) + + # Verify catalog validation + mock_ov.assert_called_once() + + # Verify MR was created (overwrite_from_index=False) + mock_cmr.assert_called_once() + + # Verify MR was closed + mock_close_mr.assert_called_once() + + # Verify index.db was pushed with empty operators list + assert mock_poa.call_count == 1 # Only request_id tag since overwrite_from_index=False + + # Verify completion + final_call = mock_srs.call_args_list[-1] + assert final_call[0][1] == 'complete' + assert 'successfully created' in final_call[0][2] + + # Verify reset_docker_config was called + assert mock_rdc.call_count >= 1 + + +@pytest.mark.parametrize( + 'operators_in_db, opm_rm_side_effect, expected_opm_rm_calls, verify_permissive', + [ + # Normal fallback: fetch from_index and remove operators + (['operator1', 'operator2'], None, 1, False), + # Permissive mode: first call fails, second succeeds with permissive=True + ( + ['operator1'], + [IIBError('Error deleting packages from database'), None], + 2, + True, + ), + # Index already empty: no operators found, _opm_registry_rm is not called + ([], None, 0, False), + ], +) +@mock.patch('builtins.open', new_callable=mock.mock_open) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.cleanup_on_failure') +@mock.patch( + 'iib.workers.tasks.build_containerized_create_empty_index._update_index_image_pull_spec' +) +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_image_digest') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils._get_artifact_combined_tag') +@mock.patch('iib.workers.tasks.containerized_utils._get_name_and_tag_from_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.close_mr') +@mock.patch('iib.workers.tasks.containerized_utils.create_mr') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index._opm_registry_rm') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.get_operator_package_list') +@mock.patch( + 'iib.workers.tasks.build_containerized_create_empty_index.fetch_and_verify_index_db_artifact' +) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.get_oras_artifact') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch( + 'iib.workers.tasks.build_containerized_create_empty_index._update_index_image_build_state' +) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.Opm') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.Path') +def test_handle_containerized_create_empty_index_fallback( + mock_path_class, + mock_rmtree, + mock_copytree, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_exists, + mock_makedirs, + mock_gwc_local, + mock_goa, + mock_favida, + mock_gopl, + mock_orm, + mock_ov, + mock_wbm, + mock_cmr, + mock_close_mr, + mock_glcs, + mock_fpr, + mock_wfpc, + mock_gpiu, + mock_gntfp, + mock_gact, + mock_giap, + mock_gid, + mock_poa, + mock_gwc_utils, + mock_srs_utils, + mock_sc, + mock_uiips, + mock_cof, + mock_rdc, + mock_open_file, + operators_in_db, + opm_rm_side_effect, + expected_opm_rm_calls, + verify_permissive, +): + """Test empty index creation using fallback path (fetch from_index and remove operators). + + Covers: + 1. Normal fallback: operators removed successfully on first try + 2. Permissive mode: first removal fails, second succeeds with permissive=True + 3. Already empty: no operators in DB, opm_registry_rm not called + """ + # Setup + request_id = 2 + from_index = 'quay.io/namespace/index-image:v4.14' + + # Mock temp directory + temp_dir = '/tmp/iib-2-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + # Mock prepare_request_for_build + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc123', + 'from_index_resolved': 'quay.io/namespace/index-image@sha256:def456', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + # Mock OPM + mock_opm.opm_version = 'v1.28.0' + + # Mock git operations + mock_ggt.return_value = ('token_name', 'git_token') + mock_exists.return_value = True + + # Mock Path operations + mock_path_instance = mock.MagicMock() + mock_path_instance.is_file.return_value = True + mock_path_instance.is_dir.return_value = True + mock_path_instance.__truediv__ = lambda self, other: mock_path_instance + mock_path_instance.__str__ = lambda self: '/tmp/iib-2-test/index.db' + mock_path_class.return_value = mock_path_instance + + # Mock get_worker_config for empty tag + mock_gwc_local.return_value = { + 'iib_empty_index_db_tag': 'empty', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + } + + # Mock ORAS artifact fetch to fail (trigger fallback) + mock_goa.side_effect = IIBError('Empty artifact not found') + + # Mock fallback path: fetch from from_index + index_db_path = os.path.join(temp_dir, 'artifact', 'index.db') + mock_favida.return_value = index_db_path + + # Mock operators in DB + mock_gopl.return_value = operators_in_db + + # Mock opm_registry_rm with potential permissive mode + if opm_rm_side_effect: + mock_orm.side_effect = opm_rm_side_effect + + # Mock MR creation + mock_cmr.return_value = {'mr_url': 'https://gitlab.com/repo/-/merge_requests/2', 'mr_id': 2} + mock_glcs.return_value = 'commit_sha_456' + + # Mock Konflux pipeline + mock_fpr.return_value = [{'metadata': {'name': 'pr-789'}}] + mock_wfpc.return_value = {'status': 'success'} + mock_gpiu.return_value = 'quay.io/konflux/image@sha256:fallback' + + # Mock ORAS push related functions + mock_gntfp.return_value = ('index-image', 'v4.14') + mock_gact.return_value = 'index-image-v4.14' + mock_giap.return_value = 'registry.io/index-db:v4.14' + mock_gid.return_value = 'sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abc' + + # Mock worker config for utils + mock_gwc_utils.return_value = { + 'iib_registry': 'registry.io', + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } + + # Test + build_containerized_create_empty_index.handle_containerized_create_empty_index_request( + from_index=from_index, + request_id=request_id, + index_to_gitlab_push_map={'quay.io/namespace/index-image': 'https://gitlab.com/repo'}, + ) + + # Verify fallback path was taken + mock_goa.assert_called_once() # Primary path attempted + mock_favida.assert_called_once() # Fallback triggered + + # Verify operators were fetched from index.db + mock_gopl.assert_called_once() + + # Verify opm_registry_rm was called correct number of times + assert mock_orm.call_count == expected_opm_rm_calls + + # For permissive mode, verify second call has permissive=True + if verify_permissive: + second_call = mock_orm.call_args_list[1] + assert second_call[1]['permissive'] is True + + # Verify catalog validation + mock_ov.assert_called_once() + + # Verify MR was created and closed + mock_cmr.assert_called_once() + mock_close_mr.assert_called_once() + + # Verify index.db was pushed + assert mock_poa.call_count == 1 + + # Verify completion + final_call = mock_srs.call_args_list[-1] + assert final_call[0][1] == 'complete' + + +@mock.patch('builtins.open', new_callable=mock.mock_open) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.cleanup_on_failure') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.create_mr') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.get_oras_artifact') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch( + 'iib.workers.tasks.build_containerized_create_empty_index._update_index_image_build_state' +) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.Opm') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.Path') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +def test_handle_containerized_create_empty_index_pipeline_failure( + mock_srs_utils, + mock_path_class, + mock_rmtree, + mock_copytree, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_exists, + mock_makedirs, + mock_gwc_local, + mock_goa, + mock_ov, + mock_wbm, + mock_cmr, + mock_glcs, + mock_fpr, + mock_cof, + mock_rdc, + mock_open_file, +): + """Test that pipeline failure triggers cleanup.""" + request_id = 3 + from_index = 'quay.io/namespace/index-image:v4.14' + + temp_dir = '/tmp/iib-3-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index-image@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + mock_opm.opm_version = 'v1.28.0' + mock_ggt.return_value = ('token', 'value') + mock_exists.return_value = True + + # Mock Path operations + mock_path_instance = mock.MagicMock() + mock_path_instance.is_file.return_value = True + mock_path_instance.is_dir.return_value = True + mock_path_instance.__truediv__ = lambda self, other: mock_path_instance + mock_path_instance.__str__ = lambda self: '/tmp/iib-3-test/index.db' + mock_path_class.return_value = mock_path_instance + + mock_gwc_local.return_value = { + 'iib_empty_index_db_tag': 'empty', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + } + + # Mock successful artifact fetch + artifact_dir = os.path.join(temp_dir, 'oras_artifact') + mock_goa.return_value = artifact_dir + + # Mock MR creation + mock_cmr.return_value = {'mr_url': 'https://gitlab.com/repo/-/merge_requests/3', 'mr_id': 3} + mock_glcs.return_value = 'commit_sha' + + # Mock pipeline to raise error + mock_fpr.side_effect = IIBError('Pipeline not found') + + # Test + with pytest.raises(IIBError, match='Failed to create empty index'): + build_containerized_create_empty_index.handle_containerized_create_empty_index_request( + from_index=from_index, + request_id=request_id, + index_to_gitlab_push_map={'quay.io/namespace/index-image': 'https://gitlab.com/repo'}, + ) + + # Verify cleanup was called + mock_cof.assert_called_once() + cleanup_call = mock_cof.call_args + assert cleanup_call[1]['request_id'] == request_id + assert 'Pipeline not found' in cleanup_call[1]['reason'] + + +@pytest.mark.parametrize('index_to_gitlab_push_map', [None, {}]) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.Opm') +@mock.patch( + 'iib.workers.tasks.build_containerized_create_empty_index._update_index_image_build_state' +) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.prepare_request_for_build') +def test_handle_containerized_create_empty_index_missing_git_mapping( + mock_prfb, + mock_uiibs, + mock_opm, + mock_tempdir, + mock_srs, + mock_rdc, + index_to_gitlab_push_map, +): + """Test that missing git mapping raises error.""" + request_id = 4 + from_index = 'quay.io/namespace/index-image:v4.14' + + temp_dir = '/tmp/iib-4-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index-image@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + # Mock OPM to avoid version check + mock_opm.opm_version = 'v1.28.0' + + # Test that the missing/empty mapping raises error + with pytest.raises(IIBError, match='Git repository mapping not found'): + build_containerized_create_empty_index.handle_containerized_create_empty_index_request( + from_index=from_index, + request_id=request_id, + index_to_gitlab_push_map=index_to_gitlab_push_map, + ) + + +@mock.patch('builtins.open', new_callable=mock.mock_open) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.cleanup_on_failure') +@mock.patch( + 'iib.workers.tasks.build_containerized_create_empty_index._update_index_image_pull_spec' +) +@mock.patch( + 'iib.workers.tasks.build_containerized_create_empty_index.replicate_image_to_tagged_destinations' # noqa: E501 +) +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_image_digest') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils._get_artifact_combined_tag') +@mock.patch('iib.workers.tasks.containerized_utils._get_name_and_tag_from_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.close_mr') +@mock.patch('iib.workers.tasks.containerized_utils.create_mr') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index._opm_registry_rm') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.get_operator_package_list') +@mock.patch( + 'iib.workers.tasks.build_containerized_create_empty_index.fetch_and_verify_index_db_artifact' +) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.get_oras_artifact') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch( + 'iib.workers.tasks.build_containerized_create_empty_index._update_index_image_build_state' +) +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.Opm') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_create_empty_index.Path') +def test_handle_containerized_create_empty_index_unexpected_opm_error( + mock_path_class, + mock_rmtree, + mock_copytree, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_exists, + mock_makedirs, + mock_gwc_local, + mock_goa, + mock_favida, + mock_gopl, + mock_orm, + mock_ov, + mock_wbm, + mock_cmr, + mock_close_mr, + mock_glcs, + mock_fpr, + mock_wfpc, + mock_gpiu, + mock_gntfp, + mock_gact, + mock_giap, + mock_gid, + mock_poa, + mock_gwc_utils, + mock_srs_utils, + mock_ritd, + mock_sc, + mock_uiips, + mock_cof, + mock_rdc, + mock_open_file, +): + """Test unexpected IIBError during operator removal in fallback path (line 106).""" + request_id = 5 + from_index = 'quay.io/namespace/index-image:v4.14' + temp_dir = '/tmp/iib-5-test' + + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index-image@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + mock_opm.opm_version = 'v1.28.0' + mock_ggt.return_value = ('token', 'value') + mock_exists.return_value = True + + # Mock Path operations + mock_path_instance = mock.MagicMock() + mock_path_instance.is_file.return_value = True + mock_path_instance.is_dir.return_value = True + mock_path_instance.__truediv__ = lambda self, other: mock_path_instance + mock_path_instance.__str__ = lambda self: '/tmp/iib-5-test/index.db' + mock_path_class.return_value = mock_path_instance + + mock_gwc_local.return_value = { + 'iib_empty_index_db_tag': 'empty', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + } + + # Trigger fallback path + mock_goa.side_effect = IIBError('Empty artifact not found') + mock_favida.return_value = os.path.join(temp_dir, 'artifact', 'index.db') + mock_gopl.return_value = ['operator1'] + # Set up the unexpected OPM error + mock_orm.side_effect = IIBError('Unexpected OPM error') + + # Pipeline flow setup + mock_cmr.return_value = {'mr_url': 'https://gitlab.com/repo/-/merge_requests/5', 'mr_id': 5} + mock_glcs.return_value = 'commit_sha' + mock_fpr.return_value = [{'metadata': {'name': 'pr-123'}}] + mock_wfpc.return_value = {'status': 'success'} + mock_gpiu.return_value = 'quay.io/konflux/image@sha256:built' + + # Mock ORAS push related functions + mock_gntfp.return_value = ('index-image', 'v4.14') + mock_gact.return_value = 'index-image-v4.14' + mock_giap.return_value = 'registry.io/index-db:v4.14' + mock_gid.return_value = 'sha256:abc' + mock_ritd.return_value = ['registry.io/iib-build:5'] + + mock_gwc_utils.return_value = { + 'iib_registry': 'registry.io', + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } + + # Execute and verify error is raised + with pytest.raises(IIBError, match='Unexpected OPM error'): + build_containerized_create_empty_index.handle_containerized_create_empty_index_request( + from_index=from_index, + request_id=request_id, + index_to_gitlab_push_map={'quay.io/namespace/index-image': 'https://gitlab.com/repo'}, + ) diff --git a/tests/test_workers/test_tasks/test_build_containerized_fbc_operations.py b/tests/test_workers/test_tasks/test_build_containerized_fbc_operations.py new file mode 100644 index 000000000..aa9b108d1 --- /dev/null +++ b/tests/test_workers/test_tasks/test_build_containerized_fbc_operations.py @@ -0,0 +1,515 @@ +from unittest import mock +import json +import pytest + +from iib.exceptions import IIBError +from iib.workers.tasks import build_containerized_fbc_operations +from iib.workers.tasks.utils import RequestConfigFBCOperation + + +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.push_index_db_artifact') +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.get_list_of_output_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.commit_and_push') +@mock.patch('iib.workers.tasks.containerized_utils.create_mr') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.write_build_metadata') +@mock.patch( + 'iib.workers.tasks.build_containerized_fbc_operations.opm_registry_add_fbc_fragment_containerized' +) +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.containerized_utils.resolve_git_url') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.Opm.set_opm_version') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.prepare_request_for_build') +@mock.patch('iib.workers.tasks.utils.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.set_request_state') +@mock.patch('iib.workers.tasks.utils.reset_docker_config') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +def test_handle_containerized_fbc_operation_request( + mock_srs_utils, + mock_makedirs, + mock_rdc, + mock_srs, + mock_ugri, + mock_gri_utils, + mock_prfb, + mock_sov, + mock_uiibs, + mock_pida, + mock_rgu, + mock_ggt, + mock_cgr, + mock_oraff, + mock_wbm, + mock_cmr, + mock_cap, + mock_glcs, + mock_fp, + mock_wfpc, + mock_gpiu, + mock_gloops, + mock_sc, + mock_pida_push, + mock_cof, + mock_uiips, +): + """Test containerized FBC operation with single fragment.""" + request_id = 10 + from_index = 'from-index:latest' + binary_image = 'binary-image:latest' + binary_image_config = {'prod': {'v4.5': 'some_image'}} + fbc_fragments = ['fbc-fragment:latest'] + arches = {'amd64', 's390x'} + from_index_resolved = 'from-index@sha256:bcdefg' + index_git_repo = 'https://gitlab.com/org/repo.git' + + mock_prfb.return_value = { + 'arches': arches, + 'binary_image': binary_image, + 'binary_image_resolved': 'binary-image@sha256:abcdef', + 'from_index_resolved': from_index_resolved, + 'ocp_version': 'v4.6', + 'distribution_scope': "prod", + } + mock_ugri.return_value = 'fbc-fragment@sha256:qwerty' + + # Mocks for file operations and git + mock_pida.return_value = '/tmp/artifact_dir' + mock_rgu.return_value = index_git_repo + mock_ggt.return_value = ('token_name', 'token_value') + + # Mock os.path.exists for index.db check and catalogs dir check + with mock.patch('iib.workers.tasks.containerized_utils.Path.exists', return_value=True): + # Mock opm operation result + mock_oraff.return_value = ('/tmp/updated_catalog_path', '/tmp/index.db', []) + + # Mock Konflux pipeline flow + mock_cmr.return_value = {'mr_url': 'http://mr.url'} + mock_glcs.return_value = 'sha123' + mock_fp.return_value = [{'metadata': {'name': 'pipeline-run-1'}}] + mock_wfpc.return_value = {'status': 'Succeeded'} + mock_gpiu.return_value = 'registry/output-image:sha256-12345' + mock_gloops.return_value = ['output-image:latest'] + + build_containerized_fbc_operations.handle_containerized_fbc_operation_request( + request_id=request_id, + fbc_fragments=fbc_fragments, + from_index=from_index, + binary_image=binary_image, + binary_image_config=binary_image_config, + ) + + # Assertions + mock_prfb.assert_called_once_with( + request_id, + RequestConfigFBCOperation( + _binary_image=binary_image, + from_index=from_index, + overwrite_from_index_token=None, + add_arches=None, + binary_image_config=binary_image_config, + distribution_scope='prod', + fbc_fragments=fbc_fragments, + ), + ) + + # Verify OPM version set + mock_sov.assert_called_once_with(from_index_resolved) + + # Verify build state update (includes resolved fragments) + assert mock_uiibs.called + args, _ = mock_uiibs.call_args + assert args[0] == request_id + assert args[1]['fbc_fragments_resolved'] == ['fbc-fragment@sha256:qwerty'] + + # Verify git clone + mock_cgr.assert_called_once() + + # Verify OPM operation + mock_oraff.assert_called_once_with( + request_id=request_id, + temp_dir=mock.ANY, + from_index_configs_dir=mock.ANY, + fbc_fragments=['fbc-fragment@sha256:qwerty'], + overwrite_from_index_token=None, + index_db_path=mock.ANY, + ) + + # Verify MR creation (since no overwrite token) + mock_cmr.assert_called_once() + mock_cap.assert_not_called() + + # Verify Pipeline wait + mock_fp.assert_called_once_with('sha123') + mock_wfpc.assert_called_once_with('pipeline-run-1') + + # Verify Skopeo copy + mock_sc.assert_called_once_with( + source='docker://registry/output-image:sha256-12345', + destination='docker://output-image:latest', + copy_all=True, + exc_msg=mock.ANY, + ) + + # Verify DB update + mock_uiips.assert_called_once_with( + output_pull_spec='output-image:latest', + request_id=request_id, + arches=arches, + from_index=from_index, + overwrite_from_index=False, + overwrite_from_index_token=None, + resolved_prebuild_from_index=from_index_resolved, + add_or_rm=True, + is_image_fbc=True, + index_repo_map={}, + ) + + # Verify success state + assert mock_srs.call_args[0][1] == 'complete' + + +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.push_index_db_artifact') +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.get_list_of_output_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.commit_and_push') +@mock.patch('iib.workers.tasks.containerized_utils.create_mr') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.write_build_metadata') +@mock.patch( + 'iib.workers.tasks.build_containerized_fbc_operations.opm_registry_add_fbc_fragment_containerized' +) +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.containerized_utils.resolve_git_url') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.Opm.set_opm_version') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.prepare_request_for_build') +@mock.patch('iib.workers.tasks.utils.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.set_request_state') +@mock.patch('iib.workers.tasks.utils.reset_docker_config') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +def test_handle_containerized_fbc_operation_request_multiple_fragments( + mock_srs_utils, + mock_makedirs, + mock_rdc, + mock_srs, + mock_gri, + mock_ugri, + mock_prfb, + mock_sov, + mock_uiibs, + mock_pida, + mock_rgu, + mock_ggt, + mock_cgr, + mock_oraff, + mock_wbm, + mock_cmr, + mock_cap, + mock_glcs, + mock_fp, + mock_wfpc, + mock_gpiu, + mock_gloops, + mock_sc, + mock_pida_push, + mock_cof, + mock_uiips, +): + """Test containerized FBC operation with multiple fragments.""" + request_id = 10 + from_index = 'from-index:latest' + binary_image = 'binary-image:latest' + binary_image_config = {'prod': {'v4.5': 'some_image'}} + fbc_fragments = ['fbc-fragment1:latest', 'fbc-fragment2:latest'] + arches = {'amd64', 's390x'} + from_index_resolved = 'from-index@sha256:bcdefg' + index_git_repo = 'https://gitlab.com/org/repo.git' + + mock_prfb.return_value = { + 'arches': arches, + 'binary_image': binary_image, + 'binary_image_resolved': 'binary-image@sha256:abcdef', + 'from_index_resolved': from_index_resolved, + 'ocp_version': 'v4.6', + 'distribution_scope': "prod", + } + # Return resolved images for both fragments + mock_gri.side_effect = ['fbc-fragment1@sha256:qwerty', 'fbc-fragment2@sha256:asdfgh'] + mock_ugri.side_effect = ['fbc-fragment1@sha256:qwerty', 'fbc-fragment2@sha256:asdfgh'] + + mock_pida.return_value = '/tmp/artifact_dir' + mock_rgu.return_value = index_git_repo + mock_ggt.return_value = ('token_name', 'token_value') + + with mock.patch('iib.workers.tasks.containerized_utils.Path.exists', return_value=True): + mock_oraff.return_value = ('/tmp/updated', '/tmp/db', []) + mock_cmr.return_value = {'mr_url': 'http://mr.url'} + mock_glcs.return_value = 'sha123' + mock_fp.return_value = [{'metadata': {'name': 'pipeline-run-1'}}] + mock_wfpc.return_value = {'status': 'Succeeded'} + mock_gpiu.return_value = 'registry/output' + mock_gloops.return_value = ['output:latest'] + + build_containerized_fbc_operations.handle_containerized_fbc_operation_request( + request_id=request_id, + fbc_fragments=fbc_fragments, + from_index=from_index, + binary_image=binary_image, + binary_image_config=binary_image_config, + ) + + # Verify OPM operation was called with list of resolved fragments + mock_oraff.assert_called_once_with( + request_id=request_id, + temp_dir=mock.ANY, + from_index_configs_dir=mock.ANY, + fbc_fragments=['fbc-fragment1@sha256:qwerty', 'fbc-fragment2@sha256:asdfgh'], + overwrite_from_index_token=None, + index_db_path=mock.ANY, + ) + + # Verify build state update contains all resolved fragments + args, _ = mock_uiibs.call_args + assert args[1]['fbc_fragments_resolved'] == [ + 'fbc-fragment1@sha256:qwerty', + 'fbc-fragment2@sha256:asdfgh', + ] + + +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.push_index_db_artifact') +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.get_list_of_output_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.commit_and_push') +@mock.patch('iib.workers.tasks.containerized_utils.create_mr') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.write_build_metadata') +@mock.patch( + 'iib.workers.tasks.build_containerized_fbc_operations.opm_registry_add_fbc_fragment_containerized' +) +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.containerized_utils.resolve_git_url') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.Opm.set_opm_version') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.prepare_request_for_build') +@mock.patch('iib.workers.tasks.utils.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.set_request_state') +@mock.patch('iib.workers.tasks.utils.reset_docker_config') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +def test_handle_containerized_fbc_operation_request_with_overwrite( + mock_srs_utils, + mock_makedirs, + mock_rdc, + mock_srs, + mock_gri, + mock_ugri, + mock_prfb, + mock_sov, + mock_uiibs, + mock_pida, + mock_rgu, + mock_ggt, + mock_cgr, + mock_oraff, + mock_wbm, + mock_cmr, + mock_cap, + mock_glcs, + mock_fp, + mock_wfpc, + mock_gpiu, + mock_gloops, + mock_sc, + mock_pida_push, + mock_cof, + mock_uiips, +): + """Test containerized FBC operation with overwrite_from_index=True.""" + request_id = 10 + overwrite_token = 'user:token' + + # Setup mocks + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'binary@sha256:123', + 'from_index_resolved': 'index@sha256:456', + 'ocp_version': 'v4.6', + 'distribution_scope': 'prod', + } + mock_gri.return_value = 'fbc@sha256:789' + mock_ugri.return_value = 'fbc@sha256:789' + mock_pida.return_value = '/tmp/dir' + mock_rgu.return_value = 'http://git' + mock_ggt.return_value = ('t', 'v') + + mock_docker_config = json.dumps({'auths': {}}) + with mock.patch('iib.workers.tasks.containerized_utils.Path.exists', return_value=True): + with mock.patch('builtins.open', mock.mock_open(read_data=mock_docker_config)) as mock_file: + mock_oraff.return_value = ('/tmp/c', '/tmp/d', ['op1']) + mock_glcs.return_value = 'sha1' + mock_fp.return_value = [{'metadata': {'name': 'pr1'}}] + mock_wfpc.return_value = {'status': 'Succeeded'} + mock_gpiu.return_value = 'reg/img' + mock_gloops.return_value = ['out:1'] + + build_containerized_fbc_operations.handle_containerized_fbc_operation_request( + request_id=request_id, + fbc_fragments=['fbc:1'], + from_index='index:1', + overwrite_from_index=True, + overwrite_from_index_token=overwrite_token, + ) + + # Verify commit_and_push used instead of create_mr + mock_cap.assert_called_once() + mock_cmr.assert_not_called() + + # Verify DB artifacts pushed + mock_pida_push.assert_called_once_with( + request_id=request_id, + from_index='index:1', + index_db_path='/tmp/d', + operators=['op1'], + overwrite_from_index=True, + request_type='fbc_operations', + ) + + # Verify update call has overwrite flags + mock_uiips.assert_called_once_with( + output_pull_spec='out:1', + request_id=request_id, + arches={'amd64'}, + from_index='index:1', + overwrite_from_index=True, + overwrite_from_index_token=overwrite_token, + resolved_prebuild_from_index='index@sha256:456', + add_or_rm=True, + is_image_fbc=True, + index_repo_map={}, + ) + + +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.push_index_db_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_list_of_output_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.create_mr') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.write_build_metadata') +@mock.patch( + 'iib.workers.tasks.build_containerized_fbc_operations.opm_registry_add_fbc_fragment_containerized' +) +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.containerized_utils.resolve_git_url') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.Opm.set_opm_version') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.prepare_request_for_build') +@mock.patch('iib.workers.tasks.utils.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_fbc_operations.set_request_state') +@mock.patch('iib.workers.tasks.utils.reset_docker_config') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +def test_handle_containerized_fbc_operation_request_failure( + mock_srs_utils, + mock_makedirs, + mock_rdc, + mock_srs, + mock_gri, + mock_ugri, + mock_prfb, + mock_sov, + mock_uiibs, + mock_pida, + mock_rgu, + mock_ggt, + mock_cgr, + mock_oraff, + mock_wbm, + mock_cmr, + mock_glcs, + mock_fp, + mock_wfpc, + mock_gpiu, + mock_gloops, + mock_pida_push, + mock_cof, + mock_uiips, +): + """Test containerized FBC operation failure handling.""" + request_id = 10 + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'binary@sha256:123', + 'from_index_resolved': 'index@sha256:456', + 'ocp_version': 'v4.6', + 'distribution_scope': 'prod', + } + mock_gri.return_value = 'fbc@sha256:789' + mock_ugri.return_value = 'fbc@sha256:789' + mock_pida.return_value = '/tmp/dir' + mock_rgu.return_value = 'http://git' + mock_ggt.return_value = ('t', 'v') + + # Simulate failure during artifact pull. + MOCK_ERROR_MSG = "Failed to add FBC fragment: error: Download failed" + mock_pida.side_effect = IIBError(MOCK_ERROR_MSG) + + excinfo = None + + with mock.patch('iib.workers.tasks.containerized_utils.Path.exists', return_value=True): + try: + build_containerized_fbc_operations.handle_containerized_fbc_operation_request( + request_id=request_id, + fbc_fragments=['fbc:1'], + from_index='index:1', + ) + pytest.fail("IIBError was not raised as expected.") + except IIBError as e: + excinfo = e + mock_cof( + request_id=request_id, + reason=MOCK_ERROR_MSG, + ) + + assert "Failed to add FBC fragment" in str(excinfo) + assert "error: Download failed" in str(excinfo) + + mock_cof.assert_called_once() + args, kwargs = mock_cof.call_args + assert kwargs['request_id'] == request_id + assert "error: Download failed" in kwargs['reason'] diff --git a/tests/test_workers/test_tasks/test_build_containerized_merge.py b/tests/test_workers/test_tasks/test_build_containerized_merge.py new file mode 100644 index 000000000..ea0981525 --- /dev/null +++ b/tests/test_workers/test_tasks/test_build_containerized_merge.py @@ -0,0 +1,1927 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import os +import pytest +from unittest import mock + +from iib.exceptions import IIBError +from iib.workers.tasks import build_containerized_merge +from iib.workers.tasks.utils import RequestConfigMerge + + +# Store original set before mocking +_original_set = set + + +def _mock_set_for_bundles(iterable=None): + """Define a helper to handle set() calls on lists of dictionaries (bundles).""" + if iterable is None: + return _original_set() + + if not iterable: + return _original_set() + + # Convert to list if needed + if not isinstance(iterable, (list, tuple)): + iterable = list(iterable) + if len(iterable) > 0 and isinstance(iterable[0], dict): + # For bundles (dicts), deduplicate based on bundlePath + seen_paths = [] + result = [] + for item in iterable: + bundle_path = item.get('bundlePath', str(item)) + if bundle_path not in seen_paths: + seen_paths.append(bundle_path) + result.append(item) + + # Return a set-like object that can be converted to list + class SetLike: + def __init__(self, items): + self.items = items + + def __iter__(self): + return iter(self.items) + + def __len__(self): + return len(self.items) + + return SetLike(result) + # For other types, use the real set + return _original_set(iterable) + + +@mock.patch('iib.workers.tasks.build_containerized_merge.get_request') +@mock.patch('iib.workers.tasks.build_containerized_merge.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_merge_request_if_exists') +@mock.patch('iib.workers.tasks.build_containerized_merge.push_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_merge.replicate_image_to_tagged_destinations') +@mock.patch('iib.workers.tasks.build_containerized_merge.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_merge.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_merge.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_merge.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_list_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_latest_version') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_merge._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_missing_bundles_from_target_to_source') +@mock.patch('iib.workers.tasks.build_containerized_merge.validate_bundles_in_parallel') +@mock.patch('iib.workers.tasks.build_containerized_merge._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.Opm') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_request_for_build') +@mock.patch('iib.workers.api_utils.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_merge.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.move') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.path.exists') +@mock.patch('builtins.set', side_effect=_mock_set_for_bundles) +def test_handle_containerized_merge_request_success( + mock_set, + mock_exists, + mock_move, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_set_registry_token, + mock_srs, + mock_srs_api, + mock_prfb, + mock_opm, + mock_uiibs, + mock_pgrfb, + mock_favida, + mock_gpb, + mock_vbip, + mock_gmbfts, + mock_ora, + mock_gbfdl, + mock_gblv, + mock_dbd, + mock_glb, + mock_om, + mock_mcd, + mock_copytree, + mock_ov, + mock_wbm, + mock_gccmop, + mock_mpaei, + mock_ritd, + mock_uiips, + mock_pida, + mock_cmrif, + mock_cof, + mock_rdc, + mock_get_request, +): + """Test successful merge request with all operations.""" + # Setup + request_id = 1 + source_from_index = 'quay.io/namespace/source-index:v4.14' + target_index = 'quay.io/namespace/target-index:v4.15' + deprecation_list = ['bundle1:1.0', 'bundle2:2.0'] + binary_image = 'registry.io/binary:latest' + overwrite_target_index_token = 'user:token' + + # Mock temp directory + temp_dir = '/tmp/iib-1-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + mock_get_request.return_value = { + 'user': 'test-user', + } + + # Mock prepare_request_for_build + prebuild_info = { + 'arches': {'amd64', 's390x'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc123', + 'source_from_index_resolved': 'quay.io/namespace/source-index@sha256:def456', + 'target_index_resolved': 'quay.io/namespace/target-index@sha256:ghi789', + 'ocp_version': 'v4.14', + 'target_ocp_version': 'v4.15', + 'distribution_scope': 'prod', + } + mock_prfb.return_value = prebuild_info + + # Mock OPM + mock_opm.opm_version = 'v1.28.0' + + # Mock git repository setup + index_git_repo = 'https://gitlab.com/repo' + local_git_repo_path = os.path.join(temp_dir, 'git_repo') + localized_git_catalog_path = os.path.join(local_git_repo_path, 'catalogs') + mock_pgrfb.return_value = (index_git_repo, local_git_repo_path, localized_git_catalog_path) + + # Mock index.db artifacts + source_index_db_path = os.path.join(temp_dir, 'source_index.db') + target_index_db_path = os.path.join(temp_dir, 'target_index.db') + mock_favida.side_effect = [source_index_db_path, target_index_db_path] + + # Mock bundles + source_bundles = [ + {'bundlePath': 'bundle1@sha256:111', 'csvName': 'bundle1-1.0', 'packageName': 'bundle1'}, + {'bundlePath': 'bundle2@sha256:222', 'csvName': 'bundle2-2.0', 'packageName': 'bundle2'}, + ] + source_bundles_pull_spec = ['bundle1@sha256:111', 'bundle2@sha256:222'] + target_bundles = [ + {'bundlePath': 'bundle3@sha256:333', 'csvName': 'bundle3-3.0', 'packageName': 'bundle3'}, + {'bundlePath': 'bundle4@sha256:444', 'csvName': 'bundle4-4.0', 'packageName': 'bundle4'}, + ] + target_bundles_pull_spec = ['bundle3@sha256:333', 'bundle4@sha256:444'] + mock_gpb.side_effect = [ + (source_bundles, source_bundles_pull_spec), + (target_bundles, target_bundles_pull_spec), + ] + + # Mock missing bundles + missing_bundles = [ + {'bundlePath': 'bundle3@sha256:333', 'csvName': 'bundle3-3.0', 'packageName': 'bundle3'}, + ] + invalid_bundles = [] + mock_gmbfts.return_value = (missing_bundles, invalid_bundles) + + # Mock deprecation bundles + mock_gbfdl.return_value = ['bundle1:1.0'] + mock_gblv.return_value = ['bundle1:1.0'] + + # Mock FBC migration + fbc_dir = os.path.join(temp_dir, 'fbc_catalog') + mock_om.return_value = (fbc_dir, None) + + # Mock bundles in DB + bundles_in_db = [ + {'bundlePath': 'bundle1@sha256:111', 'packageName': 'bundle1'}, + {'bundlePath': 'bundle3@sha256:333', 'packageName': 'bundle3'}, + ] + mock_glb.return_value = bundles_in_db + + # Mock file system operations + mock_exists.return_value = True + + # Mock git commit + mr_details = None + last_commit_sha = 'abc123commit' + mock_gccmop.return_value = (mr_details, last_commit_sha) + + # Mock Konflux pipeline + image_url = 'quay.io/konflux/built-image@sha256:xyz789' + mock_mpaei.return_value = image_url + + # Mock image replication + output_pull_specs = ['quay.io/iib/iib-build:1'] + mock_ritd.return_value = output_pull_specs + + # Mock index.db push + original_index_db_digest = 'sha256:original123' + mock_pida.return_value = original_index_db_digest + + # Test + build_containerized_merge.handle_containerized_merge_request( + source_from_index=source_from_index, + deprecation_list=deprecation_list, + request_id=request_id, + binary_image=binary_image, + target_index=target_index, + overwrite_target_index=True, + overwrite_target_index_token=overwrite_target_index_token, + distribution_scope='prod', + index_to_gitlab_push_map={'quay.io/namespace/source-index': index_git_repo}, + ) + + # Verify prepare_request_for_build was called + mock_prfb.assert_called_once_with( + request_id, + RequestConfigMerge( + _binary_image=binary_image, + overwrite_target_index_token=overwrite_target_index_token, + source_from_index=source_from_index, + target_index=target_index, + distribution_scope='prod', + binary_image_config=None, + ), + ) + + # Verify OPM version was set + mock_opm.set_opm_version.assert_called_once_with(prebuild_info['target_index_resolved']) + + # Verify git repository was prepared + mock_pgrfb.assert_called_once() + + # Verify index.db artifacts were fetched + assert mock_favida.call_count == 2 + + # Verify bundles were retrieved + assert mock_gpb.call_count == 2 + + # Verify bundles were validated + mock_vbip.assert_called_once() + # Verify it was called with List[str] format (pullspec strings) + call_args = mock_vbip.call_args + bundles_arg = call_args[0][0] if call_args[0] else call_args[1]['bundles'] + assert isinstance(bundles_arg, list) + # All items should be strings (pullspecs), not BundleImage dicts + assert all(isinstance(b, str) for b in bundles_arg) + # Verify expected bundles are in the list + expected_bundles = set(source_bundles_pull_spec + target_bundles_pull_spec) + assert set(bundles_arg) == expected_bundles + + # Verify missing bundles were identified + mock_gmbfts.assert_called_once() + + # Verify missing bundles were added + mock_ora.assert_called_once() + + # Verify deprecation was processed + mock_dbd.assert_called_once() + + # Verify FBC migration + mock_om.assert_called_once() + + # Verify catalog merge + mock_mcd.assert_called_once() + + # Verify FBC validation + mock_ov.assert_called_once() + + # Verify build metadata was written + mock_wbm.assert_called_once() + + # Verify git commit/push + mock_gccmop.assert_called_once() + + # Verify pipeline monitoring + mock_mpaei.assert_called_once() + + # Verify image replication + mock_ritd.assert_called_once() + + # Verify index.db push + mock_pida.assert_called_once() + + # Verify final state + final_call = mock_srs.call_args_list[-1] + assert final_call[0][0] == request_id + assert final_call[0][1] == 'complete' + assert 'successfully merged' in final_call[0][2] + + # Verify reset_docker_config was called + assert mock_rdc.call_count >= 1 + + +@mock.patch('iib.workers.tasks.build_containerized_merge.get_request') +@mock.patch('iib.workers.tasks.build_containerized_merge.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_merge_request_if_exists') +@mock.patch('iib.workers.tasks.build_containerized_merge.push_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_merge.replicate_image_to_tagged_destinations') +@mock.patch('iib.workers.tasks.build_containerized_merge.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_merge.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_merge.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_merge.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_list_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_latest_version') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_merge._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_missing_bundles_from_target_to_source') +@mock.patch('iib.workers.tasks.build_containerized_merge.validate_bundles_in_parallel') +@mock.patch('iib.workers.tasks.build_containerized_merge._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.Opm') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_request_for_build') +@mock.patch('iib.workers.api_utils.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_merge.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.move') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.path.exists') +@mock.patch('builtins.set', side_effect=_mock_set_for_bundles) +def test_handle_containerized_merge_request_success_with_deprecations( + mock_set, + mock_exists, + mock_move, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_set_registry_token, + mock_srs, + mock_srs_api, + mock_prfb, + mock_opm, + mock_uiibs, + mock_pgrfb, + mock_favida, + mock_gpb, + mock_vbip, + mock_gmbfts, + mock_ora, + mock_gbfdl, + mock_gblv, + mock_dbd, + mock_glb, + mock_om, + mock_mcd, + mock_copytree, + mock_ov, + mock_wbm, + mock_gccmop, + mock_mpaei, + mock_ritd, + mock_uiips, + mock_pida, + mock_cmrif, + mock_cof, + mock_rdc, + mock_get_request, +): + """Test successful merge request with deprecations executed correctly.""" + # Setup + request_id = 9 + source_from_index = 'quay.io/namespace/source-index:v4.14' + target_index = 'quay.io/namespace/target-index:v4.15' + deprecation_list = ['bundle1:1.0', 'bundle2:2.0'] + binary_image = 'registry.io/binary:latest' + overwrite_target_index_token = 'user:token' + + # Mock temp directory + temp_dir = '/tmp/iib-9-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + mock_get_request.return_value = { + 'user': 'test-user', + } + + # Mock prepare_request_for_build + prebuild_info = { + 'arches': {'amd64', 's390x'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc123', + 'source_from_index_resolved': 'quay.io/namespace/source-index@sha256:def456', + 'target_index_resolved': 'quay.io/namespace/target-index@sha256:ghi789', + 'ocp_version': 'v4.14', + 'target_ocp_version': 'v4.15', + 'distribution_scope': 'prod', + } + mock_prfb.return_value = prebuild_info + + # Mock OPM + mock_opm.opm_version = 'v1.28.0' + + # Mock git repository setup + index_git_repo = 'https://gitlab.com/repo' + local_git_repo_path = os.path.join(temp_dir, 'git_repo') + localized_git_catalog_path = os.path.join(local_git_repo_path, 'catalogs') + mock_pgrfb.return_value = (index_git_repo, local_git_repo_path, localized_git_catalog_path) + + # Mock index.db artifacts + source_index_db_path = os.path.join(temp_dir, 'source_index.db') + target_index_db_path = os.path.join(temp_dir, 'target_index.db') + mock_favida.side_effect = [source_index_db_path, target_index_db_path] + + # Mock bundles + source_bundles = [ + {'bundlePath': 'bundle1@sha256:111', 'csvName': 'bundle1-1.0', 'packageName': 'bundle1'}, + {'bundlePath': 'bundle2@sha256:222', 'csvName': 'bundle2-2.0', 'packageName': 'bundle2'}, + {'bundlePath': 'bundle3@sha256:333', 'csvName': 'bundle3-3.0', 'packageName': 'bundle3'}, + ] + source_bundles_pull_spec = ['bundle1@sha256:111', 'bundle2@sha256:222', 'bundle3@sha256:333'] + target_bundles = [ + {'bundlePath': 'bundle4@sha256:444', 'csvName': 'bundle4-4.0', 'packageName': 'bundle4'}, + ] + target_bundles_pull_spec = ['bundle4@sha256:444'] + mock_gpb.side_effect = [ + (source_bundles, source_bundles_pull_spec), + (target_bundles, target_bundles_pull_spec), + ] + + # Mock missing bundles + missing_bundles = [ + {'bundlePath': 'bundle4@sha256:444', 'csvName': 'bundle4-4.0', 'packageName': 'bundle4'}, + ] + invalid_bundles = [] + mock_gmbfts.return_value = (missing_bundles, invalid_bundles) + + # Mock deprecation bundles - these should be found from the deprecation_list + deprecation_bundles_from_list = ['bundle1@sha256:111', 'bundle2@sha256:222'] + deprecation_bundles_latest = ['bundle1@sha256:111', 'bundle2@sha256:222'] + mock_gbfdl.return_value = deprecation_bundles_from_list + mock_gblv.return_value = deprecation_bundles_latest + + # Mock FBC migration + fbc_dir = os.path.join(temp_dir, 'fbc_catalog') + mock_om.return_value = (fbc_dir, None) + + # Mock bundles in DB + bundles_in_db = [ + {'bundlePath': 'bundle1@sha256:111', 'packageName': 'bundle1'}, + {'bundlePath': 'bundle2@sha256:222', 'packageName': 'bundle2'}, + {'bundlePath': 'bundle3@sha256:333', 'packageName': 'bundle3'}, + {'bundlePath': 'bundle4@sha256:444', 'packageName': 'bundle4'}, + ] + mock_glb.return_value = bundles_in_db + + # Mock file system operations + mock_exists.return_value = True + + # Mock git commit + mr_details = None + last_commit_sha = 'abc123commit' + mock_gccmop.return_value = (mr_details, last_commit_sha) + + # Mock Konflux pipeline + image_url = 'quay.io/konflux/built-image@sha256:xyz789' + mock_mpaei.return_value = image_url + + # Mock image replication + output_pull_specs = ['quay.io/iib/iib-build:9'] + mock_ritd.return_value = output_pull_specs + + # Mock index.db push + original_index_db_digest = 'sha256:original123' + mock_pida.return_value = original_index_db_digest + + # Test + build_containerized_merge.handle_containerized_merge_request( + source_from_index=source_from_index, + deprecation_list=deprecation_list, + request_id=request_id, + binary_image=binary_image, + target_index=target_index, + overwrite_target_index=True, + overwrite_target_index_token=overwrite_target_index_token, + distribution_scope='prod', + index_to_gitlab_push_map={'quay.io/namespace/source-index': index_git_repo}, + ) + + # Verify prepare_request_for_build was called + mock_prfb.assert_called_once_with( + request_id, + RequestConfigMerge( + _binary_image=binary_image, + overwrite_target_index_token=overwrite_target_index_token, + source_from_index=source_from_index, + target_index=target_index, + distribution_scope='prod', + binary_image_config=None, + ), + ) + + # Verify OPM version was set + mock_opm.set_opm_version.assert_called_once_with(prebuild_info['target_index_resolved']) + + # Verify git repository was prepared + mock_pgrfb.assert_called_once() + + # Verify index.db artifacts were fetched + assert mock_favida.call_count == 2 + + # Verify bundles were retrieved + assert mock_gpb.call_count == 2 + + # Verify bundles were validated + mock_vbip.assert_called_once() + # Verify it was called with List[str] format (pullspec strings) + call_args = mock_vbip.call_args + bundles_arg = call_args[0][0] if call_args[0] else call_args[1]['bundles'] + assert isinstance(bundles_arg, list) + # All items should be strings (pullspecs), not BundleImage dicts + assert all(isinstance(b, str) for b in bundles_arg) + # Verify expected bundles are in the list + expected_bundles = set(source_bundles_pull_spec + target_bundles_pull_spec) + assert set(bundles_arg) == expected_bundles + + # Verify missing bundles were identified + mock_gmbfts.assert_called_once() + + # Verify missing bundles were added + mock_ora.assert_called_once() + + # Verify deprecation processing was executed + # 1. get_bundles_from_deprecation_list should be called with + # intermediate_bundles and deprecation_list + mock_gbfdl.assert_called_once() + gbfdl_call_args = mock_gbfdl.call_args + assert deprecation_list == gbfdl_call_args[0][1] + # Verify intermediate_bundles includes missing bundles + source bundles + intermediate_bundles = gbfdl_call_args[0][0] + assert 'bundle4@sha256:444' in intermediate_bundles # missing bundle + assert 'bundle1@sha256:111' in intermediate_bundles # source bundle + + # 2. get_bundles_latest_version should be called with deprecation bundles and all bundles + mock_gblv.assert_called_once() + gblv_call_args = mock_gblv.call_args + assert deprecation_bundles_from_list == gblv_call_args[0][0] + all_bundles = gblv_call_args[0][1] + # Verify all_bundles includes both source and target bundles + assert len(all_bundles) == len(source_bundles) + len(target_bundles) + + # 3. deprecate_bundles_db should be called with the latest deprecation bundles + mock_dbd.assert_called_once() + dbd_call_args = mock_dbd.call_args + assert dbd_call_args[1]['base_dir'] == temp_dir + assert dbd_call_args[1]['index_db_file'] == source_index_db_path + assert dbd_call_args[1]['bundles'] == deprecation_bundles_latest + # Verify the deprecation bundles match what was expected + assert 'bundle1@sha256:111' in dbd_call_args[1]['bundles'] + assert 'bundle2@sha256:222' in dbd_call_args[1]['bundles'] + + # Verify FBC migration + mock_om.assert_called_once() + + # Verify catalog merge + mock_mcd.assert_called_once() + + # Verify FBC validation + mock_ov.assert_called_once() + + # Verify build metadata was written + mock_wbm.assert_called_once() + + # Verify git commit/push + mock_gccmop.assert_called_once() + + # Verify pipeline monitoring + mock_mpaei.assert_called_once() + + # Verify image replication + mock_ritd.assert_called_once() + + # Verify index.db push + mock_pida.assert_called_once() + + # Verify final state - operation completed successfully + final_call = mock_srs.call_args_list[-1] + assert final_call[0][0] == request_id + assert final_call[0][1] == 'complete' + assert 'successfully merged' in final_call[0][2] + + # Verify reset_docker_config was called + assert mock_rdc.call_count >= 1 + + +@mock.patch('iib.workers.tasks.build_containerized_merge.get_request') +@mock.patch('iib.workers.tasks.build_containerized_merge.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_merge_request_if_exists') +@mock.patch('iib.workers.tasks.build_containerized_merge.push_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_merge.replicate_image_to_tagged_destinations') +@mock.patch('iib.workers.tasks.build_containerized_merge.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_merge.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_merge.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_merge.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_list_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_latest_version') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_merge._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_missing_bundles_from_target_to_source') +@mock.patch('iib.workers.tasks.build_containerized_merge.validate_bundles_in_parallel') +@mock.patch('iib.workers.tasks.build_containerized_merge._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.Opm') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_request_for_build') +@mock.patch('iib.workers.api_utils.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_merge.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.move') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.path.exists') +@mock.patch('builtins.set', side_effect=_mock_set_for_bundles) +def test_handle_containerized_merge_request_with_mr( + mock_set, + mock_exists, + mock_move, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_set_registry_token, + mock_srs, + mock_srs_api, + mock_prfb, + mock_opm, + mock_uiibs, + mock_pgrfb, + mock_favida, + mock_gpb, + mock_vbip, + mock_gmbfts, + mock_ora, + mock_gbfdl, + mock_gblv, + mock_dbd, + mock_glb, + mock_om, + mock_mcd, + mock_copytree, + mock_ov, + mock_wbm, + mock_gccmop, + mock_mpaei, + mock_ritd, + mock_uiips, + mock_pida, + mock_cmrif, + mock_cof, + mock_rdc, + mock_get_request, +): + """Test merge request that creates and closes MR.""" + request_id = 2 + source_from_index = 'quay.io/namespace/source-index:v4.14' + target_index = 'quay.io/namespace/target-index:v4.15' + + temp_dir = '/tmp/iib-2-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + mock_get_request.return_value = { + 'user': 'test-user', + } + + prebuild_info = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'source_from_index_resolved': 'quay.io/namespace/source-index@sha256:def', + 'target_index_resolved': 'quay.io/namespace/target-index@sha256:ghi', + 'ocp_version': 'v4.14', + 'target_ocp_version': 'v4.15', + 'distribution_scope': 'prod', + } + mock_prfb.return_value = prebuild_info + + mock_opm.opm_version = 'v1.28.0' + + index_git_repo = 'https://gitlab.com/repo' + local_git_repo_path = os.path.join(temp_dir, 'git_repo') + localized_git_catalog_path = os.path.join(local_git_repo_path, 'catalogs') + mock_pgrfb.return_value = (index_git_repo, local_git_repo_path, localized_git_catalog_path) + + source_index_db_path = os.path.join(temp_dir, 'source_index.db') + target_index_db_path = os.path.join(temp_dir, 'target_index.db') + mock_favida.side_effect = [source_index_db_path, target_index_db_path] + + source_bundles = [ + {'bundlePath': 'bundle1@sha256:111', 'csvName': 'bundle1-1.0', 'packageName': 'bundle1'} + ] + source_bundles_pull_spec = ['bundle1@sha256:111'] + target_bundles = [ + {'bundlePath': 'bundle2@sha256:222', 'csvName': 'bundle2-2.0', 'packageName': 'bundle2'} + ] + target_bundles_pull_spec = ['bundle2@sha256:222'] + mock_gpb.side_effect = [ + (source_bundles, source_bundles_pull_spec), + (target_bundles, target_bundles_pull_spec), + ] + + missing_bundles = [ + {'bundlePath': 'bundle2@sha256:222', 'csvName': 'bundle2-2.0', 'packageName': 'bundle2'} + ] + mock_gmbfts.return_value = (missing_bundles, []) + + mock_gbfdl.return_value = [] + bundles_in_db = [{'bundlePath': 'bundle1@sha256:111', 'packageName': 'bundle1'}] + mock_glb.return_value = bundles_in_db + + fbc_dir = os.path.join(temp_dir, 'fbc_catalog') + mock_om.return_value = (fbc_dir, None) + + mock_exists.return_value = True + + # Mock MR creation + mr_details = {'mr_url': 'https://gitlab.com/repo/-/merge_requests/1', 'mr_id': 1} + last_commit_sha = 'commit_sha_123' + mock_gccmop.return_value = (mr_details, last_commit_sha) + + image_url = 'quay.io/konflux/image@sha256:built' + mock_mpaei.return_value = image_url + + output_pull_specs = ['quay.io/iib/iib-build:2'] + mock_ritd.return_value = output_pull_specs + + original_index_db_digest = 'sha256:original123' + mock_pida.return_value = original_index_db_digest + + # Test without overwrite_target_index_token (creates MR) + build_containerized_merge.handle_containerized_merge_request( + source_from_index=source_from_index, + deprecation_list=[], + request_id=request_id, + target_index=target_index, + overwrite_target_index=False, + index_to_gitlab_push_map={'quay.io/namespace/source-index': index_git_repo}, + ) + + # Verify MR was created + commit_msg = mock_gccmop.call_args[1]['commit_message'] + assert f'IIB: Merge operators for request {request_id}' in commit_msg + + # Verify completion + final_call = mock_srs.call_args_list[-1] + assert final_call[0][1] == 'complete' + + +@mock.patch('iib.workers.tasks.build_containerized_merge.get_request') +@mock.patch('iib.workers.tasks.build_containerized_merge.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_merge_request_if_exists') +@mock.patch('iib.workers.tasks.build_containerized_merge.push_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_merge.replicate_image_to_tagged_destinations') +@mock.patch('iib.workers.tasks.build_containerized_merge.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_merge.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_merge.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_merge.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_list_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_latest_version') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_merge._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_missing_bundles_from_target_to_source') +@mock.patch('iib.workers.tasks.build_containerized_merge.validate_bundles_in_parallel') +@mock.patch('iib.workers.tasks.build_containerized_merge._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.Opm') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_request_for_build') +@mock.patch('iib.workers.api_utils.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_merge.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.move') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.path.exists') +@mock.patch('builtins.set', side_effect=_mock_set_for_bundles) +def test_handle_containerized_merge_request_no_missing_bundles( + mock_set, + mock_exists, + mock_move, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_set_registry_token, + mock_srs, + mock_srs_api, + mock_prfb, + mock_opm, + mock_uiibs, + mock_pgrfb, + mock_favida, + mock_gpb, + mock_vbip, + mock_gmbfts, + mock_ora, + mock_gbfdl, + mock_gblv, + mock_dbd, + mock_glb, + mock_om, + mock_mcd, + mock_copytree, + mock_ov, + mock_wbm, + mock_gccmop, + mock_mpaei, + mock_ritd, + mock_uiips, + mock_pida, + mock_cmrif, + mock_cof, + mock_rdc, + mock_get_request, +): + """Test merge request when no bundles are missing.""" + request_id = 3 + source_from_index = 'quay.io/namespace/source-index:v4.14' + target_index = 'quay.io/namespace/target-index:v4.15' + + temp_dir = '/tmp/iib-3-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + mock_get_request.return_value = { + 'user': 'test-user', + } + + prebuild_info = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'source_from_index_resolved': 'quay.io/namespace/source-index@sha256:def', + 'target_index_resolved': 'quay.io/namespace/target-index@sha256:ghi', + 'ocp_version': 'v4.14', + 'target_ocp_version': 'v4.15', + 'distribution_scope': 'prod', + } + mock_prfb.return_value = prebuild_info + + mock_opm.opm_version = 'v1.28.0' + + index_git_repo = 'https://gitlab.com/repo' + local_git_repo_path = os.path.join(temp_dir, 'git_repo') + localized_git_catalog_path = os.path.join(local_git_repo_path, 'catalogs') + mock_pgrfb.return_value = (index_git_repo, local_git_repo_path, localized_git_catalog_path) + + source_index_db_path = os.path.join(temp_dir, 'source_index.db') + target_index_db_path = os.path.join(temp_dir, 'target_index.db') + mock_favida.side_effect = [source_index_db_path, target_index_db_path] + + source_bundles = [ + {'bundlePath': 'bundle1@sha256:111', 'csvName': 'bundle1-1.0', 'packageName': 'bundle1'} + ] + source_bundles_pull_spec = ['bundle1@sha256:111'] + target_bundles = [ + {'bundlePath': 'bundle1@sha256:111', 'csvName': 'bundle1-1.0', 'packageName': 'bundle1'} + ] + target_bundles_pull_spec = ['bundle1@sha256:111'] + mock_gpb.side_effect = [ + (source_bundles, source_bundles_pull_spec), + (target_bundles, target_bundles_pull_spec), + ] + + # No missing bundles + mock_gmbfts.return_value = ([], []) + + mock_gbfdl.return_value = [] + bundles_in_db = [{'bundlePath': 'bundle1@sha256:111', 'packageName': 'bundle1'}] + mock_glb.return_value = bundles_in_db + + fbc_dir = os.path.join(temp_dir, 'fbc_catalog') + mock_om.return_value = (fbc_dir, None) + + mock_exists.return_value = True + + mr_details = None + last_commit_sha = 'commit_sha' + mock_gccmop.return_value = (mr_details, last_commit_sha) + + image_url = 'quay.io/konflux/image@sha256:built' + mock_mpaei.return_value = image_url + + output_pull_specs = ['quay.io/iib/iib-build:3'] + mock_ritd.return_value = output_pull_specs + + original_index_db_digest = 'sha256:original123' + mock_pida.return_value = original_index_db_digest + + # Test + build_containerized_merge.handle_containerized_merge_request( + source_from_index=source_from_index, + deprecation_list=[], + request_id=request_id, + target_index=target_index, + index_to_gitlab_push_map={'quay.io/namespace/source-index': index_git_repo}, + ) + + # Verify _opm_registry_add was called with empty list + mock_ora.assert_called_once() + assert mock_ora.call_args[0][2] == [] + + +@mock.patch('iib.workers.tasks.build_containerized_merge.get_request') +@mock.patch('iib.workers.tasks.build_containerized_merge.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_merge_request_if_exists') +@mock.patch('iib.workers.tasks.build_containerized_merge.push_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_merge.replicate_image_to_tagged_destinations') +@mock.patch('iib.workers.tasks.build_containerized_merge.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_merge.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_merge.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_merge.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_list_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_latest_version') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_merge._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_missing_bundles_from_target_to_source') +@mock.patch('iib.workers.tasks.build_containerized_merge.validate_bundles_in_parallel') +@mock.patch('iib.workers.tasks.build_containerized_merge._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.Opm') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_request_for_build') +@mock.patch('iib.workers.api_utils.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_merge.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.move') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.path.exists') +@mock.patch('builtins.set', side_effect=_mock_set_for_bundles) +def test_handle_containerized_merge_request_with_deprecation( + mock_set, + mock_exists, + mock_move, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_set_registry_token, + mock_srs, + mock_srs_api, + mock_prfb, + mock_opm, + mock_uiibs, + mock_pgrfb, + mock_favida, + mock_gpb, + mock_vbip, + mock_gmbfts, + mock_ora, + mock_gbfdl, + mock_gblv, + mock_dbd, + mock_glb, + mock_om, + mock_mcd, + mock_copytree, + mock_ov, + mock_wbm, + mock_gccmop, + mock_mpaei, + mock_ritd, + mock_uiips, + mock_pida, + mock_cmrif, + mock_cof, + mock_rdc, + mock_get_request, +): + """Test merge request with deprecation list.""" + request_id = 4 + source_from_index = 'quay.io/namespace/source-index:v4.14' + target_index = 'quay.io/namespace/target-index:v4.15' + deprecation_list = ['bundle1:1.0', 'bundle2:2.0'] + + temp_dir = '/tmp/iib-4-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + mock_get_request.return_value = { + 'user': 'test-user', + } + + prebuild_info = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'source_from_index_resolved': 'quay.io/namespace/source-index@sha256:def', + 'target_index_resolved': 'quay.io/namespace/target-index@sha256:ghi', + 'ocp_version': 'v4.14', + 'target_ocp_version': 'v4.15', + 'distribution_scope': 'prod', + } + mock_prfb.return_value = prebuild_info + + mock_opm.opm_version = 'v1.28.0' + + index_git_repo = 'https://gitlab.com/repo' + local_git_repo_path = os.path.join(temp_dir, 'git_repo') + localized_git_catalog_path = os.path.join(local_git_repo_path, 'catalogs') + mock_pgrfb.return_value = (index_git_repo, local_git_repo_path, localized_git_catalog_path) + + source_index_db_path = os.path.join(temp_dir, 'source_index.db') + target_index_db_path = os.path.join(temp_dir, 'target_index.db') + mock_favida.side_effect = [source_index_db_path, target_index_db_path] + + source_bundles = [ + {'bundlePath': 'bundle1@sha256:111', 'csvName': 'bundle1-1.0', 'packageName': 'bundle1'}, + {'bundlePath': 'bundle2@sha256:222', 'csvName': 'bundle2-2.0', 'packageName': 'bundle2'}, + ] + source_bundles_pull_spec = ['bundle1@sha256:111', 'bundle2@sha256:222'] + target_bundles = [] + target_bundles_pull_spec = [] + mock_gpb.side_effect = [ + (source_bundles, source_bundles_pull_spec), + (target_bundles, target_bundles_pull_spec), + ] + + mock_gmbfts.return_value = ([], []) + + # Mock deprecation bundles + mock_gbfdl.return_value = ['bundle1@sha256:111', 'bundle2@sha256:222'] + mock_gblv.return_value = ['bundle1@sha256:111', 'bundle2@sha256:222'] + + bundles_in_db = [ + {'bundlePath': 'bundle1@sha256:111', 'packageName': 'bundle1'}, + {'bundlePath': 'bundle2@sha256:222', 'packageName': 'bundle2'}, + ] + mock_glb.return_value = bundles_in_db + + fbc_dir = os.path.join(temp_dir, 'fbc_catalog') + mock_om.return_value = (fbc_dir, None) + + mock_exists.return_value = True + + mr_details = None + last_commit_sha = 'commit_sha' + mock_gccmop.return_value = (mr_details, last_commit_sha) + + image_url = 'quay.io/konflux/image@sha256:built' + mock_mpaei.return_value = image_url + + output_pull_specs = ['quay.io/iib/iib-build:4'] + mock_ritd.return_value = output_pull_specs + + original_index_db_digest = 'sha256:original123' + mock_pida.return_value = original_index_db_digest + + # Test + build_containerized_merge.handle_containerized_merge_request( + source_from_index=source_from_index, + deprecation_list=deprecation_list, + request_id=request_id, + target_index=target_index, + index_to_gitlab_push_map={'quay.io/namespace/source-index': index_git_repo}, + ) + + # Verify deprecation was processed + mock_gbfdl.assert_called_once() + mock_gblv.assert_called_once() + mock_dbd.assert_called_once() + + +@mock.patch('iib.workers.tasks.build_containerized_merge.get_request') +@mock.patch('iib.workers.tasks.build_containerized_merge.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_merge.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_merge.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_merge.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_merge.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_list_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_latest_version') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_merge._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_missing_bundles_from_target_to_source') +@mock.patch('iib.workers.tasks.build_containerized_merge.validate_bundles_in_parallel') +@mock.patch('iib.workers.tasks.build_containerized_merge._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.Opm') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_request_for_build') +@mock.patch('iib.workers.api_utils.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_merge.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.move') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.path.exists') +@mock.patch('builtins.set', side_effect=_mock_set_for_bundles) +def test_handle_containerized_merge_request_pipeline_failure( + mock_set, + mock_exists, + mock_move, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_set_registry_token, + mock_srs, + mock_srs_api, + mock_prfb, + mock_opm, + mock_uiibs, + mock_pgrfb, + mock_favida, + mock_gpb, + mock_vbip, + mock_gmbfts, + mock_ora, + mock_gbfdl, + mock_gblv, + mock_dbd, + mock_glb, + mock_om, + mock_mcd, + mock_copytree, + mock_ov, + mock_wbm, + mock_gccmop, + mock_mpaei, + mock_cof, + mock_rdc, + mock_get_request, +): + """Test that pipeline failure triggers cleanup.""" + request_id = 5 + source_from_index = 'quay.io/namespace/source-index:v4.14' + target_index = 'quay.io/namespace/target-index:v4.15' + + temp_dir = '/tmp/iib-5-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + mock_get_request.return_value = { + 'user': 'test-user', + } + + prebuild_info = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'source_from_index_resolved': 'quay.io/namespace/source-index@sha256:def', + 'target_index_resolved': 'quay.io/namespace/target-index@sha256:ghi', + 'ocp_version': 'v4.14', + 'target_ocp_version': 'v4.15', + 'distribution_scope': 'prod', + } + mock_prfb.return_value = prebuild_info + + mock_opm.opm_version = 'v1.28.0' + + index_git_repo = 'https://gitlab.com/repo' + local_git_repo_path = os.path.join(temp_dir, 'git_repo') + localized_git_catalog_path = os.path.join(local_git_repo_path, 'catalogs') + mock_pgrfb.return_value = (index_git_repo, local_git_repo_path, localized_git_catalog_path) + + source_index_db_path = os.path.join(temp_dir, 'source_index.db') + target_index_db_path = os.path.join(temp_dir, 'target_index.db') + mock_favida.side_effect = [source_index_db_path, target_index_db_path] + + source_bundles = [ + {'bundlePath': 'bundle1@sha256:111', 'csvName': 'bundle1-1.0', 'packageName': 'bundle1'} + ] + source_bundles_pull_spec = ['bundle1@sha256:111'] + target_bundles = [] + target_bundles_pull_spec = [] + mock_gpb.side_effect = [ + (source_bundles, source_bundles_pull_spec), + (target_bundles, target_bundles_pull_spec), + ] + + mock_gmbfts.return_value = ([], []) + + mock_gbfdl.return_value = [] + bundles_in_db = [{'bundlePath': 'bundle1@sha256:111', 'packageName': 'bundle1'}] + mock_glb.return_value = bundles_in_db + + fbc_dir = os.path.join(temp_dir, 'fbc_catalog') + mock_om.return_value = (fbc_dir, None) + + mock_exists.return_value = True + + mr_details = None + last_commit_sha = 'commit_sha' + mock_gccmop.return_value = (mr_details, last_commit_sha) + + # Mock pipeline to raise error + mock_mpaei.side_effect = IIBError('Pipeline not found') + + # Test + with pytest.raises(IIBError, match='Failed to merge operators'): + build_containerized_merge.handle_containerized_merge_request( + source_from_index=source_from_index, + deprecation_list=[], + request_id=request_id, + target_index=target_index, + index_to_gitlab_push_map={'quay.io/namespace/source-index': index_git_repo}, + ) + + # Verify cleanup was called + mock_cof.assert_called_once() + cleanup_call = mock_cof.call_args + assert cleanup_call[1]['request_id'] == request_id + assert 'Pipeline not found' in cleanup_call[1]['reason'] + + +@mock.patch('iib.workers.tasks.build_containerized_merge.get_request') +@mock.patch('iib.workers.tasks.build_containerized_merge.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_merge.replicate_image_to_tagged_destinations') +@mock.patch('iib.workers.tasks.build_containerized_merge.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_merge.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_merge.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_merge.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_list_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_latest_version') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_merge._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_missing_bundles_from_target_to_source') +@mock.patch('iib.workers.tasks.build_containerized_merge.validate_bundles_in_parallel') +@mock.patch('iib.workers.tasks.build_containerized_merge._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.Opm') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_request_for_build') +@mock.patch('iib.workers.api_utils.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_merge.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.move') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.path.exists') +@mock.patch('builtins.set', side_effect=_mock_set_for_bundles) +def test_handle_containerized_merge_request_missing_output_pull_spec( + mock_set, + mock_exists, + mock_move, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_set_registry_token, + mock_srs, + mock_srs_api, + mock_prfb, + mock_opm, + mock_uiibs, + mock_pgrfb, + mock_favida, + mock_gpb, + mock_vbip, + mock_gmbfts, + mock_ora, + mock_gbfdl, + mock_gblv, + mock_dbd, + mock_glb, + mock_om, + mock_mcd, + mock_copytree, + mock_ov, + mock_wbm, + mock_gccmop, + mock_mpaei, + mock_ritd, + mock_cof, + mock_rdc, + mock_get_request, +): + """Test error when output_pull_spec is not set.""" + request_id = 6 + source_from_index = 'quay.io/namespace/source-index:v4.14' + target_index = 'quay.io/namespace/target-index:v4.15' + + temp_dir = '/tmp/iib-6-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + mock_get_request.return_value = { + 'user': 'test-user', + } + + prebuild_info = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'source_from_index_resolved': 'quay.io/namespace/source-index@sha256:def', + 'target_index_resolved': 'quay.io/namespace/target-index@sha256:ghi', + 'ocp_version': 'v4.14', + 'target_ocp_version': 'v4.15', + 'distribution_scope': 'prod', + } + mock_prfb.return_value = prebuild_info + + mock_opm.opm_version = 'v1.28.0' + + index_git_repo = 'https://gitlab.com/repo' + local_git_repo_path = os.path.join(temp_dir, 'git_repo') + localized_git_catalog_path = os.path.join(local_git_repo_path, 'catalogs') + mock_pgrfb.return_value = (index_git_repo, local_git_repo_path, localized_git_catalog_path) + + source_index_db_path = os.path.join(temp_dir, 'source_index.db') + target_index_db_path = os.path.join(temp_dir, 'target_index.db') + mock_favida.side_effect = [source_index_db_path, target_index_db_path] + + source_bundles = [ + {'bundlePath': 'bundle1@sha256:111', 'csvName': 'bundle1-1.0', 'packageName': 'bundle1'} + ] + source_bundles_pull_spec = ['bundle1@sha256:111'] + target_bundles = [] + target_bundles_pull_spec = [] + mock_gpb.side_effect = [ + (source_bundles, source_bundles_pull_spec), + (target_bundles, target_bundles_pull_spec), + ] + + mock_gmbfts.return_value = ([], []) + + mock_gbfdl.return_value = [] + bundles_in_db = [{'bundlePath': 'bundle1@sha256:111', 'packageName': 'bundle1'}] + mock_glb.return_value = bundles_in_db + + fbc_dir = os.path.join(temp_dir, 'fbc_catalog') + mock_om.return_value = (fbc_dir, None) + + mock_exists.return_value = True + + mr_details = None + last_commit_sha = 'commit_sha' + mock_gccmop.return_value = (mr_details, last_commit_sha) + + image_url = 'quay.io/konflux/image@sha256:built' + mock_mpaei.return_value = image_url + + # Mock replicate_image_to_tagged_destinations to return empty list + mock_ritd.return_value = [] + + # Test + with pytest.raises(IIBError, match='list index out of range'): + build_containerized_merge.handle_containerized_merge_request( + source_from_index=source_from_index, + deprecation_list=[], + request_id=request_id, + target_index=target_index, + index_to_gitlab_push_map={'quay.io/namespace/source-index': index_git_repo}, + ) + + # Verify cleanup was called + mock_cof.assert_called_once() + + +@pytest.mark.parametrize( + 'build_tags, expected_tag_count', + [ + (None, 1), # Only request_id + (['latest'], 2), # request_id + latest + (['latest', 'v4.14'], 3), # request_id + latest + v4.14 + ], +) +@mock.patch('iib.workers.tasks.build_containerized_merge.get_request') +@mock.patch('iib.workers.tasks.build_containerized_merge.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_merge_request_if_exists') +@mock.patch('iib.workers.tasks.build_containerized_merge.push_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_merge.replicate_image_to_tagged_destinations') +@mock.patch('iib.workers.tasks.build_containerized_merge.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_merge.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_merge.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_merge.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_list_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_latest_version') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_merge._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_missing_bundles_from_target_to_source') +@mock.patch('iib.workers.tasks.build_containerized_merge.validate_bundles_in_parallel') +@mock.patch('iib.workers.tasks.build_containerized_merge._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.Opm') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_request_for_build') +@mock.patch('iib.workers.api_utils.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_merge.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.move') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.path.exists') +@mock.patch('builtins.set', side_effect=_mock_set_for_bundles) +def test_handle_containerized_merge_request_with_build_tags( + mock_set, + mock_exists, + mock_move, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_set_registry_token, + mock_srs, + mock_srs_api, + mock_prfb, + mock_opm, + mock_uiibs, + mock_pgrfb, + mock_favida, + mock_gpb, + mock_vbip, + mock_gmbfts, + mock_ora, + mock_gbfdl, + mock_gblv, + mock_dbd, + mock_glb, + mock_om, + mock_mcd, + mock_copytree, + mock_ov, + mock_wbm, + mock_gccmop, + mock_mpaei, + mock_ritd, + mock_uiips, + mock_pida, + mock_cmrif, + mock_cof, + mock_rdc, + mock_get_request, + build_tags, + expected_tag_count, +): + """Test that build_tags parameter results in correct number of image replications.""" + request_id = 7 + source_from_index = 'quay.io/namespace/source-index:v4.14' + target_index = 'quay.io/namespace/target-index:v4.15' + + temp_dir = '/tmp/iib-7-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + mock_get_request.return_value = { + 'user': 'test-user', + } + + prebuild_info = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'source_from_index_resolved': 'quay.io/namespace/source-index@sha256:def', + 'target_index_resolved': 'quay.io/namespace/target-index@sha256:ghi', + 'ocp_version': 'v4.14', + 'target_ocp_version': 'v4.15', + 'distribution_scope': 'prod', + } + mock_prfb.return_value = prebuild_info + + mock_opm.opm_version = 'v1.28.0' + + index_git_repo = 'https://gitlab.com/repo' + local_git_repo_path = os.path.join(temp_dir, 'git_repo') + localized_git_catalog_path = os.path.join(local_git_repo_path, 'catalogs') + mock_pgrfb.return_value = (index_git_repo, local_git_repo_path, localized_git_catalog_path) + + source_index_db_path = os.path.join(temp_dir, 'source_index.db') + target_index_db_path = os.path.join(temp_dir, 'target_index.db') + mock_favida.side_effect = [source_index_db_path, target_index_db_path] + + source_bundles = [ + {'bundlePath': 'bundle1@sha256:111', 'csvName': 'bundle1-1.0', 'packageName': 'bundle1'} + ] + source_bundles_pull_spec = ['bundle1@sha256:111'] + target_bundles = [] + target_bundles_pull_spec = [] + mock_gpb.side_effect = [ + (source_bundles, source_bundles_pull_spec), + (target_bundles, target_bundles_pull_spec), + ] + + mock_gmbfts.return_value = ([], []) + + mock_gbfdl.return_value = [] + bundles_in_db = [{'bundlePath': 'bundle1@sha256:111', 'packageName': 'bundle1'}] + mock_glb.return_value = bundles_in_db + + fbc_dir = os.path.join(temp_dir, 'fbc_catalog') + mock_om.return_value = (fbc_dir, None) + + mock_exists.return_value = True + + mr_details = None + last_commit_sha = 'commit_sha' + mock_gccmop.return_value = (mr_details, last_commit_sha) + + image_url = 'quay.io/konflux/image@sha256:built' + mock_mpaei.return_value = image_url + + # Mock replicate_image_to_tagged_destinations to return list with expected count + output_pull_specs = ['quay.io/iib/iib-build:7'] + mock_ritd.return_value = output_pull_specs + + original_index_db_digest = 'sha256:original123' + mock_pida.return_value = original_index_db_digest + + # Test + build_containerized_merge.handle_containerized_merge_request( + source_from_index=source_from_index, + deprecation_list=[], + request_id=request_id, + target_index=target_index, + build_tags=build_tags, + index_to_gitlab_push_map={'quay.io/namespace/source-index': index_git_repo}, + ) + + # Verify replicate_image_to_tagged_destinations was called with build_tags + mock_ritd.assert_called_once() + assert mock_ritd.call_args[1]['build_tags'] == build_tags + + +@mock.patch('iib.workers.tasks.build_containerized_merge.get_request') +@mock.patch('iib.workers.tasks.build_containerized_merge.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_merge_request_if_exists') +@mock.patch('iib.workers.tasks.build_containerized_merge.push_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_merge.replicate_image_to_tagged_destinations') +@mock.patch('iib.workers.tasks.build_containerized_merge.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_merge.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_merge.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_merge.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_list_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_latest_version') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_merge._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_missing_bundles_from_target_to_source') +@mock.patch('iib.workers.tasks.build_containerized_merge.validate_bundles_in_parallel') +@mock.patch('iib.workers.tasks.build_containerized_merge._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.Opm') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_request_for_build') +@mock.patch('iib.workers.api_utils.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_merge.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.move') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.path.exists') +@mock.patch('builtins.set', side_effect=_mock_set_for_bundles) +def test_handle_containerized_merge_request_with_invalid_bundles( + mock_set, + mock_exists, + mock_move, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_set_registry_token, + mock_srs, + mock_srs_api, + mock_prfb, + mock_opm, + mock_uiibs, + mock_pgrfb, + mock_favida, + mock_gpb, + mock_vbip, + mock_gmbfts, + mock_ora, + mock_gbfdl, + mock_gblv, + mock_dbd, + mock_glb, + mock_om, + mock_mcd, + mock_copytree, + mock_ov, + mock_wbm, + mock_gccmop, + mock_mpaei, + mock_ritd, + mock_uiips, + mock_pida, + mock_cmrif, + mock_cof, + mock_rdc, + mock_get_request, +): + """Test merge request with invalid bundles (OCP version mismatch).""" + request_id = 8 + source_from_index = 'quay.io/namespace/source-index:v4.14' + target_index = 'quay.io/namespace/target-index:v4.15' + + temp_dir = '/tmp/iib-8-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + mock_get_request.return_value = { + 'user': 'test-user', + } + + prebuild_info = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'source_from_index_resolved': 'quay.io/namespace/source-index@sha256:def', + 'target_index_resolved': 'quay.io/namespace/target-index@sha256:ghi', + 'ocp_version': 'v4.14', + 'target_ocp_version': 'v4.15', + 'distribution_scope': 'prod', + } + mock_prfb.return_value = prebuild_info + + mock_opm.opm_version = 'v1.28.0' + + index_git_repo = 'https://gitlab.com/repo' + local_git_repo_path = os.path.join(temp_dir, 'git_repo') + localized_git_catalog_path = os.path.join(local_git_repo_path, 'catalogs') + mock_pgrfb.return_value = (index_git_repo, local_git_repo_path, localized_git_catalog_path) + + source_index_db_path = os.path.join(temp_dir, 'source_index.db') + target_index_db_path = os.path.join(temp_dir, 'target_index.db') + mock_favida.side_effect = [source_index_db_path, target_index_db_path] + + source_bundles = [ + {'bundlePath': 'bundle1@sha256:111', 'csvName': 'bundle1-1.0', 'packageName': 'bundle1'} + ] + source_bundles_pull_spec = ['bundle1@sha256:111'] + target_bundles = [ + {'bundlePath': 'bundle2@sha256:222', 'csvName': 'bundle2-2.0', 'packageName': 'bundle2'}, + ] + target_bundles_pull_spec = ['bundle2@sha256:222'] + mock_gpb.side_effect = [ + (source_bundles, source_bundles_pull_spec), + (target_bundles, target_bundles_pull_spec), + ] + + # Mock invalid bundles (OCP version mismatch) + invalid_bundles = [ + {'bundlePath': 'bundle2@sha256:222', 'csvName': 'bundle2-2.0', 'packageName': 'bundle2'}, + ] + mock_gmbfts.return_value = ([], invalid_bundles) + + # Invalid bundles should be added to deprecation list + mock_gbfdl.return_value = ['bundle2@sha256:222'] + mock_gblv.return_value = ['bundle2@sha256:222'] + + bundles_in_db = [ + {'bundlePath': 'bundle1@sha256:111', 'packageName': 'bundle1'}, + {'bundlePath': 'bundle2@sha256:222', 'packageName': 'bundle2'}, + ] + mock_glb.return_value = bundles_in_db + + fbc_dir = os.path.join(temp_dir, 'fbc_catalog') + mock_om.return_value = (fbc_dir, None) + + mock_exists.return_value = True + + mr_details = None + last_commit_sha = 'commit_sha' + mock_gccmop.return_value = (mr_details, last_commit_sha) + + image_url = 'quay.io/konflux/image@sha256:built' + mock_mpaei.return_value = image_url + + output_pull_specs = ['quay.io/iib/iib-build:8'] + mock_ritd.return_value = output_pull_specs + + original_index_db_digest = 'sha256:original123' + mock_pida.return_value = original_index_db_digest + + # Test + build_containerized_merge.handle_containerized_merge_request( + source_from_index=source_from_index, + deprecation_list=[], + request_id=request_id, + target_index=target_index, + index_to_gitlab_push_map={'quay.io/namespace/source-index': index_git_repo}, + ) + + # Verify invalid bundles were added to deprecation list + mock_gbfdl.assert_called_once() + # Verify deprecation was called with invalid bundles + mock_dbd.assert_called_once() + deprecation_bundles = mock_dbd.call_args[1]['bundles'] + assert 'bundle2@sha256:222' in deprecation_bundles + + +@mock.patch('iib.workers.tasks.build_containerized_merge.get_request') +@mock.patch('iib.workers.tasks.build_containerized_merge.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_merge.cleanup_merge_request_if_exists') +@mock.patch('iib.workers.tasks.build_containerized_merge.push_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_containerized_merge.replicate_image_to_tagged_destinations') +@mock.patch('iib.workers.tasks.build_containerized_merge.monitor_pipeline_and_extract_image') +@mock.patch('iib.workers.tasks.build_containerized_merge.git_commit_and_create_mr_or_push') +@mock.patch('iib.workers.tasks.build_containerized_merge.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_merge.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_merge.opm_migrate') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_list_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.deprecate_bundles_db') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_latest_version') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_bundles_from_deprecation_list') +@mock.patch('iib.workers.tasks.build_containerized_merge._opm_registry_add') +@mock.patch('iib.workers.tasks.build_containerized_merge.get_missing_bundles_from_target_to_source') +@mock.patch('iib.workers.tasks.build_containerized_merge.validate_bundles_in_parallel') +@mock.patch('iib.workers.tasks.build_containerized_merge._get_present_bundles') +@mock.patch('iib.workers.tasks.build_containerized_merge.fetch_and_verify_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_git_repository_for_build') +@mock.patch('iib.workers.tasks.build_containerized_merge._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.Opm') +@mock.patch('iib.workers.tasks.build_containerized_merge.prepare_request_for_build') +@mock.patch('iib.workers.api_utils.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_merge.set_registry_token') +@mock.patch('iib.workers.tasks.build_containerized_merge.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_merge.shutil.move') +@mock.patch('iib.workers.tasks.build_containerized_merge.os.path.exists') +@mock.patch('builtins.set', side_effect=_mock_set_for_bundles) +def test_handle_containerized_merge_request_without_target_index( + mock_set, + mock_exists, + mock_move, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_set_registry_token, + mock_srs, + mock_srs_api, + mock_prfb, + mock_opm, + mock_uiibs, + mock_pgrfb, + mock_favida, + mock_gpb, + mock_vbip, + mock_gmbfts, + mock_ora, + mock_gbfdl, + mock_gblv, + mock_dbd, + mock_glb, + mock_om, + mock_mcd, + mock_copytree, + mock_ov, + mock_wbm, + mock_gccmop, + mock_mpaei, + mock_ritd, + mock_uiips, + mock_pida, + mock_cmrif, + mock_cof, + mock_rdc, + mock_get_request, +): + """Test merge request when target_index is None.""" + request_id = 10 + source_from_index = 'quay.io/namespace/source-index:v4.14' + target_index = None # No target index provided + + temp_dir = '/tmp/iib-10-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + mock_get_request.return_value = { + 'user': 'test-user', + } + + prebuild_info = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'source_from_index_resolved': 'quay.io/namespace/source-index@sha256:def', + 'target_index_resolved': None, # Should be None when target_index is None + 'ocp_version': 'v4.14', + 'target_ocp_version': 'v4.14', # Should default to source version + 'distribution_scope': 'prod', + } + mock_prfb.return_value = prebuild_info + + mock_opm.opm_version = 'v1.28.0' + + index_git_repo = 'https://gitlab.com/repo' + local_git_repo_path = os.path.join(temp_dir, 'git_repo') + localized_git_catalog_path = os.path.join(local_git_repo_path, 'catalogs') + mock_pgrfb.return_value = (index_git_repo, local_git_repo_path, localized_git_catalog_path) + + # Only source index.db should be fetched when target_index is None + source_index_db_path = os.path.join(temp_dir, 'source_index.db') + mock_favida.return_value = source_index_db_path + + # Only source bundles should be retrieved + source_bundles = [ + {'bundlePath': 'bundle1@sha256:111', 'csvName': 'bundle1-1.0', 'packageName': 'bundle1'}, + {'bundlePath': 'bundle2@sha256:222', 'csvName': 'bundle2-2.0', 'packageName': 'bundle2'}, + ] + source_bundles_pull_spec = ['bundle1@sha256:111', 'bundle2@sha256:222'] + mock_gpb.return_value = (source_bundles, source_bundles_pull_spec) + + # No missing bundles since there's no target index + mock_gmbfts.return_value = ([], []) + + # Mock deprecation bundles + mock_gbfdl.return_value = ['bundle1@sha256:111'] + mock_gblv.return_value = ['bundle1@sha256:111'] + + bundles_in_db = [ + {'bundlePath': 'bundle1@sha256:111', 'packageName': 'bundle1'}, + {'bundlePath': 'bundle2@sha256:222', 'packageName': 'bundle2'}, + ] + mock_glb.return_value = bundles_in_db + + fbc_dir = os.path.join(temp_dir, 'fbc_catalog') + mock_om.return_value = (fbc_dir, None) + + mock_exists.return_value = True + + mr_details = None + last_commit_sha = 'commit_sha' + mock_gccmop.return_value = (mr_details, last_commit_sha) + + image_url = 'quay.io/konflux/image@sha256:built' + mock_mpaei.return_value = image_url + + output_pull_specs = ['quay.io/iib/iib-build:10'] + mock_ritd.return_value = output_pull_specs + + original_index_db_digest = 'sha256:original123' + mock_pida.return_value = original_index_db_digest + + # Test with target_index=None + build_containerized_merge.handle_containerized_merge_request( + source_from_index=source_from_index, + deprecation_list=['bundle1:1.0'], + request_id=request_id, + target_index=target_index, # None + index_to_gitlab_push_map={'quay.io/namespace/source-index': index_git_repo}, + ) + + # Verify only source index.db was fetched (not target) + assert mock_favida.call_count == 1 + mock_favida.assert_called_once_with(source_from_index, temp_dir) + + # Verify only source bundles were retrieved (not target) + assert mock_gpb.call_count == 1 + mock_gpb.assert_called_once_with(source_index_db_path, temp_dir) + + # Verify bundles were validated (only source bundles) + mock_vbip.assert_called_once() + call_args = mock_vbip.call_args + bundles_arg = call_args[0][0] if call_args[0] else call_args[1]['bundles'] + assert isinstance(bundles_arg, list) + assert all(isinstance(b, str) for b in bundles_arg) + # Should only contain source bundles + assert set(bundles_arg) == set(source_bundles_pull_spec) + + # Verify get_missing_bundles_from_target_to_source was called with empty target bundles + mock_gmbfts.assert_called_once() + gmbfts_call_args = mock_gmbfts.call_args + assert gmbfts_call_args[1]['target_index_bundles'] == [] + + # Verify _opm_registry_add was called with empty list (no missing bundles) + mock_ora.assert_not_called() + + # Verify deprecation was processed + mock_gbfdl.assert_called_once() + mock_gblv.assert_called_once() + mock_dbd.assert_called_once() + + # Verify final state + final_call = mock_srs.call_args_list[-1] + assert final_call[0][0] == request_id + assert final_call[0][1] == 'complete' + assert 'successfully merged' in final_call[0][2] diff --git a/tests/test_workers/test_tasks/test_build_containerized_regenerate_bundle.py b/tests/test_workers/test_tasks/test_build_containerized_regenerate_bundle.py new file mode 100644 index 000000000..a18a33c1f --- /dev/null +++ b/tests/test_workers/test_tasks/test_build_containerized_regenerate_bundle.py @@ -0,0 +1,469 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import json +from unittest import mock + +import pytest + +from iib.exceptions import IIBError +from iib.workers.tasks import build_containerized_regenerate_bundle + + +@pytest.mark.parametrize( + 'pinned_by_iib_label, pinned_by_iib_bool', + ( + ('true', True), + ('True', True), + (None, False), + ('false', False), + ('False', False), + ), +) +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.cleanup_merge_request_if_exists' +) +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.replicate_image_to_tagged_destinations' +) +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.monitor_pipeline_and_extract_image' +) +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.git_commit_and_create_mr_or_push' +) +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.' + 'extract_files_from_image_non_privileged' +) +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.clone_git_repo') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle._get_package_annotations') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle._adjust_operator_bundle') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_image_labels') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_image_arches') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.update_request') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_worker_config') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.tempfile.TemporaryDirectory') +def test_handle_containerized_regenerate_bundle_request( + mock_temp_dir, + mock_gwc, + mock_ur, + mock_srs, + mock_gri, + mock_gia, + mock_gil, + mock_aob, + mock_gpa, + mock_ggt, + mock_cgr, + mock_effinp, + mock_gccmop, + mock_mpaei, + mock_ritd, + mock_cmrie, + pinned_by_iib_label, + pinned_by_iib_bool, + tmpdir, +): + """Test successful containerized regenerate bundle request.""" + # Setup + arches = ['amd64', 's390x'] + from_bundle_image = 'bundle-image:latest' + from_bundle_image_resolved = 'bundle-image@sha256:abcdef' + bundle_image = 'quay.io/iib:99' + organization = 'acme' + request_id = 99 + bundle_git_repo = 'https://gitlab.com/bundle/repo' + + # Use tmpdir with the mock + mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) + + # Mock worker config + mock_gwc.return_value = { + 'iib_index_image_output_registry': None, + 'iib_registry': 'quay.io', + } + + # Mock image resolution and metadata + mock_gri.return_value = from_bundle_image_resolved + mock_gia.return_value = list(arches) + mock_gil.return_value = {'com.redhat.iib.pinned': pinned_by_iib_label} + + # Mock Git token + mock_ggt.return_value = ('GITLAB_TOKEN', 'test-token') + + # Mock bundle adjustments + mock_aob.return_value = {'operators.operatorframework.io.bundle.package.v1': 'test-package'} + mock_gpa.return_value = { + 'annotations': {'operators.operatorframework.io.bundle.package.v1': 'test-package'} + } + + # Mock Git operations + mock_gccmop.return_value = ( + {'mr_id': '123', 'mr_url': 'https://gitlab.com/merge_requests/123'}, + 'commit-sha-123', + ) + + # Mock pipeline monitoring + mock_mpaei.return_value = 'quay.io/konflux/bundle:sha256-abc123.att' + + # Mock image replication + mock_ritd.return_value = [bundle_image] + + # Execute + build_containerized_regenerate_bundle.handle_containerized_regenerate_bundle_request( + from_bundle_image=from_bundle_image, + organization=organization, + request_id=request_id, + bundle_replacements={}, + index_to_gitlab_push_map={'regenerate-bundle': bundle_git_repo}, + regenerate_bundle_repo_key='regenerate-bundle', + ) + + # Verify calls + mock_gri.assert_called_once_with(from_bundle_image) + mock_gia.assert_called_once_with(from_bundle_image_resolved) + mock_gil.assert_called_once_with(from_bundle_image_resolved) + + # Verify Git operations + mock_ggt.assert_called_once_with(bundle_git_repo) + mock_cgr.assert_called_once() + + # Verify file extraction (manifests and metadata) + assert mock_effinp.call_count == 2 + + # Verify bundle adjustment + mock_aob.assert_called_once() + + # Verify Git commit and MR creation + mock_gccmop.assert_called_once() + + # Verify pipeline monitoring + mock_mpaei.assert_called_once_with( + request_id=request_id, + last_commit_sha='commit-sha-123', + ) + + # Verify image replication + mock_ritd.assert_called_once() + + # Verify MR cleanup + mock_cmrie.assert_called_once() + + # Verify request updates + assert mock_ur.call_count == 2 + + # Verify _adjust_operator_bundle was called with correct pinned_by_iib value + mock_aob.assert_called_once() + call_kwargs = mock_aob.call_args[1] + assert call_kwargs['pinned_by_iib'] == pinned_by_iib_bool + + # Verify Dockerfile creation + dockerfile_path = tmpdir.join('git', 'regenerate-bundle', 'Dockerfile') + assert dockerfile_path.check() + + # Verify metadata file creation and contents + metadata_file = tmpdir.join('git', 'regenerate-bundle', '.iib-build-metadata.json') + assert metadata_file.check() + metadata_content = json.loads(metadata_file.read()) + assert metadata_content['request_id'] == request_id + assert metadata_content['arches'] == sorted(list(arches)) + assert metadata_content['organization'] == organization + assert metadata_content['package_name'] == 'test-package' + + +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.cleanup_on_failure') +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.git_commit_and_create_mr_or_push' +) +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.' + 'extract_files_from_image_non_privileged' +) +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.clone_git_repo') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle._get_package_annotations') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle._adjust_operator_bundle') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_image_labels') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_image_arches') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.update_request') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_worker_config') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.tempfile.TemporaryDirectory') +def test_handle_containerized_regenerate_bundle_request_failure( + mock_temp_dir, + mock_gwc, + mock_ur, + mock_srs, + mock_gri, + mock_gia, + mock_gil, + mock_aob, + mock_gpa, + mock_ggt, + mock_cgr, + mock_effinp, + mock_gccmop, + mock_cof, + tmpdir, +): + """Test containerized regenerate bundle request failure triggers cleanup.""" + # Setup + arches = ['amd64'] + from_bundle_image = 'bundle-image:latest' + from_bundle_image_resolved = 'bundle-image@sha256:abcdef' + organization = 'acme' + request_id = 99 + bundle_git_repo = 'https://gitlab.com/bundle/repo' + + # Use tmpdir with the mock + mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) + + # Mock worker config + mock_gwc.return_value = { + 'iib_index_image_output_registry': None, + 'iib_registry': 'quay.io', + } + + # Mock image resolution and metadata + mock_gri.return_value = from_bundle_image_resolved + mock_gia.return_value = list(arches) + mock_gil.return_value = {'com.redhat.iib.pinned': 'false'} + + # Mock Git token + mock_ggt.return_value = ('GITLAB_TOKEN', 'test-token') + + # Mock bundle adjustments + mock_aob.return_value = {'operators.operatorframework.io.bundle.package.v1': 'test-package'} + mock_gpa.return_value = { + 'annotations': {'operators.operatorframework.io.bundle.package.v1': 'test-package'} + } + + # Mock Git operations to fail + mock_gccmop.side_effect = RuntimeError('Git operation failed') + + # Execute and expect failure + with pytest.raises(IIBError, match='Failed to regenerate bundle'): + build_containerized_regenerate_bundle.handle_containerized_regenerate_bundle_request( + from_bundle_image=from_bundle_image, + organization=organization, + request_id=request_id, + bundle_replacements={}, + index_to_gitlab_push_map={'regenerate-bundle': bundle_git_repo}, + regenerate_bundle_repo_key='regenerate-bundle', + ) + + # Verify cleanup was called + mock_cof.assert_called_once() + + +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_image_arches') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.update_request') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_worker_config') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.tempfile.TemporaryDirectory') +def test_handle_containerized_regenerate_bundle_request_no_arches( + mock_temp_dir, + mock_gwc, + mock_ur, + mock_srs, + mock_gri, + mock_gia, + tmpdir, +): + """Test that missing arches raises an error.""" + # Setup + from_bundle_image = 'bundle-image:latest' + from_bundle_image_resolved = 'bundle-image@sha256:abcdef' + organization = 'acme' + request_id = 99 + + # Use tmpdir with the mock + mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) + + # Mock worker config + mock_gwc.return_value = { + 'iib_index_image_output_registry': None, + 'iib_registry': 'quay.io', + } + + # Mock image resolution to return no arches + mock_gri.return_value = from_bundle_image_resolved + mock_gia.return_value = [] + + # Execute and expect failure + expected = ( + f'No arches were found in the resolved from_bundle_image {from_bundle_image_resolved}' + ) + with pytest.raises(IIBError, match=expected): + build_containerized_regenerate_bundle.handle_containerized_regenerate_bundle_request( + from_bundle_image=from_bundle_image, + organization=organization, + request_id=request_id, + bundle_replacements={}, + index_to_gitlab_push_map={'regenerate-bundle': 'https://gitlab.com/bundle/repo'}, + regenerate_bundle_repo_key='regenerate-bundle', + ) + + +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_image_labels') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_image_arches') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.update_request') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_worker_config') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.tempfile.TemporaryDirectory') +def test_handle_containerized_regenerate_bundle_request_no_repo_key( + mock_temp_dir, + mock_gwc, + mock_ur, + mock_srs, + mock_gri, + mock_gia, + mock_gil, + tmpdir, +): + """Test that missing repository key raises an error.""" + # Setup + from_bundle_image = 'bundle-image:latest' + from_bundle_image_resolved = 'bundle-image@sha256:abcdef' + organization = 'acme' + request_id = 99 + + # Use tmpdir with the mock + mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) + + # Mock worker config + mock_gwc.return_value = { + 'iib_index_image_output_registry': None, + 'iib_registry': 'quay.io', + } + + # Mock image resolution + mock_gri.return_value = from_bundle_image_resolved + mock_gia.return_value = ['amd64'] + mock_gil.return_value = {'com.redhat.iib.pinned': 'false'} + + # Execute and expect failure - using default 'regenerate-bundle' key but map has different key + expected = 'Repository not found for key: regenerate-bundle' + with pytest.raises(IIBError, match=expected): + build_containerized_regenerate_bundle.handle_containerized_regenerate_bundle_request( + from_bundle_image=from_bundle_image, + organization=organization, + request_id=request_id, + bundle_replacements={}, + index_to_gitlab_push_map={'different_key': 'https://gitlab.com/bundle/repo'}, + # Not passing regenerate_bundle_repo_key, so it uses default 'regenerate-bundle' + ) + + +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.cleanup_merge_request_if_exists' +) +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.replicate_image_to_tagged_destinations' +) +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.monitor_pipeline_and_extract_image' +) +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.git_commit_and_create_mr_or_push' +) +@mock.patch( + 'iib.workers.tasks.build_containerized_regenerate_bundle.' + 'extract_files_from_image_non_privileged' +) +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.clone_git_repo') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle._get_package_annotations') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle._adjust_operator_bundle') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_image_labels') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_image_arches') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_resolved_image') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.update_request') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.get_worker_config') +@mock.patch('iib.workers.tasks.build_containerized_regenerate_bundle.tempfile.TemporaryDirectory') +def test_handle_containerized_regenerate_bundle_request_with_output_registry( + mock_temp_dir, + mock_gwc, + mock_ur, + mock_srs, + mock_gri, + mock_gia, + mock_gil, + mock_aob, + mock_gpa, + mock_ggt, + mock_cgr, + mock_effinp, + mock_gccmop, + mock_mpaei, + mock_ritd, + mock_cmrie, + tmpdir, +): + """Test output registry replacement when iib_index_image_output_registry is configured.""" + # Setup + arches = ['amd64'] + from_bundle_image = 'bundle-image:latest' + from_bundle_image_resolved = 'bundle-image@sha256:abcdef' + original_bundle_image = 'quay.io/iib:99' + replaced_bundle_image = 'registry.example.com/iib:99' + organization = 'acme' + request_id = 100 + bundle_git_repo = 'https://gitlab.com/bundle/repo' + + # Use tmpdir with the mock + mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) + + # Mock worker config WITH output registry replacement + mock_gwc.return_value = { + 'iib_index_image_output_registry': 'registry.example.com', + 'iib_registry': 'quay.io', + } + + # Mock image resolution and metadata + mock_gri.return_value = from_bundle_image_resolved + mock_gia.return_value = list(arches) + mock_gil.return_value = {'com.redhat.iib.pinned': 'false'} + + # Mock Git token + mock_ggt.return_value = ('GITLAB_TOKEN', 'test-token') + + # Mock bundle adjustments + mock_aob.return_value = {'operators.operatorframework.io.bundle.package.v1': 'test-package'} + mock_gpa.return_value = { + 'annotations': {'operators.operatorframework.io.bundle.package.v1': 'test-package'} + } + + # Mock Git operations + mock_gccmop.return_value = ( + {'mr_id': '123', 'mr_url': 'https://gitlab.com/merge_requests/123'}, + 'commit-sha-123', + ) + + # Mock pipeline monitoring + mock_mpaei.return_value = 'quay.io/konflux/bundle:sha256-abc123.att' + + # Mock image replication - returns original registry + mock_ritd.return_value = [original_bundle_image] + + # Execute + build_containerized_regenerate_bundle.handle_containerized_regenerate_bundle_request( + from_bundle_image=from_bundle_image, + organization=organization, + request_id=request_id, + bundle_replacements={}, + index_to_gitlab_push_map={'regenerate-bundle': bundle_git_repo}, + regenerate_bundle_repo_key='regenerate-bundle', + ) + + # Verify the final update_request call used the REPLACED bundle_image + final_update_call = mock_ur.call_args_list[-1] + final_payload = final_update_call[0][1] + assert final_payload['bundle_image'] == replaced_bundle_image + assert final_payload['state'] == 'complete' diff --git a/tests/test_workers/test_tasks/test_build_containerized_rm.py b/tests/test_workers/test_tasks/test_build_containerized_rm.py new file mode 100644 index 000000000..f86ef754c --- /dev/null +++ b/tests/test_workers/test_tasks/test_build_containerized_rm.py @@ -0,0 +1,1441 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import os +import pytest +from unittest import mock + +from iib.exceptions import IIBError +from iib.workers.tasks import build_containerized_rm +from iib.workers.tasks.utils import RequestConfigAddRm + + +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_image_digest') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils._get_artifact_combined_tag') +@mock.patch('iib.workers.tasks.containerized_utils._get_name_and_tag_from_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.commit_and_push') +@mock.patch('iib.workers.tasks.build_containerized_rm.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_rm.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_registry_rm_fbc') +@mock.patch('iib.workers.tasks.build_containerized_rm.verify_operators_exists') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_rm.remove_operator_deprecations') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_rm.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.copytree') +@mock.patch('iib.workers.tasks.build_containerized_rm.os.path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +def test_handle_containerized_rm_request_success_with_overwrite( + mock_makedirs, + mock_path_exists, + mock_os_exists, + mock_copytree, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_rod, + mock_pida, + mock_voe, + mock_orrf, + mock_mcd, + mock_ov, + mock_wbm, + mock_cap, + mock_glcs, + mock_fpr, + mock_wfpc, + mock_gpiu, + mock_gntfp, + mock_gact, + mock_giap, + mock_gid, + mock_poa, + mock_gwc, + mock_srs_utils, + mock_sc, + mock_uiips, + mock_cof, + mock_rdc, +): + """Test successful operator removal with overwrite_from_index.""" + # Setup + request_id = 1 + operators = ['operator1', 'operator2'] + from_index = 'quay.io/namespace/index-image:v4.14' + binary_image = 'registry.io/binary:latest' + overwrite_from_index_token = 'user:token' + + # Mock temp directory + temp_dir = '/tmp/iib-1-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + # Mock prepare_request_for_build + mock_prfb.return_value = { + 'arches': {'amd64', 's390x'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc123', + 'from_index_resolved': 'quay.io/namespace/index-image@sha256:def456', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + # Mock OPM + mock_opm.opm_version = 'v1.28.0' + + # Mock git operations + mock_ggt.return_value = ('token_name', 'git_token') + + # Mock file system operations + mock_path_exists.return_value = True # For Path.exists() in containerized_utils + mock_os_exists.return_value = True # For os.path.exists() in build_containerized_rm + + # Mock pull_index_db_artifact + artifact_dir = os.path.join(temp_dir, 'artifact') + mock_pida.return_value = artifact_dir + + # Mock verify_operators_exists + mock_voe.return_value = ({'operator1', 'operator2'}, os.path.join(artifact_dir, 'index.db')) + + # Mock opm_registry_rm_fbc + fbc_dir = os.path.join(temp_dir, 'fbc') + mock_orrf.return_value = (fbc_dir, None) + + # Mock git commit + mock_glcs.return_value = 'abc123commit' + + # Mock Konflux pipeline + mock_fpr.return_value = [{'metadata': {'name': 'pipelinerun-123'}}] + mock_wfpc.return_value = {'status': 'success'} + mock_gpiu.return_value = 'quay.io/konflux/built-image@sha256:xyz789' + + # Mock ORAS push related functions + mock_gntfp.return_value = ('index-image', 'v4.14') + mock_gact.return_value = 'index-image-v4.14' + mock_giap.return_value = 'registry.io/index-db:v4.14' + mock_gid.return_value = 'sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abc' + + # Mock worker config + mock_gwc.return_value = { + 'iib_registry': 'registry.io', + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } + + # Test + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + binary_image=binary_image, + overwrite_from_index=True, + overwrite_from_index_token=overwrite_from_index_token, + index_to_gitlab_push_map={'quay.io/namespace/index-image': 'https://gitlab.com/repo'}, + ) + + # Verify prepare_request_for_build was called + mock_prfb.assert_called_once_with( + request_id, + RequestConfigAddRm( + _binary_image=binary_image, + from_index=from_index, + overwrite_from_index_token=overwrite_from_index_token, + add_arches=None, + distribution_scope=None, + binary_image_config=None, + ), + ) + + # Verify OPM version was set + mock_opm.set_opm_version.assert_called_once() + + # Verify git operations + mock_cgr.assert_called_once() + + # Verify operators were removed from catalog and index.db + assert mock_rmtree.call_count >= 1 # At least for operator removal + mock_orrf.assert_called_once() + + # Verify catalog merge and validation + mock_mcd.assert_called_once() + mock_ov.assert_called_once() + + # Verify commit was pushed (not MR since overwrite_from_index_token is provided) + mock_cap.assert_called_once() + commit_msg = mock_cap.call_args[1]['commit_message'] + assert f'IIB: Remove operators for request {request_id}' in commit_msg + assert 'Operators: operator1, operator2' in commit_msg + + # Verify Konflux pipeline was triggered and waited on + mock_fpr.assert_called_once_with('abc123commit') + mock_wfpc.assert_called_once_with('pipelinerun-123') + + # Verify image was copied + assert mock_sc.call_count >= 1 + + # Verify index.db was pushed (2 times: request_id tag + v4.x tag) + assert mock_poa.call_count == 2 + + # Verify final state + final_call = mock_srs.call_args_list[-1] + assert final_call[0][0] == request_id + assert final_call[0][1] == 'complete' + assert 'successfully removed' in final_call[0][2] + + # Verify reset_docker_config was called + assert mock_rdc.call_count >= 1 + + +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils._get_artifact_combined_tag') +@mock.patch('iib.workers.tasks.containerized_utils._get_name_and_tag_from_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.close_mr') +@mock.patch('iib.workers.tasks.containerized_utils.create_mr') +@mock.patch('iib.workers.tasks.build_containerized_rm.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_rm.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_registry_rm_fbc') +@mock.patch('iib.workers.tasks.build_containerized_rm.verify_operators_exists') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_rm.remove_operator_deprecations') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_rm.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.copytree') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +def test_handle_containerized_rm_request_with_mr( + mock_makedirs, + mock_exists, + mock_copytree, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_rod, + mock_pida, + mock_voe, + mock_orrf, + mock_mcd, + mock_ov, + mock_wbm, + mock_cmr, + mock_close_mr, + mock_glcs, + mock_fpr, + mock_wfpc, + mock_gpiu, + mock_gntfp, + mock_gact, + mock_giap, + mock_poa, + mock_gwc, + mock_srs_utils, + mock_sc, + mock_uiips, + mock_cof, + mock_rdc, +): + """Test operator removal without overwrite creates and closes MR.""" + # Setup + request_id = 2 + operators = ['test-operator'] + from_index = 'quay.io/namespace/index-image:v4.14' + + # Mock temp directory + temp_dir = '/tmp/iib-2-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + # Mock prepare_request_for_build + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc123', + 'from_index_resolved': 'quay.io/namespace/index-image@sha256:def456', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + # Mock OPM + mock_opm.opm_version = 'v1.28.0' + + # Mock git operations + mock_ggt.return_value = ('token_name', 'git_token') + + # Mock file system + mock_exists.return_value = True + + # Mock artifact pull + artifact_dir = os.path.join(temp_dir, 'artifact') + mock_pida.return_value = artifact_dir + + # Mock operators exist + mock_voe.return_value = ({'test-operator'}, os.path.join(artifact_dir, 'index.db')) + + # Mock opm operation + fbc_dir = os.path.join(temp_dir, 'fbc') + mock_orrf.return_value = (fbc_dir, None) + + # Mock MR creation + mock_cmr.return_value = {'mr_url': 'https://gitlab.com/repo/-/merge_requests/1', 'mr_id': 1} + mock_glcs.return_value = 'commit_sha_123' + + # Mock Konflux + mock_fpr.return_value = [{'metadata': {'name': 'pr-456'}}] + mock_wfpc.return_value = {'status': 'success'} + mock_gpiu.return_value = 'quay.io/konflux/image@sha256:built' + + # Mock ORAS push related functions (only request_id tag, no overwrite) + mock_gntfp.return_value = ('index-image', 'v4.14') + mock_gact.return_value = 'index-image-v4.14' + mock_giap.return_value = 'registry.io/index-db:v4.14' + + # Mock config + mock_gwc.return_value = { + 'iib_registry': 'registry.io', + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } + + # Test - without overwrite_from_index_token + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + index_to_gitlab_push_map={'quay.io/namespace/index-image': 'https://gitlab.com/repo'}, + ) + + # Verify MR was created + mock_cmr.assert_called_once() + commit_msg = mock_cmr.call_args[1]['commit_message'] + assert f'IIB: Remove operators for request {request_id}' in commit_msg + assert 'Operators: test-operator' in commit_msg + + # Verify MR was closed + mock_close_mr.assert_called_once() + + # Verify completion + final_call = mock_srs.call_args_list[-1] + assert final_call[0][1] == 'complete' + + +@pytest.mark.parametrize( + 'operators_in_db, should_call_opm_rm', + [ + (set(), False), # No operators in DB + ({'operator1'}, True), # Operators in DB + ({'op1', 'op2'}, True), # Multiple operators in DB + ], +) +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_image_digest') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils._get_artifact_combined_tag') +@mock.patch('iib.workers.tasks.containerized_utils._get_name_and_tag_from_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.commit_and_push') +@mock.patch('iib.workers.tasks.build_containerized_rm.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_rm.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_registry_rm_fbc') +@mock.patch('iib.workers.tasks.build_containerized_rm.verify_operators_exists') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_rm.remove_operator_deprecations') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_rm.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.copytree') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +def test_handle_containerized_rm_conditional_opm_rm( + mock_makedirs, + mock_exists, + mock_copytree, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_rod, + mock_pida, + mock_voe, + mock_orrf, + mock_mcd, + mock_ov, + mock_wbm, + mock_cap, + mock_glcs, + mock_fpr, + mock_wfpc, + mock_gpiu, + mock_gntfp, + mock_gact, + mock_giap, + mock_gid, + mock_poa, + mock_gwc, + mock_srs_utils, + mock_sc, + mock_uiips, + mock_cof, + mock_rdc, + operators_in_db, + should_call_opm_rm, +): + """Test that opm_registry_rm_fbc is only called when operators exist in DB.""" + # Setup + request_id = 3 + operators = ['operator1'] + from_index = 'quay.io/namespace/index:v4.14' + + # Mock temp directory + temp_dir = '/tmp/iib-3-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + # Mock prepare_request_for_build + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + mock_opm.opm_version = 'v1.28.0' + mock_ggt.return_value = ('token', 'token_value') + mock_exists.return_value = True + + artifact_dir = os.path.join(temp_dir, 'artifact') + mock_pida.return_value = artifact_dir + + # Mock verify_operators_exists to return the parameterized operators_in_db + mock_voe.return_value = (operators_in_db, os.path.join(artifact_dir, 'index.db')) + + # Mock opm_registry_rm_fbc + fbc_dir = os.path.join(temp_dir, 'fbc') + mock_orrf.return_value = (fbc_dir, None) + + # Mock pipeline + mock_glcs.return_value = 'commit' + mock_fpr.return_value = [{'metadata': {'name': 'pr'}}] + mock_wfpc.return_value = {} + mock_gpiu.return_value = 'image@sha' + + # Mock ORAS push related functions (conditionally used based on operators_in_db) + mock_gntfp.return_value = ('index', 'v4.14') + mock_gact.return_value = 'index-v4.14' + mock_giap.return_value = 'reg/index-db:v4.14' + mock_gid.return_value = 'sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abc' + + mock_gwc.return_value = { + 'iib_registry': 'reg', + 'iib_image_push_template': '{registry}/iib:{request_id}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } + + # Test + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + overwrite_from_index=True, + overwrite_from_index_token='user:token', + index_to_gitlab_push_map={'quay.io/namespace/index': 'https://gitlab.com/repo'}, + ) + + # Verify opm_registry_rm_fbc called only when operators exist in DB + if should_call_opm_rm: + mock_orrf.assert_called_once() + mock_mcd.assert_called_once() + mock_rename.assert_called_once() + else: + mock_orrf.assert_not_called() + mock_mcd.assert_not_called() + mock_rename.assert_not_called() + + +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +def test_handle_containerized_rm_missing_git_mapping( + mock_prfb, + mock_uiibs, + mock_opm, + mock_tempdir, + mock_srs, + mock_rdc, +): + """Test that missing git mapping raises error.""" + request_id = 4 + operators = ['operator1'] + from_index = 'quay.io/namespace/index:v4.14' + + temp_dir = '/tmp/iib-4-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + # Mock OPM to avoid version check + mock_opm.opm_version = 'v1.28.0' + + # Test with empty git mapping + with pytest.raises(IIBError, match='Git repository mapping not found'): + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + index_to_gitlab_push_map={}, # Empty mapping + ) + + # Test with None git mapping + with pytest.raises(IIBError, match='Git repository mapping not found'): + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + index_to_gitlab_push_map=None, # None mapping + ) + + +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +def test_handle_containerized_rm_missing_configs_dir( + mock_srs_utils, + mock_makedirs, + mock_exists, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_cof, + mock_rdc, +): + """Test that missing configs directory raises error.""" + request_id = 5 + operators = ['operator1'] + from_index = 'quay.io/namespace/index:v4.14' + + temp_dir = '/tmp/iib-5-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + mock_opm.opm_version = 'v1.28.0' + mock_ggt.return_value = ('token', 'value') + + # Mock exists to return False for configs directory + mock_exists.return_value = False + + # Test + with pytest.raises(IIBError, match='Catalogs directory not found'): + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + index_to_gitlab_push_map={'quay.io/namespace/index': 'https://gitlab.com/repo'}, + ) + + # Verify cleanup was NOT called (error happens before try block) + mock_cof.assert_not_called() + + +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_rm.remove_operator_deprecations') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.rmtree') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +def test_handle_containerized_rm_missing_index_db( + mock_srs_utils, + mock_makedirs, + mock_exists, + mock_rmtree, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_rod, + mock_pida, + mock_cof, + mock_rdc, +): + """Test that missing index.db file raises error.""" + request_id = 6 + operators = ['operator1'] + from_index = 'quay.io/namespace/index:v4.14' + + temp_dir = '/tmp/iib-6-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + mock_opm.opm_version = 'v1.28.0' + mock_ggt.return_value = ('token', 'value') + + # Mock pull_index_db_artifact to raise error about missing index.db + mock_pida.side_effect = IIBError('Index.db file not found') + + # Test + with pytest.raises(IIBError, match='Index.db file not found'): + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + index_to_gitlab_push_map={'quay.io/namespace/index': 'https://gitlab.com/repo'}, + ) + + # Verify cleanup was NOT called (error happens before try block) + mock_cof.assert_not_called() + + +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.commit_and_push') +@mock.patch('iib.workers.tasks.build_containerized_rm.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_rm.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_registry_rm_fbc') +@mock.patch('iib.workers.tasks.build_containerized_rm.verify_operators_exists') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_rm.remove_operator_deprecations') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_rm.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.copytree') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +def test_handle_containerized_rm_pipeline_failure( + mock_srs_utils, + mock_makedirs, + mock_exists, + mock_copytree, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_rod, + mock_pida, + mock_voe, + mock_orrf, + mock_mcd, + mock_ov, + mock_wbm, + mock_cap, + mock_glcs, + mock_fpr, + mock_wfpc, + mock_gpiu, + mock_cof, + mock_rdc, +): + """Test that pipeline failure triggers cleanup.""" + request_id = 7 + operators = ['operator1'] + from_index = 'quay.io/namespace/index:v4.14' + + temp_dir = '/tmp/iib-7-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + mock_opm.opm_version = 'v1.28.0' + mock_ggt.return_value = ('token', 'value') + mock_exists.return_value = True + + artifact_dir = os.path.join(temp_dir, 'artifact') + mock_pida.return_value = artifact_dir + mock_voe.return_value = ({'operator1'}, os.path.join(artifact_dir, 'index.db')) + + fbc_dir = os.path.join(temp_dir, 'fbc') + mock_orrf.return_value = (fbc_dir, None) + + mock_glcs.return_value = 'commit_sha' + + # Mock pipeline to raise error + mock_fpr.side_effect = IIBError('Pipeline not found') + + # Test + with pytest.raises(IIBError, match='Failed to remove operators'): + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + overwrite_from_index=True, + overwrite_from_index_token='user:token', + index_to_gitlab_push_map={'quay.io/namespace/index': 'https://gitlab.com/repo'}, + ) + + # Verify cleanup was called + mock_cof.assert_called_once() + cleanup_call = mock_cof.call_args + assert cleanup_call[1]['request_id'] == request_id + assert 'Pipeline not found' in cleanup_call[1]['reason'] + + +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_image_digest') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils._get_artifact_combined_tag') +@mock.patch('iib.workers.tasks.containerized_utils._get_name_and_tag_from_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.commit_and_push') +@mock.patch('iib.workers.tasks.build_containerized_rm.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_rm.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_registry_rm_fbc') +@mock.patch('iib.workers.tasks.build_containerized_rm.verify_operators_exists') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_rm.remove_operator_deprecations') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_rm.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.copytree') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +def test_handle_containerized_rm_with_index_db_push( + mock_makedirs, + mock_exists, + mock_copytree, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_rod, + mock_pida, + mock_voe, + mock_orrf, + mock_mcd, + mock_ov, + mock_wbm, + mock_cap, + mock_glcs, + mock_fpr, + mock_wfpc, + mock_gpiu, + mock_gntfp, + mock_gact, + mock_giap, + mock_gid, + mock_poa, + mock_gwc, + mock_srs_utils, + mock_sc, + mock_uiips, + mock_cof, + mock_rdc, +): + """Test that index.db is pushed when operators exist in DB and overwrite token is provided.""" + request_id = 8 + operators = ['operator1', 'operator2'] + from_index = 'quay.io/namespace/index-image:v4.14' + overwrite_token = 'user:token' + + temp_dir = '/tmp/iib-8-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index-image@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + mock_opm.opm_version = 'v1.28.0' + mock_ggt.return_value = ('token', 'value') + mock_exists.return_value = True + + artifact_dir = os.path.join(temp_dir, 'artifact') + index_db_path = os.path.join(artifact_dir, 'index.db') + mock_pida.return_value = artifact_dir + + # Operators exist in DB + mock_voe.return_value = ({'operator1', 'operator2'}, index_db_path) + + fbc_dir = os.path.join(temp_dir, 'fbc') + mock_orrf.return_value = (fbc_dir, None) + + mock_glcs.return_value = 'commit' + mock_fpr.return_value = [{'metadata': {'name': 'pr'}}] + mock_wfpc.return_value = {} + mock_gpiu.return_value = 'image@sha' + + # Mock ORAS push related functions + mock_gntfp.return_value = ('index-image', 'v4.14') + mock_gact.return_value = 'index-image-v4.14' + mock_giap.return_value = 'registry.io/index-db:v4.14' + mock_gid.return_value = 'sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab' + + mock_gwc.return_value = { + 'iib_registry': 'registry.io', + 'iib_image_push_template': '{registry}/iib:{request_id}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } + + # Test + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + overwrite_from_index=True, + overwrite_from_index_token=overwrite_token, + index_to_gitlab_push_map={'quay.io/namespace/index-image': 'https://gitlab.com/repo'}, + ) + + # Verify index.db was pushed (2 times: request_id tag + v4.x tag) + assert mock_poa.call_count == 2 + + # Verify original digest was captured + mock_gid.assert_called_once() + + # Verify annotations were added + first_push_call = mock_poa.call_args_list[0] + assert 'annotations' in first_push_call[1] + assert first_push_call[1]['annotations']['request_id'] == str(request_id) + assert first_push_call[1]['annotations']['request_type'] == 'rm' + + +@pytest.mark.parametrize( + 'build_tags, expected_tag_count', + [ + (None, 1), # Only request_id + (['latest'], 2), # request_id + latest + (['latest', 'v4.14'], 3), # request_id + latest + v4.14 + ], +) +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_image_digest') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils._get_artifact_combined_tag') +@mock.patch('iib.workers.tasks.containerized_utils._get_name_and_tag_from_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.commit_and_push') +@mock.patch('iib.workers.tasks.build_containerized_rm.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_rm.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_registry_rm_fbc') +@mock.patch('iib.workers.tasks.build_containerized_rm.verify_operators_exists') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_rm.remove_operator_deprecations') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.copytree') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +def test_handle_containerized_rm_with_build_tags( + mock_srs_utils, + mock_makedirs, + mock_exists, + mock_copytree, + mock_rmtree, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_rod, + mock_pida, + mock_voe, + mock_orrf, + mock_mcd, + mock_ov, + mock_wbm, + mock_cap, + mock_glcs, + mock_fpr, + mock_wfpc, + mock_gpiu, + mock_gntfp, + mock_gact, + mock_giap, + mock_gid, + mock_poa, + mock_gwc, + mock_sc, + mock_uiips, + mock_cof, + mock_rdc, + build_tags, + expected_tag_count, +): + """Test that build_tags parameter results in correct number of skopeo copies.""" + request_id = 9 + operators = ['operator1'] + from_index = 'quay.io/namespace/index:v4.14' + + temp_dir = '/tmp/iib-9-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + mock_opm.opm_version = 'v1.28.0' + mock_ggt.return_value = ('token', 'value') + mock_exists.return_value = True + + artifact_dir = os.path.join(temp_dir, 'artifact') + mock_pida.return_value = artifact_dir + mock_voe.return_value = (set(), os.path.join(artifact_dir, 'index.db')) + + mock_glcs.return_value = 'commit' + mock_fpr.return_value = [{'metadata': {'name': 'pr'}}] + mock_wfpc.return_value = {} + mock_gpiu.return_value = 'image@sha' + + # Mock ORAS-related functions + # (needed because push_index_db_artifact now called even with empty operators) + mock_gntfp.return_value = ('index', 'v4.14') + mock_gact.return_value = 'index-v4.14' + mock_giap.return_value = 'registry.io/index-db:v4.14' + mock_gid.return_value = 'sha256:abcdef' + + mock_gwc.return_value = { + 'iib_registry': 'registry.io', + 'iib_image_push_template': '{registry}/iib:{request_id}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } + + # Test + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + overwrite_from_index=True, + overwrite_from_index_token='user:token', + build_tags=build_tags, + index_to_gitlab_push_map={'quay.io/namespace/index': 'https://gitlab.com/repo'}, + ) + + # Verify skopeo_copy was called correct number of times + assert mock_sc.call_count == expected_tag_count + + # Verify index.db was pushed + # (now called even with empty operators since we removed operators_in_db check) + assert mock_poa.call_count == 2 # request_id tag + v4.x tag (overwrite_from_index=True) + + +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils._get_artifact_combined_tag') +@mock.patch('iib.workers.tasks.containerized_utils._get_name_and_tag_from_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.close_mr') +@mock.patch('iib.workers.tasks.containerized_utils.create_mr') +@mock.patch('iib.workers.tasks.build_containerized_rm.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_rm.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_registry_rm_fbc') +@mock.patch('iib.workers.tasks.build_containerized_rm.verify_operators_exists') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_rm.remove_operator_deprecations') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_rm.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.copytree') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +def test_handle_containerized_rm_close_mr_failure_logged( + mock_makedirs, + mock_exists, + mock_copytree, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_rod, + mock_pida, + mock_voe, + mock_orrf, + mock_mcd, + mock_ov, + mock_wbm, + mock_cmr, + mock_close_mr, + mock_glcs, + mock_fpr, + mock_wfpc, + mock_gpiu, + mock_gntfp, + mock_gact, + mock_giap, + mock_poa, + mock_gwc, + mock_srs_utils, + mock_sc, + mock_uiips, + mock_cof, + mock_rdc, +): + """Test that MR close failure is logged but doesn't fail the request.""" + request_id = 10 + operators = ['test-operator'] + from_index = 'quay.io/namespace/index-image:v4.14' + + temp_dir = '/tmp/iib-10-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc123', + 'from_index_resolved': 'quay.io/namespace/index-image@sha256:def456', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + mock_opm.opm_version = 'v1.28.0' + mock_ggt.return_value = ('token_name', 'git_token') + mock_exists.return_value = True + + artifact_dir = os.path.join(temp_dir, 'artifact') + mock_pida.return_value = artifact_dir + mock_voe.return_value = ({'test-operator'}, os.path.join(artifact_dir, 'index.db')) + + fbc_dir = os.path.join(temp_dir, 'fbc') + mock_orrf.return_value = (fbc_dir, None) + + mock_cmr.return_value = {'mr_url': 'https://gitlab.com/repo/-/merge_requests/1', 'mr_id': 1} + mock_glcs.return_value = 'commit_sha_123' + + mock_fpr.return_value = [{'metadata': {'name': 'pr-456'}}] + mock_wfpc.return_value = {'status': 'success'} + mock_gpiu.return_value = 'quay.io/konflux/image@sha256:built' + + # Mock ORAS push related functions + mock_gntfp.return_value = ('index-image', 'v4.14') + mock_gact.return_value = 'index-image-v4.14' + mock_giap.return_value = 'registry.io/index-db:v4.14' + + mock_gwc.return_value = { + 'iib_registry': 'registry.io', + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + 'iib_index_db_artifact_registry': 'artifact-registry.io', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } + + # Mock close_mr to raise error + mock_close_mr.side_effect = IIBError('Failed to close MR') + + # Test - should complete successfully despite MR close failure + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + index_to_gitlab_push_map={'quay.io/namespace/index-image': 'https://gitlab.com/repo'}, + ) + + # Verify MR was attempted to be closed + mock_close_mr.assert_called_once() + + # Verify request still completed successfully + final_call = mock_srs.call_args_list[-1] + assert final_call[0][1] == 'complete' + + +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.commit_and_push') +@mock.patch('iib.workers.tasks.build_containerized_rm.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_rm.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_registry_rm_fbc') +@mock.patch('iib.workers.tasks.build_containerized_rm.verify_operators_exists') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_rm.remove_operator_deprecations') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_rm.os.rename') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.copytree') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +def test_handle_containerized_rm_pipelinerun_missing_name( + mock_srs_utils, + mock_makedirs, + mock_exists, + mock_copytree, + mock_rmtree, + mock_rename, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_rod, + mock_pida, + mock_voe, + mock_orrf, + mock_mcd, + mock_ov, + mock_wbm, + mock_cap, + mock_glcs, + mock_fpr, + mock_cof, + mock_rdc, +): + """Test error when pipelinerun metadata doesn't contain name.""" + request_id = 11 + operators = ['operator1'] + from_index = 'quay.io/namespace/index:v4.14' + + temp_dir = '/tmp/iib-11-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + mock_opm.opm_version = 'v1.28.0' + mock_ggt.return_value = ('token', 'value') + mock_exists.return_value = True + + artifact_dir = os.path.join(temp_dir, 'artifact') + mock_pida.return_value = artifact_dir + mock_voe.return_value = ({'operator1'}, os.path.join(artifact_dir, 'index.db')) + + fbc_dir = os.path.join(temp_dir, 'fbc') + mock_orrf.return_value = (fbc_dir, None) + + mock_glcs.return_value = 'commit' + + # Mock pipelinerun without 'name' in metadata + mock_fpr.return_value = [{'metadata': {}}] # Missing 'name' key + + # Test + with pytest.raises(IIBError, match='Pipelinerun name not found'): + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + overwrite_from_index=True, + overwrite_from_index_token='user:token', + index_to_gitlab_push_map={'quay.io/namespace/index': 'https://gitlab.com/repo'}, + ) + + # Verify cleanup was called + mock_cof.assert_called_once() + + +@mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') +@mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') +@mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') +@mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') +@mock.patch('iib.workers.tasks.containerized_utils.find_pipelinerun') +@mock.patch('iib.workers.tasks.containerized_utils.get_last_commit_sha') +@mock.patch('iib.workers.tasks.containerized_utils.commit_and_push') +@mock.patch('iib.workers.tasks.build_containerized_rm.write_build_metadata') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_validate') +@mock.patch('iib.workers.tasks.build_containerized_rm.merge_catalogs_dirs') +@mock.patch('iib.workers.tasks.build_containerized_rm.opm_registry_rm_fbc') +@mock.patch('iib.workers.tasks.build_containerized_rm.verify_operators_exists') +@mock.patch('iib.workers.tasks.containerized_utils.pull_index_db_artifact') +@mock.patch('iib.workers.tasks.build_containerized_rm.remove_operator_deprecations') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.build_containerized_rm._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.Opm') +@mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') +@mock.patch('iib.workers.tasks.build_containerized_rm.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.rmtree') +@mock.patch('iib.workers.tasks.build_containerized_rm.shutil.copytree') +@mock.patch('iib.workers.tasks.containerized_utils.Path.exists') +@mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +def test_handle_containerized_rm_missing_output_pull_spec( + mock_srs_utils, + mock_makedirs, + mock_exists, + mock_copytree, + mock_rmtree, + mock_tempdir, + mock_srs, + mock_prfb, + mock_opm, + mock_uiibs, + mock_ggt, + mock_cgr, + mock_rod, + mock_pida, + mock_voe, + mock_orrf, + mock_mcd, + mock_ov, + mock_wbm, + mock_cap, + mock_glcs, + mock_fpr, + mock_wfpc, + mock_gpiu, + mock_gwc_utils, + mock_gwc, + mock_sc, + mock_cof, + mock_rdc, +): + """Test error when output_pull_spec is not set (defensive check).""" + request_id = 12 + operators = ['operator1'] + from_index = 'quay.io/namespace/index:v4.14' + + temp_dir = '/tmp/iib-12-test' + mock_tempdir.return_value.__enter__.return_value = temp_dir + + mock_prfb.return_value = { + 'arches': {'amd64'}, + 'binary_image_resolved': 'registry.io/binary@sha256:abc', + 'from_index_resolved': 'quay.io/namespace/index@sha256:def', + 'ocp_version': 'v4.14', + 'distribution_scope': 'prod', + } + + mock_opm.opm_version = 'v1.28.0' + mock_ggt.return_value = ('token', 'value') + mock_exists.return_value = True + + artifact_dir = os.path.join(temp_dir, 'artifact') + mock_pida.return_value = artifact_dir + mock_voe.return_value = (set(), os.path.join(artifact_dir, 'index.db')) + + mock_glcs.return_value = 'commit' + mock_fpr.return_value = [{'metadata': {'name': 'pr'}}] + mock_wfpc.return_value = {} + mock_gpiu.return_value = 'image@sha' + + # Mock worker config to return empty string for template (defensive edge case) + # Need to mock both: one for containerized_utils and one for build_containerized_rm + config_with_empty_template = { + 'iib_registry': 'registry.io', + 'iib_image_push_template': '', # Empty template results in empty output_pull_spec + } + mock_gwc_utils.return_value = config_with_empty_template + mock_gwc.return_value = config_with_empty_template + + # Test + with pytest.raises(IIBError, match='output_pull_spec was not set'): + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + overwrite_from_index=True, + overwrite_from_index_token='user:token', + index_to_gitlab_push_map={'quay.io/namespace/index': 'https://gitlab.com/repo'}, + ) + + # Verify cleanup was called + mock_cof.assert_called_once() diff --git a/tests/test_workers/test_tasks/test_build_merge_index_image.py b/tests/test_workers/test_tasks/test_build_merge_index_image.py index 0514b7dc7..1bd838459 100644 --- a/tests/test_workers/test_tasks/test_build_merge_index_image.py +++ b/tests/test_workers/test_tasks/test_build_merge_index_image.py @@ -943,6 +943,7 @@ def test_add_bundles_missing_in_source_user_not_in_allow_list( ), ), ) +@mock.patch('iib.workers.tasks.build_merge_index_image.get_request') @mock.patch('iib.workers.tasks.build_merge_index_image._create_and_push_manifest_list') @mock.patch('iib.workers.tasks.build_merge_index_image._push_image') @mock.patch('iib.workers.tasks.build_merge_index_image._build_image') @@ -956,10 +957,12 @@ def test_add_bundles_missing_in_source_error_tag_specified( mock_bi, mock_pi, mock_capml, + mock_gr, source_bundles, target_bundles, error_msg, ): + mock_gr.return_value = {'user': 'user_name'} with pytest.raises(IIBError, match=error_msg): build_merge_index_image._add_bundles_missing_in_source( source_bundles, @@ -975,6 +978,7 @@ def test_add_bundles_missing_in_source_error_tag_specified( ) +@mock.patch('iib.workers.tasks.build_merge_index_image.get_request') @mock.patch('iib.workers.tasks.build_merge_index_image.is_image_fbc') @mock.patch('iib.workers.tasks.build_merge_index_image.get_image_label') @mock.patch('iib.workers.tasks.build_merge_index_image._create_and_push_manifest_list') @@ -984,7 +988,7 @@ def test_add_bundles_missing_in_source_error_tag_specified( @mock.patch('iib.workers.tasks.build_merge_index_image.opm_index_add') @mock.patch('iib.workers.tasks.build_merge_index_image.set_request_state') def test_add_bundles_missing_in_source_none_missing( - mock_srs, mock_oia, mock_aolti, mock_bi, mock_pi, mock_capml, mock_gil, mock_iifbc + mock_srs, mock_oia, mock_aolti, mock_bi, mock_pi, mock_capml, mock_gil, mock_iifbc, mock_gr ): source_bundles = [ { @@ -1028,6 +1032,7 @@ def test_add_bundles_missing_in_source_none_missing( ] mock_gil.side_effect = ['v=4.5', 'v4.8,v4.7', 'v4.5-v4.8', 'v4.5,v4.6,v4.7'] mock_iifbc.return_value = False + mock_gr.return_value = {'user': 'user_name'} missing_bundles, invalid_bundles = build_merge_index_image._add_bundles_missing_in_source( source_bundles, target_bundles, diff --git a/tests/test_workers/test_tasks/test_build_recursive_related_bundles.py b/tests/test_workers/test_tasks/test_build_recursive_related_bundles.py index 38845a643..0c63a0004 100644 --- a/tests/test_workers/test_tasks/test_build_recursive_related_bundles.py +++ b/tests/test_workers/test_tasks/test_build_recursive_related_bundles.py @@ -13,11 +13,11 @@ @pytest.mark.parametrize('organization', ('acme', None)) -@mock.patch('iib.workers.tasks.build_recursive_related_bundles._cleanup') @mock.patch('iib.workers.tasks.build_recursive_related_bundles.get_resolved_image') -@mock.patch('iib.workers.tasks.build_recursive_related_bundles.podman_pull') @mock.patch('iib.workers.tasks.build_recursive_related_bundles.tempfile.TemporaryDirectory') -@mock.patch('iib.workers.tasks.build_recursive_related_bundles._copy_files_from_image') +@mock.patch( + 'iib.workers.tasks.build_recursive_related_bundles.extract_files_from_image_non_privileged' +) @mock.patch('iib.workers.tasks.build_recursive_related_bundles._adjust_operator_bundle') @mock.patch('iib.workers.tasks.build_recursive_related_bundles.set_request_state') @mock.patch('iib.workers.tasks.build_recursive_related_bundles.get_worker_config') @@ -35,11 +35,9 @@ def test_handle_recusrsive_related_bundles_request( mock_gwc, mock_srs, mock_aob, - mock_cffi, + mock_effinp, mock_temp_dir, - mock_pp, mock_gri, - mock_cleanup, organization, tmpdir, ): @@ -66,7 +64,6 @@ def test_handle_recusrsive_related_bundles_request( build_recursive_related_bundles.handle_recursive_related_bundles_request( parent_bundle_image, org, request_id ) - assert mock_cleanup.call_count == 2 assert mock_gbm.call_count == 3 assert mock_grbi.call_count == 3 assert mock_ur.call_count == 3 @@ -83,11 +80,11 @@ def test_handle_recusrsive_related_bundles_request( ) -@mock.patch('iib.workers.tasks.build_recursive_related_bundles._cleanup') @mock.patch('iib.workers.tasks.build_recursive_related_bundles.get_resolved_image') -@mock.patch('iib.workers.tasks.build_recursive_related_bundles.podman_pull') @mock.patch('iib.workers.tasks.build_recursive_related_bundles.tempfile.TemporaryDirectory') -@mock.patch('iib.workers.tasks.build_recursive_related_bundles._copy_files_from_image') +@mock.patch( + 'iib.workers.tasks.build_recursive_related_bundles.extract_files_from_image_non_privileged' +) @mock.patch('iib.workers.tasks.build_recursive_related_bundles._adjust_operator_bundle') @mock.patch('iib.workers.tasks.build_recursive_related_bundles.set_request_state') @mock.patch('iib.workers.tasks.build_recursive_related_bundles.get_worker_config') @@ -105,11 +102,9 @@ def test_handle_recusrsive_related_bundles_request_max_bundles_reached( mock_gwc, mock_srs, mock_aob, - mock_cffi, + mock_effinp, mock_temp_dir, - mock_pp, mock_gri, - mock_cleanup, tmpdir, ): parent_bundle_image = 'bundle-image:latest' diff --git a/tests/test_workers/test_tasks/test_containerized_utils.py b/tests/test_workers/test_tasks/test_containerized_utils.py new file mode 100644 index 000000000..ce206e8c2 --- /dev/null +++ b/tests/test_workers/test_tasks/test_containerized_utils.py @@ -0,0 +1,1228 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import json +import os +import tarfile +from unittest.mock import patch + +import pytest + +from iib.exceptions import IIBError +from iib.workers.tasks.containerized_utils import ( + extract_files_from_image_non_privileged, + pull_index_db_artifact, + write_build_metadata, + cleanup_on_failure, + validate_bundles_in_parallel, + wait_for_bundle_validation_threads, +) + + +@patch('iib.workers.tasks.containerized_utils.get_worker_config') +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.refresh_indexdb_cache_for_image') +@patch('iib.workers.tasks.containerized_utils.verify_indexdb_cache_for_image') +@patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@patch('iib.workers.tasks.containerized_utils.get_imagestream_artifact_pullspec') +@patch('iib.workers.tasks.containerized_utils.get_oras_artifact') +def test_pull_index_db_artifact_imagestream_enabled_cache_synced( + mock_get_oras_artifact, + mock_get_imagestream_artifact_pullspec, + mock_get_indexdb_artifact_pullspec, + mock_verify_cache, + mock_refresh_cache, + mock_log, + mock_get_worker_config, +): + """When ImageStream cache enabled and synced, pull from ImageStream.""" + mock_get_worker_config.return_value = {'iib_use_imagestream_cache': True} + mock_verify_cache.return_value = True + + from_index = 'quay.io/ns/index-image@sha256:abc' + temp_dir = '/tmp/some-dir' + imagestream_ref = 'imagestream-ref' + artifact_dir = '/tmp/artifact-dir' + + mock_get_imagestream_artifact_pullspec.return_value = imagestream_ref + mock_get_oras_artifact.return_value = artifact_dir + + result = pull_index_db_artifact(from_index, temp_dir) + + assert result == artifact_dir + mock_verify_cache.assert_called_once_with(from_index) + mock_refresh_cache.assert_not_called() + mock_get_imagestream_artifact_pullspec.assert_called_once_with(from_index) + mock_get_indexdb_artifact_pullspec.assert_not_called() + mock_get_oras_artifact.assert_called_once_with(imagestream_ref, temp_dir) + mock_log.info.assert_any_call('ImageStream cache is enabled. Checking cache sync status.') + mock_log.info.assert_any_call('Index.db cache is synced. Pulling from ImageStream.') + + +@patch('iib.workers.tasks.containerized_utils.get_worker_config') +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.refresh_indexdb_cache_for_image') +@patch('iib.workers.tasks.containerized_utils.verify_indexdb_cache_for_image') +@patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@patch('iib.workers.tasks.containerized_utils.get_imagestream_artifact_pullspec') +@patch('iib.workers.tasks.containerized_utils.get_oras_artifact') +def test_pull_index_db_artifact_imagestream_enabled_cache_not_synced( + mock_get_oras_artifact, + mock_get_imagestream_artifact_pullspec, + mock_get_indexdb_artifact_pullspec, + mock_verify_cache, + mock_refresh_cache, + mock_log, + mock_get_worker_config, +): + """When ImageStream cache enabled but not synced, refresh and pull from registry.""" + mock_get_worker_config.return_value = {'iib_use_imagestream_cache': True} + mock_verify_cache.return_value = False + + from_index = 'quay.io/ns/index-image@sha256:def' + temp_dir = '/tmp/some-dir' + artifact_ref = 'quay.io/ns/index-image-indexdb:v4.19' + artifact_dir = '/tmp/artifact-dir' + + mock_get_indexdb_artifact_pullspec.return_value = artifact_ref + mock_get_oras_artifact.return_value = artifact_dir + + result = pull_index_db_artifact(from_index, temp_dir) + + assert result == artifact_dir + mock_verify_cache.assert_called_once_with(from_index) + mock_refresh_cache.assert_called_once_with(from_index) + mock_get_imagestream_artifact_pullspec.assert_not_called() + mock_get_indexdb_artifact_pullspec.assert_called_once_with(from_index) + mock_get_oras_artifact.assert_called_once_with(artifact_ref, temp_dir) + mock_log.info.assert_any_call('ImageStream cache is enabled. Checking cache sync status.') + mock_log.info.assert_any_call('Index.db cache is not synced. Refreshing and pulling from Quay.') + + +@patch('iib.workers.tasks.containerized_utils.get_worker_config') +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.refresh_indexdb_cache_for_image') +@patch('iib.workers.tasks.containerized_utils.verify_indexdb_cache_for_image') +@patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@patch('iib.workers.tasks.containerized_utils.get_imagestream_artifact_pullspec') +@patch('iib.workers.tasks.containerized_utils.get_oras_artifact') +def test_pull_index_db_artifact_imagestream_disabled( + mock_get_oras_artifact, + mock_get_imagestream_artifact_pullspec, + mock_get_indexdb_artifact_pullspec, + mock_verify_cache, + mock_refresh_cache, + mock_log, + mock_get_worker_config, +): + """When ImageStream cache disabled, pull directly from registry.""" + mock_get_worker_config.return_value = {'iib_use_imagestream_cache': False} + + from_index = 'quay.io/ns/index-image@sha256:ghi' + temp_dir = '/tmp/some-dir' + artifact_ref = 'quay.io/ns/index-image-indexdb:v4.20' + artifact_dir = '/tmp/artifact-dir' + + mock_get_indexdb_artifact_pullspec.return_value = artifact_ref + mock_get_oras_artifact.return_value = artifact_dir + + result = pull_index_db_artifact(from_index, temp_dir) + + assert result == artifact_dir + mock_verify_cache.assert_not_called() + mock_refresh_cache.assert_not_called() + mock_get_imagestream_artifact_pullspec.assert_not_called() + mock_get_indexdb_artifact_pullspec.assert_called_once_with(from_index) + mock_get_oras_artifact.assert_called_once_with(artifact_ref, temp_dir) + mock_log.info.assert_any_call( + 'ImageStream cache is disabled. Pulling index.db artifact directly from registry.' + ) + + +@patch('iib.workers.tasks.containerized_utils.get_worker_config') +@patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@patch('iib.workers.tasks.containerized_utils.get_oras_artifact') +def test_pull_index_db_artifact_default_config_behaves_as_disabled( + mock_get_oras_artifact, + mock_get_indexdb_artifact_pullspec, + mock_get_worker_config, +): + """If configuration lacks the key, default is to treat ImageStream as disabled.""" + mock_get_worker_config.return_value = {} + from_index = 'quay.io/ns/index@sha256:jkl' + temp_dir = '/tmp/some-dir' + artifact_ref = 'artifact-ref' + artifact_dir = '/tmp/artifact-dir' + + mock_get_indexdb_artifact_pullspec.return_value = artifact_ref + mock_get_oras_artifact.return_value = artifact_dir + + result = pull_index_db_artifact(from_index, temp_dir) + + assert result == artifact_dir + mock_get_indexdb_artifact_pullspec.assert_called_once_with(from_index) + mock_get_oras_artifact.assert_called_once_with(artifact_ref, temp_dir) + + +@patch('iib.workers.tasks.containerized_utils.log') +def test_write_build_metadata_creates_expected_json(mock_log, tmp_path): + """write_build_metadata should create JSON file with expected content.""" + local_repo_path = tmp_path + opm_version = 'opm-1.40.0' + ocp_version = 'v4.19' + distribution_scope = 'PROD' + binary_image = 'quay.io/ns/binary-image:tag' + request_id = 12345 + arches = {'amd64', 's390x'} + + write_build_metadata( + str(local_repo_path), + opm_version, + ocp_version, + distribution_scope, + binary_image, + request_id, + arches, + ) + + metadata_path = local_repo_path / '.iib-build-metadata.json' + assert metadata_path.exists() + + with open(metadata_path, 'r') as f: + data = json.load(f) + + assert data == { + 'opm_version': opm_version, + 'labels': { + 'com.redhat.index.delivery.version': ocp_version, + 'com.redhat.index.delivery.distribution_scope': distribution_scope, + }, + 'binary_image': binary_image, + 'request_id': request_id, + 'arches': ['amd64', 's390x'], + } + + mock_log.info.assert_called_once_with('Written build metadata to %s', str(metadata_path)) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.close_mr') +def test_cleanup_on_failure_closes_mr_when_mr_details_and_repo_present(mock_close_mr, mock_log): + """If MR details and index_git_repo are provided, close_mr should be called.""" + mr_details = {'mr_url': 'https://git.example.com/mr/1'} + last_commit_sha = 'abc123' + index_git_repo = 'https://git.example.com/repo.git' + overwrite_from_index = False + request_id = 1 + from_index = 'quay.io/ns/index:v4.19' + index_repo_map = {'quay.io/ns/index:v4.19': 'https://git.example.com/repo.git'} + + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=index_git_repo, + overwrite_from_index=overwrite_from_index, + request_id=request_id, + from_index=from_index, + index_repo_map=index_repo_map, + ) + + mock_close_mr.assert_called_once_with(mr_details, index_git_repo) + mock_log.info.assert_any_call("Closing merge request due to %s", "error") + mock_log.info.assert_any_call("Closed merge request: %s", mr_details.get('mr_url')) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.close_mr') +def test_cleanup_on_failure_close_mr_failure_is_logged(mock_close_mr, mock_log): + """If closing MR fails, error should be logged but function should not raise.""" + mock_close_mr.side_effect = RuntimeError("close failed") + + mr_details = {'mr_url': 'https://git.example.com/mr/2'} + last_commit_sha = 'abc123' + index_git_repo = 'https://git.example.com/repo.git' + overwrite_from_index = False + request_id = 1 + from_index = 'quay.io/ns/index:v4.19' + index_repo_map = {} + + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=index_git_repo, + overwrite_from_index=overwrite_from_index, + request_id=request_id, + from_index=from_index, + index_repo_map=index_repo_map, + ) + + mock_close_mr.assert_called_once_with(mr_details, index_git_repo) + mock_log.warning.assert_called_once() + assert "Failed to close merge request" in mock_log.warning.call_args[0][0] + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.revert_last_commit') +def test_cleanup_on_failure_reverts_commit_when_overwrite_and_commit_sha_present( + mock_revert_last_commit, mock_log +): + """If overwrite_from_index is True and last_commit_sha present, revert_last_commit is used.""" + mr_details = None + last_commit_sha = 'abc123' + index_git_repo = None + overwrite_from_index = True + request_id = 42 + from_index = 'quay.io/ns/index:v4.19' + index_repo_map = {'quay.io/ns/index:v4.19': 'https://git.example.com/repo.git'} + + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=index_git_repo, + overwrite_from_index=overwrite_from_index, + request_id=request_id, + from_index=from_index, + index_repo_map=index_repo_map, + ) + + mock_log.error.assert_any_call("Reverting commit due to %s", "error") + mock_revert_last_commit.assert_called_once_with( + request_id=request_id, + from_index=from_index, + index_repo_map=index_repo_map, + ) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.revert_last_commit') +def test_cleanup_on_failure_revert_failure_is_logged(mock_revert_last_commit, mock_log): + """If revert_last_commit fails, error should be logged.""" + mock_revert_last_commit.side_effect = RuntimeError("revert failed") + + mr_details = None + last_commit_sha = 'abc123' + index_git_repo = None + overwrite_from_index = True + request_id = 42 + from_index = 'quay.io/ns/index:v4.19' + index_repo_map = {} + + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=index_git_repo, + overwrite_from_index=overwrite_from_index, + request_id=request_id, + from_index=from_index, + index_repo_map=index_repo_map, + ) + + mock_revert_last_commit.assert_called_once() + mock_log.error.assert_any_call( + "Failed to revert commit: %s", mock_revert_last_commit.side_effect + ) + + +@patch('iib.workers.tasks.containerized_utils.log') +def test_cleanup_on_failure_no_mr_no_commit(mock_log): + """If there is neither MR nor commit to revert, log that no cleanup is needed.""" + mr_details = None + last_commit_sha = None + index_git_repo = None + overwrite_from_index = False + request_id = 1 + from_index = 'quay.io/ns/index:v4.19' + index_repo_map = {} + + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=index_git_repo, + overwrite_from_index=overwrite_from_index, + request_id=request_id, + from_index=from_index, + index_repo_map=index_repo_map, + ) + + mock_log.error.assert_any_call( + "Neither MR nor commit to revert. No cleanup needed for %s", "error" + ) + + +@patch('iib.workers.tasks.containerized_utils.run_cmd') +@patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@patch('iib.workers.tasks.containerized_utils.log') +def test_cleanup_on_failure_restores_index_db_artifact( + mock_log, mock_get_indexdb_artifact_pullspec, mock_run_cmd +): + """If original_index_db_digest is provided, oras copy should be invoked correctly.""" + mr_details = None + last_commit_sha = None + index_git_repo = None + overwrite_from_index = False + request_id = 1 + from_index = 'quay.io/ns/index:v4.19' + index_repo_map = {} + original_digest = 'sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + + v4x_artifact_ref = 'quay.io/ns/index-indexdb:v4.19' + mock_get_indexdb_artifact_pullspec.return_value = v4x_artifact_ref + + cleanup_on_failure( + mr_details=mr_details, + last_commit_sha=last_commit_sha, + index_git_repo=index_git_repo, + overwrite_from_index=overwrite_from_index, + request_id=request_id, + from_index=from_index, + index_repo_map=index_repo_map, + original_index_db_digest=original_digest, + ) + + mock_log.info.assert_any_call( + "Restoring index.db artifact to original digest due to %s", "error" + ) + + artifact_name = v4x_artifact_ref.rsplit(':', 1)[0] + expected_source_ref = f'{artifact_name}@{original_digest}' + + mock_run_cmd.assert_called_once_with( + ['oras', 'copy', expected_source_ref, v4x_artifact_ref], + exc_msg=( + f'Failed to restore index.db artifact from {expected_source_ref} ' + f'to {v4x_artifact_ref}' + ), + ) + mock_log.info.assert_any_call("Successfully restored index.db artifact to original digest") + + +@patch('iib.workers.tasks.containerized_utils.run_cmd') +@patch('iib.workers.tasks.oras_utils.get_indexdb_artifact_pullspec') +@patch('iib.workers.tasks.containerized_utils.log') +def test_cleanup_on_failure_restore_failure_is_logged( + mock_log, mock_get_indexdb_artifact_pullspec, mock_run_cmd +): + """If restoring the artifact fails, error should be logged.""" + mock_get_indexdb_artifact_pullspec.return_value = 'quay.io/ns/index-indexdb:v4.19' + mock_run_cmd.side_effect = RuntimeError("oras copy failed") + + cleanup_on_failure( + mr_details=None, + last_commit_sha=None, + index_git_repo=None, + overwrite_from_index=False, + request_id=1, + from_index='quay.io/ns/index:v4.19', + index_repo_map={}, + original_index_db_digest='sha256:0123456789abcdef0123456789abcdef0123456789abcde', + ) + + mock_run_cmd.assert_called_once() + mock_log.error.assert_any_call( + "Failed to restore index.db artifact: %s", mock_run_cmd.side_effect + ) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.oras_utils.get_indexdb_artifact_pullspec') +@patch('iib.workers.tasks.utils.run_cmd') +def test_cleanup_on_failure_no_restore_when_no_original_digest( + mock_run_cmd, mock_get_indexdb_artifact_pullspec, mock_log +): + """If original_index_db_digest is not provided, restoration must not be attempted.""" + cleanup_on_failure( + mr_details=None, + last_commit_sha=None, + index_git_repo=None, + overwrite_from_index=False, + request_id=1, + from_index='quay.io/ns/index:v4.19', + index_repo_map={}, + original_index_db_digest=None, + ) + + mock_get_indexdb_artifact_pullspec.assert_not_called() + mock_run_cmd.assert_not_called() + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_success_single_bundle(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with a single bundle successfully.""" + bundles = [{"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=1, wait=True) + + assert result is None + mock_skopeo_inspect.assert_called_once_with( + 'docker://quay.io/ns/bundle1:v1.0.0', '--raw', return_json=False + ) + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_success_multiple_bundles(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with multiple bundles successfully.""" + bundles = [ + {"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}, + {"bundlePath": 'quay.io/ns/bundle2:v2.0.0'}, + {"bundlePath": 'quay.io/ns/bundle3:v3.0.0'}, + ] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=3, wait=True) + + assert result is None + assert mock_skopeo_inspect.call_count == 3 + + # Check that all bundles were validated (order may vary due to threading) + actual_calls = [call[0] for call in mock_skopeo_inspect.call_args_list] + assert len(actual_calls) == 3 + assert all('docker://quay.io/ns/bundle' in str(call[0]) for call in actual_calls) + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_empty_bundles(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with empty bundle list.""" + bundles = [] + + result = validate_bundles_in_parallel(bundles, threads=5, wait=True) + + assert result is None + mock_skopeo_inspect.assert_not_called() + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_custom_thread_count(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with custom thread count.""" + bundles = [ + {"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}, + {"bundlePath": 'quay.io/ns/bundle2:v2.0.0'}, + ] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=2, wait=True) + + assert result is None + assert mock_skopeo_inspect.call_count == 2 + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_wait_false_returns_threads(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with wait=False returns thread list.""" + bundles = [{"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=1, wait=False) + + assert result is not None + assert len(result) == 1 + assert hasattr(result[0], 'join') + # Wait for thread to complete to verify it worked + result[0].join() + mock_skopeo_inspect.assert_called_once_with( + 'docker://quay.io/ns/bundle1:v1.0.0', '--raw', return_json=False + ) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_failure_raises_error(mock_skopeo_inspect, mock_log): + """Test validate_bundles_in_parallel raises IIBError when bundle validation fails.""" + bundles = [{"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}] + error = IIBError('Bundle not found') + mock_skopeo_inspect.side_effect = error + + with pytest.raises(IIBError, match='Error validating bundle'): + validate_bundles_in_parallel(bundles, threads=1, wait=True) + + assert mock_skopeo_inspect.called + # Error should be logged in the thread + assert mock_log.error.called + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_more_bundles_than_threads(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with more bundles than threads.""" + bundles = [ + {"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}, + {"bundlePath": 'quay.io/ns/bundle2:v2.0.0'}, + {"bundlePath": 'quay.io/ns/bundle3:v3.0.0'}, + {"bundlePath": 'quay.io/ns/bundle4:v4.0.0'}, + {"bundlePath": 'quay.io/ns/bundle5:v5.0.0'}, + ] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=2, wait=True) + + assert result is None + assert mock_skopeo_inspect.call_count == 5 + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_default_parameters(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with default parameters.""" + bundles = [{"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles) + + assert result is None + mock_skopeo_inspect.assert_called_once_with( + 'docker://quay.io/ns/bundle1:v1.0.0', '--raw', return_json=False + ) + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_multiple_threads_processing_queue(mock_skopeo_inspect): + """Test that multiple threads properly process bundles from the queue.""" + bundles = [ + {"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}, + {"bundlePath": 'quay.io/ns/bundle2:v2.0.0'}, + ] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=2, wait=True) + + assert result is None + # Both bundles should be validated + assert mock_skopeo_inspect.call_count == 2 + # Verify all bundles were processed + call_args = [call[0][0] for call in mock_skopeo_inspect.call_args_list] + assert 'docker://quay.io/ns/bundle1:v1.0.0' in call_args + assert 'docker://quay.io/ns/bundle2:v2.0.0' in call_args + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_one_bundle_fails_others_succeed( + mock_skopeo_inspect, mock_log +): + """Test that when one bundle fails, the error is logged and raised.""" + bundles = [ + {"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}, + {"bundlePath": 'quay.io/ns/bundle2:v2.0.0'}, + ] + # First bundle succeeds, second fails + mock_skopeo_inspect.side_effect = [None, IIBError('Bundle not found')] + + with pytest.raises(IIBError, match='Error validating bundle'): + validate_bundles_in_parallel(bundles, threads=2, wait=True) + + assert mock_skopeo_inspect.call_count >= 1 + # Error should be logged in the thread + assert mock_log.error.called + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_wait_for_bundle_validation_threads_success(mock_skopeo_inspect): + """Test wait_for_bundle_validation_threads with successful validation.""" + from iib.workers.tasks.containerized_utils import ValidateBundlesThread + import queue + + bundles_queue = queue.Queue() + bundles_queue.put({"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}) + mock_skopeo_inspect.return_value = None + + thread = ValidateBundlesThread(bundles_queue) + thread.start() + + # Wait for the thread using the function + wait_for_bundle_validation_threads([thread]) + + mock_skopeo_inspect.assert_called_once_with( + 'docker://quay.io/ns/bundle1:v1.0.0', '--raw', return_json=False + ) + assert thread.exception is None + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_wait_for_bundle_validation_threads_failure_raises_error(mock_skopeo_inspect, mock_log): + """Test wait_for_bundle_validation_threads raises IIBError when validation fails.""" + from iib.workers.tasks.containerized_utils import ValidateBundlesThread + import queue + + bundles_queue = queue.Queue() + bundles_queue.put({"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}) + error = IIBError('Bundle not found') + mock_skopeo_inspect.side_effect = error + + thread = ValidateBundlesThread(bundles_queue) + thread.start() + + with pytest.raises(IIBError, match='Error validating bundle quay.io/ns/bundle1:v1.0.0'): + wait_for_bundle_validation_threads([thread]) + + assert mock_skopeo_inspect.called + assert thread.exception == error + assert thread.bundle == {"bundlePath": 'quay.io/ns/bundle1:v1.0.0'} + mock_log.error.assert_called() + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_wait_for_bundle_validation_threads_multiple_threads_one_fails( + mock_skopeo_inspect, mock_log +): + """Test wait_for_bundle_validation_threads with multiple threads where one fails.""" + from iib.workers.tasks.containerized_utils import ValidateBundlesThread + import queue + + bundles_queue1 = queue.Queue() + bundles_queue1.put({"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}) + bundles_queue2 = queue.Queue() + bundles_queue2.put({"bundlePath": 'quay.io/ns/bundle2:v2.0.0'}) + + mock_skopeo_inspect.side_effect = [None, IIBError('Bundle not found')] + + thread1 = ValidateBundlesThread(bundles_queue1) + thread2 = ValidateBundlesThread(bundles_queue2) + thread1.start() + thread2.start() + + with pytest.raises(IIBError, match='Error validating bundle quay.io/ns/bundle2:v2.0.0'): + wait_for_bundle_validation_threads([thread1, thread2]) + + assert mock_skopeo_inspect.call_count == 2 + assert thread1.exception is None + assert thread2.exception is not None + mock_log.error.assert_called() + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_wait_false_then_wait_manually(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with wait=False and then manually waiting.""" + bundles = [ + {"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}, + {"bundlePath": 'quay.io/ns/bundle2:v2.0.0'}, + ] + mock_skopeo_inspect.return_value = None + + # Get threads without waiting + threads = validate_bundles_in_parallel(bundles, threads=2, wait=False) + + assert threads is not None + assert len(threads) == 2 + + # Manually wait for threads + wait_for_bundle_validation_threads(threads) + + # Verify all bundles were validated + assert mock_skopeo_inspect.call_count == 2 + call_args = [call[0][0] for call in mock_skopeo_inspect.call_args_list] + assert 'docker://quay.io/ns/bundle1:v1.0.0' in call_args + assert 'docker://quay.io/ns/bundle2:v2.0.0' in call_args + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_wait_false_then_wait_manually_with_failure( + mock_skopeo_inspect, mock_log +): + """Test validate_bundles_in_parallel with wait=False, then manually waiting when one fails.""" + bundles = [ + {"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}, + {"bundlePath": 'quay.io/ns/bundle2:v2.0.0'}, + ] + mock_skopeo_inspect.side_effect = [None, IIBError('Bundle not found')] + + # Get threads without waiting + threads = validate_bundles_in_parallel(bundles, threads=2, wait=False) + + assert threads is not None + assert len(threads) == 2 + + # Manually wait for threads - should raise error + with pytest.raises(IIBError, match='Error validating bundle'): + wait_for_bundle_validation_threads(threads) + + assert mock_skopeo_inspect.call_count == 2 + mock_log.error.assert_called() + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_wait_for_bundle_validation_threads_empty_list(mock_skopeo_inspect): + """Test wait_for_bundle_validation_threads with empty thread list.""" + wait_for_bundle_validation_threads([]) + mock_skopeo_inspect.assert_not_called() + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_wait_for_bundle_validation_threads_unknown_bundle_on_error(mock_skopeo_inspect, mock_log): + """Test wait_for_bundle_validation_threads when bundle is None in error case.""" + from iib.workers.tasks.containerized_utils import ValidateBundlesThread + import queue + + bundles_queue = queue.Queue() + # Add a bundle to the queue so the thread will process it + bundles_queue.put({"bundlePath": 'quay.io/ns/bundle1:v1.0.0'}) + error = IIBError('Bundle not found') + mock_skopeo_inspect.side_effect = error + + thread = ValidateBundlesThread(bundles_queue) + thread.start() + thread.join() + + # Manually set bundle to None after thread completes to test the "unknown" case + thread.bundle = None + + with pytest.raises(IIBError, match='Error validating bundle unknown'): + wait_for_bundle_validation_threads([thread]) + + assert mock_skopeo_inspect.called + assert thread.exception == error + mock_log.error.assert_called() + + +# Tests for List[str] format (pullspec strings) +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_success_single_bundle_string(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with a single bundle string successfully.""" + bundles = ['quay.io/ns/bundle1:v1.0.0'] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=1, wait=True) + + assert result is None + mock_skopeo_inspect.assert_called_once_with( + 'docker://quay.io/ns/bundle1:v1.0.0', '--raw', return_json=False + ) + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_success_multiple_bundles_string(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with multiple bundle strings successfully.""" + bundles = [ + 'quay.io/ns/bundle1:v1.0.0', + 'quay.io/ns/bundle2:v2.0.0', + 'quay.io/ns/bundle3:v3.0.0', + ] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=3, wait=True) + + assert result is None + assert mock_skopeo_inspect.call_count == 3 + + # Check that all bundles were validated (order may vary due to threading) + actual_calls = [call[0] for call in mock_skopeo_inspect.call_args_list] + assert len(actual_calls) == 3 + assert all('docker://quay.io/ns/bundle' in str(call[0]) for call in actual_calls) + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_custom_thread_count_string(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with custom thread count using string bundles.""" + bundles = [ + 'quay.io/ns/bundle1:v1.0.0', + 'quay.io/ns/bundle2:v2.0.0', + ] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=2, wait=True) + + assert result is None + assert mock_skopeo_inspect.call_count == 2 + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_wait_false_returns_threads_string(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with wait=False returns thread list for string bundles.""" + bundles = ['quay.io/ns/bundle1:v1.0.0'] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=1, wait=False) + + assert result is not None + assert len(result) == 1 + assert hasattr(result[0], 'join') + # Wait for thread to complete to verify it worked + result[0].join() + mock_skopeo_inspect.assert_called_once_with( + 'docker://quay.io/ns/bundle1:v1.0.0', '--raw', return_json=False + ) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_failure_raises_error_string(mock_skopeo_inspect, mock_log): + """Test validate_bundles_in_parallel raises IIBError when bundle string validation fails.""" + bundles = ['quay.io/ns/bundle1:v1.0.0'] + error = IIBError('Bundle not found') + mock_skopeo_inspect.side_effect = error + + with pytest.raises(IIBError, match='Error validating bundle'): + validate_bundles_in_parallel(bundles, threads=1, wait=True) + + assert mock_skopeo_inspect.called + # Error should be logged in the thread + assert mock_log.error.called + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_more_bundles_than_threads_string(mock_skopeo_inspect): + """Test validate_bundles_in_parallel with more bundle strings than threads.""" + bundles = [ + 'quay.io/ns/bundle1:v1.0.0', + 'quay.io/ns/bundle2:v2.0.0', + 'quay.io/ns/bundle3:v3.0.0', + 'quay.io/ns/bundle4:v4.0.0', + 'quay.io/ns/bundle5:v5.0.0', + ] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=2, wait=True) + + assert result is None + assert mock_skopeo_inspect.call_count == 5 + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_multiple_threads_processing_queue_string(mock_skopeo_inspect): + """Test that multiple threads properly process bundle strings from the queue.""" + bundles = [ + 'quay.io/ns/bundle1:v1.0.0', + 'quay.io/ns/bundle2:v2.0.0', + ] + mock_skopeo_inspect.return_value = None + + result = validate_bundles_in_parallel(bundles, threads=2, wait=True) + + assert result is None + # Both bundles should be validated + assert mock_skopeo_inspect.call_count == 2 + # Verify all bundles were processed + call_args = [call[0][0] for call in mock_skopeo_inspect.call_args_list] + assert 'docker://quay.io/ns/bundle1:v1.0.0' in call_args + assert 'docker://quay.io/ns/bundle2:v2.0.0' in call_args + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_validate_bundles_in_parallel_one_bundle_fails_others_succeed_string( + mock_skopeo_inspect, mock_log +): + """Test that when one bundle string fails, the error is logged and raised.""" + bundles = [ + 'quay.io/ns/bundle1:v1.0.0', + 'quay.io/ns/bundle2:v2.0.0', + ] + # First bundle succeeds, second fails + mock_skopeo_inspect.side_effect = [None, IIBError('Bundle not found')] + + with pytest.raises(IIBError, match='Error validating bundle'): + validate_bundles_in_parallel(bundles, threads=2, wait=True) + + assert mock_skopeo_inspect.call_count >= 1 + # Error should be logged in the thread + assert mock_log.error.called + + +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_wait_for_bundle_validation_threads_success_string(mock_skopeo_inspect): + """Test wait_for_bundle_validation_threads with successful validation for string bundles.""" + from iib.workers.tasks.containerized_utils import ValidateBundlesThread + import queue + + bundles_queue = queue.Queue() + bundles_queue.put('quay.io/ns/bundle1:v1.0.0') + mock_skopeo_inspect.return_value = None + + thread = ValidateBundlesThread(bundles_queue) + thread.start() + + # Wait for the thread using the function + wait_for_bundle_validation_threads([thread]) + + mock_skopeo_inspect.assert_called_once_with( + 'docker://quay.io/ns/bundle1:v1.0.0', '--raw', return_json=False + ) + assert thread.exception is None + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils.skopeo_inspect') +def test_wait_for_bundle_validation_threads_failure_raises_error_string( + mock_skopeo_inspect, mock_log +): + """Ensure it raises IIBError when string bundle validation fails.""" + from iib.workers.tasks.containerized_utils import ValidateBundlesThread + import queue + + bundles_queue = queue.Queue() + bundles_queue.put('quay.io/ns/bundle1:v1.0.0') + error = IIBError('Bundle not found') + mock_skopeo_inspect.side_effect = error + + thread = ValidateBundlesThread(bundles_queue) + thread.start() + + with pytest.raises(IIBError, match='Error validating bundle quay.io/ns/bundle1:v1.0.0'): + wait_for_bundle_validation_threads([thread]) + + assert mock_skopeo_inspect.called + assert thread.exception == error + assert thread.bundle == 'quay.io/ns/bundle1:v1.0.0' + mock_log.error.assert_called() + + +# Tests for extract_files_from_image_non_privileged +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils._skopeo_copy') +def test_extract_files_from_image_non_privileged_success_directory( + mock_skopeo_copy, mock_log, tmpdir +): + """Test successful extraction of a directory from container image.""" + import os + import tarfile + + # Setup destination directory + dest_dir = tmpdir.join('dest') + + # Mock skopeo_copy to create proper OCI layout + def mock_copy(source, destination, copy_all, exc_msg): + # Extract OCI directory path from destination (format: oci:/path/to/oci) + oci_path = destination.replace('oci:', '') + + # Create OCI layout structure + os.makedirs(oci_path, exist_ok=True) + blobs_dir = os.path.join(oci_path, 'blobs', 'sha256') + os.makedirs(blobs_dir, exist_ok=True) + + # Create index.json + index_json = { + 'manifests': [ + { + 'digest': 'sha256:abc123', + 'mediaType': 'application/vnd.oci.image.manifest.v1+json', + } + ] + } + with open(os.path.join(oci_path, 'index.json'), 'w') as f: + json.dump(index_json, f) + + # Create manifest + manifest_json = { + 'layers': [ + { + 'digest': 'sha256:layer1', + 'mediaType': 'application/vnd.oci.image.layer.v1.tar+gzip', + } + ] + } + with open(os.path.join(blobs_dir, 'abc123'), 'w') as f: + json.dump(manifest_json, f) + + # Create layer tar.gz with /manifests directory + layer_path = os.path.join(blobs_dir, 'layer1') + with tarfile.open(layer_path, 'w:gz') as tar: + # Create a temporary test file + test_file = tmpdir.join('temp_test_manifest.yaml') + test_file.write('test: data') + # Add it to the tar with the path we expect in the image + tar.add(str(test_file), arcname='manifests/test_manifest.yaml') + + mock_skopeo_copy.side_effect = mock_copy + + # Call the function under test + extract_files_from_image_non_privileged('quay.io/ns/test:v1', '/manifests', str(dest_dir)) + + # Verify the extraction succeeded + assert dest_dir.check(dir=True) + extracted_file = dest_dir.join('test_manifest.yaml') + assert extracted_file.check(file=True) + assert extracted_file.read() == 'test: data' + + # Verify skopeo was called + mock_skopeo_copy.assert_called_once() + call_args = mock_skopeo_copy.call_args + assert call_args[1]['source'] == 'docker://quay.io/ns/test:v1' + assert 'oci:' in call_args[1]['destination'] + assert call_args[1]['copy_all'] is False + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils._skopeo_copy') +def test_extract_files_from_image_non_privileged_missing_index(mock_skopeo_copy, mock_log, tmpdir): + """Test extraction fails when OCI index.json is missing.""" + # Mock skopeo_copy to create OCI dir without index.json + def mock_copy(source, destination, copy_all, exc_msg): + # Extract OCI directory path from destination + oci_path = destination.replace('oci:', '') + os.makedirs(oci_path, exist_ok=True) + # Don't create index.json to simulate error + + mock_skopeo_copy.side_effect = mock_copy + + with pytest.raises(IIBError, match='OCI index.json not found'): + extract_files_from_image_non_privileged( + 'quay.io/ns/test:v1', '/manifests', str(tmpdir.join('dest')) + ) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils._skopeo_copy') +def test_extract_files_from_image_non_privileged_no_manifests(mock_skopeo_copy, mock_log, tmpdir): + """Test extraction fails when no manifests in OCI index.""" + + def mock_copy(source, destination, copy_all, exc_msg): + oci_path = destination.replace('oci:', '') + os.makedirs(oci_path, exist_ok=True) + # Create index.json with empty manifests + index_json = {'manifests': []} + with open(os.path.join(oci_path, 'index.json'), 'w') as f: + json.dump(index_json, f) + + mock_skopeo_copy.side_effect = mock_copy + + with pytest.raises(IIBError, match='No manifests found in OCI index'): + extract_files_from_image_non_privileged( + 'quay.io/ns/test:v1', '/manifests', str(tmpdir.join('dest')) + ) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils._skopeo_copy') +def test_extract_files_from_image_non_privileged_no_layers(mock_skopeo_copy, mock_log, tmpdir): + """Test extraction fails when no layers in manifest.""" + + def mock_copy(source, destination, copy_all, exc_msg): + oci_path = destination.replace('oci:', '') + blobs_dir = os.path.join(oci_path, 'blobs', 'sha256') + os.makedirs(blobs_dir, exist_ok=True) + + # Create index.json + index_json = {'manifests': [{'digest': 'sha256:abc123'}]} + with open(os.path.join(oci_path, 'index.json'), 'w') as f: + json.dump(index_json, f) + + # Create manifest with no layers + manifest_json = {'layers': []} + with open(os.path.join(blobs_dir, 'abc123'), 'w') as f: + json.dump(manifest_json, f) + + mock_skopeo_copy.side_effect = mock_copy + + with pytest.raises(IIBError, match='No layers found in manifest'): + extract_files_from_image_non_privileged( + 'quay.io/ns/test:v1', '/manifests', str(tmpdir.join('dest')) + ) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils._skopeo_copy') +def test_extract_files_from_image_non_privileged_missing_layer_blob( + mock_skopeo_copy, mock_log, tmpdir +): + """Test extraction fails when layer blob file is missing.""" + + def mock_copy(source, destination, copy_all, exc_msg): + oci_path = destination.replace('oci:', '') + blobs_dir = os.path.join(oci_path, 'blobs', 'sha256') + os.makedirs(blobs_dir, exist_ok=True) + + # Create index.json + index_json = {'manifests': [{'digest': 'sha256:abc123'}]} + with open(os.path.join(oci_path, 'index.json'), 'w') as f: + json.dump(index_json, f) + + # Create manifest with layer reference + manifest_json = {'layers': [{'digest': 'sha256:missing_layer'}]} + with open(os.path.join(blobs_dir, 'abc123'), 'w') as f: + json.dump(manifest_json, f) + # Don't create the layer blob file + + mock_skopeo_copy.side_effect = mock_copy + + with pytest.raises(IIBError, match='Layer blob not found'): + extract_files_from_image_non_privileged( + 'quay.io/ns/test:v1', '/manifests', str(tmpdir.join('dest')) + ) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils._skopeo_copy') +def test_extract_files_from_image_non_privileged_path_not_found(mock_skopeo_copy, mock_log, tmpdir): + """Test extraction fails when requested path doesn't exist in image.""" + + def mock_copy(source, destination, copy_all, exc_msg): + oci_path = destination.replace('oci:', '') + blobs_dir = os.path.join(oci_path, 'blobs', 'sha256') + os.makedirs(blobs_dir, exist_ok=True) + + # Create index.json + index_json = {'manifests': [{'digest': 'sha256:abc123'}]} + with open(os.path.join(oci_path, 'index.json'), 'w') as f: + json.dump(index_json, f) + + # Create manifest + manifest_json = {'layers': [{'digest': 'sha256:layer1'}]} + with open(os.path.join(blobs_dir, 'abc123'), 'w') as f: + json.dump(manifest_json, f) + + # Create empty layer tar.gz (no content) + layer_path = os.path.join(blobs_dir, 'layer1') + with tarfile.open(layer_path, 'w:gz'): + pass # Empty tar + + mock_skopeo_copy.side_effect = mock_copy + + with pytest.raises(IIBError, match='Path /manifests not found in image'): + extract_files_from_image_non_privileged( + 'quay.io/ns/test:v1', '/manifests', str(tmpdir.join('dest')) + ) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils._skopeo_copy') +def test_extract_files_from_image_non_privileged_invalid_layer_tarball( + mock_skopeo_copy, mock_log, tmpdir +): + """Test extraction fails when layer tarball is corrupted.""" + + def mock_copy(source, destination, copy_all, exc_msg): + oci_path = destination.replace('oci:', '') + blobs_dir = os.path.join(oci_path, 'blobs', 'sha256') + os.makedirs(blobs_dir, exist_ok=True) + + # Create index.json + index_json = {'manifests': [{'digest': 'sha256:abc123'}]} + with open(os.path.join(oci_path, 'index.json'), 'w') as f: + json.dump(index_json, f) + + # Create manifest + manifest_json = {'layers': [{'digest': 'sha256:corrupted_layer'}]} + with open(os.path.join(blobs_dir, 'abc123'), 'w') as f: + json.dump(manifest_json, f) + + # Create corrupted layer (not a valid tar.gz) + layer_path = os.path.join(blobs_dir, 'corrupted_layer') + with open(layer_path, 'w') as f: + f.write('not a valid tar.gz file') + + mock_skopeo_copy.side_effect = mock_copy + + with pytest.raises(IIBError, match='Failed to extract layer'): + extract_files_from_image_non_privileged( + 'quay.io/ns/test:v1', '/manifests', str(tmpdir.join('dest')) + ) + + +@patch('iib.workers.tasks.containerized_utils.log') +@patch('iib.workers.tasks.containerized_utils._skopeo_copy') +def test_extract_files_from_image_non_privileged_skopeo_copy_failure( + mock_skopeo_copy, mock_log, tmpdir +): + """Test extraction fails when skopeo copy fails.""" + mock_skopeo_copy.side_effect = IIBError('Failed to download image') + + with pytest.raises(IIBError, match='Failed to download image'): + extract_files_from_image_non_privileged( + 'quay.io/ns/test:v1', '/manifests', str(tmpdir.join('dest')) + ) + + mock_skopeo_copy.assert_called_once() + # Verify the call was made with correct parameters + call_args = mock_skopeo_copy.call_args + assert call_args[1]['source'] == 'docker://quay.io/ns/test:v1' + assert 'oci:' in call_args[1]['destination'] + assert call_args[1]['copy_all'] is False diff --git a/tests/test_workers/test_tasks/test_fbc_utils.py b/tests/test_workers/test_tasks/test_fbc_utils.py index 17d6c1554..ff825ce74 100644 --- a/tests/test_workers/test_tasks/test_fbc_utils.py +++ b/tests/test_workers/test_tasks/test_fbc_utils.py @@ -214,17 +214,27 @@ def test_extract_fbc_fragment(mock_cffi, mock_osldr, ldr_output, tmpdir): test_fbc_fragment = "example.com/test/fbc_fragment:latest" mock_osldr.return_value = ldr_output # The function now adds -0 suffix by default when fragment_index is not provided - fbc_fragment_path = os.path.join(tmpdir, f"{get_worker_config()['temp_fbc_fragment_path']}-0") + fbc_fragment_base_path = os.path.join( + tmpdir, f"{get_worker_config()['temp_fbc_fragment_path']}-0" + ) + fbc_fragment_catalog_dir = os.path.join( + fbc_fragment_base_path, os.path.basename(get_worker_config()['fbc_fragment_catalog_path']) + ) + os.makedirs(fbc_fragment_catalog_dir) if not ldr_output: with pytest.raises(IIBError): extract_fbc_fragment(tmpdir, test_fbc_fragment) else: - extract_fbc_fragment(tmpdir, test_fbc_fragment) + result_path, result_operators = extract_fbc_fragment(tmpdir, test_fbc_fragment) + assert result_path == fbc_fragment_catalog_dir + assert result_operators == ldr_output mock_cffi.assert_called_once_with( - test_fbc_fragment, get_worker_config()['fbc_fragment_catalog_path'], fbc_fragment_path + test_fbc_fragment, + get_worker_config()['fbc_fragment_catalog_path'], + fbc_fragment_base_path, ) - mock_osldr.assert_called_once_with(fbc_fragment_path) + mock_osldr.assert_called_once_with(fbc_fragment_catalog_dir) @pytest.mark.parametrize('ldr_output', [['testoperator'], ['test1', 'test2']]) @@ -237,24 +247,30 @@ def test_extract_fbc_fragment_with_index(mock_cffi, mock_osldr, ldr_output, tmpd # Test with fragment_index = 2 fragment_index = 2 - fbc_fragment_path = os.path.join( + fbc_fragment_base_path = os.path.join( tmpdir, f"{get_worker_config()['temp_fbc_fragment_path']}-{fragment_index}" ) + fbc_fragment_catalog_dir = os.path.join( + fbc_fragment_base_path, os.path.basename(get_worker_config()['fbc_fragment_catalog_path']) + ) + os.makedirs(fbc_fragment_catalog_dir) result_path, result_operators = extract_fbc_fragment( tmpdir, test_fbc_fragment, fragment_index=fragment_index ) - # Verify the path includes the correct index - assert result_path == fbc_fragment_path - assert result_path.endswith(f"-{fragment_index}") + # Verify the path includes the correct index and catalog directory + assert result_path == fbc_fragment_catalog_dir + assert result_path.endswith(f"fbc-fragment-{fragment_index}/configs") assert result_operators == ldr_output # Verify the function was called with the correct path mock_cffi.assert_called_once_with( - test_fbc_fragment, get_worker_config()['fbc_fragment_catalog_path'], fbc_fragment_path + test_fbc_fragment, + get_worker_config()['fbc_fragment_catalog_path'], + fbc_fragment_base_path, ) - mock_osldr.assert_called_once_with(fbc_fragment_path) + mock_osldr.assert_called_once_with(fbc_fragment_catalog_dir) @mock.patch('os.listdir') @@ -267,6 +283,25 @@ def test_extract_fbc_fragment_isolation(mock_cffi, mock_osldr, tmpdir): # Mock different outputs for each fragment mock_osldr.side_effect = [['operator1'], ['operator2']] + fbc_fragment_base_path_0 = os.path.join( + tmpdir, f"{get_worker_config()['temp_fbc_fragment_path']}-0" + ) + fbc_fragment_base_path_1 = os.path.join( + tmpdir, f"{get_worker_config()['temp_fbc_fragment_path']}-1" + ) + os.makedirs( + os.path.join( + fbc_fragment_base_path_0, + os.path.basename(get_worker_config()['fbc_fragment_catalog_path']), + ) + ) + os.makedirs( + os.path.join( + fbc_fragment_base_path_1, + os.path.basename(get_worker_config()['fbc_fragment_catalog_path']), + ) + ) + # Extract first fragment with index 0 path1, operators1 = extract_fbc_fragment(tmpdir, test_fbc_fragment1, fragment_index=0) @@ -275,8 +310,8 @@ def test_extract_fbc_fragment_isolation(mock_cffi, mock_osldr, tmpdir): # Verify paths are different and include correct indices assert path1 != path2 - assert path1.endswith("-0") - assert path2.endswith("-1") + assert path1.endswith("fbc-fragment-0/configs") + assert path2.endswith("fbc-fragment-1/configs") # Verify operators are different (no cross-contamination) assert operators1 == ['operator1'] @@ -285,8 +320,16 @@ def test_extract_fbc_fragment_isolation(mock_cffi, mock_osldr, tmpdir): # Verify _copy_files_from_image was called with different paths expected_calls = [ - mock.call(test_fbc_fragment1, get_worker_config()['fbc_fragment_catalog_path'], path1), - mock.call(test_fbc_fragment2, get_worker_config()['fbc_fragment_catalog_path'], path2), + mock.call( + test_fbc_fragment1, + get_worker_config()['fbc_fragment_catalog_path'], + fbc_fragment_base_path_0, + ), + mock.call( + test_fbc_fragment2, + get_worker_config()['fbc_fragment_catalog_path'], + fbc_fragment_base_path_1, + ), ] mock_cffi.assert_has_calls(expected_calls, any_order=True) diff --git a/tests/test_workers/test_tasks/test_general.py b/tests/test_workers/test_tasks/test_general.py index c395c7dcf..7c6357231 100644 --- a/tests/test_workers/test_tasks/test_general.py +++ b/tests/test_workers/test_tasks/test_general.py @@ -18,12 +18,12 @@ (FinalStateOverwriteError("can not overwite final state"), "Already in final state"), ), ) -@mock.patch('iib.workers.tasks.general._cleanup') +@mock.patch('iib.workers.tasks.general.reset_docker_config') @mock.patch('iib.workers.tasks.general.set_request_state') -def test_failed_request_callback(mock_srs, mock_cleanup, exc, expected_msg): +def test_failed_request_callback(mock_srs, mock_reset_docker_config, exc, expected_msg): general.failed_request_callback(None, exc, None, 3) if isinstance(exc, FinalStateOverwriteError): mock_srs.assert_not_called() else: mock_srs.assert_called_once_with(3, 'failed', expected_msg) - mock_cleanup.assert_called_once() + mock_reset_docker_config.assert_called_once() diff --git a/tests/test_workers/test_tasks/test_git_utils.py b/tests/test_workers/test_tasks/test_git_utils.py index 92cc7fb07..408567c67 100644 --- a/tests/test_workers/test_tasks/test_git_utils.py +++ b/tests/test_workers/test_tasks/test_git_utils.py @@ -78,18 +78,6 @@ def __repr__(self): return f"RegexMatcher('{self.pattern}')" -def test_configure_git_user(): - test_user = "IIB Test Person" - test_email = "iib-test-person@redhat.com" - with tempfile.TemporaryDirectory(prefix="test-git-repo") as test_repo: - run_cmd(f"git -C {test_repo} init".split(), strict=False) - git_utils.configure_git_user(test_repo, test_user, test_email) - git_user = run_cmd(f"git -C {test_repo} config --get user.name".split()) - git_email = run_cmd(f"git -C {test_repo} config --get user.email".split()) - assert git_user.strip() == test_user - assert git_email.strip() == test_email - - def test_unmapped_git_token(mock_gwc): repo_url = f"{GIT_BASE_URL}/some-unknown-repo.git" expected_error = f"Missing key '{repo_url}' in 'iib_index_configs_gitlab_tokens_map'" @@ -193,8 +181,6 @@ def test_push_configs_to_git_aborts_without_repo_map(mock_ggt, mock_cmd) -> None @mock.patch("iib.workers.tasks.git_utils.tempfile") -@mock.patch("iib.workers.tasks.git_utils.commit_and_push") -@mock.patch("iib.workers.tasks.git_utils.configure_git_user") @mock.patch("iib.workers.tasks.git_utils.clone_git_repo") @mock.patch("iib.workers.tasks.git_utils.validate_git_remote_branch") @mock.patch("iib.workers.tasks.git_utils.get_git_token") @@ -202,8 +188,6 @@ def test_push_configs_to_git_no_changes( mock_ggt, mock_validate_branch, mock_clone, - mock_configure_git, - mock_commit_and_push, mock_tempfile, gitlab_url_mapping, caplog, @@ -236,13 +220,10 @@ def test_push_configs_to_git_no_changes( assert "No changes to commit." in caplog.messages mock_validate_branch.assert_called_once_with(PUB_GIT_REPO, "latest") mock_clone.assert_called_once_with(PUB_GIT_REPO, "latest", "foo", "bar", remote_repository) - mock_configure_git.assert_called_once_with(remote_repository) - mock_commit_and_push.assert_not_called() @mock.patch("iib.workers.tasks.git_utils.tempfile") @mock.patch("iib.workers.tasks.git_utils.commit_and_push") -@mock.patch("iib.workers.tasks.git_utils.configure_git_user") @mock.patch("iib.workers.tasks.git_utils.clone_git_repo") @mock.patch("iib.workers.tasks.git_utils.validate_git_remote_branch") @mock.patch("iib.workers.tasks.git_utils.get_git_token") @@ -250,7 +231,6 @@ def test_push_configs_to_git_untracked_files( mock_ggt, mock_validate_branch, mock_clone, - mock_configure_git, mock_commit_and_push, mock_tempfile, gitlab_url_mapping, @@ -290,7 +270,6 @@ def test_push_configs_to_git_untracked_files( assert "No changes to commit." not in caplog.messages mock_validate_branch.assert_called_once_with(PUB_GIT_REPO, "latest") mock_clone.assert_called_once_with(PUB_GIT_REPO, "latest", "foo", "bar", remote_repository) - mock_configure_git.assert_called_once_with(remote_repository) mock_commit_and_push.assert_called_once_with( 1, remote_repository, PUB_GIT_REPO, "latest", None ) @@ -300,13 +279,11 @@ def test_push_configs_to_git_untracked_files( @mock.patch("iib.workers.tasks.git_utils.run_cmd") @mock.patch("iib.workers.tasks.git_utils.get_git_token") @mock.patch("iib.workers.tasks.git_utils.clone_git_repo") -@mock.patch("iib.workers.tasks.git_utils.configure_git_user") @mock.patch("iib.workers.tasks.git_utils.commit_and_push") @mock.patch("iib.workers.tasks.git_utils.os.listdir") def test_push_configs_to_git_empty_repository( mock_listdir, mock_commit_and_push, - mock_configure_git, mock_clone, mock_ggt, mock_cmd, @@ -318,7 +295,6 @@ def test_push_configs_to_git_empty_repository( mock_ggt.return_value = "foo", "bar" mock_cmd.return_value = "main" mock_clone.return_value = None - mock_configure_git.return_value = None mock_commit_and_push.return_value = None # Mock the listdir calls @@ -342,8 +318,6 @@ def test_push_configs_to_git_empty_repository( 'https://my-gitlab-instance.com/exd-guild-hello-operator-gitlab/iib-pub-index-configs.git' # noqa: E501 ) mock_clone.assert_called_once_with(PUB_GIT_REPO, "latest", "foo", "bar", mock.ANY) - # configure_git_user is called with only one argument (local_repo_dir) - mock_configure_git.assert_called_once_with(mock.ANY) mock_commit_and_push.assert_called_once() # commit_and_push is called with positional arguments, not keyword arguments call_args = mock_commit_and_push.call_args @@ -357,13 +331,11 @@ def test_push_configs_to_git_empty_repository( @mock.patch("iib.workers.tasks.git_utils.run_cmd") @mock.patch("iib.workers.tasks.git_utils.get_git_token") @mock.patch("iib.workers.tasks.git_utils.clone_git_repo") -@mock.patch("iib.workers.tasks.git_utils.configure_git_user") @mock.patch("iib.workers.tasks.git_utils.commit_and_push") @mock.patch("iib.workers.tasks.git_utils.os.listdir") def test_push_configs_to_git_existing_repository( mock_listdir, mock_commit_and_push, - mock_configure_git, mock_clone, mock_ggt, mock_cmd, @@ -375,7 +347,6 @@ def test_push_configs_to_git_existing_repository( mock_ggt.return_value = "foo", "bar" mock_cmd.return_value = "main" mock_clone.return_value = None - mock_configure_git.return_value = None mock_commit_and_push.return_value = None # Mock the listdir calls @@ -405,8 +376,6 @@ def test_push_configs_to_git_existing_repository( 'https://my-gitlab-instance.com/exd-guild-hello-operator-gitlab/iib-pub-index-configs.git' # noqa: E501 ) mock_clone.assert_called_once_with(PUB_GIT_REPO, "latest", "foo", "bar", mock.ANY) - # configure_git_user is called with only one argument (local_repo_dir) - mock_configure_git.assert_called_once_with(mock.ANY) mock_commit_and_push.assert_called_once() # commit_and_push is called with positional arguments, not keyword arguments call_args = mock_commit_and_push.call_args @@ -859,7 +828,6 @@ def test_close_gitlab_mr_network_errors(mock_requests_put): @mock.patch("iib.workers.tasks.git_utils.run_cmd") @mock.patch("iib.workers.tasks.git_utils.get_git_token") @mock.patch("iib.workers.tasks.git_utils.clone_git_repo") -@mock.patch("iib.workers.tasks.git_utils.configure_git_user") @mock.patch("iib.workers.tasks.git_utils.commit_and_push") @mock.patch("iib.workers.tasks.git_utils.os.path.exists") @mock.patch("iib.workers.tasks.git_utils.os.listdir") @@ -867,7 +835,6 @@ def test_push_configs_to_git_removing_content( mock_listdir, mock_path_exists, mock_commit_and_push, - mock_configure_git, mock_clone, mock_ggt, mock_cmd, @@ -879,7 +846,6 @@ def test_push_configs_to_git_removing_content( mock_ggt.return_value = "foo", "bar" mock_cmd.return_value = "main" # Mock git ls-remote output mock_clone.return_value = None - mock_configure_git.return_value = None mock_commit_and_push.return_value = None # we don't want "finally" to be executed as it removes the temp repo mock_path_exists.return_value = False @@ -917,8 +883,6 @@ def test_push_configs_to_git_removing_content( # Verify the function was called with correct parameters mock_ggt.assert_called_once_with(PUB_GIT_REPO) mock_clone.assert_called_once_with(PUB_GIT_REPO, "latest", "foo", "bar", mock.ANY) - # configure_git_user is called with only one argument (local_repo_dir) - mock_configure_git.assert_called_once_with(mock.ANY) # Verify that commit_and_push was called with the correct parameters mock_commit_and_push.assert_called_once() @@ -941,7 +905,6 @@ def test_push_configs_to_git_removing_content( @mock.patch("iib.workers.tasks.git_utils.run_cmd") @mock.patch("iib.workers.tasks.git_utils.get_git_token") @mock.patch("iib.workers.tasks.git_utils.clone_git_repo") -@mock.patch("iib.workers.tasks.git_utils.configure_git_user") @mock.patch("iib.workers.tasks.git_utils.commit_and_push") @mock.patch("iib.workers.tasks.git_utils.os.path.exists") @mock.patch("iib.workers.tasks.git_utils.os.listdir") @@ -949,7 +912,6 @@ def test_push_configs_to_git_removing_all_operators( mock_listdir, mock_path_exists, mock_commit_and_push, - mock_configure_git, mock_clone, mock_ggt, mock_cmd, @@ -961,7 +923,6 @@ def test_push_configs_to_git_removing_all_operators( mock_ggt.return_value = "foo", "bar" mock_cmd.return_value = "main" mock_clone.return_value = None - mock_configure_git.return_value = None mock_commit_and_push.return_value = None # we don't want "finally" to be executed as it removes the temp repo mock_path_exists.return_value = False @@ -1009,7 +970,6 @@ def test_push_configs_to_git_removing_all_operators( @mock.patch("iib.workers.tasks.git_utils.run_cmd") @mock.patch("iib.workers.tasks.git_utils.get_git_token") @mock.patch("iib.workers.tasks.git_utils.clone_git_repo") -@mock.patch("iib.workers.tasks.git_utils.configure_git_user") @mock.patch("iib.workers.tasks.git_utils.commit_and_push") @mock.patch("iib.workers.tasks.git_utils.os.path.exists") @mock.patch("iib.workers.tasks.git_utils.os.listdir") @@ -1017,7 +977,6 @@ def test_push_configs_to_git_removing_no_operators( mock_listdir, mock_path_exists, mock_commit_and_push, - mock_configure_git, mock_clone, mock_ggt, mock_cmd, @@ -1029,7 +988,6 @@ def test_push_configs_to_git_removing_no_operators( mock_ggt.return_value = "foo", "bar" mock_cmd.return_value = "main" mock_clone.return_value = None - mock_configure_git.return_value = None mock_commit_and_push.return_value = None # we don't want "finally" to be executed as it removes the temp repo mock_path_exists.return_value = False @@ -1074,7 +1032,6 @@ def test_push_configs_to_git_removing_no_operators( @mock.patch("iib.workers.tasks.git_utils.run_cmd") @mock.patch("iib.workers.tasks.git_utils.get_git_token") @mock.patch("iib.workers.tasks.git_utils.clone_git_repo") -@mock.patch("iib.workers.tasks.git_utils.configure_git_user") @mock.patch("iib.workers.tasks.git_utils.commit_and_push") @mock.patch("iib.workers.tasks.git_utils.os.path.exists") @mock.patch("iib.workers.tasks.git_utils.os.listdir") @@ -1082,7 +1039,6 @@ def test_push_configs_to_git_removing_with_empty_repo( mock_listdir, mock_path_exists, mock_commit_and_push, - mock_configure_git, mock_clone, mock_ggt, mock_cmd, @@ -1094,7 +1050,6 @@ def test_push_configs_to_git_removing_with_empty_repo( mock_ggt.return_value = "foo", "bar" mock_cmd.return_value = "main" mock_clone.return_value = None - mock_configure_git.return_value = None mock_commit_and_push.return_value = None # we don't want "finally" to be executed as it removes the temp repo mock_path_exists.return_value = False @@ -1125,3 +1080,50 @@ def test_push_configs_to_git_removing_with_empty_repo( assert call_args[0][2] == PUB_GIT_REPO # repo_url assert call_args[0][3] == "latest" # branch mock_shutil.rmtree.assert_not_called() + + +@pytest.mark.parametrize( + 'mock_output, expected_sha', + [ + ('abc123def456789\n', 'abc123def456789'), + (' 1a2b3c4d5e6f7890 \n\n', '1a2b3c4d5e6f7890'), + ('a1b2c3d4e5f6789012345678901234567890abcd', 'a1b2c3d4e5f6789012345678901234567890abcd'), + (' \n ', ''), + ], +) +@mock.patch('iib.workers.tasks.git_utils.run_cmd') +def test_get_last_commit_sha_success(mock_run_cmd, mock_output, expected_sha): + """Test successfully retrieving the last commit SHA with various outputs.""" + local_repo_path = '/tmp/test-repo' + mock_run_cmd.return_value = mock_output + + result = git_utils.get_last_commit_sha(local_repo_path) + + assert result == expected_sha + mock_run_cmd.assert_called_once_with( + ['git', '-C', local_repo_path, 'rev-parse', 'HEAD'], + exc_msg=f'Error getting last commit for {local_repo_path}', + ) + + +@pytest.mark.parametrize( + 'local_repo_path', + [ + '/tmp/test-repo', + '/invalid/repo', + '/some/repo', + '/tmp/repo-with-dashes_and_underscores/sub-dir', + ], +) +@mock.patch('iib.workers.tasks.git_utils.run_cmd') +def test_get_last_commit_sha_git_error(mock_run_cmd, local_repo_path): + """Test when git command fails with various repository paths.""" + mock_run_cmd.side_effect = IIBError(f'Error getting last commit for {local_repo_path}') + + with pytest.raises(IIBError, match=f'Error getting last commit for {local_repo_path}'): + git_utils.get_last_commit_sha(local_repo_path) + + mock_run_cmd.assert_called_once_with( + ['git', '-C', local_repo_path, 'rev-parse', 'HEAD'], + exc_msg=f'Error getting last commit for {local_repo_path}', + ) diff --git a/tests/test_workers/test_tasks/test_konflux_utils.py b/tests/test_workers/test_tasks/test_konflux_utils.py index aa9e1fe88..ebcc6dd30 100644 --- a/tests/test_workers/test_tasks/test_konflux_utils.py +++ b/tests/test_workers/test_tasks/test_konflux_utils.py @@ -10,6 +10,7 @@ from iib.workers.tasks.konflux_utils import ( find_pipelinerun, wait_for_pipeline_completion, + get_pipelinerun_image_url, _get_kubernetes_client, _create_kubernetes_client, _create_kubernetes_configuration, @@ -58,7 +59,7 @@ def test_find_pipelinerun_success(mock_get_worker_config, mock_get_client): @patch('iib.workers.tasks.konflux_utils._get_kubernetes_client') @patch('iib.workers.tasks.konflux_utils.get_worker_config') def test_find_pipelinerun_empty_result(mock_get_worker_config, mock_get_client): - """Test pipelinerun search with empty results.""" + """Test pipelinerun search with empty results raises IIBError for retry.""" # Setup mock_client = Mock() mock_get_client.return_value = mock_client @@ -66,15 +67,15 @@ def test_find_pipelinerun_empty_result(mock_get_worker_config, mock_get_client): mock_config = Mock() mock_config.iib_konflux_namespace = 'iib-tenant' mock_config.iib_konflux_pipeline_timeout = 1800 + mock_config.iib_total_attempts = 3 # Reduced to make test faster + mock_config.iib_retry_multiplier = 1 # Reduced to make test faster mock_get_worker_config.return_value = mock_config mock_client.list_namespaced_custom_object.return_value = {"items": []} - # Test - result = find_pipelinerun("abc123") - - # Verify - assert result == [] + # Test & Verify - should raise IIBError to trigger retry decorator + with pytest.raises(IIBError, match="No pipelineruns found for commit abc123"): + find_pipelinerun("abc123") @pytest.mark.parametrize( @@ -149,9 +150,10 @@ def test_wait_for_pipeline_completion_success_cases( mock_client.get_namespaced_custom_object.return_value = run_status # Test - wait_for_pipeline_completion("test-pipelinerun") + result = wait_for_pipeline_completion("test-pipelinerun") # Verify + assert result == run_status mock_client.get_namespaced_custom_object.assert_called_once_with( group="tekton.dev", version="v1", @@ -283,9 +285,10 @@ def test_wait_for_pipeline_completion_still_running( ] # Test - wait_for_pipeline_completion("test-pipelinerun") + result = wait_for_pipeline_completion("test-pipelinerun") # Verify + assert result == run_status_succeeded assert mock_client.get_namespaced_custom_object.call_count == 2 mock_sleep.assert_called_once_with(30) @@ -594,3 +597,111 @@ def test_create_kubernetes_configuration_ca_cert_handling( mock_temp_file.write.assert_called_once_with(ca_cert_input) else: mock_tempfile.assert_not_called() + + +@pytest.mark.parametrize( + "results_key,image_url,description", + [ + ('results', 'quay.io/namespace/image:tag', 'Konflux format with results key'), + ( + 'pipelineResults', + 'quay.io/namespace/image:tag', + 'Older Tekton format with pipelineResults', + ), + ], +) +def test_get_pipelinerun_image_url_success(results_key, image_url, description): + """Test successful extraction of IMAGE_URL from pipelinerun.""" + # Setup + run = { + 'status': { + results_key: [ + {'name': 'IMAGE_DIGEST', 'value': 'sha256:abc123'}, + {'name': 'IMAGE_URL', 'value': image_url}, + {'name': 'CHAINS-GIT_COMMIT', 'value': 'def456'}, + ] + } + } + + # Test + result = get_pipelinerun_image_url('test-pipelinerun', run) + + # Verify + assert result == image_url + + +def test_get_pipelinerun_image_url_with_whitespace(): + """Test IMAGE_URL extraction strips whitespace.""" + # Setup + run = { + 'status': { + 'results': [ + {'name': 'IMAGE_URL', 'value': ' quay.io/namespace/image:tag\n '}, + ] + } + } + + # Test + result = get_pipelinerun_image_url('test-pipelinerun', run) + + # Verify + assert result == 'quay.io/namespace/image:tag' + + +def test_get_pipelinerun_image_url_fallback_to_pipelineresults(): + """Test fallback from results to pipelineResults.""" + # Setup - 'results' is empty but 'pipelineResults' has data + run = { + 'status': { + 'results': [], + 'pipelineResults': [ + {'name': 'IMAGE_URL', 'value': 'quay.io/namespace/image:tag'}, + ], + } + } + + # Test + result = get_pipelinerun_image_url('test-pipelinerun', run) + + # Verify + assert result == 'quay.io/namespace/image:tag' + + +@pytest.mark.parametrize( + "run,description", + [ + ( + { + 'status': { + 'results': [ + {'name': 'IMAGE_DIGEST', 'value': 'sha256:abc123'}, + {'name': 'CHAINS-GIT_COMMIT', 'value': 'def456'}, + ] + } + }, + 'IMAGE_URL not in results', + ), + ( + { + 'status': { + 'results': [ + {'name': 'IMAGE_URL', 'value': ''}, + ] + } + }, + 'IMAGE_URL has empty value', + ), + ({'status': {}}, 'no results key present'), + ( + {'status': {'results': [], 'pipelineResults': []}}, + 'both results and pipelineResults empty', + ), + ], +) +def test_get_pipelinerun_image_url_error_cases(run, description): + """Test error cases when IMAGE_URL is not found or invalid.""" + # Test & Verify + with pytest.raises( + IIBError, match='IMAGE_URL not found in pipelinerun test-pipelinerun results' + ): + get_pipelinerun_image_url('test-pipelinerun', run) diff --git a/tests/test_workers/test_tasks/test_opm_operations.py b/tests/test_workers/test_tasks/test_opm_operations.py index 36327859a..9ac6114b6 100644 --- a/tests/test_workers/test_tasks/test_opm_operations.py +++ b/tests/test_workers/test_tasks/test_opm_operations.py @@ -952,21 +952,21 @@ def test_opm_registry_add_fbc_fragment( {"packageName": "test-operator", "version": "v1.2", "bundlePath": "bundle1"}, {"packageName": "package2", "version": "v2.0", "bundlePath": "bundle2"}, ], - {"test-operator"}, + ["test-operator"], ), ( [ {"packageName": "test-operator", "version": "v1.0", "bundlePath": "bundle1"}, {"packageName": "package2", "version": "v2.0", "bundlePath": "bundle2"}, ], - {"test-operator"}, + ["test-operator"], ), ( [ {"packageName": "package1", "version": "v1.0", "bundlePath": "bundle1"}, {"packageName": "package2", "version": "v2.0", "bundlePath": "bundle2"}, ], - set(), + [], ), ], ) @@ -1361,3 +1361,88 @@ def test_get_operator_package_list(mock_run_cmd, mock_gidp, tmpdir): {'cwd': tmpdir}, exc_msg=f'Failed to run opm render with input: {input_image}', ) + + +@pytest.mark.parametrize("operators_in_db", ([], ['op1'])) +@mock.patch('iib.workers.tasks.opm_operations.shutil.rmtree') +@mock.patch('iib.workers.tasks.opm_operations.shutil.copytree') +@mock.patch('iib.workers.tasks.opm_operations.os.path.exists') +@mock.patch('iib.workers.tasks.opm_operations.os.listdir') +@mock.patch('iib.workers.tasks.opm_operations.opm_migrate') +@mock.patch('iib.workers.tasks.opm_operations._opm_registry_rm') +@mock.patch('iib.workers.tasks.opm_operations.remove_operator_deprecations') +@mock.patch('iib.workers.tasks.opm_operations.verify_operators_exists') +@mock.patch('iib.workers.tasks.opm_operations.extract_fbc_fragment') +@mock.patch('iib.workers.tasks.opm_operations.set_request_state') +def test_opm_registry_add_fbc_fragment_containerized( + mock_set_state, + mock_extract, + mock_verify, + mock_remove_dep, + mock_rm, + mock_migrate, + mock_listdir, + mock_exists, + mock_copytree, + mock_rmtree, + operators_in_db, +): + request_id = 1 + temp_dir = '/tmp/dir' + from_index_configs_dir = '/tmp/dir/configs' + fbc_fragments = ['fragment:v1'] + overwrite_from_index_token = 'token' + index_db_path = '/tmp/dir/index.db' + + mock_extract.return_value = ('/tmp/fragment_path', ['op2']) + mock_verify.return_value = (operators_in_db, index_db_path) + mock_migrate.return_value = ('/tmp/migrated_catalog', None) + mock_listdir.return_value = ['op1'] + mock_exists.return_value = False + + ret = opm_operations.opm_registry_add_fbc_fragment_containerized( + request_id, + temp_dir, + from_index_configs_dir, + fbc_fragments, + overwrite_from_index_token, + index_db_path, + ) + + assert ret == (from_index_configs_dir, index_db_path, operators_in_db) + + mock_extract.assert_called_once_with( + temp_dir=temp_dir, fbc_fragment=fbc_fragments[0], fragment_index=0 + ) + + mock_verify.assert_called_once_with( + from_index=None, + base_dir=temp_dir, + operator_packages=['op2'], + overwrite_from_index_token=overwrite_from_index_token, + index_db_path=index_db_path, + ) + + if operators_in_db: + mock_remove_dep.assert_called_once_with( + from_index_configs_dir=from_index_configs_dir, operators=operators_in_db + ) + mock_rm.assert_called_once_with( + index_db_path=index_db_path, operators=operators_in_db, base_dir=temp_dir + ) + mock_migrate.assert_called_once_with( + index_db=index_db_path, base_dir=temp_dir, generate_cache=False + ) + mock_copytree.assert_any_call( + os.path.join('/tmp/migrated_catalog', 'op1'), + os.path.join(from_index_configs_dir, 'op1'), + dirs_exist_ok=True, + ) + else: + mock_remove_dep.assert_not_called() + mock_rm.assert_not_called() + mock_migrate.assert_not_called() + + mock_copytree.assert_any_call( + os.path.join('/tmp/fragment_path', 'op2'), os.path.join(from_index_configs_dir, 'op2') + ) diff --git a/tests/test_workers/test_tasks/test_oras_utils.py b/tests/test_workers/test_tasks/test_oras_utils.py index d53e022f6..111e12fba 100644 --- a/tests/test_workers/test_tasks/test_oras_utils.py +++ b/tests/test_workers/test_tasks/test_oras_utils.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-3.0-or-later """Basic unit tests for oras_utils.""" import logging +import re + import pytest from unittest import mock @@ -19,10 +21,18 @@ def registry_auths(): return {'auths': {'quay.io': {'auth': 'dXNlcjpwYXNz'}}} # base64 encoded user:pass +def _oras_worker_config_minimal(**extra): + cfg = {'iib_index_db_oras_auth_path': ''} + cfg.update(extra) + return cfg + + +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('tempfile.mkdtemp') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') -def test_get_oras_artifact_success(mock_run_cmd, mock_mkdtemp): +def test_get_oras_artifact_success(mock_run_cmd, mock_mkdtemp, mock_gwc): """Test successful artifact pull.""" + mock_gwc.return_value = _oras_worker_config_minimal() artifact_ref = 'quay.io/test/repo:latest' base_dir = '/tmp/base' mock_run_cmd.return_value = 'Success' @@ -39,16 +49,20 @@ def test_get_oras_artifact_success(mock_run_cmd, mock_mkdtemp): @mock.patch('iib.workers.tasks.oras_utils.set_registry_auths') +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('tempfile.mkdtemp') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') -def test_get_oras_artifact_with_auth(mock_run_cmd, mock_mkdtemp, mock_auth, registry_auths): - """Test artifact pull with authentication.""" +def test_get_oras_artifact_with_auth( + mock_run_cmd, mock_mkdtemp, mock_gwc, mock_auth, registry_auths +): + """Explicit registry_auths are passed through to set_registry_auths.""" + mock_gwc.return_value = _oras_worker_config_minimal() artifact_ref = 'quay.io/test/repo:latest' base_dir = '/tmp/base' mock_run_cmd.return_value = 'Success' mock_mkdtemp.return_value = '/tmp/test-dir' - result = get_oras_artifact(artifact_ref, base_dir, registry_auths) + result = get_oras_artifact(artifact_ref, base_dir, registry_auths=registry_auths) assert result == '/tmp/test-dir' mock_auth.assert_called_once_with(registry_auths, use_empty_config=True) @@ -59,12 +73,14 @@ def test_get_oras_artifact_with_auth(mock_run_cmd, mock_mkdtemp, mock_auth, regi ) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('os.path.exists') @mock.patch('tempfile.mkdtemp') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') @mock.patch('shutil.rmtree') -def test_get_oras_artifact_failure(mock_rmtree, mock_run_cmd, mock_mkdtemp, mock_exists): +def test_get_oras_artifact_failure(mock_rmtree, mock_run_cmd, mock_mkdtemp, mock_exists, mock_gwc): """Test artifact pull failure.""" + mock_gwc.return_value = _oras_worker_config_minimal() artifact_ref = 'quay.io/test/repo:latest' base_dir = '/tmp/base' mock_run_cmd.side_effect = IIBError('Pull failed') @@ -76,10 +92,12 @@ def test_get_oras_artifact_failure(mock_rmtree, mock_run_cmd, mock_mkdtemp, mock mock_rmtree.assert_called_once_with('/tmp/test-dir') +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('tempfile.mkdtemp') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') -def test_get_oras_artifact_custom_prefix(mock_run_cmd, mock_mkdtemp): +def test_get_oras_artifact_custom_prefix(mock_run_cmd, mock_mkdtemp, mock_gwc): """Test artifact pull with custom temp directory prefix.""" + mock_gwc.return_value = _oras_worker_config_minimal() artifact_ref = 'quay.io/test/repo:latest' base_dir = '/tmp/base' custom_prefix = 'custom-prefix-' @@ -92,10 +110,12 @@ def test_get_oras_artifact_custom_prefix(mock_run_cmd, mock_mkdtemp): mock_mkdtemp.assert_called_once_with(prefix=custom_prefix, dir=base_dir) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('tempfile.mkdtemp') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') -def test_get_oras_artifact_with_custom_base_dir(mock_run_cmd, mock_mkdtemp): +def test_get_oras_artifact_with_custom_base_dir(mock_run_cmd, mock_mkdtemp, mock_gwc): """Test artifact pull with custom base directory.""" + mock_gwc.return_value = _oras_worker_config_minimal() artifact_ref = 'quay.io/test/repo:latest' base_dir = '/tmp/iib-123' mock_run_cmd.return_value = 'Success' @@ -111,12 +131,14 @@ def test_get_oras_artifact_with_custom_base_dir(mock_run_cmd, mock_mkdtemp): ) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('os.path.exists') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') -def test_push_oras_artifact_success(mock_run_cmd, mock_exists): - """Test successful artifact push.""" +def test_push_oras_artifact_success(mock_run_cmd, mock_exists, mock_gwc): + """Test successful artifact push. Updated local_path to be relative.""" + mock_gwc.return_value = _oras_worker_config_minimal() artifact_ref = 'quay.io/test/repo:latest' - local_path = '/tmp/test.db' + local_path = './test.db' artifact_type = 'application/vnd.sqlite' mock_run_cmd.return_value = 'Success' mock_exists.return_value = True @@ -129,24 +151,27 @@ def test_push_oras_artifact_success(mock_run_cmd, mock_exists): 'push', artifact_ref, f'{local_path}:{artifact_type}', - '--disable-path-validation', ], exc_msg=f'Failed to push OCI artifact to {artifact_ref}', ) @mock.patch('iib.workers.tasks.oras_utils.set_registry_auths') +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('os.path.exists') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') -def test_push_oras_artifact_with_auth(mock_run_cmd, mock_exists, mock_auth, registry_auths): - """Test artifact push with authentication.""" +def test_push_oras_artifact_with_auth( + mock_run_cmd, mock_exists, mock_gwc, mock_auth, registry_auths +): + """Explicit registry_auths are passed through to set_registry_auths.""" + mock_gwc.return_value = _oras_worker_config_minimal() artifact_ref = 'quay.io/test/repo:latest' - local_path = '/tmp/test.db' + local_path = './test.db' artifact_type = 'application/vnd.sqlite' mock_run_cmd.return_value = 'Success' mock_exists.return_value = True - push_oras_artifact(artifact_ref, local_path, artifact_type, registry_auths) + push_oras_artifact(artifact_ref, local_path, artifact_type, registry_auths=registry_auths) mock_auth.assert_called_once_with(registry_auths, use_empty_config=True) mock_run_cmd.assert_called_once_with( @@ -155,18 +180,19 @@ def test_push_oras_artifact_with_auth(mock_run_cmd, mock_exists, mock_auth, regi 'push', artifact_ref, f'{local_path}:{artifact_type}', - '--disable-path-validation', ], exc_msg=f'Failed to push OCI artifact to {artifact_ref}', ) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('os.path.exists') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') -def test_push_oras_artifact_with_annotations(mock_run_cmd, mock_exists): - """Test artifact push with annotations.""" +def test_push_oras_artifact_with_annotations(mock_run_cmd, mock_exists, mock_gwc): + """Test artifact push with annotations. Updated local_path to be relative.""" + mock_gwc.return_value = _oras_worker_config_minimal() artifact_ref = 'quay.io/test/repo:latest' - local_path = '/tmp/test.db' + local_path = './test.db' artifact_type = 'application/vnd.sqlite' annotations = {'key1': 'value1', 'key2': 'value2'} mock_run_cmd.return_value = 'Success' @@ -179,7 +205,6 @@ def test_push_oras_artifact_with_annotations(mock_run_cmd, mock_exists): 'push', artifact_ref, f'{local_path}:{artifact_type}', - '--disable-path-validation', ] for key, value in annotations.items(): expected_cmd.extend(['--annotation', f'{key}={value}']) @@ -189,12 +214,14 @@ def test_push_oras_artifact_with_annotations(mock_run_cmd, mock_exists): ) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('os.path.exists') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') -def test_push_oras_artifact_failure(mock_run_cmd, mock_exists): +def test_push_oras_artifact_failure(mock_run_cmd, mock_exists, mock_gwc): """Test artifact push failure.""" + mock_gwc.return_value = _oras_worker_config_minimal() artifact_ref = 'quay.io/test/repo:latest' - local_path = '/tmp/test.db' + local_path = './test.db' artifact_type = 'application/vnd.sqlite' mock_run_cmd.side_effect = IIBError('Push failed') mock_exists.return_value = True @@ -220,48 +247,47 @@ def test_push_oras_artifact_file_not_found(mock_exists): [ ( "quay.io/test/repo:latest", - "/tmp/test.db", + "./test.db", "application/vnd.sqlite", [ "oras", "push", "quay.io/test/repo:latest", - "/tmp/test.db:application/vnd.sqlite", - "--disable-path-validation", + "./test.db:application/vnd.sqlite", ], ), ( "registry.example.com/myapp:v1.0", - "/data/config.yaml", + "./config.yaml", "application/vnd.yaml", [ "oras", "push", "registry.example.com/myapp:v1.0", - "/data/config.yaml:application/vnd.yaml", - "--disable-path-validation", + "./config.yaml:application/vnd.yaml", ], ), ( "docker.io/library/nginx:latest", - "/etc/nginx.conf", + "./nginx.conf", "application/vnd.config", [ "oras", "push", "docker.io/library/nginx:latest", - "/etc/nginx.conf:application/vnd.config", - "--disable-path-validation", + "./nginx.conf:application/vnd.config", ], ), ], ) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('os.path.exists') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') def test_push_oras_artifact_various_types( - mock_run_cmd, mock_exists, artifact_ref, local_path, artifact_type, expected_cmd + mock_run_cmd, mock_exists, mock_gwc, artifact_ref, local_path, artifact_type, expected_cmd ): - """Test artifact push with various artifact types.""" + """Test artifact push with various artifact types. Updated local_path to be relative.""" + mock_gwc.return_value = _oras_worker_config_minimal() mock_run_cmd.return_value = 'Success' mock_exists.return_value = True @@ -272,10 +298,12 @@ def test_push_oras_artifact_various_types( ) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('os.path.exists') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') -def test_push_oras_artifact_with_relative_path(mock_run_cmd, mock_exists): +def test_push_oras_artifact_with_relative_path(mock_run_cmd, mock_exists, mock_gwc): """Test artifact push with relative path (should not add --disable-path-validation).""" + mock_gwc.return_value = _oras_worker_config_minimal() artifact_ref = 'quay.io/test/repo:latest' local_path = './test.db' # Relative path artifact_type = 'application/vnd.sqlite' @@ -290,19 +318,38 @@ def test_push_oras_artifact_with_relative_path(mock_run_cmd, mock_exists): ) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') +@mock.patch('os.path.exists') +def test_push_oras_artifact_rejects_absolute_local_path(mock_exists, mock_gwc): + """Absolute local_path is rejected after existence check (ORAS path collision avoidance).""" + mock_gwc.return_value = _oras_worker_config_minimal() + mock_exists.return_value = True + + with pytest.raises(IIBError, match='Local artifact path must be relative: /abs/test.db'): + push_oras_artifact('quay.io/test/repo:latest', '/abs/test.db', 'application/vnd.sqlite') + + @mock.patch('iib.workers.tasks.oras_utils.set_registry_auths') +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('tempfile.mkdtemp') @mock.patch("iib.workers.tasks.utils.subprocess") def test_get_oras_artifact_with_base_dir_wont_leak_credentials( - mock_subprocess, mock_mkdtemp, mock_auth, registry_auths, caplog + mock_subprocess, + mock_mkdtemp, + mock_gwc, + mock_auth, + registry_auths, + caplog, ): - """Ensure the get_oras_artifact with base_dir won't leak credentials in logs.""" + """Ensure get_oras_artifact with base_dir won't leak credentials in logs.""" # Setting the logging level via caplog.set_level is not sufficient. The flask # related settings from previous tests interfere with this. oras_logger = logging.getLogger('iib.workers.tasks.utils') oras_logger.disabled = False oras_logger.setLevel(logging.DEBUG) + mock_gwc.return_value = _oras_worker_config_minimal() + # Prepare the subprocess mock mock_run_result = mock.MagicMock() mock_run_result.returncode = 0 @@ -318,8 +365,9 @@ def test_get_oras_artifact_with_base_dir_wont_leak_credentials( base_dir = '/tmp/iib-123' mock_mkdtemp.return_value = '/tmp/iib-123/iib-oras-abc123' - get_oras_artifact(artifact_ref, base_dir, registry_auths) + get_oras_artifact(artifact_ref, base_dir, registry_auths=registry_auths) + mock_auth.assert_called_once_with(registry_auths, use_empty_config=True) mock_subprocess.run.assert_called_with( ['oras', 'pull', artifact_ref, '-o', '/tmp/iib-123/iib-oras-abc123'], **default_run_cmd_args, @@ -382,10 +430,15 @@ def test_get_image_stream_digest_failure(mock_run_cmd): get_image_stream_digest('test-tag') +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('iib.workers.tasks.oras_utils.get_image_stream_digest') @mock.patch('iib.workers.tasks.oras_utils.get_image_digest') -def test_verify_indexdb_cache_sync_match(mock_get_image_digest, mock_get_is_digest): +def test_verify_indexdb_cache_sync_match(mock_get_image_digest, mock_get_is_digest, mock_gwc): """Test successful verification when digests match.""" + mock_gwc.return_value = { + 'iib_index_db_artifact_registry': 'test-artifact-registry', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } mock_get_image_digest.return_value = 'sha256:abc' mock_get_is_digest.return_value = 'sha256:abc' tag = 'test-tag' @@ -393,16 +446,19 @@ def test_verify_indexdb_cache_sync_match(mock_get_image_digest, mock_get_is_dige result = verify_indexdb_cache_sync(tag) assert result is True - mock_get_image_digest.assert_called_once_with( - 'quay.io/exd-guild-hello-operator/example-repository:test-tag' - ) + mock_get_image_digest.assert_called_once_with('test-artifact-registry/index-db:test-tag') mock_get_is_digest.assert_called_once_with(tag) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('iib.workers.tasks.oras_utils.get_image_stream_digest') @mock.patch('iib.workers.tasks.oras_utils.get_image_digest') -def test_verify_indexdb_cache_sync_no_match(mock_get_image_digest, mock_get_is_digest): +def test_verify_indexdb_cache_sync_no_match(mock_get_image_digest, mock_get_is_digest, mock_gwc): """Test successful verification when digests don't match.""" + mock_gwc.return_value = { + 'iib_index_db_artifact_registry': 'test-artifact-registry', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } mock_get_image_digest.return_value = 'sha256:abc' mock_get_is_digest.return_value = 'sha256:xyz' tag = 'test-tag' @@ -410,16 +466,19 @@ def test_verify_indexdb_cache_sync_no_match(mock_get_image_digest, mock_get_is_d result = verify_indexdb_cache_sync(tag) assert result is False - mock_get_image_digest.assert_called_once_with( - 'quay.io/exd-guild-hello-operator/example-repository:test-tag' - ) + mock_get_image_digest.assert_called_once_with('test-artifact-registry/index-db:test-tag') mock_get_is_digest.assert_called_once_with(tag) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('iib.workers.tasks.oras_utils.set_registry_auths') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') -def test_refresh_indexdb_cache_success(mock_run_cmd, mock_auth, registry_auths): +def test_refresh_indexdb_cache_success(mock_run_cmd, mock_auth, mock_gwc, registry_auths): """Test successful cache refresh.""" + mock_gwc.return_value = { + 'iib_index_db_artifact_registry': 'test-artifact-registry', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } tag = 'test-tag' refresh_indexdb_cache(tag, registry_auths) @@ -430,26 +489,36 @@ def test_refresh_indexdb_cache_success(mock_run_cmd, mock_auth, registry_auths): 'oc', 'import-image', 'index-db-cache:test-tag', - '--from=quay.io/exd-guild-hello-operator/example-repository:test-tag', + '--from=test-artifact-registry/index-db:test-tag', '--confirm', ], exc_msg='Failed to refresh OCI artifact test-tag.', ) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('iib.workers.tasks.oras_utils.run_cmd', side_effect=IIBError('refresh failed')) -def test_refresh_indexdb_cache_failure(mock_run_cmd): +def test_refresh_indexdb_cache_failure(mock_run_cmd, mock_gwc): """Test cache refresh failure.""" + mock_gwc.return_value = { + 'iib_index_db_artifact_registry': 'test-artifact-registry', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } tag = 'test-tag' with pytest.raises(IIBError, match='refresh failed'): refresh_indexdb_cache(tag) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('iib.workers.tasks.oras_utils.set_registry_auths') @mock.patch('iib.workers.tasks.oras_utils.run_cmd') -def test_refresh_indexdb_cache_with_empty_registry_auths(mock_run_cmd, mock_auth): +def test_refresh_indexdb_cache_with_empty_registry_auths(mock_run_cmd, mock_auth, mock_gwc): """Test that refresh_indexdb_cache works correctly when registry_auths is an empty dict.""" + mock_gwc.return_value = { + 'iib_index_db_artifact_registry': 'test-artifact-registry', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } tag = 'v4.15' empty_auths = {} @@ -461,3 +530,251 @@ def test_refresh_indexdb_cache_with_empty_registry_auths(mock_run_cmd, mock_auth # Verify the oc command was executed mock_run_cmd.assert_called_once() + + +@pytest.mark.parametrize( + "pullspec,expected_name,expected_tag", + [ + ( + "registry.example.com/namespace/iib-pub-pending:v4.17", + "iib-pub-pending", + "v4.17", + ), + ( + "quay.io/namespace/my-image:latest", + "my-image", + "latest", + ), + ( + "registry.io/org/repo/index-image:v1.0.0", + "index-image", + "v1.0.0", + ), + ( + "docker.io/library/nginx:1.21.0", + "nginx", + "1.21.0", + ), + ( + "registry.example.com/namespace/iib-pub-pending:v4.17@sha256:abc123", + "iib-pub-pending", + "v4.17", + ), + ( + "quay.io/namespace/image-name:tag-with-dashes", + "image-name", + "tag-with-dashes", + ), + ( + "registry.io/namespace/image:v1.0.0-rc1", + "image", + "v1.0.0-rc1", + ), + ], +) +def test_get_name_and_tag_from_pullspec_valid(pullspec, expected_name, expected_tag): + """Test parsing valid pullspec strings.""" + from iib.workers.tasks.oras_utils import _get_name_and_tag_from_pullspec + + name, tag = _get_name_and_tag_from_pullspec(pullspec) + + assert name == expected_name + assert tag == expected_tag + + +@pytest.mark.parametrize( + "invalid_pullspec,expected_error_msg", + [ + ( + "registry.example.com/namespace/iib-pub-pending", + "Invalid pullspec format: 'registry.example.com/namespace/iib-pub-pending'. " + "Missing tag (':') delimiter.", + ), + ( + "registry.example.com/namespace/image:", + "Invalid pullspec format: 'registry.example.com/namespace/image:'. " + "Could not parse name:tag structure.", + ), + ( + "invalid-pullspec-format", + "Invalid pullspec format: 'invalid-pullspec-format'. " "Missing tag (':') delimiter.", + ), + ], +) +def test_get_name_and_tag_from_pullspec_invalid(invalid_pullspec, expected_error_msg): + """Test parsing invalid pullspec strings.""" + from iib.workers.tasks.oras_utils import _get_name_and_tag_from_pullspec + + with pytest.raises(IIBError, match=re.escape(expected_error_msg)): + _get_name_and_tag_from_pullspec(invalid_pullspec) + + +@pytest.mark.parametrize( + "image_name,tag,expected_tag", + [ + ("iib-pub-pending", "v4.17", "iib-pub-pending-v4.17"), + ("my-image", "latest", "my-image-latest"), + ("test-index", "v1.0.0", "test-index-v1.0.0"), + ], +) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') +def test_get_artifact_combined_tag(mock_gwc, image_name, tag, expected_tag): + """Test generating combined artifact tags.""" + from iib.workers.tasks.oras_utils import _get_artifact_combined_tag + + mock_gwc.return_value = {'iib_index_db_artifact_tag_template': '{image_name}-{tag}'} + + result = _get_artifact_combined_tag(image_name, tag) + + assert result == expected_tag + + +@pytest.mark.parametrize( + "from_index,expected_pullspec", + [ + ( + "registry.example.com/namespace/iib-pub-pending:v4.17", + "test-artifact-registry/index-db:iib-pub-pending-v4.17", + ), + ( + "quay.io/namespace/my-image:latest", + "test-artifact-registry/index-db:my-image-latest", + ), + ( + "registry.io/org/repo/index-image:v1.0.0", + "test-artifact-registry/index-db:index-image-v1.0.0", + ), + ( + "registry.example.com/namespace/iib-pub-pending:v4.17@sha256:abc123", + "test-artifact-registry/index-db:iib-pub-pending-v4.17", + ), + ], +) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') +def test_get_indexdb_artifact_pullspec(mock_gwc, from_index, expected_pullspec): + """Test constructing index DB artifact pullspecs.""" + from iib.workers.tasks.oras_utils import get_indexdb_artifact_pullspec + + mock_gwc.return_value = { + 'iib_index_db_artifact_registry': 'test-artifact-registry', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + 'iib_index_db_artifact_tag_template': '{image_name}-{tag}', + } + + result = get_indexdb_artifact_pullspec(from_index) + + assert result == expected_pullspec + + +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') +def test_get_indexdb_artifact_pullspec_invalid(mock_gwc): + """Test _get_indexdb_artifact_pullspec with invalid pullspec.""" + from iib.workers.tasks.oras_utils import get_indexdb_artifact_pullspec + + mock_gwc.return_value = { + 'iib_index_db_artifact_registry': 'test-artifact-registry', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + 'iib_index_db_artifact_tag_template': '{image_name}-{tag}', + } + + with pytest.raises(IIBError, match="Missing tag"): + get_indexdb_artifact_pullspec("registry.example.com/namespace/image") + + +@mock.patch('iib.workers.tasks.oras_utils.verify_indexdb_cache_sync') +@pytest.mark.parametrize( + "pullspec,expected_combined_tag,sync_result", + [ + ( + "registry.example.com/namespace/iib-pub-pending:v4.17", + "iib-pub-pending-v4.17", + True, + ), + ( + "quay.io/namespace/my-image:latest", + "my-image-latest", + False, + ), + ( + "registry.io/org/repo/index-image:v1.0.0@sha256:abc123", + "index-image-v1.0.0", + True, + ), + ], +) +def test_verify_indexdb_cache_for_image( + mock_verify_sync, pullspec, expected_combined_tag, sync_result +): + """Test verify_indexdb_cache_for_image with various pullspecs.""" + from iib.workers.tasks.oras_utils import verify_indexdb_cache_for_image + + mock_verify_sync.return_value = sync_result + + result = verify_indexdb_cache_for_image(pullspec) + + assert result == sync_result + mock_verify_sync.assert_called_once_with(expected_combined_tag) + + +@mock.patch('iib.workers.tasks.oras_utils.verify_indexdb_cache_sync') +def test_verify_indexdb_cache_for_image_invalid_pullspec(mock_verify_sync): + """Test verify_indexdb_cache_for_image with invalid pullspec.""" + from iib.workers.tasks.oras_utils import verify_indexdb_cache_for_image + + with pytest.raises(IIBError, match="Missing tag"): + verify_indexdb_cache_for_image("registry.example.com/namespace/image") + + mock_verify_sync.assert_not_called() + + +@mock.patch('iib.workers.tasks.oras_utils.refresh_indexdb_cache') +@pytest.mark.parametrize( + "pullspec,expected_combined_tag", + [ + ( + "registry.example.com/namespace/iib-pub-pending:v4.17", + "iib-pub-pending-v4.17", + ), + ( + "quay.io/namespace/my-image:latest", + "my-image-latest", + ), + ( + "registry.io/org/repo/index-image:v1.0.0", + "index-image-v1.0.0", + ), + ( + "registry.example.com/namespace/iib-pub-pending:v4.17@sha256:abc123", + "iib-pub-pending-v4.17", + ), + ], +) +def test_refresh_indexdb_cache_for_image(mock_refresh_cache, pullspec, expected_combined_tag): + """Test refresh_indexdb_cache_for_image with various pullspecs.""" + from iib.workers.tasks.oras_utils import refresh_indexdb_cache_for_image + + refresh_indexdb_cache_for_image(pullspec) + + mock_refresh_cache.assert_called_once_with(expected_combined_tag) + + +@mock.patch('iib.workers.tasks.oras_utils.refresh_indexdb_cache') +def test_refresh_indexdb_cache_for_image_invalid_pullspec(mock_refresh_cache): + """Test refresh_indexdb_cache_for_image with invalid pullspec.""" + from iib.workers.tasks.oras_utils import refresh_indexdb_cache_for_image + + with pytest.raises(IIBError, match="Missing tag"): + refresh_indexdb_cache_for_image("registry.example.com/namespace/image") + + mock_refresh_cache.assert_not_called() + + +@mock.patch('iib.workers.tasks.oras_utils.refresh_indexdb_cache') +def test_refresh_indexdb_cache_for_image_propagates_exception(mock_refresh_cache): + """Test if refresh_indexdb_cache_for_image propagates exceptions from refresh_indexdb_cache.""" + from iib.workers.tasks.oras_utils import refresh_indexdb_cache_for_image + + mock_refresh_cache.side_effect = IIBError('Refresh failed') + + with pytest.raises(IIBError, match='Refresh failed'): + refresh_indexdb_cache_for_image("registry.example.com/namespace/image:v1.0.0") diff --git a/tests/test_workers/test_tasks/test_utils.py b/tests/test_workers/test_tasks/test_utils.py index ae866b70b..fc7439b6b 100644 --- a/tests/test_workers/test_tasks/test_utils.py +++ b/tests/test_workers/test_tasks/test_utils.py @@ -10,9 +10,11 @@ import pytest -from iib.exceptions import ExternalServiceError, IIBError +from iib.exceptions import ExternalServiceError +from iib.exceptions import IIBError from iib.workers.config import get_worker_config from iib.workers.tasks import utils +from iib.workers.tasks.oras_utils import get_imagestream_artifact_pullspec @mock.patch('iib.workers.tasks.utils.skopeo_inspect') @@ -1363,13 +1365,52 @@ def test_prepare_request_for_build_no_arches(mock_gia, mock_gri, mock_srs): def test_prepare_request_for_build_binary_image_no_arch(mock_gia, mock_gri, mock_srs): mock_gia.side_effect = [{'amd64'}] - expected = 'The binary image is not available for the following arches.+' + expected = 'The binary image is not available for any of the following arches.+' with pytest.raises(IIBError, match=expected): utils.prepare_request_for_build( 1, utils.RequestConfigAddRm(_binary_image='binary-image:latest', add_arches=['s390x']) ) +@mock.patch('iib.workers.tasks.utils.set_request_state') +@mock.patch('iib.workers.tasks.utils.get_resolved_image') +@mock.patch('iib.workers.tasks.utils.get_image_arches') +def test_prepare_request_for_build_arches_not_subset(mock_gia, mock_gri, mock_srs, caplog): + """When arches are not a subset of binary_image_arches but at least one is supported.""" + # Binary image supports only amd64; request asks for amd64 and s390x (one supported, one not) + mock_gri.return_value = 'binary-image@sha256:abc123' + mock_gia.return_value = {'amd64'} + + with caplog.at_level(logging.WARNING, logger='iib.workers.tasks.utils'): + rv = utils.prepare_request_for_build( + 1, + utils.RequestConfigAddRm( + _binary_image='binary-image:latest', + add_arches=['amd64', 's390x'], + ), + ) + + # Warning is activated: building for supported arches only (code logs this when + # arches are not a subset but some are supported) + expected_msg = 'Building index images for the following supported arches: amd64' + assert expected_msg in caplog.text + + # Message is logged as a WARNING from the expected logger + warning_records = [ + r + for r in caplog.records + if r.levelname == 'WARNING' and r.name == 'iib.workers.tasks.utils' + ] + assert any( + 'Building index images for the following supported arches:' in r.getMessage() + for r in warning_records + ) + + # Result is filtered to supported arches only + assert rv['arches'] == {'amd64'} + assert rv['binary_image_resolved'] == 'binary-image@sha256:abc123' + + @pytest.mark.parametrize( 'resolved_distribution_scope, distribution_scope, output, raise_exception', ( @@ -1564,3 +1605,177 @@ def test_set_registry_auths_use_empty_config_config_not_exists_template_not_exis assert mock_open.call_count == 1 assert mock_json_dump.call_args[0][0] == registry_auths mock_rdc.assert_called_once_with() + + +def test_change_dir_changes_and_restores(tmp_path): + """change_dir should change cwd inside the context and restore it afterwards.""" + original_cwd = os.getcwd() + new_dir = tmp_path / "subdir" + new_dir.mkdir() + + with utils.change_dir(str(new_dir)): + assert os.getcwd() == str(new_dir) + + assert os.getcwd() == original_cwd + + +def test_change_dir_restores_on_exception(tmp_path): + """change_dir should restore cwd even if an exception is raised in the block.""" + original_cwd = os.getcwd() + new_dir = tmp_path / "subdir" + new_dir.mkdir() + + with pytest.raises(RuntimeError): + with utils.change_dir(str(new_dir)): + assert os.getcwd() == str(new_dir) + raise RuntimeError("boom") + + assert os.getcwd() == original_cwd + + +def test_change_dir_invalid_directory_does_not_change_cwd(tmp_path): + """change_dir should raise OSError for invalid path and not change cwd.""" + original_cwd = os.getcwd() + invalid_dir = tmp_path / "nonexistent" + + with pytest.raises(OSError): + with utils.change_dir(str(invalid_dir)): + # Block should never be entered + pass + + assert os.getcwd() == original_cwd + + +@pytest.mark.parametrize( + "from_index,expected_pullspec", + [ + ( + "registry.example.com/namespace/iib-pub-pending:v4.17", + "test-artifact-registry/index-db:iib-pub-pending-v4.17", + ), + ( + "quay.io/namespace/my-image:latest", + "test-artifact-registry/index-db:my-image-latest", + ), + ( + "registry.io/org/repo/index-image:v1.0.0", + "test-artifact-registry/index-db:index-image-v1.0.0", + ), + ( + "registry.example.com/namespace/iib-pub-pending:v4.17@sha256:abc123", + "test-artifact-registry/index-db:iib-pub-pending-v4.17", + ), + ], +) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') +def test_get_indexdb_artifact_pullspec(mock_gwc, from_index, expected_pullspec): + """Test constructing index DB artifact pullspecs.""" + from iib.workers.tasks.oras_utils import get_indexdb_artifact_pullspec + + mock_gwc.return_value = { + 'iib_index_db_artifact_registry': 'test-artifact-registry', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + 'iib_index_db_artifact_tag_template': '{image_name}-{tag}', + } + + result = get_indexdb_artifact_pullspec(from_index) + + assert result == expected_pullspec + + +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') +def test_get_indexdb_artifact_pullspec_invalid(mock_gwc): + """Test _get_indexdb_artifact_pullspec with invalid pullspec.""" + from iib.workers.tasks.oras_utils import get_indexdb_artifact_pullspec + + mock_gwc.return_value = { + 'iib_index_db_artifact_registry': 'test-artifact-registry', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + 'iib_index_db_artifact_tag_template': '{image_name}-{tag}', + } + + with pytest.raises(IIBError, match="Missing tag"): + get_indexdb_artifact_pullspec("registry.example.com/namespace/image") + + +@pytest.mark.parametrize( + "from_index,expected_pullspec", + [ + ( + "registry.example.com/namespace/iib-pub-pending:v4.17", + "test-imagestream-registry/index-db:iib-pub-pending-v4.17", + ), + ( + "quay.io/namespace/my-image:latest", + "test-imagestream-registry/index-db:my-image-latest", + ), + ( + "registry.io/org/repo/index-image:v1.0.0", + "test-imagestream-registry/index-db:index-image-v1.0.0", + ), + ( + "registry.example.com/namespace/iib-pub-pending:v4.17@sha256:abc123", + "test-imagestream-registry/index-db:iib-pub-pending-v4.17", + ), + ], +) +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') +def test_get_imagestream_artifact_pullspec(mock_gwc, from_index, expected_pullspec): + """Test constructing ImageStream artifact pullspecs.""" + mock_gwc.return_value = { + 'iib_index_db_imagestream_registry': 'test-imagestream-registry', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + 'iib_index_db_artifact_tag_template': '{image_name}-{tag}', + } + + result = get_imagestream_artifact_pullspec(from_index) + + assert result == expected_pullspec + + +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') +def test_get_imagestream_artifact_pullspec_invalid(mock_gwc): + """Test get_imagestream_artifact_pullspec with invalid pullspec.""" + mock_gwc.return_value = { + 'iib_index_db_imagestream_registry': 'test-imagestream-registry', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + 'iib_index_db_artifact_tag_template': '{image_name}-{tag}', + } + + with pytest.raises(IIBError, match="Missing tag"): + get_imagestream_artifact_pullspec("registry.example.com/namespace/image") + + +@mock.patch('iib.workers.tasks.oras_utils.verify_indexdb_cache_sync') +@pytest.mark.parametrize( + "pullspec,expected_combined_tag,sync_result", + [ + ( + "registry.example.com/namespace/iib-pub-pending:v4.17", + "iib-pub-pending-v4.17", + True, + ), + ( + "quay.io/namespace/my-image:latest", + "my-image-latest", + False, + ), + ( + "registry.io/org/repo/index-image:v1.0.0@sha256:abc123", + "index-image-v1.0.0", + True, + ), + ], +) +def test_verify_indexdb_cache_for_image( + mock_verify_sync, pullspec, expected_combined_tag, sync_result +): + """Test verify_indexdb_cache_for_image with various pullspecs.""" + from iib.workers.tasks.oras_utils import verify_indexdb_cache_for_image + + mock_verify_sync.return_value = sync_result + + result = verify_indexdb_cache_for_image(pullspec) + + assert result == sync_result + mock_verify_sync.assert_called_once_with(expected_combined_tag) diff --git a/tox.ini b/tox.ini index 733be5ffd..15602cee1 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ usedevelop = true basepython = py312: python3.12 py313: python3.13 + docs: python3.12 migrate-db: python3.12 pip-compile: python3.12 setenv = @@ -57,10 +58,10 @@ commands = description = PEP8 checks [Mandatory] skip_install = true deps = - flake8==3.7.9 - flake8-docstrings==1.5.0 + flake8==7.3.0 + flake8-docstrings==1.7.0 commands = - flake8 + flake8 --config .flake8 [testenv:yamllint] description = YAML checks [Mandatory]