Fixing extract_fbc_fragment

.env.containerized.template

@@ -0,0 +1,94 @@
+# IIB Containerized Workflow Environment Configuration
+# Copy this file to .env.containerized and fill in your values
+# DO NOT commit .env.containerized to git (it's already in .gitignore)
+
+# ===================================================================
+# Konflux Cluster Configuration
+# ===================================================================
+# These settings are required for the worker to connect to your Konflux dev cluster
+
+# Konflux cluster API URL (e.g., https://api.konflux-dev.example.com:6443)
+IIB_KONFLUX_CLUSTER_URL=
+
+# Konflux cluster service account token
+# To create a token:
+# 1. Create a service account: kubectl create serviceaccount iib-worker -n <namespace>
+# 2. Create a role with permissions to read/list pipelineruns
+# 3. Create a rolebinding to bind the role to the service account
+# 4. Get the token: kubectl create token iib-worker -n <namespace> --duration=720h
+IIB_KONFLUX_CLUSTER_TOKEN=
+
+# Konflux cluster CA certificate path (relative to this file)
+# This should point to the file mounted at /etc/iib/konflux-ca.crt
+# You can get the CA cert with: kubectl config view --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d > docker/containerized/konflux-ca.crt
+IIB_KONFLUX_CLUSTER_CA_CERT=/etc/iib/konflux-ca.crt
+
+# Namespace where Konflux pipelines run
+IIB_KONFLUX_NAMESPACE=
+
+# Pipeline timeout in seconds (default: 1800 = 30 minutes)
+IIB_KONFLUX_PIPELINE_TIMEOUT=1800
+
+# ===================================================================
+# GitLab Configuration
+# ===================================================================
+# Required for pushing commits and creating merge requests
+
+# GitLab tokens for different repositories
+# Format: {"repo_url": {"token_name": "ENV_VAR_NAME", "token": "actual_token"}}
+# Example:
+# IIB_INDEX_CONFIGS_GITLAB_TOKENS_MAP='{"https://gitlab.example.com/catalogs/v4.19": {"token_name": "GITLAB_TOKEN_V419", "token": "glpat-xxxxxxxxxxxxx"}}'
+IIB_INDEX_CONFIGS_GITLAB_TOKENS_MAP=
+
+# ===================================================================
+# Registry Configuration
+# ===================================================================
+# Configuration for the IIB output registry
+
+# Registry where built images will be pushed
+IIB_REGISTRY=registry:8443
+
+# Template for pushing built images
+# Available placeholders: {registry}, {request_id}
+IIB_IMAGE_PUSH_TEMPLATE={registry}/iib-build:{request_id}
+
+# ===================================================================
+# Index DB Artifact Configuration
+# ===================================================================
+# Configuration for index.db artifact storage
+
+# Registry for index.db artifacts (usually Quay.io)
+IIB_INDEX_DB_ARTIFACT_REGISTRY=
+
+# Registry for index.db ImageStream cache
+IIB_INDEX_DB_IMAGESTREAM_REGISTRY=
+
+# Template for index.db artifact storage
+IIB_INDEX_DB_ARTIFACT_TEMPLATE={registry}/index-db:{tag}
+
+# ===================================================================
+# Optional Configuration
+# ===================================================================
+
+# AWS S3 bucket for storing artifacts (optional)
+# IIB_AWS_S3_BUCKET_NAME=
+
+# Greenwave URL for gating (optional)
+# IIB_GREENWAVE_URL=
+
+# Log level (DEBUG, INFO, WARNING, ERROR)
+IIB_LOG_LEVEL=DEBUG
+
+# Request logs directory (inside container)
+IIB_REQUEST_LOGS_DIR=/var/log/iib/requests
+
+# Skopeo timeout
+IIB_SKOPEO_TIMEOUT=300s
+
+# Total retry attempts for operations
+IIB_TOTAL_ATTEMPTS=5
+
+# Retry configuration
+IIB_RETRY_DELAY=10
+IIB_RETRY_JITTER=10
+IIB_RETRY_MULTIPLIER=5

.flake8

@@ -6,17 +6,19 @@ exclude =
     venv
 per-file-ignores =
     ./iib/workers/tasks/build_regenerate_bundle.py: E713
-    ./iib/workers/tasks/utils.py: E203,E702
+    ./iib/workers/tasks/utils.py: E203,E702,E721
     ./iib/workers/tasks/build_add_deprecations.py: E713
     ./iib/workers/tasks/opm_operations.py: E203
     ./iib/web/api_v1.py: E226
     ./iib/web/migrations/versions/*: D103
     ./tests/*: D103
     ./tests/test_web/test_models.py: D103
     ./tests/test_web/test_s3_utils.py: D103
-    ./tests/test_web/test_api_v1.py: D103
+    ./tests/test_web/test_api_v1.py: D103,F541
     ./tests/test_workers/test_tasks/test_build.py: D103,E231
+    ./tests/test_workers/test_tasks/test_build_containerized_fbc_operations.py: F841,E501
     ./tests/test_workers/test_tasks/test_build_regenerate_bundle.py: D103,E241,E222
     ./tests/test_workers/test_tasks/test_opm_operations.py: D103, E203
     ./tests/test_web/test_migrations.py: E231,D103
-extend-ignore = E231
+extend-ignore = E231, D104, D100, D105
+max-line-length = 100

.github/workflows/run_tox.yml

@@ -7,6 +7,7 @@ on:
   push:
     branches:
       - "master"
+      - "main"
 
 jobs:
   build:

README.md

@@ -199,6 +199,10 @@ The custom configuration options for the REST API are listed below:
   to another dictionary mapping ocp_version label to a binary image pull specification.
   This is useful in setting up customized binary image for different index image images thus
   reducing complexity for the end user. This defaults to `{}`.
+* `IIB_BINARY_IMAGE_LESS_ARCHES_ALLOWED_VERSIONS` - an optional `list(<str>)` to specify the OPM
+  versions which are allowed to build index images with lesser arches than the configured 
+  on `iib_supported_archs`. When a certain version is set it will allow building only to the
+  available arches supported by the binary image.
 * `IIB_INDEX_TO_GITLAB_PUSH_MAP` - the mapping, `dict(<str>:<str>)`, to specify which index
   images (keys) which should have its catalog pushed into a GitLab repository (value). This defaults to {}.
 * `IIB_GRAPH_MODE_INDEX_ALLOW_LIST` - the list of index image pull specs on which using the
@@ -313,6 +317,24 @@ The custom configuration options for the Celery workers are listed below:
   and related_bundles if specified. `iib_request_logs_dir` and `iib_request_related_bundles_dir`
   are required when this variable is specified. This defaults to `None` which means IIB will try to store
   the files locally if `iib_request_logs_dir` and `iib_request_related_bundles_dir` are configured.
+* `iib_index_db_imagestream_registry` - the default container registry where the `index.db`
+   ImageStream is pushed. This is typically an internal OpenShift registry or another registry
+   dedicated to hosting ImageStreams for `index.db` artifacts. If unset, caching of index.db
+  artifacts will be disabled.
+* `iib_index_db_artifact_registry` - the container registry where `index.db` artifact images
+  (for example `index-db:<tag>`) are stored and from which they are distributed. This is often
+  a central or dedicated registry for artifacts generated by IIB. This value **must be set** in
+  order for `index.db` artifacts to be pushed and for configuration validation to succeed.
+  When `iib_index_db_oras_auth_path` is unset and this is set together with the artifact registry,
+  ORAS uses `set_registry_auths` with an isolated Docker config (not the worker’s default `config.json`).
+* `iib_index_db_oras_auth_path` - path to a JSON Docker config file for ORAS. When set, ORAS uses
+  `oras --registry-config` with this file and does not apply the inline secret above.
+* `iib_empty_index_db_tag` - the tag used to identify pre-created empty `index.db` artifacts in the
+  registry. When creating an empty index, IIB will first attempt to fetch an artifact tagged with
+  this value. If not found, it falls back to fetching the `from_index` and removing all operators.
+  This defaults to `'empty'`.
+* `iib_use_imagestream_cache` - whether to use OpenShift ImageStream cache for `index.db` artifacts.
+  Requires an OpenShift cluster with ImageStream configured. This defaults to `False`.
 * `iib_docker_config_template` - the path to the Docker config.json file for IIB to use as a
   template. IIB will symlink this file to `~/.docker/config.json` at the beginning of every request.
   Additionally, it will use this file as a base and set the `overwrite_from_index_token` for the
@@ -341,6 +363,11 @@ The custom configuration options for the Celery workers are listed below:
 * `iib_index_configs_gitlab_tokens_map` - A map of index image addresses to GitLab tokens.
   These Gitlab repositories are intended to store image `/configs` directories.
   Its format should be the full repository URL as keys and `token-name:token-value` as value.
+* `iib_regenerate_bundle_repo_key` - The key used to look up the GitLab repository URL from
+  `iib_index_to_gitlab_push_map` for containerized bundle regeneration workflow. This defaults
+  to `'regenerate-bundle'`. The actual repository URL should be configured in
+  `iib_index_to_gitlab_push_map` with this key, and the token must be configured in
+  `iib_index_configs_gitlab_tokens_map`.
 * `iib_log_level` - the Python log level for `iib.workers` logger. This defaults to `INFO`.
 * `iib_max_recursive_related_bundles` - the maximum number of recursive related bundles IIB will
   recurse through. This is to avoid DOS attacks.

docker/Dockerfile-workers

@@ -11,6 +11,9 @@ RUN dnf -y install \
     buildah \
     fuse-overlayfs \
     gcc \
+    curl \
+    tar \
+    gzip \
     git \
     krb5-devel \
     libffi-devel \
@@ -25,17 +28,40 @@ RUN dnf -y install \
     && dnf update -y \
     && dnf clean all
 
-ADD https://github.com/operator-framework/operator-registry/releases/download/v1.26.4/linux-amd64-opm /usr/bin/opm-v1.26.4
-RUN chmod +x /usr/bin/opm-v1.26.4
-ADD https://github.com/operator-framework/operator-registry/releases/download/v1.40.0/linux-amd64-opm /usr/bin/opm-v1.40.0
-RUN chmod +x /usr/bin/opm-v1.40.0
-# Create a link for default opm
-RUN ln -s /usr/bin/opm-v1.26.4 /usr/bin/opm
-RUN chmod +x /usr/bin/opm
+# Install all opm variants,
+# then expose the default opm via symlink.
+RUN set -eux; \
+    install_binary() { \
+      local name="$1"; local url="$2"; local sha="$3"; \
+      curl -fsSL "$url" -o "/usr/local/bin/${name}"; \
+      echo "${sha}  /usr/local/bin/${name}" | sha256sum -c -; \
+      chmod 0555 "/usr/local/bin/${name}"; \
+    }; \
+    install_binary "opm-v1.26.4" "https://github.com/operator-framework/operator-registry/releases/download/v1.26.4/linux-amd64-opm" "cf94e9dbd58c338e1eed03ca50af847d24724b99b40980812abbe540e8c7ff8e"; \
+    install_binary "opm-v1.28.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.28.0/linux-amd64-opm" "e18e5abc8febb63c9dc76db0f33475553d98495465bd2dca81c39dcdbc875c08"; \
+    install_binary "opm-v1.40.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.40.0/linux-amd64-opm" "33eb929264a69f31895e1973248b7e97e3b6a862d7ca27f6892e158f79ad6aeb"; \
+    install_binary "opm-v1.44.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.44.0/linux-amd64-opm" "21f0a423dfbfcddcffdde98266307a08d87b4db980be859b9e252a5a24df51bf"; \
+    install_binary "opm-v1.48.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.48.0/linux-amd64-opm" "0a301826baff730489162caff13e04f7dc16c1a79072cbcbdfc5379d95caef40"; \
+    install_binary "opm-v1.50.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.50.0/linux-amd64-opm" "d9bfdc08dd9640c1d9085d191f10f884f2ef29370db1ac097a73a0e23e803f95"; \
+    install_binary "opm-v1.57.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.57.0/linux-amd64-opm" "8d2f51f166f47f76eb6906c4de9af90462b7163cbacef6c932bda4829ec086c7"; \
+    install_binary "opm-v1.61.0" "https://github.com/operator-framework/operator-registry/releases/download/v1.61.0/linux-amd64-opm" "c5701ef59e12c930337a9a9363cd44c2a4d9f64f6d4f96513d3511a36f81cf5d"; \
+    install_binary "operator-sdk" "https://github.com/operator-framework/operator-sdk/releases/download/v1.15.0/operator-sdk_linux_amd64" "d2065f1f7a0d03643ad71e396776dac0ee809ef33195e0f542773b377bab1b2a"; \
+    # set default opm \
+    ln -sfn /usr/local/bin/opm-v1.26.4 /usr/local/bin/opm
+
 ADD https://github.com/fullstorydev/grpcurl/releases/download/v1.8.5/grpcurl_1.8.5_linux_x86_64.tar.gz /src/grpcurl_1.8.5_linux_x86_64.tar.gz
 RUN cd /usr/bin && tar -xf /src/grpcurl_1.8.5_linux_x86_64.tar.gz grpcurl && rm -f /src/grpcurl_1.8.5_linux_x86_64.tar.gz
-ADD https://github.com/operator-framework/operator-sdk/releases/download/v1.15.0/operator-sdk_linux_amd64 /usr/bin/operator-sdk
-RUN chmod +x /usr/bin/operator-sdk
+
+RUN curl -L "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.10/openshift-client-linux.tar.gz" -o /tmp/oc_client.tar.gz && \
+    tar -xvzf /tmp/oc_client.tar.gz -C /usr/bin/ && \
+    rm /tmp/oc_client.tar.gz /usr/bin/README.md
+
+RUN curl -L "https://github.com/oras-project/oras/releases/download/v1.3.0/oras_1.3.0_linux_amd64.tar.gz" -o /tmp/oras.tar.gz && \
+    tar -xvzf /tmp/oras.tar.gz -C /usr/bin/ && \
+    rm /tmp/oras.tar.gz /usr/bin/LICENSE
+
+RUN git config --global user.email "exd-guild-hello-operator+iib-dev-env@redhat.com"
+RUN git config --global user.name "IIB dev-env"
 
 RUN update-alternatives --set python3 $(which python3.12)
 
@@ -45,9 +71,16 @@ COPY docker/libpod.conf /usr/share/containers/libpod.conf
 
 COPY . .
 
+# Prepare writable HOME for OpenShift random UID runtime.
+RUN mkdir -p /home/iib-worker/.docker \
+    && chgrp -R 0 /home/iib-worker \
+    && chmod -R g=u /home/iib-worker
+ENV HOME=/home/iib-worker
+ENV KRB5CCNAME=FILE:/home/iib-worker/krb5cc_iib_worker
+
 # default python3-pip version for rhel8 python3.6 is 9.0.3 and it can't be updated by dnf
 # we have to update it by pip to version above 21.0.0
 RUN pip3 install --upgrade pip
 RUN pip3 install -r requirements.txt --no-deps --require-hashes
 RUN pip3 install . --no-deps
-CMD ["/bin/celery-3", "-A", "iib.workers.tasks", "worker", "--loglevel=info"]
+CMD ["/usr/local/bin/celery", "-A", "iib.workers.tasks", "worker", "--loglevel=debug"]