diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 90b0079..20cdcfa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,19 +42,19 @@ repos: - id: detect-secrets - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.12 + rev: v0.15.15 hooks: - id: ruff args: [--fix] - id: ruff-format - repo: https://github.com/gitleaks/gitleaks - rev: v8.28.0 + rev: v8.30.0 hooks: - id: gitleaks - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.17.1 + rev: v2.1.0 hooks: - id: mypy additional_dependencies: [types-croniter, types-PyYAML, types-requests] diff --git a/ci_jobs_trigger/app.py b/ci_jobs_trigger/app.py index fee00b3..c11dfb8 100644 --- a/ci_jobs_trigger/app.py +++ b/ci_jobs_trigger/app.py @@ -1,20 +1,19 @@ import os import tempfile -from flask import Flask -from flask import request -from simple_logger.logger import get_logger +from flask import Flask, request from flask.logging import default_handler +from simple_logger.logger import get_logger from ci_jobs_trigger.libs.addons_webhook_trigger.addons_webhook_trigger import ( - process_hook, ADDONS_WEBHOOK_JOBS_TRIGGER_CONFIG_STR, + process_hook, ) from ci_jobs_trigger.libs.openshift_ci.re_trigger.re_trigger import JobTriggering from ci_jobs_trigger.libs.openshift_ci.zstream_trigger.zstream_trigger import ( OPENSHIFT_CI_ZSTREAM_TRIGGER_CONFIG_OS_ENV_STR, - process_and_trigger_jobs, monitor_and_trigger, + process_and_trigger_jobs, ) from ci_jobs_trigger.libs.operators_iib_trigger.iib_trigger import run_iib_update from ci_jobs_trigger.utils.general import ( diff --git a/ci_jobs_trigger/libs/addons_webhook_trigger/addons_webhook_trigger.py b/ci_jobs_trigger/libs/addons_webhook_trigger/addons_webhook_trigger.py index 0d13aeb..ea92960 100644 --- a/ci_jobs_trigger/libs/addons_webhook_trigger/addons_webhook_trigger.py +++ b/ci_jobs_trigger/libs/addons_webhook_trigger/addons_webhook_trigger.py @@ -1,7 +1,7 @@ import re from ci_jobs_trigger.libs.utils.general import trigger_ci_job -from ci_jobs_trigger.utils.general import get_config, get_gitlab_api, AddonsWebhookTriggerError +from ci_jobs_trigger.utils.general import AddonsWebhookTriggerError, get_config, get_gitlab_api ADDONS_WEBHOOK_JOBS_TRIGGER_CONFIG_STR = "ADDONS_WEBHOOK_JOBS_TRIGGER_CONFIG" diff --git a/ci_jobs_trigger/libs/openshift_ci/zstream_trigger/zstream_trigger.py b/ci_jobs_trigger/libs/openshift_ci/zstream_trigger/zstream_trigger.py index 2e6b803..837b819 100644 --- a/ci_jobs_trigger/libs/openshift_ci/zstream_trigger/zstream_trigger.py +++ b/ci_jobs_trigger/libs/openshift_ci/zstream_trigger/zstream_trigger.py @@ -4,7 +4,6 @@ import json import logging import time -from typing import Dict, List import packaging.version import yaml @@ -23,7 +22,7 @@ LOG_PREFIX: str = "Zstream trigger:" -def processed_versions_file(processed_versions_file_path: str, logger: logging.Logger) -> Dict: +def processed_versions_file(processed_versions_file_path: str, logger: logging.Logger) -> dict: try: with open(processed_versions_file_path) as fd: return json.load(fd) @@ -57,7 +56,7 @@ def already_processed_version( return False -def get_gitlab_project_file(config: Dict, ocm_env: str) -> Dict: +def get_gitlab_project_file(config: dict, ocm_env: str) -> dict: api = get_gitlab_api(url=config["gitlab_url"], token=config["gitlab_token"]) project = api.projects.get(config["gitlab_project"]) project_file_content = project.files.get( @@ -66,7 +65,7 @@ def get_gitlab_project_file(config: Dict, ocm_env: str) -> Dict: return yaml.safe_load(project_file_content.decode().decode("utf-8")) -def is_rosa_version_enabled(config: Dict, version: str, channel: str, ocm_env: str, logger: logging.Logger) -> bool: +def is_rosa_version_enabled(config: dict, version: str, channel: str, ocm_env: str, logger: logging.Logger) -> bool: processed_versions_file_path = config["processed_versions_file_path"] processed_versions_file_content = processed_versions_file( processed_versions_file_path=processed_versions_file_path, logger=logger @@ -87,8 +86,8 @@ def is_rosa_version_enabled(config: Dict, version: str, channel: str, ocm_env: s return False -def filter_rosa_versions_by_channel(all_rosa_versions: Dict, rosa_channel: str, version_channel: str) -> Dict: - filtered_rosa_dict: Dict[str, Dict[str, List[str]]] = {version_channel: {}} +def filter_rosa_versions_by_channel(all_rosa_versions: dict, rosa_channel: str, version_channel: str) -> dict: + filtered_rosa_dict: dict[str, dict[str, list[str]]] = {version_channel: {}} for version_key, versions in all_rosa_versions[rosa_channel].items(): filtered_rosa_dict[version_channel][version_key] = [ver for ver in versions if version_channel in ver] @@ -97,7 +96,7 @@ def filter_rosa_versions_by_channel(all_rosa_versions: Dict, rosa_channel: str, def get_all_rosa_versions( ocm_token: str, ocm_env: str, rosa_channel: str, version_channel: str, aws_region: str -) -> Dict[str, Dict[str, List[str]]]: +) -> dict[str, dict[str, list[str]]]: ocm_client = OCMPythonClient( token=ocm_token, endpoint="https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token", @@ -115,9 +114,9 @@ def get_all_rosa_versions( return _all_rosa_versions -def trigger_jobs(config: Dict, jobs: List, logger: logging.Logger, zstream_version: str) -> bool: - failed_triggers_jobs: List = [] - successful_triggers_jobs: List = [] +def trigger_jobs(config: dict, jobs: list, logger: logging.Logger, zstream_version: str) -> bool: + failed_triggers_jobs: list = [] + successful_triggers_jobs: list = [] if not jobs: no_jobs_mgs: str = f"{LOG_PREFIX} No jobs to trigger" logger.info(no_jobs_mgs) @@ -160,8 +159,8 @@ def trigger_jobs(config: Dict, jobs: List, logger: logging.Logger, zstream_versi return False -def process_and_trigger_jobs(logger: logging.Logger, version: str | None = None) -> Dict: - trigger_res: Dict = {} +def process_and_trigger_jobs(logger: logging.Logger, version: str | None = None) -> dict: + trigger_res: dict = {} config = get_config( os_environ=OPENSHIFT_CI_ZSTREAM_TRIGGER_CONFIG_OS_ENV_STR, logger=logger, @@ -305,7 +304,7 @@ def monitor_and_trigger(logger: logging.Logger) -> None: time.sleep(DAYS_TO_SECONDS) -def get_cron_iter(cron_schedule: str, config: Dict, logger: logging.Logger) -> croniter | None: +def get_cron_iter(cron_schedule: str, config: dict, logger: logging.Logger) -> croniter | None: try: return croniter(cron_schedule, start_time=datetime.datetime.now(), day_or=False) except CroniterBadCronError: diff --git a/ci_jobs_trigger/libs/operators_iib_trigger/iib_trigger.py b/ci_jobs_trigger/libs/operators_iib_trigger/iib_trigger.py index a2127eb..33085e2 100644 --- a/ci_jobs_trigger/libs/operators_iib_trigger/iib_trigger.py +++ b/ci_jobs_trigger/libs/operators_iib_trigger/iib_trigger.py @@ -5,15 +5,15 @@ from time import sleep import requests +from clouds.aws.session_clients import s3_client from ci_jobs_trigger.libs.utils.general import trigger_ci_job from ci_jobs_trigger.utils.constant import DAYS_TO_SECONDS from ci_jobs_trigger.utils.general import ( - send_slack_message, - get_config, AddonsWebhookTriggerError, + get_config, + send_slack_message, ) -from clouds.aws.session_clients import s3_client LOG_PREFIX = "iib-trigger:" @@ -247,15 +247,14 @@ def fetch_update_iib_and_trigger_jobs(logger, tmp_dir, config_dict=None): logger.info(f"{LOG_PREFIX} Created temp dir: {local_operators_latest_iib_filepath}") config_data["local_operators_latest_iib_filepath"] = local_operators_latest_iib_filepath - if s3_bucket_operators_latest_iib_path: - if not download_iib_file_from_s3_bucket( - s3_bucket_operators_latest_iib_path=s3_bucket_operators_latest_iib_path, - aws_region=config_data.get("aws_region"), - slack_errors_webhook_url=config_data.get("slack_errors_webhook_url"), - logger=logger, - target_file_path=local_operators_latest_iib_filepath, - ): - return False + if s3_bucket_operators_latest_iib_path and not download_iib_file_from_s3_bucket( + s3_bucket_operators_latest_iib_path=s3_bucket_operators_latest_iib_path, + aws_region=config_data.get("aws_region"), + slack_errors_webhook_url=config_data.get("slack_errors_webhook_url"), + logger=logger, + target_file_path=local_operators_latest_iib_filepath, + ): + return False if (ci_jobs := config_data.get("ci_jobs", {})) is None: logger.error(f"{LOG_PREFIX} No ci_jobs found in config") diff --git a/ci_jobs_trigger/libs/utils/general.py b/ci_jobs_trigger/libs/utils/general.py index 4229ac3..a556f2c 100644 --- a/ci_jobs_trigger/libs/utils/general.py +++ b/ci_jobs_trigger/libs/utils/general.py @@ -1,7 +1,7 @@ +from ci_jobs_trigger.libs.jenkins.utils.general import jenkins_trigger_job from ci_jobs_trigger.libs.openshift_ci.utils.constants import GANGWAY_API_URL from ci_jobs_trigger.libs.openshift_ci.utils.general import openshift_ci_trigger_job -from ci_jobs_trigger.libs.jenkins.utils.general import jenkins_trigger_job -from ci_jobs_trigger.utils.general import send_slack_message, AddonsWebhookTriggerError +from ci_jobs_trigger.utils.general import AddonsWebhookTriggerError, send_slack_message def dict_to_str(_dict): diff --git a/ci_jobs_trigger/tests/addons_webhook_trigger/test_addons_webhook_trigger.py b/ci_jobs_trigger/tests/addons_webhook_trigger/test_addons_webhook_trigger.py index 52ebddf..58f13f0 100644 --- a/ci_jobs_trigger/tests/addons_webhook_trigger/test_addons_webhook_trigger.py +++ b/ci_jobs_trigger/tests/addons_webhook_trigger/test_addons_webhook_trigger.py @@ -1,7 +1,6 @@ import pytest from gitlab import Gitlab from gitlab.v4.objects import ProjectManager, ProjectMergeRequestManager - from simple_logger.logger import get_logger from ci_jobs_trigger.libs.addons_webhook_trigger.addons_webhook_trigger import process_hook diff --git a/ci_jobs_trigger/tests/operators_iib_trigger/test_operators_iib_trigger.py b/ci_jobs_trigger/tests/operators_iib_trigger/test_operators_iib_trigger.py index eae3b26..b3e1d52 100644 --- a/ci_jobs_trigger/tests/operators_iib_trigger/test_operators_iib_trigger.py +++ b/ci_jobs_trigger/tests/operators_iib_trigger/test_operators_iib_trigger.py @@ -6,9 +6,9 @@ from ci_jobs_trigger.libs.operators_iib_trigger.iib_trigger import ( fetch_update_iib_and_trigger_jobs, + get_new_iib, upload_download_s3_bucket_file, verify_s3_or_local_file, - get_new_iib, ) LOGGER = get_logger("test_operators_iib_trigger") @@ -33,11 +33,11 @@ def json(): class MockS3Client: @staticmethod - def download_file(Filename, Bucket, Key): # noqa N803 + def download_file(Filename, Bucket, Key): return True @staticmethod - def upload_file(Filename, Bucket, Key): # noqa N803 + def upload_file(Filename, Bucket, Key): return True diff --git a/ci_jobs_trigger/utils/general.py b/ci_jobs_trigger/utils/general.py index 28b12f4..db68975 100644 --- a/ci_jobs_trigger/utils/general.py +++ b/ci_jobs_trigger/utils/general.py @@ -2,8 +2,8 @@ import os from multiprocessing import Process -import requests import gitlab +import requests from pyaml_env import parse_config