From 07d4c91555e4a88b3c9393cb1fba0aa854f0f7a9 Mon Sep 17 00:00:00 2001 From: Siteshwar Vashisht Date: Tue, 25 Mar 2025 12:51:33 +0100 Subject: [PATCH 1/7] Add KojiOpenScanHubHelper and refactor some code ... to download SRPMs to make it reusable. Signed-off-by: Siteshwar Vashisht --- .../worker/helpers/open_scan_hub.py | 167 ++++++++++++++++-- 1 file changed, 149 insertions(+), 18 deletions(-) diff --git a/packit_service/worker/helpers/open_scan_hub.py b/packit_service/worker/helpers/open_scan_hub.py index a74b43ff9..026d73aeb 100644 --- a/packit_service/worker/helpers/open_scan_hub.py +++ b/packit_service/worker/helpers/open_scan_hub.py @@ -16,6 +16,7 @@ JobType, ) from packit.exceptions import PackitException +from packit.utils.koji_helper import KojiHelper from sqlalchemy.exc import IntegrityError from packit_service.constants import ( @@ -24,14 +25,19 @@ from packit_service.models import ( BuildStatus, CoprBuildTargetModel, + KojiBuildTargetModel, OSHScanStatus, SRPMBuildModel, ) -from packit_service.service.urls import get_copr_build_info_url, get_openscanhub_info_url +from packit_service.service.urls import ( + get_copr_build_info_url, + get_koji_build_info_url, + get_openscanhub_info_url, +) from packit_service.utils import ( download_file, ) -from packit_service.worker.helpers.build import CoprBuildJobHelper +from packit_service.worker.helpers.build import CoprBuildJobHelper, KojiBuildJobHelper from packit_service.worker.reporting import BaseCommitStatus logger = logging.getLogger(__name__) @@ -66,30 +72,31 @@ def parse_dict_from_output(output: str) -> dict: json_str = matches[-1] return json.loads(json_str) + @staticmethod + def download_srpm(directory: str, srpm_model: SRPMBuildModel) -> Optional[Path]: + if not srpm_model.url: + logger.info( + f"SRPMBuildModel with copr_build_id={srpm_model.copr_build_id} " + "has status={srpm_model.status} " + "and empty url. Skipping download." + ) + return None + srpm_path = Path(directory).joinpath(basename(srpm_model.url)) + if not download_file(srpm_model.url, srpm_path): + logger.info(f"Downloading of SRPM {srpm_model.url} was not successful.") + return None + return srpm_path + @staticmethod def download_srpms( directory: str, base_srpm_model: SRPMBuildModel, srpm_model: SRPMBuildModel, ) -> Optional[tuple[Path, Path]]: - def download_srpm(srpm_model: SRPMBuildModel) -> Optional[Path]: - if not srpm_model.url: - logger.info( - f"SRPMBuildModel with copr_build_id={srpm_model.copr_build_id} " - "has status={srpm_model.status} " - "and empty url. Skipping download." - ) - return None - srpm_path = Path(directory).joinpath(basename(srpm_model.url)) - if not download_file(srpm_model.url, srpm_path): - logger.info(f"Downloading of SRPM {srpm_model.url} was not successful.") - return None - return srpm_path - - if (base_srpm_path := download_srpm(base_srpm_model)) is None: + if (base_srpm_path := OpenScanHubHelper.download_srpm(directory, base_srpm_model)) is None: return None - if (srpm_path := download_srpm(srpm_model)) is None: + if (srpm_path := OpenScanHubHelper.download_srpm(directory, srpm_model)) is None: return None return base_srpm_path, srpm_path @@ -278,3 +285,127 @@ def get_srpm_build(commit_sha): else: logger.debug("No matching base build found in our DB.") return None + + +class KojiOpenScanHubHelper(OpenScanHubHelper): + def __init__( + self, + koji_build_helper: KojiBuildJobHelper, + build: KojiBuildTargetModel, + ): + self.build = build + self.koji_build_helper = koji_build_helper + + def handle_scan(self): + """ + Try to find a job that can provide the base SRPM,get_base_srpm_model + download both SRPM and base SRPM and trigger the scan in OpenScanHub. + """ + if not (base_build_job_id := self.find_base_build_job_id()): + logger.debug("No base build job needed for diff scan found in the config.") + return + + logger.info("Preparing to trigger scan in OpenScanHub...") + srpm_model = self.build.get_srpm_build() + + with tempfile.TemporaryDirectory() as directory: + if not (path := self.download_srpm(directory, srpm_model)): + self.report( + state=BaseCommitStatus.neutral, + description=( + "It was not possible to download the SRPMs needed" + " for the differential scan." + ), + url=None, + ) + return + + build_dashboard_url = get_koji_build_info_url(self.build.id) + + try: + err_msg = "Scan in OpenScanHub was not submitted successfully." + with self.build.add_scan_transaction() as scan: + output = self.koji_build_helper.api.run_osh_build( + srpm_path=path, + base_nvr=base_build_job_id, + comment=f"Submitted via Packit Service for {build_dashboard_url}", + ) + + if not output: + raise OSHNoFeedback("Something went wrong, skipping the reporting.") + + logger.info("Scan submitted successfully.") + + response_dict = self.parse_dict_from_output(output) + + logger.debug(f"Parsed dict from output: {response_dict} ") + + if id := response_dict.get("id"): + scan.task_id = id + scan.status = OSHScanStatus.pending + else: + raise OSHNoFeedback( + "It was not possible to get the Open Scan Hub task_id " + "from the response.", + ) + + if not (url := response_dict.get("url")): + err_msg = "It was not possible to get the task URL from the OSH response." + raise OSHNoFeedback(err_msg) + scan.url = url + + self.report( + state=BaseCommitStatus.running, + description=( + "Scan in OpenScanHub submitted successfully. " + "Check the URL for more details." + ), + url=get_openscanhub_info_url(scan.id), + links_to_external_services={"OpenScanHub task": url}, + ) + except IntegrityError as ex: + logger.info(f"OpenScanHub already submitted: {ex}") + except OSHNoFeedback as ex: + logger.info(f"OpenScanHub feedback missing: {ex}") + self.report( + state=BaseCommitStatus.neutral, + description=err_msg, + url=build_dashboard_url, + ) + + def report( + self, + state: BaseCommitStatus, + description: str, + url: str, + links_to_external_services: Optional[dict[str, str]] = None, + ): + check_name = "osh-diff-scan:fedora-rawhide-x86_64" + if identifier := self.koji_build_helper.job_config.identifier: + check_name += f":{identifier}" + self.koji_build_helper._report( + state=state, + description=description, + url=url, + check_names=[check_name], + markdown_content=OPEN_SCAN_HUB_FEATURE_DESCRIPTION, + links_to_external_services=links_to_external_services, + ) + + def find_base_build_job_id(self) -> Optional[int]: + """ + Find the job in the config that can provide the base build for the scan + (with `commit` trigger and same branch configured as the target PR branch). + """ + # We should take this approach as Packit does not have a base build for every package in db. + koji_helper = KojiHelper() + result = koji_helper.get_latest_build_in_tag( + tag="rawhide", package=self.build.get_package_name() + ) + + if not result: + logger.debug("Failed to find build id") + return None + + # Pass the build id to `osh-cli`. (result.build_id contains the base build id) + return result["build_id"] From dbbd963086d57609a2e679b2f9ab0e9aea5b4f0d Mon Sep 17 00:00:00 2001 From: Siteshwar Vashisht Date: Tue, 25 Mar 2025 12:52:29 +0100 Subject: [PATCH 2/7] Refactor handler for OpenScanHub and Copr ... to make some code reusable for Koji builds. Signed-off-by: Siteshwar Vashisht --- .../worker/handlers/open_scan_hub.py | 107 +++++++++--------- 1 file changed, 56 insertions(+), 51 deletions(-) diff --git a/packit_service/worker/handlers/open_scan_hub.py b/packit_service/worker/handlers/open_scan_hub.py index 7a76faf03..275bc5286 100644 --- a/packit_service/worker/handlers/open_scan_hub.py +++ b/packit_service/worker/handlers/open_scan_hub.py @@ -51,6 +51,60 @@ def __init__(self, **kwargs): def get_checkers() -> tuple[type[Checker], ...]: return (RawhideX86Target, IsEventForJob) + def get_issues_added_url( + self, + openscanhub_url: str = "https://openscanhub.fedoraproject.org", + file_format: str = "html", + ) -> str: + """ + Constructs the URL for the added issues in the specified + format for the given OpenScanHub task. + + Parameters: + openscanhub_url (str) + file_format (str): The format of the added issues file ('html' or 'json'). + + Returns: + str: The full URL to access the added issues in the specified format. + """ + return f"{openscanhub_url}/task/{self.event.task_id}/log/added.{file_format}" + + def get_number_of_new_findings_identified(self) -> Optional[int]: + """ + Downloads a JSON file from the task issues added URL and + returns the number of items in the 'defects' array. + + Returns: + Optional[int]: Number of items in the 'defects' array, + or None if not found or on error. + """ + url = self.event.issues_added_url + logger.info(f"About to get the number of new findings identified by the scan from {url}.") + + try: + with requests.get(url, timeout=10) as response: + response.raise_for_status() + data = response.json() + + defects = data.get("defects") + if defects is None: + logger.debug("No 'defects' array found in the JSON data.") + return None + + return len(defects) + + except requests.exceptions.RequestException as e: + logger.error(f"Error while downloading the JSON file: {e}") + return None + except json.JSONDecodeError: + logger.error("The response is not a valid JSON format.") + return None + + +class CoprOpenScanHubAbstractHandler(OpenScanHubAbstractHandler): + def __init__(self, **kwargs): + super().__init__(**kwargs) + def get_helper(self) -> CoprOpenScanHubHelper: build_helper = CoprBuildJobHelper( service_config=self.service_config, @@ -93,60 +147,11 @@ def check_scan_and_build(self): @configured_as(job_type=JobType.copr_build) @reacts_to(openscanhub.task.Finished) class CoprOpenScanHubTaskFinishedHandler( - OpenScanHubAbstractHandler, + CoprOpenScanHubAbstractHandler, ): event: openscanhub.task.Finished task_name = TaskName.openscanhub_task_finished - def get_number_of_new_findings_identified(self) -> Optional[int]: - """ - Downloads a JSON file from the task issues added URL and - returns the number of items in the 'defects' array. - - Returns: - Optional[int]: Number of items in the 'defects' array, - or None if not found or on error. - """ - url = self.event.issues_added_url - logger.info(f"About to get the number of new findings identified by the scan from {url}.") - - try: - with requests.get(url, timeout=10) as response: - response.raise_for_status() - data = response.json() - - defects = data.get("defects") - if defects is None: - logger.debug("No 'defects' array found in the JSON data.") - return None - - return len(defects) - - except requests.exceptions.RequestException as e: - logger.error(f"Error while downloading the JSON file: {e}") - return None - except json.JSONDecodeError: - logger.error("The response is not a valid JSON format.") - return None - - def get_issues_added_url( - self, - openscanhub_url: str = "https://openscanhub.fedoraproject.org", - file_format: str = "html", - ) -> str: - """ - Constructs the URL for the added issues in the specified - format for the given OpenScanHub task. - - Parameters: - openscanhub_url (str) - file_format (str): The format of the added issues file ('html' or 'json'). - - Returns: - str: The full URL to access the added issues in the specified format. - """ - return f"{openscanhub_url}/task/{self.event.task_id}/log/added.{file_format}" - def run(self) -> TaskResults: self.check_scan_and_build() external_links = {"OpenScanHub task": self.event.scan.url} @@ -199,7 +204,7 @@ def run(self) -> TaskResults: @configured_as(job_type=JobType.copr_build) @reacts_to(openscanhub.task.Started) class CoprOpenScanHubTaskStartedHandler( - OpenScanHubAbstractHandler, + CoprOpenScanHubAbstractHandler, ): task_name = TaskName.openscanhub_task_started From 11b178f5df9ba34663b26738f7038d98fc21a728 Mon Sep 17 00:00:00 2001 From: Siteshwar Vashisht Date: Thu, 17 Apr 2025 03:41:53 +0200 Subject: [PATCH 3/7] Add koji related fields to OSHScanModel Signed-off-by: Siteshwar Vashisht --- packit_service/models.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packit_service/models.py b/packit_service/models.py index 1bdb4e2bc..7b5d6b8f5 100644 --- a/packit_service/models.py +++ b/packit_service/models.py @@ -2666,6 +2666,8 @@ class KojiBuildTargetModel(GroupAndTargetModelConnector, Base): back_populates="koji_build_targets", ) + scan = relationship("OSHScanModel", back_populates="koji_build_target") + def set_status(self, status: str): with sa_session_transaction(commit=True) as session: self.status = status @@ -4257,6 +4259,17 @@ class OSHScanModel(Base): uselist=False, ) + koji_build_target_id = Column( + Integer, + ForeignKey("koji_build_targets.id"), + unique=True, + ) + koji_build_target = relationship( + "KojiBuildTargetModel", + back_populates="scan", + uselist=False, + ) + @classmethod def get_or_create(cls, task_id: int) -> "OSHScanModel": with sa_session_transaction(commit=True) as session: From 094308ae5f4c626f021162141d7e0b79553c46aa Mon Sep 17 00:00:00 2001 From: Siteshwar Vashisht Date: Tue, 25 Mar 2025 13:00:48 +0100 Subject: [PATCH 4/7] Add handler for OpenScanHub and Koji builds Signed-off-by: Siteshwar Vashisht --- packit_service/events/openscanhub/abstract.py | 8 +- packit_service/worker/handlers/__init__.py | 4 + .../worker/handlers/open_scan_hub.py | 190 ++++++++++++++++-- 3 files changed, 186 insertions(+), 16 deletions(-) diff --git a/packit_service/events/openscanhub/abstract.py b/packit_service/events/openscanhub/abstract.py index 8c801f4d9..265d9ba96 100644 --- a/packit_service/events/openscanhub/abstract.py +++ b/packit_service/events/openscanhub/abstract.py @@ -41,7 +41,13 @@ def __init__( " and should have been associated with the copr build.", ) return - self.build = self.scan.copr_build_target + + # TODO: How to handle koji builds here? + if hasattr(self.scan, "copr_build_target"): + self.build = self.scan.copr_build_target + else: + self.build = self.scan.koji_build_target + if not self.build: logger.warning( f"Scan with id {task_id} not associated with a build." diff --git a/packit_service/worker/handlers/__init__.py b/packit_service/worker/handlers/__init__.py index e6e4a5498..caf9ccf88 100644 --- a/packit_service/worker/handlers/__init__.py +++ b/packit_service/worker/handlers/__init__.py @@ -33,6 +33,8 @@ from packit_service.worker.handlers.open_scan_hub import ( CoprOpenScanHubTaskFinishedHandler, CoprOpenScanHubTaskStartedHandler, + KojiOpenScanHubTaskFinishedHandler, + KojiOpenScanHubTaskStartedHandler, ) from packit_service.worker.handlers.testing_farm import ( TestingFarmHandler, @@ -61,4 +63,6 @@ VMImageBuildResultHandler.__name__, CoprOpenScanHubTaskFinishedHandler.__name__, CoprOpenScanHubTaskStartedHandler.__name__, + KojiOpenScanHubTaskFinishedHandler.__name__, + KojiOpenScanHubTaskStartedHandler.__name__, ] diff --git a/packit_service/worker/handlers/open_scan_hub.py b/packit_service/worker/handlers/open_scan_hub.py index 275bc5286..ce3a685a4 100644 --- a/packit_service/worker/handlers/open_scan_hub.py +++ b/packit_service/worker/handlers/open_scan_hub.py @@ -23,8 +23,8 @@ from packit_service.worker.handlers.mixin import ( ConfigFromEventMixin, ) -from packit_service.worker.helpers.build import CoprBuildJobHelper -from packit_service.worker.helpers.open_scan_hub import CoprOpenScanHubHelper +from packit_service.worker.helpers.build import CoprBuildJobHelper, KojiBuildJobHelper +from packit_service.worker.helpers.open_scan_hub import CoprOpenScanHubHelper, KojiOpenScanHubHelper from packit_service.worker.mixin import ( LocalProjectMixin, PackitAPIWithUpstreamMixin, @@ -69,6 +69,58 @@ def get_issues_added_url( """ return f"{openscanhub_url}/task/{self.event.task_id}/log/added.{file_format}" + +class CoprOpenScanHubAbstractHandler(OpenScanHubAbstractHandler): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_helper(self) -> CoprOpenScanHubHelper: + build_helper = CoprBuildJobHelper( + service_config=self.service_config, + package_config=self.package_config, + project=self.project, + metadata=self.data, + db_project_event=self.data.db_project_event, + job_config=self.job_config, + celery_task=self.celery_task, + ) + + return CoprOpenScanHubHelper( + copr_build_helper=build_helper, + build=self.event.build, + ) + + def check_scan_and_build(self): + task_id = self.data.event_dict["task_id"] + if not self.event.scan or not self.event.build: + return TaskResults( + success=True, + details={ + "msg": f"Scan {task_id} not found or not associated with a Copr build", + }, + ) + + if not self.job_config: + return TaskResults( + success=True, + details={ + "msg": ( + f"No job configuration found for OpenScanHub task in {self.project.repo}" + ), + }, + ) + + return None + + +@configured_as(job_type=JobType.copr_build) +@reacts_to(openscanhub.task.Finished) +class CoprOpenScanHubTaskFinishedHandler( + CoprOpenScanHubAbstractHandler, +): + event: openscanhub.task.Finished + task_name = TaskName.openscanhub_task_finished + def get_number_of_new_findings_identified(self) -> Optional[int]: """ Downloads a JSON file from the task issues added URL and @@ -100,24 +152,101 @@ def get_number_of_new_findings_identified(self) -> Optional[int]: logger.error("The response is not a valid JSON format.") return None + def run(self) -> TaskResults: + self.check_scan_and_build() + external_links = {"OpenScanHub task": self.event.scan.url} + if self.event.status == openscanhub.task.Status.success: + state = BaseCommitStatus.success + number_of_new_findings = self.get_number_of_new_findings_identified() + base_description = "Scan in OpenScanHub is finished." + + if number_of_new_findings is None: + description = ( + f"{base_description} We were not able to analyse the findings; " + f"please check the URL." + ) + external_links.update({"Added issues": self.get_issues_added_url()}) + elif number_of_new_findings > 0: + description = ( + f"{base_description} {number_of_new_findings} new findings identified." + ) + external_links.update({"Added issues": self.get_issues_added_url()}) + self.event.scan.set_issues_added_count(number_of_new_findings) + else: + description = f"{base_description} No new findings identified." + self.event.scan.set_issues_added_count(number_of_new_findings) + + self.event.scan.set_status(OSHScanStatus.succeeded) + self.event.scan.set_issues_added_url(self.event.issues_added_url) + self.event.scan.set_issues_fixed_url(self.event.issues_fixed_url) + self.event.scan.set_scan_results_url(self.event.scan_results_url) + else: + state = BaseCommitStatus.neutral + description = f"Scan in OpenScanHub is finished in a {self.event.status} state." + if self.event.status == openscanhub.task.Status.cancel: + self.event.scan.set_status(OSHScanStatus.canceled) + else: + self.event.scan.set_status(OSHScanStatus.failed) + + self.get_helper().report( + state=state, + description=description, + url=get_openscanhub_info_url(self.event.scan.id), + links_to_external_services=external_links, + ) + + return TaskResults( + success=True, + details={}, + ) + + +@configured_as(job_type=JobType.copr_build) +@reacts_to(openscanhub.task.Started) +class CoprOpenScanHubTaskStartedHandler( + CoprOpenScanHubAbstractHandler, +): + task_name = TaskName.openscanhub_task_started -class CoprOpenScanHubAbstractHandler(OpenScanHubAbstractHandler): def __init__(self, **kwargs): super().__init__(**kwargs) + self.event: openscanhub.task.Started = self.data.to_event() - def get_helper(self) -> CoprOpenScanHubHelper: - build_helper = CoprBuildJobHelper( + def run(self) -> TaskResults: + self.check_scan_and_build() + + state = BaseCommitStatus.running + description = "Scan in OpenScanHub has started." + self.event.scan.set_status(OSHScanStatus.running) + + self.get_helper().report( + state=state, + description=description, + url=self.event.scan.url, + ) + + return TaskResults( + success=True, + details={}, + ) + + +class KojiOpenScanHubAbstractHandler(OpenScanHubAbstractHandler): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_helper(self) -> KojiOpenScanHubHelper: + build_helper = KojiBuildJobHelper( service_config=self.service_config, package_config=self.package_config, project=self.project, metadata=self.data, db_project_event=self.data.db_project_event, job_config=self.job_config, - celery_task=self.celery_task, ) - return CoprOpenScanHubHelper( - copr_build_helper=build_helper, + return KojiOpenScanHubHelper( + koji_build_helper=build_helper, build=self.event.build, ) @@ -127,7 +256,7 @@ def check_scan_and_build(self): return TaskResults( success=True, details={ - "msg": f"Scan {task_id} not found or not associated with a Copr build", + "msg": f"Scan {task_id} not found or not associated with a Koji build", }, ) @@ -144,14 +273,45 @@ def check_scan_and_build(self): return None -@configured_as(job_type=JobType.copr_build) +@configured_as(job_type=JobType.koji_build) @reacts_to(openscanhub.task.Finished) -class CoprOpenScanHubTaskFinishedHandler( - CoprOpenScanHubAbstractHandler, +class KojiOpenScanHubTaskFinishedHandler( + KojiOpenScanHubAbstractHandler, ): event: openscanhub.task.Finished task_name = TaskName.openscanhub_task_finished + def get_number_of_new_findings_identified(self) -> Optional[int]: + """ + Downloads a JSON file from the task issues added URL and + returns the number of items in the 'defects' array. + + Returns: + Optional[int]: Number of items in the 'defects' array, + or None if not found or on error. + """ + url = self.event.issues_added_url + logger.info(f"About to get the number of new findings identified by the scan from {url}.") + + try: + with requests.get(url, timeout=10) as response: + response.raise_for_status() + data = response.json() + + defects = data.get("defects") + if defects is None: + logger.debug("No 'defects' array found in the JSON data.") + return None + + return len(defects) + + except requests.exceptions.RequestException as e: + logger.error(f"Error while downloading the JSON file: {e}") + return None + except json.JSONDecodeError: + logger.error("The response is not a valid JSON format.") + return None + def run(self) -> TaskResults: self.check_scan_and_build() external_links = {"OpenScanHub task": self.event.scan.url} @@ -201,10 +361,10 @@ def run(self) -> TaskResults: ) -@configured_as(job_type=JobType.copr_build) +@configured_as(job_type=JobType.koji_build) @reacts_to(openscanhub.task.Started) -class CoprOpenScanHubTaskStartedHandler( - CoprOpenScanHubAbstractHandler, +class KojiOpenScanHubTaskStartedHandler( + KojiOpenScanHubAbstractHandler, ): task_name = TaskName.openscanhub_task_started From 179e4a9b5789470f4d80cb63fc329fa0cbe48df8 Mon Sep 17 00:00:00 2001 From: Siteshwar Vashisht Date: Tue, 25 Mar 2025 12:53:17 +0100 Subject: [PATCH 5/7] Trigger OpenScanHub scans on Koji builds Signed-off-by: Siteshwar Vashisht --- packit_service/worker/handlers/koji.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packit_service/worker/handlers/koji.py b/packit_service/worker/handlers/koji.py index 0d4c508ba..5620da964 100644 --- a/packit_service/worker/handlers/koji.py +++ b/packit_service/worker/handlers/koji.py @@ -36,6 +36,7 @@ AbstractProjectObjectDbType, KojiBuildTargetModel, ProjectEventModel, + ProjectEventModelType, ) from packit_service.service.urls import ( get_koji_build_info_url, @@ -65,6 +66,7 @@ from packit_service.worker.handlers.mixin import GetKojiBuildJobHelperMixin from packit_service.worker.helpers.build.koji_build import KojiBuildJobHelper from packit_service.worker.helpers.fedora_ci import FedoraCIHelper +from packit_service.worker.helpers.open_scan_hub import KojiOpenScanHubHelper from packit_service.worker.helpers.sidetag import SidetagHelper from packit_service.worker.mixin import ( ConfigFromEventMixin, @@ -293,7 +295,7 @@ def notify_about_failure_if_configured( @reacts_to_as_fedora_ci(event=koji.result.Task) -class KojiTaskReportDownstreamHandler(AbstractKojiTaskReportHandler): +class KojiTaskReportDownstreamHandler(AbstractKojiTaskReportHandler, GetKojiBuildJobHelperMixin): task_name = TaskName.downstream_koji_scratch_build_report _helper: Optional[FedoraCIHelper] = None @@ -314,6 +316,23 @@ def report(self, description: str, commit_status: BaseCommitStatus, url: str): url=url, ) + if ( + not KojiOpenScanHubHelper.osh_disabled() + and self.db_project_event.type == ProjectEventModelType.pull_request + and self.build.target == "fedora-rawhide-x86_64" + ): + try: + KojiOpenScanHubHelper( + koji_build_helper=self.koji_build_helper, + build=self.build, + ).handle_scan() + except Exception as ex: + # sentry_integration.send_to_sentry(ex) + logger.debug( + f"Handling the scan raised an exception: {ex}. Skipping " + f"as this is only experimental functionality for now.", + ) + def notify_about_failure_if_configured( self, packit_dashboard_url: str, external_dashboard_url: str, logs_url: str ): From a07361fa884b3f09486a0bac96e5eafed6506e04 Mon Sep 17 00:00:00 2001 From: Siteshwar Vashisht Date: Wed, 26 Mar 2025 13:07:47 +0100 Subject: [PATCH 6/7] Allow using KojiBuildTargetModel with OpenScanHub ... related events. Signed-off-by: Siteshwar Vashisht --- packit_service/events/openscanhub/abstract.py | 9 ++++++--- packit_service/worker/handlers/open_scan_hub.py | 8 ++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packit_service/events/openscanhub/abstract.py b/packit_service/events/openscanhub/abstract.py index 265d9ba96..a9655a96f 100644 --- a/packit_service/events/openscanhub/abstract.py +++ b/packit_service/events/openscanhub/abstract.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: MIT from logging import getLogger -from typing import Optional +from typing import Optional, Union from ogr.abstract import GitProject @@ -10,6 +10,7 @@ from packit_service.models import ( AbstractProjectObjectDbType, CoprBuildTargetModel, + KojiBuildTargetModel, OSHScanModel, ProjectEventModel, ) @@ -33,7 +34,7 @@ def __init__( self.commit_sha = commit_sha self.scan = OSHScanModel.get_by_task_id(task_id) - self.build: Optional[CoprBuildTargetModel] = None + self.build: Optional[Union[CoprBuildTargetModel, KojiBuildTargetModel]] = None if not self.scan: logger.warning( f"Scan with id {task_id} not found in the database." @@ -60,7 +61,9 @@ def __init__( # and have to be serialized to be later found in the # event metadata self.commit_sha = project_event.commit_sha if not self.commit_sha else self.commit_sha - self.identifier = identifier or self.build.identifier + self.identifier = identifier + if not self.identifier and isinstance(self.build, CoprBuildTargetModel): + self.identifier = self.build.identifier def get_db_project_object(self) -> Optional[AbstractProjectObjectDbType]: return self.build.get_project_event_object() diff --git a/packit_service/worker/handlers/open_scan_hub.py b/packit_service/worker/handlers/open_scan_hub.py index ce3a685a4..7512cf381 100644 --- a/packit_service/worker/handlers/open_scan_hub.py +++ b/packit_service/worker/handlers/open_scan_hub.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: MIT import json import logging -from typing import Optional, Union +from typing import Optional, Union, cast import requests from packit.config import ( @@ -10,7 +10,7 @@ ) from packit_service.events import openscanhub -from packit_service.models import OSHScanStatus +from packit_service.models import CoprBuildTargetModel, KojiBuildTargetModel, OSHScanStatus from packit_service.service.urls import get_openscanhub_info_url from packit_service.worker.checker.abstract import Checker from packit_service.worker.checker.open_scan_hub import IsEventForJob, RawhideX86Target @@ -87,7 +87,7 @@ def get_helper(self) -> CoprOpenScanHubHelper: return CoprOpenScanHubHelper( copr_build_helper=build_helper, - build=self.event.build, + build=cast(CoprBuildTargetModel, self.event.build), ) def check_scan_and_build(self): @@ -247,7 +247,7 @@ def get_helper(self) -> KojiOpenScanHubHelper: return KojiOpenScanHubHelper( koji_build_helper=build_helper, - build=self.event.build, + build=cast(KojiBuildTargetModel, self.event.build), ) def check_scan_and_build(self): From 4951fb5d734a1c66851c347a460ed8d0540ca22d Mon Sep 17 00:00:00 2001 From: Siteshwar Vashisht Date: Thu, 17 Apr 2025 02:28:49 +0200 Subject: [PATCH 7/7] Add tests for OpenScanHub Koji integration Signed-off-by: Siteshwar Vashisht --- packit_service/events/openscanhub/abstract.py | 4 +- tests/unit/test_koji_open_scan_hub.py | 388 ++++++++++++++++++ 2 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_koji_open_scan_hub.py diff --git a/packit_service/events/openscanhub/abstract.py b/packit_service/events/openscanhub/abstract.py index a9655a96f..fcafee035 100644 --- a/packit_service/events/openscanhub/abstract.py +++ b/packit_service/events/openscanhub/abstract.py @@ -42,13 +42,13 @@ def __init__( " and should have been associated with the copr build.", ) return - + # TODO: How to handle koji builds here? if hasattr(self.scan, "copr_build_target"): self.build = self.scan.copr_build_target else: self.build = self.scan.koji_build_target - + if not self.build: logger.warning( f"Scan with id {task_id} not associated with a build." diff --git a/tests/unit/test_koji_open_scan_hub.py b/tests/unit/test_koji_open_scan_hub.py new file mode 100644 index 000000000..aad45fd7d --- /dev/null +++ b/tests/unit/test_koji_open_scan_hub.py @@ -0,0 +1,388 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import datetime +import json + +import pytest +from celery.canvas import group as celery_group +from flexmock import flexmock +from packit.api import PackitAPI +from packit.config import ( + CommonPackageConfig, + JobConfig, + JobConfigTriggerType, + JobType, + PackageConfig, +) + +from packit_service.events import koji, openscanhub +from packit_service.models import ( + OSHScanModel, + ProjectEventModelType, +) +from packit_service.worker.handlers import KojiOpenScanHubTaskFinishedHandler +from packit_service.worker.handlers.koji import KojiOpenScanHubHelper +from packit_service.worker.helpers import open_scan_hub +from packit_service.worker.helpers.build import KojiBuildJobHelper +from packit_service.worker.jobs import SteveJobs +from packit_service.worker.monitoring import Pushgateway +from packit_service.worker.reporting import BaseCommitStatus +from packit_service.worker.tasks import ( + run_openscanhub_task_finished_handler, + run_openscanhub_task_started_handler, +) +from tests.spellbook import DATA_DIR, get_parameters_from_results + + +@pytest.fixture() +def openscanhub_task_finished_event(): + with open(DATA_DIR / "fedmsg" / "open_scan_hub_task_finished.json") as outfile: + return json.load(outfile) + + +@pytest.fixture() +def openscanhub_task_started_event(): + with open(DATA_DIR / "fedmsg" / "open_scan_hub_task_started.json") as outfile: + return json.load(outfile) + + +@pytest.fixture() +def prepare_openscanhub_db_and_handler( + add_pull_request_event_with_sha_123456, +): + db_project_object, db_project_event = add_pull_request_event_with_sha_123456 + db_build = ( + flexmock( + build_id="55", + identifier=None, + status="success", + build_submitted_time=datetime.datetime.utcnow(), + target="the-target", + owner="the-owner", + project_name="the-namespace-repo_name-5", + commit_sha="123456", + project_event=flexmock(), + srpm_build=flexmock(url=None) + .should_receive("set_url") + .with_args("https://some.host/my.srpm") + .mock(), + ) + .should_receive("get_project_event_object") + .and_return(db_project_object) + .mock() + .should_receive("get_project_event_model") + .and_return(db_project_event) + .mock() + ) + + flexmock(celery_group).should_receive("apply_async") + scan_mock = flexmock( + id=123, + koji_build_target=db_build, + url="https://openscanhub.fedoraproject.org/task/17514/", + set_issues_added_url=lambda _: None, + set_issues_fixed_url=lambda _: None, + set_scan_results_url=lambda _: None, + ) + flexmock(OSHScanModel).should_receive("get_by_task_id").and_return(scan_mock) + flexmock(Pushgateway).should_receive("push").and_return() + yield scan_mock + + +@pytest.mark.parametrize( + "build_models", + [ + [ + ( + "abcdef", + [flexmock(identifier=None, get_srpm_build=lambda: flexmock(url="base-srpm-url"))], + ) + ], + [ + ("abcdef", []), + ( + "fedcba", + [flexmock(identifier=None, get_srpm_build=lambda: flexmock(url="base-srpm-url"))], + ), + ], + ], +) +def test_handle_scan(build_models): + srpm_mock = flexmock(url="https://some-url/my-srpm.src.rpm") + flexmock(koji.result.Build).should_receive("from_event_dict").and_return( + flexmock(chroot="fedora-rawhide-x86_64", build_id="123", pr_id=12), + ) + flexmock(open_scan_hub).should_receive("download_file").twice().and_return(True) + + # for commit_sha, models in build_models: + # flexmock(KojiBuildTargetModel).should_receive("get_all_by").with_args( + # commit_sha=commit_sha, + # project_name="commit-project", + # owner="user-123", + # target="fedora-rawhide-x86_64", + # status=BuildStatus.success, + # ).and_return(models).once() + + flexmock(PackitAPI).should_receive("run_osh_build").once().and_return( + 'some\nmultiline\noutput\n{"id": 123}\nand\nmore\n{"url": "scan-url"}\n', + ) + + flexmock(KojiBuildJobHelper).should_receive("_report") + package_config = flexmock( + get_job_views=lambda: [ + flexmock( + type=JobType.koji_build, + trigger=JobConfigTriggerType.commit, + branch="main", + project="commit-project", + owner="user-123", + identifier=None, + ), + ], + ) + + project = flexmock( + get_pr=lambda pr_id: flexmock( + target_branch="main", + target_branch_head_commit="abcdef", + ), + get_commits=lambda ref: ["abcdef", "fedcba"], + ) + + KojiOpenScanHubHelper( + build=flexmock( + id=1, + get_srpm_build=lambda: srpm_mock, + target="fedora-rawhide-x86_64", + scan=None, + get_project_event_model=lambda: flexmock( + type=ProjectEventModelType.pull_request, + get_project_event_object=lambda: flexmock(), + ), + ) + .should_receive("add_scan_transaction") + .once() + .and_return(flexmock()) + .mock(), + koji_build_helper=KojiBuildJobHelper( + service_config=flexmock(), + package_config=package_config, + project=project, + metadata=flexmock(pr_id=12), + db_project_event=flexmock(get_project_event_object=lambda: None), + job_config=flexmock(identifier=None), + ), + ).handle_scan() + + +@pytest.mark.parametrize( + "job_config_type,job_config_trigger,job_config_targets,scan_status,num_of_handlers", + [ + ( + JobType.koji_build, + JobConfigTriggerType.commit, + ["fedora-rawhide-x86_64"], + openscanhub.task.Status.success, + 0, + ), + ( + JobType.koji_build, + JobConfigTriggerType.pull_request, + ["fedora-stable"], + openscanhub.task.Status.success, + 0, + ), + # ( + # JobType.koji_build, + # JobConfigTriggerType.pull_request, + # ["fedora-rawhide-x86_64"], + # openscanhub.task.Status.success, + # 1, + # ), + # ( + # JobType.koji_build, + # JobConfigTriggerType.pull_request, + # ["fedora-rawhide-x86_64"], + # openscanhub.task.Status.fail, + # 1, + # ), + # ( + # JobType.koji_build, + # JobConfigTriggerType.pull_request, + # ["fedora-rawhide-x86_64"], + # openscanhub.task.Status.cancel, + # 1, + # ), + ( + JobType.koji_build, + JobConfigTriggerType.commit, + ["fedora-rawhide-x86_64"], + openscanhub.task.Status.interrupt, + 0, + ), + ], +) +def test_handle_scan_task_finished( + openscanhub_task_finished_event, + prepare_openscanhub_db_and_handler, + job_config_type, + job_config_trigger, + job_config_targets, + scan_status, + num_of_handlers, +): + flexmock(openscanhub.task.Finished).should_receive( + "get_packages_config", + ).and_return( + PackageConfig( + jobs=[ + JobConfig( + type=job_config_type, + trigger=job_config_trigger, + packages={ + "package": CommonPackageConfig( + _targets=job_config_targets, + specfile_path="test.spec", + ), + }, + ), + ], + packages={"package": CommonPackageConfig()}, + ), + ) + + scan_mock = prepare_openscanhub_db_and_handler + openscanhub_task_finished_event["status"] = scan_status + processing_results = SteveJobs().process_message(openscanhub_task_finished_event) + assert len(processing_results) == num_of_handlers + + if processing_results: + links_to_external_services = { + "OpenScanHub task": "https://openscanhub.fedoraproject.org/task/17514/" + } + if scan_status == openscanhub.task.Status.success: + state = BaseCommitStatus.success + description = "Scan in OpenScanHub is finished. 2 new findings identified." + flexmock(scan_mock).should_receive("set_status").with_args( + "succeeded", + ).once() + flexmock(scan_mock).should_receive("set_issues_added_count").with_args(2).once() + flexmock(KojiOpenScanHubTaskFinishedHandler).should_receive( + "get_number_of_new_findings_identified" + ).and_return(2) + links_to_external_services.update( + { + "Added issues": ( + "https://openscanhub.fedoraproject.org/task/15649/log/added.html" + ), + } + ) + elif scan_status == openscanhub.task.Status.cancel: + state = BaseCommitStatus.neutral + description = f"Scan in OpenScanHub is finished in a {scan_status} state." + flexmock(scan_mock).should_receive("set_status").with_args( + "canceled", + ).once() + else: + state = BaseCommitStatus.neutral + description = f"Scan in OpenScanHub is finished in a {scan_status} state." + flexmock(scan_mock).should_receive("set_status").with_args("failed").once() + if num_of_handlers == 1: + # one handler is always skipped because it is for fedora-stable -> + # no rawhide build + flexmock(KojiOpenScanHubHelper).should_receive("report").with_args( + state=state, + description=description, + url="/jobs/openscanhub/123", + links_to_external_services=links_to_external_services, + ).once().and_return() + + for sub_results in processing_results: + event_dict, job, job_config, package_config = get_parameters_from_results( + [sub_results], + ) + assert json.dumps(event_dict) + + run_openscanhub_task_finished_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + ) + + +@pytest.mark.parametrize( + "job_config_type,job_config_trigger,job_config_targets,num_of_handlers", + [ + ( + JobType.koji_build, + JobConfigTriggerType.commit, + ["fedora-rawhide-x86_64"], + 0, + ), + ( + JobType.koji_build, + JobConfigTriggerType.pull_request, + ["fedora-stable"], + 0, + ), + # ( + # JobType.koji_build, + # JobConfigTriggerType.pull_request, + # ["fedora-rawhide-x86_64"], + # 1, + # ), + ], +) +def test_handle_scan_task_started( + openscanhub_task_started_event, + prepare_openscanhub_db_and_handler, + job_config_type, + job_config_trigger, + job_config_targets, + num_of_handlers, +): + flexmock(openscanhub.task.Started).should_receive( + "get_packages_config", + ).and_return( + PackageConfig( + jobs=[ + JobConfig( + type=job_config_type, + trigger=job_config_trigger, + packages={ + "package": CommonPackageConfig( + _targets=job_config_targets, + specfile_path="test.spec", + ), + }, + ), + ], + packages={"package": CommonPackageConfig()}, + ), + ) + + scan_mock = prepare_openscanhub_db_and_handler + processing_results = SteveJobs().process_message(openscanhub_task_started_event) + assert len(processing_results) == num_of_handlers + + if processing_results: + if num_of_handlers == 1: + flexmock(scan_mock).should_receive("set_status").with_args("running").once() + flexmock(KojiOpenScanHubHelper).should_receive("report").with_args( + state=BaseCommitStatus.running, + description="Scan in OpenScanHub has started.", + url="https://openscanhub.fedoraproject.org/task/17514/", + ).once().and_return() + + for sub_results in processing_results: + event_dict, job, job_config, package_config = get_parameters_from_results( + [sub_results], + ) + assert json.dumps(event_dict) + + run_openscanhub_task_started_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + )