From fcdcc4bc8d69b0b08fa5cfe49f421b0718b028e3 Mon Sep 17 00:00:00 2001 From: Jan Dobes Date: Fri, 4 Sep 2026 16:38:23 +0200 Subject: [PATCH] feat(database): prepare tables for storing affected package information - enum of all current package architectures - evr table for unique epoch+version+release tuples, same design as in vmaas - new partitioned table for system_vulnerabilities details - extension of system_vulnerable_package where the schema already works well with the new feature RHINENG-28511 --- .../upgrade_scripts/168-add-arch-enum.sql | 1 + .../upgrade_scripts/169-add-evr-table.sql | 8 +++ ...-add-system-vulnerabilities-pkg-detail.sql | 43 ++++++++++++ .../171-add-system-vulnerable-package-evr.sql | 25 +++++++ ...validate-system-vulnerable-package-evr.sql | 18 +++++ database/schema/ve_db_postgresql.sql | 68 ++++++++++++++++++- tests/data/truncate_dev_data.sql | 2 +- tests/taskomatic_tests/test_db_metrics.py | 2 +- 8 files changed, 164 insertions(+), 3 deletions(-) create mode 100644 database/schema/upgrade_scripts/168-add-arch-enum.sql create mode 100644 database/schema/upgrade_scripts/169-add-evr-table.sql create mode 100644 database/schema/upgrade_scripts/170-add-system-vulnerabilities-pkg-detail.sql create mode 100644 database/schema/upgrade_scripts/171-add-system-vulnerable-package-evr.sql create mode 100644 database/schema/upgrade_scripts/172-validate-system-vulnerable-package-evr.sql diff --git a/database/schema/upgrade_scripts/168-add-arch-enum.sql b/database/schema/upgrade_scripts/168-add-arch-enum.sql new file mode 100644 index 000000000..1c01e4ca7 --- /dev/null +++ b/database/schema/upgrade_scripts/168-add-arch-enum.sql @@ -0,0 +1 @@ +CREATE TYPE arch AS ENUM ('aarch64', 'i386', 'i686', 'noarch', 'ppc', 'ppc64', 'ppc64le', 's390', 's390x', 'src', 'x86_64'); diff --git a/database/schema/upgrade_scripts/169-add-evr-table.sql b/database/schema/upgrade_scripts/169-add-evr-table.sql new file mode 100644 index 000000000..b60a3fb07 --- /dev/null +++ b/database/schema/upgrade_scripts/169-add-evr-table.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS evr ( + id SERIAL, + epoch INTEGER NOT NULL, + version TEXT NOT NULL, + release TEXT NOT NULL, + PRIMARY KEY (id), + UNIQUE (epoch, version, release) +) TABLESPACE pg_default; diff --git a/database/schema/upgrade_scripts/170-add-system-vulnerabilities-pkg-detail.sql b/database/schema/upgrade_scripts/170-add-system-vulnerabilities-pkg-detail.sql new file mode 100644 index 000000000..ac4ea1f94 --- /dev/null +++ b/database/schema/upgrade_scripts/170-add-system-vulnerabilities-pkg-detail.sql @@ -0,0 +1,43 @@ +CREATE TABLE IF NOT EXISTS system_vulnerabilities_pkg_detail ( + rh_account_id BIGINT NOT NULL, + system_vulnerability_id BIGINT NOT NULL, + package_name_id BIGINT NOT NULL, + affected_evr_id INTEGER NOT NULL, + affected_arch arch NOT NULL, + fixed_evr_id INTEGER, + fixed_arch arch, + CONSTRAINT package_name_id + FOREIGN KEY (package_name_id) + REFERENCES package_name (id), + CONSTRAINT affected_evr_id + FOREIGN KEY (affected_evr_id) + REFERENCES evr (id), + CONSTRAINT fixed_evr_id + FOREIGN KEY (fixed_evr_id) + REFERENCES evr (id) +) PARTITION BY HASH (rh_account_id); + +CREATE OR REPLACE FUNCTION create_sys_vuln_pkg_detail_partitions(parts INTEGER) + RETURNS VOID AS +$$ +DECLARE + I INTEGER; +BEGIN + I := 0; + WHILE I < parts + LOOP + EXECUTE 'CREATE TABLE IF NOT EXISTS system_vulnerabilities_pkg_detail_' || text(I) || + ' PARTITION OF system_vulnerabilities_pkg_detail FOR VALUES WITH ( MODULUS ' || text(parts) || + ', REMAINDER ' || text(I) || ') TABLESPACE pg_default;'; + EXECUTE 'ALTER TABLE system_vulnerabilities_pkg_detail_' || text(I) || + ' ADD PRIMARY KEY (system_vulnerability_id, package_name_id);'; + EXECUTE 'ALTER TABLE system_vulnerabilities_pkg_detail_' || text(I) || ' ADD CONSTRAINT system_vulnerabilities_pkg_detail_' || + text(I) || '_system_vulnerability_id_fkey FOREIGN KEY (system_vulnerability_id) REFERENCES ' + 'system_vulnerabilities_active_' || text(I) || '(id) ON DELETE CASCADE;'; + I = I + 1; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +SELECT create_sys_vuln_pkg_detail_partitions(256); +DROP FUNCTION create_sys_vuln_pkg_detail_partitions; diff --git a/database/schema/upgrade_scripts/171-add-system-vulnerable-package-evr.sql b/database/schema/upgrade_scripts/171-add-system-vulnerable-package-evr.sql new file mode 100644 index 000000000..5c02b7356 --- /dev/null +++ b/database/schema/upgrade_scripts/171-add-system-vulnerable-package-evr.sql @@ -0,0 +1,25 @@ +ALTER TABLE system_vulnerable_package ADD COLUMN affected_evr_id INTEGER; + +ALTER TABLE system_vulnerable_package ADD COLUMN affected_arch arch; + +-- cannot add NOT VALID foreign key on partitioned table "system_vulnerable_package" referencing relation "evr" +-- DETAIL: This feature is not yet supported on partitioned tables. +-- This is why adding it for each partition separately +CREATE OR REPLACE FUNCTION add_sys_vuln_pkg_affected_evr_constraints(parts INTEGER) + RETURNS VOID AS +$$ +DECLARE + I INTEGER; +BEGIN + I := 0; + WHILE I < parts + LOOP + EXECUTE 'ALTER TABLE system_vulnerable_package_' || text(I) || ' ADD CONSTRAINT system_vulnerable_package_' || text(I) || + '_affected_evr_id_fkey FOREIGN KEY (affected_evr_id) REFERENCES evr (id) NOT VALID;'; + I = I + 1; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +SELECT add_sys_vuln_pkg_affected_evr_constraints(256); +DROP FUNCTION add_sys_vuln_pkg_affected_evr_constraints; diff --git a/database/schema/upgrade_scripts/172-validate-system-vulnerable-package-evr.sql b/database/schema/upgrade_scripts/172-validate-system-vulnerable-package-evr.sql new file mode 100644 index 000000000..7befd83e2 --- /dev/null +++ b/database/schema/upgrade_scripts/172-validate-system-vulnerable-package-evr.sql @@ -0,0 +1,18 @@ +CREATE OR REPLACE FUNCTION validate_sys_vuln_pkg_affected_evr_constraints(parts INTEGER) + RETURNS VOID AS +$$ +DECLARE + I INTEGER; +BEGIN + I := 0; + WHILE I < parts + LOOP + EXECUTE 'ALTER TABLE system_vulnerable_package_' || text(I) || ' VALIDATE CONSTRAINT system_vulnerable_package_' || + text(I) || '_affected_evr_id_fkey;'; + I = I + 1; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +SELECT validate_sys_vuln_pkg_affected_evr_constraints(256); +DROP FUNCTION validate_sys_vuln_pkg_affected_evr_constraints; diff --git a/database/schema/ve_db_postgresql.sql b/database/schema/ve_db_postgresql.sql index f86093e9e..83a171a3a 100644 --- a/database/schema/ve_db_postgresql.sql +++ b/database/schema/ve_db_postgresql.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS db_version ( ) TABLESPACE pg_default; -- set the schema version directly in the insert statement here!! -INSERT INTO db_version (name, version) VALUES ('schema_version', 167); +INSERT INTO db_version (name, version) VALUES ('schema_version', 172); -- INSERT INTO db_version (name, version) VALUES ('schema_version', :schema_version); @@ -123,6 +123,13 @@ $delete_system$ CREATE COLLATION IF NOT EXISTS numeric (provider = icu, locale = 'en-u-kn-true'); +-- ---------------------------------------------------------------------------- +-- Enums +-- ---------------------------------------------------------------------------- + +CREATE TYPE arch AS ENUM ('aarch64', 'i386', 'i686', 'noarch', 'ppc', 'ppc64', 'ppc64le', 's390', 's390x', 'src', 'x86_64'); + + -- ---------------------------------------------------------------------------- -- Tables -- ---------------------------------------------------------------------------- @@ -654,6 +661,61 @@ CREATE TABLE IF NOT EXISTS package_name ( PRIMARY KEY (id) ) TABLESPACE pg_default; +-- epoch, version, release table +CREATE TABLE IF NOT EXISTS evr ( + id SERIAL, + epoch INTEGER NOT NULL, + version TEXT NOT NULL, + release TEXT NOT NULL, + PRIMARY KEY (id), + UNIQUE (epoch, version, release) +) TABLESPACE pg_default; + +-- system vulnerability package detail table +CREATE TABLE IF NOT EXISTS system_vulnerabilities_pkg_detail ( + rh_account_id BIGINT NOT NULL, + system_vulnerability_id BIGINT NOT NULL, + package_name_id BIGINT NOT NULL, + affected_evr_id INTEGER NOT NULL, + affected_arch arch NOT NULL, + fixed_evr_id INTEGER, + fixed_arch arch, + CONSTRAINT package_name_id + FOREIGN KEY (package_name_id) + REFERENCES package_name (id), + CONSTRAINT affected_evr_id + FOREIGN KEY (affected_evr_id) + REFERENCES evr (id), + CONSTRAINT fixed_evr_id + FOREIGN KEY (fixed_evr_id) + REFERENCES evr (id) +) PARTITION BY HASH (rh_account_id); + +CREATE OR REPLACE FUNCTION create_sys_vuln_pkg_detail_partitions(parts INTEGER) + RETURNS VOID AS +$$ +DECLARE + I INTEGER; +BEGIN + I := 0; + WHILE I < parts + LOOP + EXECUTE 'CREATE TABLE IF NOT EXISTS system_vulnerabilities_pkg_detail_' || text(I) || + ' PARTITION OF system_vulnerabilities_pkg_detail FOR VALUES WITH ( MODULUS ' || text(parts) || + ', REMAINDER ' || text(I) || ') TABLESPACE pg_default;'; + EXECUTE 'ALTER TABLE system_vulnerabilities_pkg_detail_' || text(I) || + ' ADD PRIMARY KEY (system_vulnerability_id, package_name_id);'; + EXECUTE 'ALTER TABLE system_vulnerabilities_pkg_detail_' || text(I) || ' ADD CONSTRAINT system_vulnerabilities_pkg_detail_' || + text(I) || '_system_vulnerability_id_fkey FOREIGN KEY (system_vulnerability_id) REFERENCES ' + 'system_vulnerabilities_active_' || text(I) || '(id) ON DELETE CASCADE;'; + I = I + 1; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +SELECT create_sys_vuln_pkg_detail_partitions(256); +DROP FUNCTION create_sys_vuln_pkg_detail_partitions; + -- cpe table CREATE TABLE IF NOT EXISTS cpe ( id BIGSERIAL, @@ -714,6 +776,8 @@ CREATE TABLE IF NOT EXISTS system_vulnerable_package ( rh_account_id BIGINT NOT NULL, system_id BIGINT NOT NULL, vulnerable_package_id BIGINT NOT NULL, + affected_evr_id INTEGER, + affected_arch arch, first_reported TIMESTAMP WITH TIME ZONE NOT NULL, CONSTRAINT rh_account_id FOREIGN KEY (rh_account_id) @@ -741,6 +805,8 @@ BEGIN EXECUTE 'ALTER TABLE system_vulnerable_package_' || text(I) || ' ADD PRIMARY KEY (id);'; EXECUTE 'ALTER TABLE system_vulnerable_package_' || text(I) || ' ADD CONSTRAINT system_vulnerable_package_' || text(I) || '_system_id_vuln_pkg_id_key UNIQUE (system_id, vulnerable_package_id);'; + EXECUTE 'ALTER TABLE system_vulnerable_package_' || text(I) || ' ADD CONSTRAINT system_vulnerable_package_' || text(I) || + '_affected_evr_id_fkey FOREIGN KEY (affected_evr_id) REFERENCES evr (id);'; EXECUTE 'CREATE INDEX ON system_vulnerable_package_' || text(I) || '(vulnerable_package_id);'; EXECUTE 'CREATE INDEX ON system_vulnerable_package_' || text(I) || '(rh_account_id);'; EXECUTE 'CREATE TRIGGER system_vulnerable_package_set_first_reported BEFORE INSERT ON system_vulnerable_package_' || text(I) || diff --git a/tests/data/truncate_dev_data.sql b/tests/data/truncate_dev_data.sql index 8e6f3a8c5..2b2d5c410 100644 --- a/tests/data/truncate_dev_data.sql +++ b/tests/data/truncate_dev_data.sql @@ -1,4 +1,4 @@ -TRUNCATE TABLE repo, system_repo, timestamp_kv, cve_account_data, cve_rule_mapping, playbook, notified_accounts, package_name, cpe, vulnerable_package, vulnerable_package_cve, system_vulnerable_package, system_cve_data; +TRUNCATE TABLE repo, system_repo, timestamp_kv, cve_account_data, cve_rule_mapping, playbook, notified_accounts, package_name, evr, cpe, vulnerable_package, vulnerable_package_cve, system_vulnerable_package, system_vulnerabilities_pkg_detail, system_cve_data; DELETE FROM system_vulnerabilities WHERE rh_account_id in (0, 1); -- truncating can't be restricted to selected partitions and truncating all takes too long DELETE FROM insights_rule; DELETE FROM system_platform; diff --git a/tests/taskomatic_tests/test_db_metrics.py b/tests/taskomatic_tests/test_db_metrics.py index a315724f0..804aeec12 100644 --- a/tests/taskomatic_tests/test_db_metrics.py +++ b/tests/taskomatic_tests/test_db_metrics.py @@ -20,4 +20,4 @@ def test_db_metrics(pg_db_conn, monkeypatch): # pylint: disable=unused-argument dm.run() assert dm.METRIC_SYSTEMS.collect()[0].samples[0].value == 33 # there are 33 systems in DB - assert len(dm.METRIC_TABLE_SIZE.collect()[0].samples) == 547 # number of tables in DB + assert len(dm.METRIC_TABLE_SIZE.collect()[0].samples) == 805 # number of tables in DB