diff --git a/config/jobs/cert-manager/cert-manager/master/cert-manager-master.yaml b/config/jobs/cert-manager/cert-manager/master/cert-manager-master.yaml index 0884bec2..30c59ae4 100644 --- a/config/jobs/cert-manager/cert-manager/master/cert-manager-master.yaml +++ b/config/jobs/cert-manager/cert-manager/master/cert-manager-master.yaml @@ -894,9 +894,10 @@ periodics: annotations: description: Runs a Trivy scan against the controller container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-master + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -927,16 +928,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: master - cron: 40 04-23/24 * * * + cron: 40 02-23/12 * * * - name: ci-cert-manager-master-trivy-test-acmesolver max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the acmesolver container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-master + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -967,16 +969,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: master - cron: 44 11-23/24 * * * + cron: 44 04-23/12 * * * - name: ci-cert-manager-master-trivy-test-startupapicheck max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the startupapicheck container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-master + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1007,16 +1010,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: master - cron: 48 18-23/24 * * * + cron: 48 06-23/12 * * * - name: ci-cert-manager-master-trivy-test-cainjector max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the cainjector container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-master + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1047,16 +1051,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: master - cron: 52 01-23/24 * * * + cron: 52 08-23/12 * * * - name: ci-cert-manager-master-trivy-test-webhook max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the webhook container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-master + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1087,4 +1092,4 @@ periodics: - org: cert-manager repo: cert-manager base_ref: master - cron: 56 08-23/24 * * * + cron: 56 10-23/12 * * * diff --git a/config/jobs/cert-manager/cert-manager/release-1.18/cert-manager-release-1.18.yaml b/config/jobs/cert-manager/cert-manager/release-1.18/cert-manager-release-1.18.yaml index 800561c0..7559f289 100644 --- a/config/jobs/cert-manager/cert-manager/release-1.18/cert-manager-release-1.18.yaml +++ b/config/jobs/cert-manager/cert-manager/release-1.18/cert-manager-release-1.18.yaml @@ -1123,9 +1123,10 @@ periodics: annotations: description: Runs a Trivy scan against the controller container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-release-1.18 + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1156,16 +1157,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: release-1.18 - cron: 57 19-23/24 * * * + cron: 57 03-23/12 * * * - name: ci-cert-manager-release-1.18-trivy-test-acmesolver max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the acmesolver container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-release-1.18 + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1196,16 +1198,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: release-1.18 - cron: 01 02-23/24 * * * + cron: 01 05-23/12 * * * - name: ci-cert-manager-release-1.18-trivy-test-startupapicheck max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the startupapicheck container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-release-1.18 + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1236,16 +1239,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: release-1.18 - cron: 05 09-23/24 * * * + cron: 05 07-23/12 * * * - name: ci-cert-manager-release-1.18-trivy-test-cainjector max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the cainjector container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-release-1.18 + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1276,16 +1280,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: release-1.18 - cron: 09 16-23/24 * * * + cron: 09 09-23/12 * * * - name: ci-cert-manager-release-1.18-trivy-test-webhook max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the webhook container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-release-1.18 + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1316,4 +1321,4 @@ periodics: - org: cert-manager repo: cert-manager base_ref: release-1.18 - cron: 13 23-23/24 * * * + cron: 13 11-23/12 * * * diff --git a/config/jobs/cert-manager/cert-manager/release-1.19/cert-manager-release-1.19.yaml b/config/jobs/cert-manager/cert-manager/release-1.19/cert-manager-release-1.19.yaml index d2b32725..e34c5507 100644 --- a/config/jobs/cert-manager/cert-manager/release-1.19/cert-manager-release-1.19.yaml +++ b/config/jobs/cert-manager/cert-manager/release-1.19/cert-manager-release-1.19.yaml @@ -992,9 +992,10 @@ periodics: annotations: description: Runs a Trivy scan against the controller container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-release-1.19 + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1025,16 +1026,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: release-1.19 - cron: 50 13-23/24 * * * + cron: 50 04-23/12 * * * - name: ci-cert-manager-release-1.19-trivy-test-acmesolver max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the acmesolver container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-release-1.19 + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1065,16 +1067,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: release-1.19 - cron: 54 20-23/24 * * * + cron: 54 06-23/12 * * * - name: ci-cert-manager-release-1.19-trivy-test-startupapicheck max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the startupapicheck container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-release-1.19 + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1105,16 +1108,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: release-1.19 - cron: 58 03-23/24 * * * + cron: 58 08-23/12 * * * - name: ci-cert-manager-release-1.19-trivy-test-cainjector max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the cainjector container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-release-1.19 + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1145,16 +1149,17 @@ periodics: - org: cert-manager repo: cert-manager base_ref: release-1.19 - cron: 02 10-23/24 * * * + cron: 02 10-23/12 * * * - name: ci-cert-manager-release-1.19-trivy-test-webhook max_concurrency: 2 decorate: true annotations: description: Runs a Trivy scan against the webhook container testgrid-alert-email: cert-manager-dev-alerts@googlegroups.com - testgrid-alert-stale-results-hours: "36" + testgrid-alert-stale-results-hours: "18" testgrid-create-job-group: "true" testgrid-dashboards: cert-manager-periodics-release-1.19 + testgrid-num-columns-recent: "1" testgrid-num-failures-to-alert: "1" labels: preset-dind-enabled: "true" @@ -1185,4 +1190,4 @@ periodics: - org: cert-manager repo: cert-manager base_ref: release-1.19 - cron: 06 17-23/24 * * * + cron: 06 00-23/12 * * * diff --git a/config/prowgen/pkg/configurers.go b/config/prowgen/pkg/configurers.go index c1b0c790..d04eefda 100644 --- a/config/prowgen/pkg/configurers.go +++ b/config/prowgen/pkg/configurers.go @@ -147,6 +147,15 @@ func addTestGridCustomFailuresToAlert(failuresToAlert int) JobConfigurer { } } +// addTestGridNumColumnsRecent changes the number of test results to considered when testgrid +// decides whether a test is "flaky" +// See https://github.com/kubernetes/test-infra/blob/737791c6e2ee79bdc8efce2195eb6d20ebb6eb04/testgrid/config.md#prow-job-configuration +func addTestGridNumColumnsRecent(numColumnsRecent int) JobConfigurer { + return func(job *Job) { + job.Annotations["testgrid-num-columns-recent"] = fmt.Sprintf("%d", numColumnsRecent) + } +} + // addTestGridStaleResultsAlert sets, in hours, the length of time before a job should be // considered stale. This guards against a job not running for whatever reason. func addTestGridStaleResultsAlert(hoursUntilStale int) JobConfigurer { diff --git a/config/prowgen/pkg/generators.go b/config/prowgen/pkg/generators.go index afc2bcd4..cb5a2f88 100644 --- a/config/prowgen/pkg/generators.go +++ b/config/prowgen/pkg/generators.go @@ -19,6 +19,7 @@ package pkg import ( "fmt" + "math" "strings" ) @@ -328,9 +329,12 @@ func UpgradeTest(ctx *ProwContext, k8sVersion string) *Job { // so e.g. if there's a vuln in the "controller" container we might never scan "ctl" container. // Instead, we generate a test for each container so it's obvious which ones have failures and it's easier to get results // for each container -func TrivyTest(ctx *ProwContext, containerName string) *Job { +// periodicity is the number of hours between runs of this job; this is used to calculate when the job should be considered stale +func TrivyTest(ctx *ProwContext, containerName string, periodicity int) *Job { containerName = strings.ToLower(containerName) + stale := math.Round(float64(periodicity) * 1.5) + job := jobTemplate( fmt.Sprintf("trivy-test-%s", containerName), fmt.Sprintf("Runs a Trivy scan against the %s container", containerName), @@ -342,9 +346,13 @@ func TrivyTest(ctx *ProwContext, containerName string) *Job { // Need to ensure that trivy tests send a failure email as soon as they fail since // they tend to be run relatively infrequently and a failure is important to address addTestGridCustomFailuresToAlert(1), + // Trivy tests are quite binary - either the scan passes or fails. + // Having a fixed test report as "flaky" isn't helpful, so set "num columns recent" to 1 + // so that the test should report as either passing or failing but not flaky. + addTestGridNumColumnsRecent(1), // Ask TestGrid to alert us if the job hasn't run in the last 36 hours. Sets // an upper limit on how regularly the job can be scheduled. - addTestGridStaleResultsAlert(36), + addTestGridStaleResultsAlert(int(stale)), ) makeJobs, cpuRequest := calculateMakeConcurrency("1000m") diff --git a/config/prowgen/prowspecs/specs.go b/config/prowgen/prowspecs/specs.go index 96963447..68d442a9 100644 --- a/config/prowgen/prowspecs/specs.go +++ b/config/prowgen/prowspecs/specs.go @@ -167,7 +167,8 @@ func (m *BranchSpec) GenerateJobFile() *pkg.JobFile { } for _, container := range m.containerNames { - m.prowContext.Periodics(pkg.TrivyTest(m.prowContext, container), 24) + periodicity := 12 + m.prowContext.Periodics(pkg.TrivyTest(m.prowContext, container, periodicity), periodicity) } return m.prowContext.JobFile()