Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,16 @@ schemas and the `paramify` CLI — not the internal code.

### Added

- **`azure_velero_backups` fetcher.** Reads Velero backup storage locations in Azure Blob Storage, from Azure APIs
only: each backup's phase, schedule, completion time and expiry, missed runs of configured expected schedules,
schedules past the recovery point objective, each restore's result, and whether the backup
storage is geo-replicated and encrypted with a customer-managed key. Missed runs come from the gaps between
Completed backups, with 10% grace, and only over the part of the lookback the backups are kept for, so a backup
Velero deleted at the end of its retention is not a missed run. A listed schedule is judged against its own
interval. Failed backups, incomplete snapshots and restores with errors are counted in full and again as
current (unresolved) failures, with planned failure tests matched by name and counted apart. Blob reads go only
to endpoints Resource Manager returned for in-scope accounts in the target subscription, with redirects refused,
retry dropped connections, and parse XML with defusedxml. Unread locations leave counts null rather than 0.
- `paramify upload --no-waf-check` turns off the check that drops a file holding `../`, `/etc/passwd`, `/bin/sh`,
`/bin/bash` or `<script` before the upload. Those strings were confirmed blocked on one host's WAF, but some hosts
exempt the upload routes, so CVE text in a findings file was dropped though the WAF would accept it. With the flag
Expand Down
1 change: 1 addition & 0 deletions fetchers/_categories/azure.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ requires:
- azure-mgmt-subscription
- azure-mgmt-web
- cryptography
- defusedxml
- msgraph-sdk
- microsoft-kiota-http

Expand Down
567 changes: 567 additions & 0 deletions fetchers/azure/velero_backups/fetcher.py

Large diffs are not rendered by default.

199 changes: 199 additions & 0 deletions fetchers/azure/velero_backups/fetcher.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
name: azure_velero_backups
version: 0.1.0
description: >-
Shows whether Velero backups stored in Azure Blob Storage succeed on schedule, expire on time and restore cleanly, and whether their storage is geo-replicated and encrypted with a customer key.
category: azure

supports_targets: true

runtime:
type: python
entry: fetcher.py

output:
type: json
path: azure_velero_backups.json
aggregation: per_target

config_schema:
container_names:
type: string
required: false
default: velero
env: AZURE_VELERO_CONTAINER_NAMES
description: >
Comma-separated blob container names to treat as Velero backup storage locations when backup_locations is not
set. Every in-scope storage account in the subscription is searched for containers with these names.
backup_locations:
type: string
required: false
env: AZURE_VELERO_BACKUP_LOCATIONS
description: >
Comma-separated Velero backup storage locations as account/container or account/container/prefix (the prefix
configured on the BackupStorageLocation). When set, only these are read and container_names is ignored. An
entry not in that form, or an account that is not an in-scope account in the subscription, fails the run.
success_window_hours:
type: integer
required: false
default: 26
env: AZURE_VELERO_SUCCESS_WINDOW_HOURS
description: >
How recent a schedule's newest Completed backup must be to count as a success within the window, for a schedule
not listed in expected_schedules. A listed schedule is judged against its own interval plus 10% (at least 2
hours), so a weekly schedule is not judged against a daily window. The default suits a daily schedule.
expiry_grace_hours:
type: integer
required: false
default: 24
env: AZURE_VELERO_EXPIRY_GRACE_HOURS
description: >
How long past its expiration a backup may stay in storage before it counts as expired but not deleted. Velero
removes expired backups on its own garbage-collection cycle.
expected_schedules:
type: string
required: false
env: AZURE_VELERO_EXPECTED_SCHEDULES
description: >
Comma-separated schedules that must be producing backups, as schedule:hours (the Velero schedule name and how
often it should produce one, such as daily:24). Velero does not write schedules to backup storage, so a schedule
that has never produced a backup is only visible when it is listed here. There is no default: while it is unset,
the expected-schedule and missed-run counts are null. An entry not in that form fails the run as bad_config.
rpo_hours:
type: integer
required: false
env: AZURE_VELERO_RPO_HOURS
description: >
The recovery point objective in hours: the longest a schedule may go without a Completed backup. There is no
default: while it is unset, schedules_exceeding_rpo is null. A value that is not a whole number above 0 fails the
run as bad_config and leaves it unset.
missed_run_lookback_days:
type: integer
required: false
default: 7
env: AZURE_VELERO_MISSED_RUN_LOOKBACK_DAYS
description: >
How many days of each expected schedule's periods to check for missed runs, up to 90. A value that is not a
whole number above 0 fails the run as bad_config.
deliberate_failure_tests:
type: string
required: false
env: AZURE_VELERO_DELIBERATE_FAILURE_TESTS
description: >
Comma-separated name patterns (* and ? wildcards) for backups and restores that are planned failure tests. A
failed backup or a restore with errors whose name matches is counted in deliberate_failure_tests and never as a
current failure. It stays in the totals, so history is kept.
restore_test_interval_days:
type: integer
required: false
env: AZURE_VELERO_RESTORE_TEST_INTERVAL_DAYS
description: >
The adopted interval, in days, within which a clean restore must have been recorded. There is no default:
while it is unset, clean_restore_within_interval is null rather than judged against an interval nobody adopted.
A value that is not a whole number above 0 fails the run as bad_config and leaves it unset.
resource_exclusions:
type: patterns
prefixes: [rg, location, id]
bare: false
required: false
env: AZURE_RESOURCE_EXCLUSIONS
description: >
Patterns that take storage accounts out of scope, as for the other Azure fetchers.

target_schema:
subscription_id:
type: string
required: false
env: AZURE_SUBSCRIPTION_ID
description: >
Azure subscription id to collect from. Optional — omit to use the first
enabled subscription the ambient credential can see; set one target per
subscription to fan out.
environment:
type: string
required: false
env: AZURE_ENVIRONMENT
description: Deployment environment label (e.g. prod) written into the evidence metadata.

secrets: [] # DefaultAzureCredential only — see _categories/azure.yaml

ksis:
- KSI-RPL-ABO # each schedule's newest successful backup against the window, and failed or partial backups
- KSI-RPL-TRC # restore results, and how long since the last clean restore
- KSI-RPL-ARP # whether backup storage is geo-replicated
- KSI-SVC-SIN # whether backup storage is encrypted with a customer-managed key
- KSI-SVC-RUD # backups kept past their expiration

evidence_set:
reference_id: EVD-AZURE-VELERO-BACKUPS
name: Azure Velero Backups and Restores
instructions: |
Script: azure/velero_backups/fetcher.py

Access: Reader on the subscription, and Storage Blob Data Reader on each Velero backup container, from a network the storage account's firewall allows. Without the data role or network access, the blob reads fail and the backup and restore counts read as null.

Commands Executed:
- subscriptions.list() — picks a subscription when none is set
- GET Microsoft.Storage/storageAccounts — each account's blob endpoint, replication and encryption key source
- GET storageAccounts/{account}/blobServices/default/containers (per account) — finds the Velero containers by name
- Blob List Blobs on backups/ and restores/ (per location) — every backup and restore Velero wrote
- Blob Get velero-backup.json (per backup) — the backup's phase, schedule, completion time, expiration and volume snapshot counts
- Blob Get restore-<name>-results.gz (per restore) — the restore's errors and warnings

Output: azure_velero_backups_<subscription>.json — each Velero backup storage location with its storage settings, every backup's outcome and expiry, and every restore's result. Schedules and restore status are not stored in backup storage, so schedules come from each backup's schedule label and a restore is judged by its results file.

Summary fields:

What was checked?
- backup_locations — Number of Velero backup storage locations found (a container with a configured name in an in-scope account, or an entry in backup_locations); null if the account or container lists could not be read
- backup_locations_read — Number of backup storage locations whose backups could be listed
- resource_exclusion_patterns — List of the resource_exclusions patterns as given
- resource_exclusion_patterns_unmatched — List of the patterns that matched nothing, such as a mistyped name; null when the storage account list could not be read
- excluded_storage_accounts — Number of storage accounts a pattern took out of scope
- in_scope_storage_accounts — Number of storage accounts searched for backup storage locations

Do backups run on schedule?
- backups_total — Number of backups in the backup storage locations; null unless every location was read
- backups_completed — Number of backups whose phase is Completed
- backups_partially_failed — Number of backups whose phase is PartiallyFailed
- backups_failed — Number of backups whose phase is Failed
- backups_other_phase — Number of backups in any other phase, such as InProgress
- backups_metadata_unreadable — Number of backups whose velero-backup.json could not be read or parsed
- backups_volume_snapshots_incomplete — Number of backups that completed fewer volume snapshots (native and CSI) than they attempted
- backups_failed_unresolved — Number of Failed or PartiallyFailed backups that are not deliberate failure tests and have no later Completed backup of the same schedule (a current failure); a backup with no schedule label stays unresolved, since a later manual backup may cover other namespaces; resolved failures stay in backups_failed and backups_partially_failed
- backups_volume_snapshots_incomplete_unresolved — Number of backups with incomplete volume snapshots that are not deliberate failure tests and have no later Completed backup of the same schedule that completed every snapshot it attempted; a backup with no schedule label stays unresolved
- schedules_observed — Number of schedules named by a backup's velero.io/schedule-name label
- schedules_with_success_within_window — Number of schedules whose newest Completed backup finished within the schedule's window (a success within the window): its interval plus 10% (at least 2 hours) when listed in expected_schedules, otherwise success_window_hours
- schedules_without_success_within_window — List of schedule names without a success within the window
- max_hours_since_scheduled_success — Hours since the newest Completed backup of the schedule that has gone longest without one; null if any schedule has no Completed backup
- success_window_hours — Hours a schedule not listed in expected_schedules may go since its newest Completed backup and still count as a success within the window

Is every expected backup happening?
- expected_schedules — Number of schedules listed in expected_schedules; null when none are listed
- expected_schedules_without_backups — Number of expected schedules with no backup at all in storage; null when expected_schedules is not set or any location could not be read
- expected_schedules_without_backups_names — List of the expected schedules with no backup at all
- missed_scheduled_runs — Number of missed runs across expected schedules, from the gaps between Completed backups and from the newest one to now: each gap longer than the interval plus 10% adds one missed run per whole interval. Only the part of missed_run_lookback_days that the schedule's backups are kept for is checked, so a backup Velero deleted when its retention ended is not a missed run. Null when expected_schedules is not set or any location could not be read
- schedules_with_missed_runs — Number of expected schedules with at least one missed run
- missed_run_lookback_days — Days of each expected schedule's periods checked for missed runs
- schedules_exceeding_rpo — Number of schedules (the expected schedules when listed, otherwise every schedule a backup names) whose newest Completed backup is older than rpo_hours, or that have none; null while rpo_hours is not set
- rpo_hours — The recovery point objective in hours, as configured; null when not set

Are expired backups removed?
- expired_backups_not_deleted — Number of backups still in storage more than expiry_grace_hours past their expiration
- expiry_grace_hours — Hours past its expiration a backup may stay before it counts as expired but not deleted

Do restores work?
- restores_total — Number of restores Velero recorded in the backup storage locations; null unless every location's restores were read
- restores_without_errors — Number of clean restores (a results file that lists no errors)
- restores_with_errors — Number of restores whose results file lists at least one error
- restores_with_errors_unresolved — Number of restores with errors that are not deliberate failure tests and have no later restore without errors (a current failure); a restore with errors and no readable date is never resolved
- restores_results_missing — Number of restores whose results file is missing or could not be read
- deliberate_failure_tests — Number of failed backups and restores with errors whose name matches a deliberate_failure_tests pattern; null unless every location's backups and restores were read
- deliberate_failure_test_patterns — List of the deliberate_failure_tests patterns as given
- restores_date_unknown — Number of restores whose results file was read but has no readable date; they never count as recent
- days_since_last_clean_restore — Days since the newest clean restore's results file was written; null if there is no clean restore
- restore_test_interval_days — The adopted restore-test interval in days, as configured; null when not set
- clean_restore_within_interval — Whether the newest clean restore is no older than restore_test_interval_days (false when there is none); null while no interval is set or any location's restores could not be read

Is backup storage protected?
- backup_locations_geo_replicated — Number of backup storage locations whose storage account is geo-replicated (GRS, RA-GRS, GZRS or RA-GZRS)
- backup_locations_customer_managed_key — Number of backup storage locations whose storage account encrypts with a customer-managed key from Key Vault
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ azure = [
"azure-keyvault-certificates>=4.11.3,<5",
# <7 is load-bearing: 7.0.0 ships no diagnostic_settings operations at all.
"azure-mgmt-monitor>=6.0.2,<7",
# Blob list responses (azure_velero_backups) are XML; parsed with entities and external references refused.
"defusedxml",
"azure-mgmt-loganalytics>=14.0.0,<15",
"azure-mgmt-recoveryservices>=4.1.0,<5",
"azure-mgmt-recoveryservicesbackup>=10.0.0,<12",
Expand Down
108 changes: 108 additions & 0 deletions tests/fixtures/velero/backups/manual-1/velero-backup.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
{
"kind": "Backup",
"apiVersion": "velero.io/v1",
"metadata": {
"name": "manual-1",
"namespace": "velero",
"uid": "b279ec05-9089-4661-a82d-095bf49f93db",
"resourceVersion": "2117",
"generation": 5,
"creationTimestamp": "2026-11-16T23:51:36Z",
"labels": {
"velero.io/storage-location": "default"
},
"annotations": {
"velero.io/resource-timeout": "10m0s",
"velero.io/source-cluster-k8s-gitversion": "v1.36.5",
"velero.io/source-cluster-k8s-major-version": "1",
"velero.io/source-cluster-k8s-minor-version": "36"
},
"managedFields": [
{
"manager": "velero",
"operation": "Update",
"apiVersion": "velero.io/v1",
"time": "2026-11-16T23:51:36Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:spec": {
".": {},
"f:hooks": {},
"f:includedNamespaces": {},
"f:metadata": {}
},
"f:status": {}
}
},
{
"manager": "velero-server",
"operation": "Update",
"apiVersion": "velero.io/v1",
"time": "2026-11-16T23:51:37Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:metadata": {
"f:annotations": {
".": {},
"f:velero.io/resource-timeout": {},
"f:velero.io/source-cluster-k8s-gitversion": {},
"f:velero.io/source-cluster-k8s-major-version": {},
"f:velero.io/source-cluster-k8s-minor-version": {}
},
"f:labels": {
".": {},
"f:velero.io/storage-location": {}
}
},
"f:spec": {
"f:csiSnapshotTimeout": {},
"f:defaultVolumesToFsBackup": {},
"f:itemOperationTimeout": {},
"f:snapshotMoveData": {},
"f:storageLocation": {},
"f:ttl": {}
},
"f:status": {
"f:expiration": {},
"f:formatVersion": {},
"f:hookStatus": {},
"f:phase": {},
"f:progress": {
".": {},
"f:itemsBackedUp": {},
"f:totalItems": {}
},
"f:startTimestamp": {},
"f:version": {}
}
}
}
]
},
"spec": {
"metadata": {},
"includedNamespaces": [
"demo"
],
"ttl": "720h0m0s",
"hooks": {},
"storageLocation": "default",
"defaultVolumesToFsBackup": false,
"csiSnapshotTimeout": "10m0s",
"itemOperationTimeout": "4h0m0s",
"snapshotMoveData": false
},
"status": {
"version": 1,
"formatVersion": "1.1.0",
"expiration": "2026-12-16T23:51:36Z",
"phase": "Completed",
"startTimestamp": "2026-11-16T23:51:36Z",
"completionTimestamp": "2026-11-16T23:51:37Z",
"progress": {
"totalItems": 10,
"itemsBackedUp": 10
},
"hookStatus": {}
}
}
Loading
Loading