diff --git a/CHANGELOG.md b/CHANGELOG.md index 01dee889..8a9a079d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,16 @@ schemas and the `paramify` CLI — not the internal code. ### Added +- **`azure_velero_backups` fetcher.** Reads Velero backup storage locations in Azure Blob Storage, from Azure APIs + only: each backup's phase, schedule, completion time and expiry, missed runs of configured expected schedules, + schedules past the recovery point objective, each restore's result, and whether the backup + storage is geo-replicated and encrypted with a customer-managed key. Missed runs come from the gaps between + Completed backups, with 10% grace, and only over the part of the lookback the backups are kept for, so a backup + Velero deleted at the end of its retention is not a missed run. A listed schedule is judged against its own + interval. Failed backups, incomplete snapshots and restores with errors are counted in full and again as + current (unresolved) failures, with planned failure tests matched by name and counted apart. Blob reads go only + to endpoints Resource Manager returned for in-scope accounts in the target subscription, with redirects refused, + retry dropped connections, and parse XML with defusedxml. Unread locations leave counts null rather than 0. - `paramify upload --no-waf-check` turns off the check that drops a file holding `../`, `/etc/passwd`, `/bin/sh`, `/bin/bash` or ` List[str]: + return [part.strip() for part in (value or "").split(",") if part.strip()] + + +def parse_locations(value: Optional[str]) -> Tuple[List[Dict[str, str]], List[str]]: + """`account/container[/prefix]` entries, and the entries that are not in that form.""" + locations, invalid = [], [] + for entry in _csv(value): + parts = entry.strip("/").split("/", 2) + if len(parts) < 2 or not all(parts[:2]): + invalid.append(entry) + continue + locations.append({"account": parts[0].lower(), "container": parts[1], "prefix": parts[2].strip("/") if len(parts) > 2 else ""}) + return locations, invalid + + +def parse_expected(value: Optional[str]) -> Tuple[Optional[List[Dict[str, Any]]], List[str]]: + """`name:hours` entries (the schedule and how often it should produce a backup); None when unset.""" + if not (value or "").strip(): + return None, [] + expected, invalid = [], [] + for entry in _csv(value): + name, _, hours = entry.rpartition(":") + if name and hours.isdigit() and int(hours) > 0: + expected.append({"name": name, "interval_hours": int(hours)}) + else: + invalid.append(entry) + return expected, invalid + + +def retention_hours(backups: List[Dict[str, Any]]) -> Optional[float]: + """The shortest retention among a schedule's backups (expiration minus completion), in hours; None when unknown.""" + spans = [(parse_time(b["expiration"]) - parse_time(b["completed"])).total_seconds() / 3600 + for b in backups if parse_time(b["expiration"]) and parse_time(b["completed"])] + spans = [s for s in spans if s > 0] + return min(spans) if spans else None + + +def count_missed(done: List[datetime], interval_hours: int, window_start: datetime, now: datetime) -> int: + """Missed runs from the gaps between Completed backups, and from the newest one to now. + + Each gap longer than the interval plus RUN_GRACE_FRACTION of it adds floor((gap - grace) / interval) missed runs. + The first gap starts at the newest Completed backup at or before window_start, or at window_start when there is + none, so a run due just inside the window is not lost; runs that fell due before window_start in that gap are + subtracted, so an outage that ended before the lookback does not count. A schedule with no Completed backup at + all misses every interval in the window. + """ + step = interval_hours * 3600 + window = (now - window_start).total_seconds() + if not done: + return int(window // step) + grace = step * RUN_GRACE_FRACTION + before = [t for t in done if t <= window_start] + points = [before[-1] if before else window_start] + [t for t in done if t > window_start] + [now] + missed = 0 + for i, (a, b) in enumerate(zip(points, points[1:])): + runs = math.floor(((b - a).total_seconds() - grace) / step) + if i == 0 and before: + runs -= math.floor((window_start - a).total_seconds() / step) # due before the window started + missed += max(0, runs) + return missed + + +def success_window_hours(interval_hours: Optional[int], settings: Dict[str, Any]) -> float: + """An expected schedule is judged against its own interval plus slack; any other schedule against success_window_hours.""" + if interval_hours: + return interval_hours + max(MIN_WINDOW_SLACK_HOURS, interval_hours // 10) + return settings["success_window_hours"] + + +def schedule_check(name: str, interval_hours: Optional[int], backups: List[Dict[str, Any]], settings: Dict[str, Any], + now: datetime) -> Dict[str, Any]: + """One schedule: its backups, newest success, missed runs over the lookback, and the recovery point objective. + + Missed runs are judged only over the part of the lookback the schedule's backups are still kept for, because Velero + deletes each backup when its retention ends and a deleted backup is not a missed run. + """ + mine = [b for b in backups if b["schedule"] == name] + done = sorted(t for t in (parse_time(b["completed"]) for b in mine if b["phase"] == COMPLETED) if t) + hours = round((now - done[-1]).total_seconds() / 3600, 1) if done else None + missed = periods = None + retention = retention_hours(mine) + lookback = settings["missed_run_lookback_days"] * 24 + checked = min(lookback, retention) if retention else lookback + if interval_hours: + periods = int(checked // interval_hours) + missed = count_missed(done, interval_hours, now - timedelta(hours=checked), now) + rpo = settings["rpo_hours"] + window = success_window_hours(interval_hours, settings) + return {"name": name, "expected_interval_hours": interval_hours, "backups": len(mine), "backups_completed": len(done), + "newest_success": done[-1].strftime("%Y-%m-%dT%H:%M:%SZ") if done else None, "hours_since_success": hours, + "success_window_hours": window, "success_within_window": hours is not None and hours <= window, + "retention_hours": round(retention, 1) if retention else None, "hours_checked_for_missed_runs": round(checked, 1), + "periods_checked": periods, "missed_runs": missed, + "exceeds_rpo": (hours is None or hours > rpo) if rpo else None} + + +def schedule_checks(backups: List[Dict[str, Any]], settings: Dict[str, Any], now: datetime) -> List[Dict[str, Any]]: + """Expected schedules when listed (so one with no backups still appears); otherwise every schedule a backup names.""" + if settings["expected_schedules"] is not None: + return [schedule_check(e["name"], e["interval_hours"], backups, settings, now) for e in settings["expected_schedules"]] + return [schedule_check(n, None, backups, settings, now) for n in sorted({b["schedule"] for b in backups if b["schedule"]})] + + +def account_record(raw: Dict[str, Any]) -> Dict[str, Any]: + props = raw.get("properties") or {} + sku = ((raw.get("sku") or {}).get("name") or "") + key_source = (props.get("encryption") or {}).get("keySource") + return { + "id": raw.get("id"), + "name": (raw.get("name") or "").lower(), + "resource_group": resource_group_from_id(raw.get("id")), + "location": raw.get("location"), + "sku": sku or None, + "geo_replicated": sku.lower() in GEO_REPLICATED_SKUS if sku else None, + "encryption_key_source": key_source, + "customer_managed_key": (key_source or "").lower() == "microsoft.keyvault" if key_source else None, + "blob_endpoint": (props.get("primaryEndpoints") or {}).get("blob"), + } + + +def backup_record(name: str, raw: Dict[str, Any], now: datetime, grace_hours: int) -> Dict[str, Any]: + status = raw.get("status") or {} + labels = (raw.get("metadata") or {}).get("labels") or {} + expiration = parse_time(status.get("expiration")) + attempted = sum(int(status.get(k) or 0) for k in ("volumeSnapshotsAttempted", "csiVolumeSnapshotsAttempted")) + completed = sum(int(status.get(k) or 0) for k in ("volumeSnapshotsCompleted", "csiVolumeSnapshotsCompleted")) + return { + "name": name, + "schedule": labels.get(SCHEDULE_LABEL) or None, + "phase": status.get("phase"), + "started": status.get("startTimestamp"), + "completed": status.get("completionTimestamp"), + "expiration": status.get("expiration"), + "errors": status.get("errors") or 0, + "warnings": status.get("warnings") or 0, + "volume_snapshots_attempted": attempted, + "volume_snapshots_completed": completed, + "volume_snapshots_incomplete": completed < attempted, + "expired_not_deleted": bool(expiration and (now - expiration).total_seconds() > grace_hours * 3600), + } + + +def count_messages(section: Any) -> int: + """Messages in a Velero results section: lists under 'velero' and 'cluster', and per namespace.""" + if not isinstance(section, dict): + return 0 + total = sum(len(section.get(k) or []) for k in ("velero", "cluster")) + return total + sum(len(v or []) for v in (section.get("namespaces") or {}).values()) + + +def restore_record(name: str, results: Optional[Dict[str, Any]], written: Optional[str]) -> Dict[str, Any]: + if results is None: + return {"name": name, "results_read": False, "errors": None, "warnings": None, "clean": None, "results_written": written} + errors, warnings = count_messages(results.get("errors")), count_messages(results.get("warnings")) + return {"name": name, "results_read": True, "errors": errors, "warnings": warnings, "clean": errors == 0, "results_written": written} + + +def is_deliberate(name: str, patterns: List[str]) -> bool: + """A backup or restore whose name matches a deliberate_failure_tests pattern is a planned failure test.""" + return any(fnmatch.fnmatchcase(name, p) for p in patterns) + + +def classify_history(backups: List[Dict[str, Any]], restores: List[Dict[str, Any]], patterns: List[str]) -> None: + """Mark each failure as a deliberate test, resolved by a later success, or current; records are updated in place. + + A Failed or PartiallyFailed backup is resolved once its schedule has a Completed backup that finished after it; a + backup with no schedule is never resolved, since a later manual backup may cover other namespaces. A scheduled + backup with incomplete volume snapshots is + resolved once a later Completed backup of the same schedule completed every snapshot it attempted. A restore with + errors is resolved once a later restore completed without errors. A failure without a readable time is never + resolved. History is kept: the totals still count every record in storage. + """ + def when(b): + return parse_time(b["completed"]) or parse_time(b["started"]) + + newest_done: Dict[Optional[str], datetime] = {} + newest_full: Dict[Optional[str], datetime] = {} + for x in backups: + done = parse_time(x["completed"]) if x["phase"] == COMPLETED else None + if done: + newest_done[x["schedule"]] = max(done, newest_done.get(x["schedule"], done)) + if not x["volume_snapshots_incomplete"]: + newest_full[x["schedule"]] = max(done, newest_full.get(x["schedule"], done)) + for b in backups: + b["deliberate_test"] = is_deliberate(b["name"], patterns) + t = when(b) + later_done, later_full = newest_done.get(b["schedule"]), newest_full.get(b["schedule"]) + # A backup with no schedule label is never resolved: a later manual backup may cover other namespaces entirely. + scheduled = b["schedule"] is not None + b["failure_resolved"] = bool(scheduled and t and later_done and later_done > t) if b["phase"] in (FAILED, PARTIALLY_FAILED) else None + b["snapshots_resolved"] = bool(scheduled and t and later_full and later_full > t) if b["volume_snapshots_incomplete"] else None + newest_clean = max((c for c in (parse_time(r["results_written"]) for r in restores if r["clean"]) if c), default=None) + for r in restores: + r["deliberate_test"] = is_deliberate(r["name"], patterns) + t = parse_time(r["results_written"]) + r["error_resolved"] = bool(t and newest_clean and newest_clean > t) if r["clean"] is False else None + + +def summarize(locations: List[Dict[str, Any]], settings: Dict[str, Any], now: datetime, discovered: bool = True) -> Dict[str, Any]: + """Counts are null, not 0, unless discovery finished and every location's backups were read.""" + read = [loc for loc in locations if loc["backups_read"]] + complete = discovered and len(read) == len(locations) + backups = [b for loc in read for b in loc["backups"]] + restores = [r for loc in locations if loc["restores_read"] for r in loc["restores"]] + restores_complete = discovered and all(loc["restores_read"] for loc in locations) + patterns = settings.get("deliberate_failure_tests") or [] + classify_history(backups, restores, patterns) + latest: Dict[str, datetime] = {} + for b in backups: + when = parse_time(b["completed"]) + if b["schedule"] and b["phase"] == COMPLETED and when and (b["schedule"] not in latest or when > latest[b["schedule"]]): + latest[b["schedule"]] = when + schedules = sorted({b["schedule"] for b in backups if b["schedule"]}) + hours = {s: (now - latest[s]).total_seconds() / 3600 for s in latest} + expected = settings["expected_schedules"] + windows = {e["name"]: success_window_hours(e["interval_hours"], settings) for e in expected or []} + recent = [s for s in schedules if s in hours and hours[s] <= windows.get(s, settings["success_window_hours"])] + clean_times = [parse_time(r["results_written"]) for r in restores if r["clean"]] + clean_times = [t for t in clean_times if t] + checks = schedule_checks(backups, settings, now) + + def known(value): + return value if complete else None + + return { + "backup_locations": len(locations) if discovered else None, + "backup_locations_read": len(read), + "backup_locations_geo_replicated": sum(1 for loc in locations if loc["geo_replicated"]), + "backup_locations_customer_managed_key": sum(1 for loc in locations if loc["customer_managed_key"]), + "backups_total": known(len(backups)), + "backups_completed": known(sum(1 for b in backups if b["phase"] == COMPLETED)), + "backups_partially_failed": known(sum(1 for b in backups if b["phase"] == PARTIALLY_FAILED)), + "backups_failed": known(sum(1 for b in backups if b["phase"] == FAILED)), + "backups_other_phase": known(sum(1 for b in backups if b["phase"] not in (COMPLETED, PARTIALLY_FAILED, FAILED))), + "backups_metadata_unreadable": known(sum(loc["backups_unreadable"] for loc in read)), + "backups_volume_snapshots_incomplete": known(sum(1 for b in backups if b["volume_snapshots_incomplete"])), + "backups_failed_unresolved": known(sum(1 for b in backups if b["failure_resolved"] is False and not b["deliberate_test"])), + "backups_volume_snapshots_incomplete_unresolved": known(sum(1 for b in backups + if b["snapshots_resolved"] is False and not b["deliberate_test"])), + "expired_backups_not_deleted": known(sum(1 for b in backups if b["expired_not_deleted"])), + "schedules_observed": known(len(schedules)), + "schedules_with_success_within_window": known(len(recent)), + "schedules_without_success_within_window": known(sorted(set(schedules) - set(recent))), + "max_hours_since_scheduled_success": known(round(max(hours.values()), 1) if hours and set(hours) == set(schedules) else None), + "success_window_hours": settings["success_window_hours"], + "expected_schedules": len(expected) if expected is not None else None, + "expected_schedules_without_backups": known(sum(1 for c in checks if c["backups"] == 0)) if expected is not None else None, + "expected_schedules_without_backups_names": known(sorted(c["name"] for c in checks if c["backups"] == 0)) if expected is not None else None, + "missed_scheduled_runs": known(sum(c["missed_runs"] for c in checks)) if expected is not None else None, + "schedules_with_missed_runs": known(sum(1 for c in checks if c["missed_runs"])) if expected is not None else None, + "missed_run_lookback_days": settings["missed_run_lookback_days"], + "rpo_hours": settings["rpo_hours"], + "schedules_exceeding_rpo": known(sum(1 for c in checks if c["exceeds_rpo"])) if settings["rpo_hours"] else None, + "expiry_grace_hours": settings["expiry_grace_hours"], + "restores_total": len(restores) if restores_complete else None, + "restores_without_errors": sum(1 for r in restores if r["clean"]) if restores_complete else None, + "restores_with_errors": sum(1 for r in restores if r["clean"] is False) if restores_complete else None, + "restores_with_errors_unresolved": sum(1 for r in restores if r["error_resolved"] is False and not r["deliberate_test"]) + if restores_complete else None, + "restores_results_missing": sum(1 for r in restores if not r["results_read"]) if restores_complete else None, + "deliberate_failure_tests": sum(1 for b in backups if b["deliberate_test"] and (b["failure_resolved"] is not None or b["snapshots_resolved"] is not None)) + + sum(1 for r in restores if r["deliberate_test"] and r["clean"] is False) if complete and restores_complete else None, + "deliberate_failure_test_patterns": patterns, + "days_since_last_clean_restore": int((now - max(clean_times)).total_seconds() // 86400) if restores_complete and clean_times else None, + "restores_date_unknown": sum(1 for r in restores if r["results_read"] and not parse_time(r["results_written"])) if restores_complete else None, + "restore_test_interval_days": settings.get("restore_test_interval_days"), + "clean_restore_within_interval": (bool(clean_times) and (now - max(clean_times)).days <= settings["restore_test_interval_days"]) + if restores_complete and settings.get("restore_test_interval_days") else None, + } + + +def settings_from_env(collector: Optional[Collector] = None) -> Dict[str, Any]: + locations, invalid = parse_locations(os.environ.get(LOCATIONS_ENV)) + if collector is not None and invalid: + collector.record(f"config {LOCATIONS_ENV}", ValueError(f"{LOCATIONS_ENV} entries must be account/container[/prefix]: {invalid!r}")) + raw = (os.environ.get(INTERVAL_ENV) or "").strip() + interval = int(raw) if raw.isdigit() and int(raw) > 0 else None # unset until the offering adopts a restore-test interval + if collector is not None and raw and interval is None: + collector.record(f"config {INTERVAL_ENV}", ValueError(f"{INTERVAL_ENV}={raw!r} is not a whole number above 0; left unset")) + expected, bad = parse_expected(os.environ.get(EXPECTED_ENV)) + if collector is not None and bad: + collector.record(f"config {EXPECTED_ENV}", ValueError(f"{EXPECTED_ENV} entries must be schedule:hours: {bad!r}")) + expected = None # a malformed list is unknown, never an empty one that reads as nothing expected + raw_rpo = (os.environ.get(RPO_ENV) or "").strip() + rpo = int(raw_rpo) if raw_rpo.isdigit() and int(raw_rpo) > 0 else None # unset until the offering's objective is configured + if collector is not None and raw_rpo and rpo is None: + collector.record(f"config {RPO_ENV}", ValueError(f"{RPO_ENV}={raw_rpo!r} is not a whole number above 0; left unset")) + raw_lb = (os.environ.get(MISSED_LOOKBACK_ENV) or "").strip() + lookback = min(int(raw_lb), MAX_MISSED_LOOKBACK_DAYS) if raw_lb.isdigit() and int(raw_lb) > 0 else DEFAULT_MISSED_LOOKBACK_DAYS + if collector is not None and raw_lb and not (raw_lb.isdigit() and int(raw_lb) > 0): + collector.record(f"config {MISSED_LOOKBACK_ENV}", ValueError(f"{MISSED_LOOKBACK_ENV}={raw_lb!r} is not a whole number above 0")) + return { + "deliberate_failure_tests": _csv(os.environ.get(DELIBERATE_ENV)), + "expected_schedules": expected, + "rpo_hours": rpo, + "missed_run_lookback_days": lookback, + "restore_test_interval_days": interval, + "container_names": _csv(os.environ.get(CONTAINERS_ENV)) or list(DEFAULT_CONTAINERS), + "explicit_locations": locations, + "success_window_hours": int_config(WINDOW_ENV, DEFAULT_WINDOW_HOURS), + "expiry_grace_hours": int_config(GRACE_ENV, DEFAULT_GRACE_HOURS), + } + + +# --- blob data plane --- + +class BlobReader: + """Read-only Blob REST calls, only to blob endpoints ARM returned for this subscription's accounts.""" + + def __init__(self, client, allowed_endpoints: List[str]): + self.session, self.cred = client.session, client.cred + self.allowed = {urlsplit(e).netloc.lower() for e in allowed_endpoints if e} + self._token: Any = None + + def token(self) -> str: + if self._token is None or self._token.expires_on - 120 < time.time(): + self._token = self.cred.get_token(f"{STORAGE_RESOURCE}/.default") + return self._token.token + + def _get(self, endpoint: str, container: str, path: str = "", params: Optional[Dict[str, str]] = None): + url = endpoint.rstrip("/") + "/" + quote(container) + (("/" + quote(path)) if path else "") + target = urlsplit(url) + if (target.scheme != "https" or target.netloc.lower() not in self.allowed or target.username or target.password + or target.fragment or not target.path.startswith("/" + quote(container))): + raise ValueError("Blob read origin is untrusted") + timeouts = 0 + for attempt in range(MAX_RETRIES + 1): + try: + response = self.session.get(url, params=params, timeout=60, allow_redirects=False, headers={ + "Authorization": f"Bearer {self.token()}", "x-ms-version": BLOB_VERSION}) + except requests.ConnectionError: + if attempt == MAX_RETRIES: + raise + time.sleep(float(min(2 ** attempt, MAX_RETRY_WAIT))) + continue + except requests.Timeout: # reads only, so a resend is safe; bounded as in azure_rest + if timeouts == MAX_TIMEOUT_RETRIES or attempt == MAX_RETRIES: + raise + timeouts += 1 + time.sleep(float(min(2 ** attempt, MAX_RETRY_WAIT))) + continue + if response.status_code not in RETRY_STATUSES or attempt == MAX_RETRIES: + break + time.sleep(retry_wait(response, attempt)) + if response.status_code >= 300: + raise ArmError(response.status_code, response.headers.get("x-ms-error-code") or "BlobError", (response.text or "")[:300]) + return response + + def list(self, endpoint: str, container: str, prefix: str) -> List[Dict[str, Optional[str]]]: + """Every blob under `prefix`: name and last-modified time, across every page.""" + blobs, marker = [], None + while True: + params = {"restype": "container", "comp": "list", "prefix": prefix, "maxresults": str(PAGE_SIZE)} + if marker: + params["marker"] = marker + root = ET.fromstring(self._get(endpoint, container, params=params).content) + for blob in root.iter("Blob"): + blobs.append({"name": blob.findtext("Name"), "last_modified": blob.findtext("Properties/Last-Modified")}) + marker = root.findtext("NextMarker") + if not marker: + return blobs + + def read_json(self, endpoint: str, container: str, name: str) -> Any: + body = self._get(endpoint, container, name).content + return json.loads(gzip.decompress(body) if name.endswith(".gz") else body) + + +def _http_time(value: Optional[str]) -> Optional[str]: + """RFC 1123 Last-Modified to ISO 8601 UTC.""" + if not value: + return None + try: + return datetime.strptime(value, "%a, %d %b %Y %H:%M:%S %Z").replace(tzinfo=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + except ValueError: + return None + + +# --- collection --- + +def read_location(reader: BlobReader, account: Dict[str, Any], container: str, prefix: str, settings: Dict[str, Any], + now: datetime, collector: Collector) -> Dict[str, Any]: + base = (prefix + "/") if prefix else "" + label = f"{account['name']}/{container}{'/' + prefix if prefix else ''}" + loc = {"storage_account_id": account["id"], "storage_account": account["name"], "container": container, "prefix": prefix or None, + "sku": account["sku"], "geo_replicated": account["geo_replicated"], "encryption_key_source": account["encryption_key_source"], + "customer_managed_key": account["customer_managed_key"], "backups_read": False, "backups_unreadable": 0, "backups": [], + "restores_read": False, "restores": []} + listed = collector.guard(f"blob.list backups ({label})", lambda: reader.list(account["blob_endpoint"], container, base + "backups/"), default=None) + if listed is not None: + names = sorted({b["name"][len(base + "backups/"):].split("/", 1)[0] for b in listed + if b["name"].endswith("/velero-backup.json")}) + + def one(n): + try: + return backup_record(n, reader.read_json(account["blob_endpoint"], container, f"{base}backups/{n}/velero-backup.json"), + now, settings["expiry_grace_hours"]) + except Exception as exc: # noqa: BLE001 - recorded; the backup counts as unreadable + collector.record(f"blob.get velero-backup.json ({label}/{n})", exc) + return None + with ThreadPoolExecutor(max_workers=WORKERS) as pool: + records = list(pool.map(one, names)) + loc["backups"] = [r for r in records if r is not None] + loc["backups_unreadable"] = sum(1 for r in records if r is None) + loc["backups_read"] = True + restored = collector.guard(f"blob.list restores ({label})", lambda: reader.list(account["blob_endpoint"], container, base + "restores/"), default=None) + if restored is not None: + by_name: Dict[str, Optional[Dict[str, Optional[str]]]] = {} + for b in restored: + n = b["name"][len(base + "restores/"):].split("/", 1)[0] + by_name.setdefault(n, None) + if b["name"].endswith(f"/restore-{n}-results.gz"): + by_name[n] = b + + def restore(item): + n, blob = item + if blob is None: + return restore_record(n, None, None) + try: + return restore_record(n, reader.read_json(account["blob_endpoint"], container, blob["name"]), _http_time(blob["last_modified"])) + except Exception as exc: # noqa: BLE001 - recorded; the restore result stays unknown + collector.record(f"blob.get restore results ({label}/{n})", exc) + return restore_record(n, None, _http_time(blob["last_modified"])) + with ThreadPoolExecutor(max_workers=WORKERS) as pool: + loc["restores"] = list(pool.map(restore, sorted(by_name.items()))) + loc["restores_read"] = True + return loc + + +def collect(client, subscription_id: str, collector: Collector, exclusions: Optional[ResourceExclusions] = None, + now: Optional[datetime] = None) -> tuple: + exclusions = exclusions if exclusions is not None else ResourceExclusions() + now = now or datetime.now(timezone.utc) + settings = settings_from_env(collector) + raw = collector.guard("storage.storageAccounts.list", lambda: client.list( + f"/subscriptions/{subscription_id}/providers/Microsoft.Storage/storageAccounts", STORAGE_API), default=None) + accounts_listed = raw is not None + discovered = accounts_listed + accounts = sorted((account_record(a) for a in raw or []), key=lambda a: arm_id_key(a["id"])) + accounts, excluded = exclusions.split(accounts, "storage_account") + by_name = {a["name"]: a for a in accounts} + targets: List[Tuple[Dict[str, Any], str, str]] = [] + if settings["explicit_locations"]: + for want in settings["explicit_locations"]: + account = by_name.get(want["account"]) + if account is None: + discovered = False + collector.record(f"config {LOCATIONS_ENV}", ValueError( + f"storage account {want['account']!r} is not an in-scope account in this subscription")) + continue + targets.append((account, want["container"], want["prefix"])) + else: + wanted = {c.lower() for c in settings["container_names"]} + for account in accounts: + containers = collector.guard(f"storage.blobContainers.list ({account['name']})", lambda a=account: client.list( + f"{a['id']}/blobServices/default/containers", STORAGE_API), default=None) + discovered = discovered and containers is not None + for c in containers or []: + if (c.get("name") or "").lower() in wanted: + targets.append((account, c["name"], "")) + reader = BlobReader(client, [a["blob_endpoint"] for a in accounts]) + locations = [] + for account, container, prefix in targets: + if not account["blob_endpoint"]: + discovered = False + collector.record(f"storage.blobEndpoint ({account['name']})", ValueError("the account reports no blob endpoint")) + continue + locations.append(read_location(reader, account, container, prefix, settings, now, collector)) + exclusions.record_invalid(collector) + if not accounts_listed: + exclusions.mark_unread() + exclusions.warn_unmatched(logger) + all_backups = [b for loc in locations if loc["backups_read"] for b in loc["backups"]] + results = {"backup_locations": locations, "evaluated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "schedules": schedule_checks(all_backups, settings, now), + "container_names": settings["container_names"], "explicit_locations": settings["explicit_locations"], + "excluded": sorted(excluded, key=lambda e: arm_id_key(e.get("id")))} + summary = {**summarize(locations, settings, now, discovered), + **exclusions.summary(excluded, ("storage_account",), {"storage_accounts": len(accounts) if accounts_listed else None})} + return results, summary + + +def collect_subscription(client, subscription_id: str, collector: Collector) -> tuple: + return collect(client, subscription_id, collector, ResourceExclusions.from_env()) + + +def main() -> int: + return run_subscription(fetcher=NAME, logger=logger, collect=collect_subscription) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/fetchers/azure/velero_backups/fetcher.yaml b/fetchers/azure/velero_backups/fetcher.yaml new file mode 100644 index 00000000..4f575391 --- /dev/null +++ b/fetchers/azure/velero_backups/fetcher.yaml @@ -0,0 +1,199 @@ +name: azure_velero_backups +version: 0.1.0 +description: >- + Shows whether Velero backups stored in Azure Blob Storage succeed on schedule, expire on time and restore cleanly, and whether their storage is geo-replicated and encrypted with a customer key. +category: azure + +supports_targets: true + +runtime: + type: python + entry: fetcher.py + +output: + type: json + path: azure_velero_backups.json + aggregation: per_target + +config_schema: + container_names: + type: string + required: false + default: velero + env: AZURE_VELERO_CONTAINER_NAMES + description: > + Comma-separated blob container names to treat as Velero backup storage locations when backup_locations is not + set. Every in-scope storage account in the subscription is searched for containers with these names. + backup_locations: + type: string + required: false + env: AZURE_VELERO_BACKUP_LOCATIONS + description: > + Comma-separated Velero backup storage locations as account/container or account/container/prefix (the prefix + configured on the BackupStorageLocation). When set, only these are read and container_names is ignored. An + entry not in that form, or an account that is not an in-scope account in the subscription, fails the run. + success_window_hours: + type: integer + required: false + default: 26 + env: AZURE_VELERO_SUCCESS_WINDOW_HOURS + description: > + How recent a schedule's newest Completed backup must be to count as a success within the window, for a schedule + not listed in expected_schedules. A listed schedule is judged against its own interval plus 10% (at least 2 + hours), so a weekly schedule is not judged against a daily window. The default suits a daily schedule. + expiry_grace_hours: + type: integer + required: false + default: 24 + env: AZURE_VELERO_EXPIRY_GRACE_HOURS + description: > + How long past its expiration a backup may stay in storage before it counts as expired but not deleted. Velero + removes expired backups on its own garbage-collection cycle. + expected_schedules: + type: string + required: false + env: AZURE_VELERO_EXPECTED_SCHEDULES + description: > + Comma-separated schedules that must be producing backups, as schedule:hours (the Velero schedule name and how + often it should produce one, such as daily:24). Velero does not write schedules to backup storage, so a schedule + that has never produced a backup is only visible when it is listed here. There is no default: while it is unset, + the expected-schedule and missed-run counts are null. An entry not in that form fails the run as bad_config. + rpo_hours: + type: integer + required: false + env: AZURE_VELERO_RPO_HOURS + description: > + The recovery point objective in hours: the longest a schedule may go without a Completed backup. There is no + default: while it is unset, schedules_exceeding_rpo is null. A value that is not a whole number above 0 fails the + run as bad_config and leaves it unset. + missed_run_lookback_days: + type: integer + required: false + default: 7 + env: AZURE_VELERO_MISSED_RUN_LOOKBACK_DAYS + description: > + How many days of each expected schedule's periods to check for missed runs, up to 90. A value that is not a + whole number above 0 fails the run as bad_config. + deliberate_failure_tests: + type: string + required: false + env: AZURE_VELERO_DELIBERATE_FAILURE_TESTS + description: > + Comma-separated name patterns (* and ? wildcards) for backups and restores that are planned failure tests. A + failed backup or a restore with errors whose name matches is counted in deliberate_failure_tests and never as a + current failure. It stays in the totals, so history is kept. + restore_test_interval_days: + type: integer + required: false + env: AZURE_VELERO_RESTORE_TEST_INTERVAL_DAYS + description: > + The adopted interval, in days, within which a clean restore must have been recorded. There is no default: + while it is unset, clean_restore_within_interval is null rather than judged against an interval nobody adopted. + A value that is not a whole number above 0 fails the run as bad_config and leaves it unset. + resource_exclusions: + type: patterns + prefixes: [rg, location, id] + bare: false + required: false + env: AZURE_RESOURCE_EXCLUSIONS + description: > + Patterns that take storage accounts out of scope, as for the other Azure fetchers. + +target_schema: + subscription_id: + type: string + required: false + env: AZURE_SUBSCRIPTION_ID + description: > + Azure subscription id to collect from. Optional — omit to use the first + enabled subscription the ambient credential can see; set one target per + subscription to fan out. + environment: + type: string + required: false + env: AZURE_ENVIRONMENT + description: Deployment environment label (e.g. prod) written into the evidence metadata. + +secrets: [] # DefaultAzureCredential only — see _categories/azure.yaml + +ksis: + - KSI-RPL-ABO # each schedule's newest successful backup against the window, and failed or partial backups + - KSI-RPL-TRC # restore results, and how long since the last clean restore + - KSI-RPL-ARP # whether backup storage is geo-replicated + - KSI-SVC-SIN # whether backup storage is encrypted with a customer-managed key + - KSI-SVC-RUD # backups kept past their expiration + +evidence_set: + reference_id: EVD-AZURE-VELERO-BACKUPS + name: Azure Velero Backups and Restores + instructions: | + Script: azure/velero_backups/fetcher.py + + Access: Reader on the subscription, and Storage Blob Data Reader on each Velero backup container, from a network the storage account's firewall allows. Without the data role or network access, the blob reads fail and the backup and restore counts read as null. + + Commands Executed: + - subscriptions.list() — picks a subscription when none is set + - GET Microsoft.Storage/storageAccounts — each account's blob endpoint, replication and encryption key source + - GET storageAccounts/{account}/blobServices/default/containers (per account) — finds the Velero containers by name + - Blob List Blobs on backups/ and restores/ (per location) — every backup and restore Velero wrote + - Blob Get velero-backup.json (per backup) — the backup's phase, schedule, completion time, expiration and volume snapshot counts + - Blob Get restore--results.gz (per restore) — the restore's errors and warnings + + Output: azure_velero_backups_.json — each Velero backup storage location with its storage settings, every backup's outcome and expiry, and every restore's result. Schedules and restore status are not stored in backup storage, so schedules come from each backup's schedule label and a restore is judged by its results file. + + Summary fields: + + What was checked? + - backup_locations — Number of Velero backup storage locations found (a container with a configured name in an in-scope account, or an entry in backup_locations); null if the account or container lists could not be read + - backup_locations_read — Number of backup storage locations whose backups could be listed + - resource_exclusion_patterns — List of the resource_exclusions patterns as given + - resource_exclusion_patterns_unmatched — List of the patterns that matched nothing, such as a mistyped name; null when the storage account list could not be read + - excluded_storage_accounts — Number of storage accounts a pattern took out of scope + - in_scope_storage_accounts — Number of storage accounts searched for backup storage locations + + Do backups run on schedule? + - backups_total — Number of backups in the backup storage locations; null unless every location was read + - backups_completed — Number of backups whose phase is Completed + - backups_partially_failed — Number of backups whose phase is PartiallyFailed + - backups_failed — Number of backups whose phase is Failed + - backups_other_phase — Number of backups in any other phase, such as InProgress + - backups_metadata_unreadable — Number of backups whose velero-backup.json could not be read or parsed + - backups_volume_snapshots_incomplete — Number of backups that completed fewer volume snapshots (native and CSI) than they attempted + - backups_failed_unresolved — Number of Failed or PartiallyFailed backups that are not deliberate failure tests and have no later Completed backup of the same schedule (a current failure); a backup with no schedule label stays unresolved, since a later manual backup may cover other namespaces; resolved failures stay in backups_failed and backups_partially_failed + - backups_volume_snapshots_incomplete_unresolved — Number of backups with incomplete volume snapshots that are not deliberate failure tests and have no later Completed backup of the same schedule that completed every snapshot it attempted; a backup with no schedule label stays unresolved + - schedules_observed — Number of schedules named by a backup's velero.io/schedule-name label + - schedules_with_success_within_window — Number of schedules whose newest Completed backup finished within the schedule's window (a success within the window): its interval plus 10% (at least 2 hours) when listed in expected_schedules, otherwise success_window_hours + - schedules_without_success_within_window — List of schedule names without a success within the window + - max_hours_since_scheduled_success — Hours since the newest Completed backup of the schedule that has gone longest without one; null if any schedule has no Completed backup + - success_window_hours — Hours a schedule not listed in expected_schedules may go since its newest Completed backup and still count as a success within the window + + Is every expected backup happening? + - expected_schedules — Number of schedules listed in expected_schedules; null when none are listed + - expected_schedules_without_backups — Number of expected schedules with no backup at all in storage; null when expected_schedules is not set or any location could not be read + - expected_schedules_without_backups_names — List of the expected schedules with no backup at all + - missed_scheduled_runs — Number of missed runs across expected schedules, from the gaps between Completed backups and from the newest one to now: each gap longer than the interval plus 10% adds one missed run per whole interval. Only the part of missed_run_lookback_days that the schedule's backups are kept for is checked, so a backup Velero deleted when its retention ended is not a missed run. Null when expected_schedules is not set or any location could not be read + - schedules_with_missed_runs — Number of expected schedules with at least one missed run + - missed_run_lookback_days — Days of each expected schedule's periods checked for missed runs + - schedules_exceeding_rpo — Number of schedules (the expected schedules when listed, otherwise every schedule a backup names) whose newest Completed backup is older than rpo_hours, or that have none; null while rpo_hours is not set + - rpo_hours — The recovery point objective in hours, as configured; null when not set + + Are expired backups removed? + - expired_backups_not_deleted — Number of backups still in storage more than expiry_grace_hours past their expiration + - expiry_grace_hours — Hours past its expiration a backup may stay before it counts as expired but not deleted + + Do restores work? + - restores_total — Number of restores Velero recorded in the backup storage locations; null unless every location's restores were read + - restores_without_errors — Number of clean restores (a results file that lists no errors) + - restores_with_errors — Number of restores whose results file lists at least one error + - restores_with_errors_unresolved — Number of restores with errors that are not deliberate failure tests and have no later restore without errors (a current failure); a restore with errors and no readable date is never resolved + - restores_results_missing — Number of restores whose results file is missing or could not be read + - deliberate_failure_tests — Number of failed backups and restores with errors whose name matches a deliberate_failure_tests pattern; null unless every location's backups and restores were read + - deliberate_failure_test_patterns — List of the deliberate_failure_tests patterns as given + - restores_date_unknown — Number of restores whose results file was read but has no readable date; they never count as recent + - days_since_last_clean_restore — Days since the newest clean restore's results file was written; null if there is no clean restore + - restore_test_interval_days — The adopted restore-test interval in days, as configured; null when not set + - clean_restore_within_interval — Whether the newest clean restore is no older than restore_test_interval_days (false when there is none); null while no interval is set or any location's restores could not be read + + Is backup storage protected? + - backup_locations_geo_replicated — Number of backup storage locations whose storage account is geo-replicated (GRS, RA-GRS, GZRS or RA-GZRS) + - backup_locations_customer_managed_key — Number of backup storage locations whose storage account encrypts with a customer-managed key from Key Vault diff --git a/pyproject.toml b/pyproject.toml index de66f1f0..4c142a20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,8 @@ azure = [ "azure-keyvault-certificates>=4.11.3,<5", # <7 is load-bearing: 7.0.0 ships no diagnostic_settings operations at all. "azure-mgmt-monitor>=6.0.2,<7", + # Blob list responses (azure_velero_backups) are XML; parsed with entities and external references refused. + "defusedxml", "azure-mgmt-loganalytics>=14.0.0,<15", "azure-mgmt-recoveryservices>=4.1.0,<5", "azure-mgmt-recoveryservicesbackup>=10.0.0,<12", diff --git a/tests/fixtures/velero/backups/manual-1/velero-backup.json b/tests/fixtures/velero/backups/manual-1/velero-backup.json new file mode 100644 index 00000000..2246044f --- /dev/null +++ b/tests/fixtures/velero/backups/manual-1/velero-backup.json @@ -0,0 +1,108 @@ +{ + "kind": "Backup", + "apiVersion": "velero.io/v1", + "metadata": { + "name": "manual-1", + "namespace": "velero", + "uid": "b279ec05-9089-4661-a82d-095bf49f93db", + "resourceVersion": "2117", + "generation": 5, + "creationTimestamp": "2026-11-16T23:51:36Z", + "labels": { + "velero.io/storage-location": "default" + }, + "annotations": { + "velero.io/resource-timeout": "10m0s", + "velero.io/source-cluster-k8s-gitversion": "v1.36.5", + "velero.io/source-cluster-k8s-major-version": "1", + "velero.io/source-cluster-k8s-minor-version": "36" + }, + "managedFields": [ + { + "manager": "velero", + "operation": "Update", + "apiVersion": "velero.io/v1", + "time": "2026-11-16T23:51:36Z", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:spec": { + ".": {}, + "f:hooks": {}, + "f:includedNamespaces": {}, + "f:metadata": {} + }, + "f:status": {} + } + }, + { + "manager": "velero-server", + "operation": "Update", + "apiVersion": "velero.io/v1", + "time": "2026-11-16T23:51:37Z", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:annotations": { + ".": {}, + "f:velero.io/resource-timeout": {}, + "f:velero.io/source-cluster-k8s-gitversion": {}, + "f:velero.io/source-cluster-k8s-major-version": {}, + "f:velero.io/source-cluster-k8s-minor-version": {} + }, + "f:labels": { + ".": {}, + "f:velero.io/storage-location": {} + } + }, + "f:spec": { + "f:csiSnapshotTimeout": {}, + "f:defaultVolumesToFsBackup": {}, + "f:itemOperationTimeout": {}, + "f:snapshotMoveData": {}, + "f:storageLocation": {}, + "f:ttl": {} + }, + "f:status": { + "f:expiration": {}, + "f:formatVersion": {}, + "f:hookStatus": {}, + "f:phase": {}, + "f:progress": { + ".": {}, + "f:itemsBackedUp": {}, + "f:totalItems": {} + }, + "f:startTimestamp": {}, + "f:version": {} + } + } + } + ] + }, + "spec": { + "metadata": {}, + "includedNamespaces": [ + "demo" + ], + "ttl": "720h0m0s", + "hooks": {}, + "storageLocation": "default", + "defaultVolumesToFsBackup": false, + "csiSnapshotTimeout": "10m0s", + "itemOperationTimeout": "4h0m0s", + "snapshotMoveData": false + }, + "status": { + "version": 1, + "formatVersion": "1.1.0", + "expiration": "2026-12-16T23:51:36Z", + "phase": "Completed", + "startTimestamp": "2026-11-16T23:51:36Z", + "completionTimestamp": "2026-11-16T23:51:37Z", + "progress": { + "totalItems": 10, + "itemsBackedUp": 10 + }, + "hookStatus": {} + } +} \ No newline at end of file diff --git a/tests/fixtures/velero/backups/short-lived/velero-backup.json b/tests/fixtures/velero/backups/short-lived/velero-backup.json new file mode 100644 index 00000000..ca169e5a --- /dev/null +++ b/tests/fixtures/velero/backups/short-lived/velero-backup.json @@ -0,0 +1,108 @@ +{ + "kind": "Backup", + "apiVersion": "velero.io/v1", + "metadata": { + "name": "short-lived", + "namespace": "velero", + "uid": "57483cdd-9af4-499a-94b3-586355fd3547", + "resourceVersion": "2134", + "generation": 5, + "creationTimestamp": "2026-11-16T23:51:38Z", + "labels": { + "velero.io/storage-location": "default" + }, + "annotations": { + "velero.io/resource-timeout": "10m0s", + "velero.io/source-cluster-k8s-gitversion": "v1.36.5", + "velero.io/source-cluster-k8s-major-version": "1", + "velero.io/source-cluster-k8s-minor-version": "36" + }, + "managedFields": [ + { + "manager": "velero", + "operation": "Update", + "apiVersion": "velero.io/v1", + "time": "2026-11-16T23:51:38Z", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:spec": { + ".": {}, + "f:hooks": {}, + "f:includedNamespaces": {}, + "f:metadata": {}, + "f:ttl": {} + }, + "f:status": {} + } + }, + { + "manager": "velero-server", + "operation": "Update", + "apiVersion": "velero.io/v1", + "time": "2026-11-16T23:51:39Z", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:annotations": { + ".": {}, + "f:velero.io/resource-timeout": {}, + "f:velero.io/source-cluster-k8s-gitversion": {}, + "f:velero.io/source-cluster-k8s-major-version": {}, + "f:velero.io/source-cluster-k8s-minor-version": {} + }, + "f:labels": { + ".": {}, + "f:velero.io/storage-location": {} + } + }, + "f:spec": { + "f:csiSnapshotTimeout": {}, + "f:defaultVolumesToFsBackup": {}, + "f:itemOperationTimeout": {}, + "f:snapshotMoveData": {}, + "f:storageLocation": {} + }, + "f:status": { + "f:expiration": {}, + "f:formatVersion": {}, + "f:hookStatus": {}, + "f:phase": {}, + "f:progress": { + ".": {}, + "f:itemsBackedUp": {}, + "f:totalItems": {} + }, + "f:startTimestamp": {}, + "f:version": {} + } + } + } + ] + }, + "spec": { + "metadata": {}, + "includedNamespaces": [ + "demo" + ], + "ttl": "5m0s", + "hooks": {}, + "storageLocation": "default", + "defaultVolumesToFsBackup": false, + "csiSnapshotTimeout": "10m0s", + "itemOperationTimeout": "4h0m0s", + "snapshotMoveData": false + }, + "status": { + "version": 1, + "formatVersion": "1.1.0", + "expiration": "2026-11-16T23:56:38Z", + "phase": "Completed", + "startTimestamp": "2026-11-16T23:51:38Z", + "completionTimestamp": "2026-11-16T23:51:39Z", + "progress": { + "totalItems": 10, + "itemsBackedUp": 10 + }, + "hookStatus": {} + } +} \ No newline at end of file diff --git a/tests/fixtures/velero/blob-names.txt b/tests/fixtures/velero/blob-names.txt new file mode 100644 index 00000000..e365cde2 --- /dev/null +++ b/tests/fixtures/velero/blob-names.txt @@ -0,0 +1,29 @@ +backups/manual-1/manual-1-csi-volumesnapshotclasses.json.gz +backups/manual-1/manual-1-csi-volumesnapshotcontents.json.gz +backups/manual-1/manual-1-csi-volumesnapshots.json.gz +backups/manual-1/manual-1-itemoperations.json.gz +backups/manual-1/manual-1-logs.gz +backups/manual-1/manual-1-podvolumebackups.json.gz +backups/manual-1/manual-1-resource-list.json.gz +backups/manual-1/manual-1-results.gz +backups/manual-1/manual-1-volumeinfo.json.gz +backups/manual-1/manual-1-volumesnapshots.json.gz +backups/manual-1/manual-1.tar.gz +backups/manual-1/velero-backup.json +backups/short-lived/short-lived-csi-volumesnapshotclasses.json.gz +backups/short-lived/short-lived-csi-volumesnapshotcontents.json.gz +backups/short-lived/short-lived-csi-volumesnapshots.json.gz +backups/short-lived/short-lived-itemoperations.json.gz +backups/short-lived/short-lived-logs.gz +backups/short-lived/short-lived-podvolumebackups.json.gz +backups/short-lived/short-lived-resource-list.json.gz +backups/short-lived/short-lived-results.gz +backups/short-lived/short-lived-volumeinfo.json.gz +backups/short-lived/short-lived-volumesnapshots.json.gz +backups/short-lived/short-lived.tar.gz +backups/short-lived/velero-backup.json +restores/r1/r1-volumeinfo.json.gz +restores/r1/restore-r1-itemoperations.json.gz +restores/r1/restore-r1-logs.gz +restores/r1/restore-r1-resource-list.json.gz +restores/r1/restore-r1-results.gz diff --git a/tests/fixtures/velero/restores/r1/restore-r1-results.gz b/tests/fixtures/velero/restores/r1/restore-r1-results.gz new file mode 100644 index 00000000..daf09de7 Binary files /dev/null and b/tests/fixtures/velero/restores/r1/restore-r1-results.gz differ diff --git a/tests/test_azure_velero_backups.py b/tests/test_azure_velero_backups.py new file mode 100644 index 00000000..aa7d66de --- /dev/null +++ b/tests/test_azure_velero_backups.py @@ -0,0 +1,507 @@ +"""azure_velero_backups: location parsing, backup and restore judgement, null-not-zero summaries, the blob read boundary, and a full run against synthetic ARM and Blob responses. No SDK or network.""" + +from __future__ import annotations + +import gzip +import importlib.util +import json +import sys +from collections import namedtuple +from datetime import datetime, timedelta, timezone +from pathlib import Path +from urllib.parse import urlparse + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +AZURE = REPO_ROOT / "fetchers" / "azure" +sys.path.insert(0, str(REPO_ROOT / "fetchers" / "_lib")) +sys.path.insert(0, str(AZURE / "_shared")) + +_spec = importlib.util.spec_from_file_location("azure_velero_backups_under_test", AZURE / "velero_backups" / "fetcher.py") +vb = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(vb) +azure_rest = sys.modules["azure_rest"] + +NOW = datetime(2026, 11, 16, 12, 0, tzinfo=timezone.utc) +SETTINGS = {"success_window_hours": 26, "expiry_grace_hours": 24, "container_names": ["velero"], "explicit_locations": [], "restore_test_interval_days": None, + "expected_schedules": None, "rpo_hours": None, "missed_run_lookback_days": 7, "deliberate_failure_tests": []} +SUB = "/subscriptions/s1" +ACCOUNT_ID = f"{SUB}/resourceGroups/rg1/providers/Microsoft.Storage/storageAccounts/backupsa" +ENDPOINT = "https://backupsa.blob.core.windows.net/" + + +def iso(dt): + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def backup(name, phase="Completed", schedule="daily", done_hours_ago=6, expires_in_hours=700, attempted=1, completed=1, now=NOW): + done = now - timedelta(hours=done_hours_ago) + return {"kind": "Backup", "metadata": {"name": name, "labels": {"velero.io/schedule-name": schedule} if schedule else {}}, + "status": {"phase": phase, "startTimestamp": iso(done - timedelta(minutes=5)), "completionTimestamp": iso(done), + "expiration": iso(now + timedelta(hours=expires_in_hours)), + "volumeSnapshotsAttempted": attempted, "volumeSnapshotsCompleted": completed}} + + +def results(errors=0, warnings=0): + return {"errors": {"velero": ["e"] * errors, "cluster": [], "namespaces": {}}, + "warnings": {"velero": [], "cluster": [], "namespaces": {"ns1": ["w"] * warnings}}} + + +def location(backups=(), restores=(), read=True, restores_read=True, **kw): + return {"backups_read": read, "backups_unreadable": 0, "backups": list(backups), "restores_read": restores_read, + "restores": list(restores), "geo_replicated": kw.get("geo", True), "customer_managed_key": kw.get("cmk", False)} + + +# --- pure transforms --- + +def test_locations_parse_account_container_and_optional_prefix(): + locs, invalid = vb.parse_locations("BackupSA/velero, other/vel/cluster-a/ , bad") + assert locs == [{"account": "backupsa", "container": "velero", "prefix": ""}, + {"account": "other", "container": "vel", "prefix": "cluster-a"}] + assert invalid == ["bad"] + + +def test_account_record_reads_replication_and_key_source(): + raw = {"id": ACCOUNT_ID, "name": "BackupSA", "sku": {"name": "Standard_RAGRS"}, + "properties": {"encryption": {"keySource": "Microsoft.Keyvault"}, "primaryEndpoints": {"blob": ENDPOINT}}} + rec = vb.account_record(raw) + assert (rec["name"], rec["geo_replicated"], rec["customer_managed_key"], rec["blob_endpoint"]) == ("backupsa", True, True, ENDPOINT) + lrs = vb.account_record({"id": ACCOUNT_ID, "name": "x", "sku": {"name": "Standard_LRS"}, "properties": {"encryption": {"keySource": "Microsoft.Storage"}}}) + assert (lrs["geo_replicated"], lrs["customer_managed_key"]) == (False, False) + + +def test_backup_past_expiry_beyond_grace_counts_and_snapshot_shortfall_is_flagged(): + old = vb.backup_record("b1", backup("b1", expires_in_hours=-30, attempted=3, completed=2), NOW, 24) + assert (old["expired_not_deleted"], old["volume_snapshots_incomplete"], old["schedule"]) == (True, True, "daily") + inside_grace = vb.backup_record("b2", backup("b2", expires_in_hours=-10), NOW, 24) + assert inside_grace["expired_not_deleted"] is False + + +def test_restore_is_clean_only_when_its_results_list_no_errors(): + assert vb.restore_record("r1", results(warnings=2), "2026-11-10T00:00:00Z")["clean"] is True + assert vb.restore_record("r2", results(errors=1), None)["clean"] is False + assert vb.restore_record("r3", None, None) == {"name": "r3", "results_read": False, "errors": None, "warnings": None, + "clean": None, "results_written": None} + + +def test_summary_judges_each_schedule_by_its_newest_completed_backup(): + bk = [vb.backup_record(n, b, NOW, 24) for n, b in ( + ("d1", backup("d1", done_hours_ago=6)), ("d0", backup("d0", done_hours_ago=30)), + ("w1", backup("w1", schedule="weekly", done_hours_ago=100)), ("f1", backup("f1", phase="Failed", schedule="weekly", done_hours_ago=2)), + ("m1", backup("m1", schedule=None, phase="PartiallyFailed")))] + rs = [vb.restore_record("r1", results(), iso(NOW - timedelta(days=3, hours=1))), vb.restore_record("r2", results(errors=1), iso(NOW))] + s = vb.summarize([location(bk, rs)], SETTINGS, NOW) + assert (s["backups_total"], s["backups_completed"], s["backups_failed"], s["backups_partially_failed"]) == (5, 3, 1, 1) + assert (s["schedules_observed"], s["schedules_with_success_within_window"], s["schedules_without_success_within_window"]) == (2, 1, ["weekly"]) + assert s["max_hours_since_scheduled_success"] == 100.0 + assert (s["restores_total"], s["restores_without_errors"], s["restores_with_errors"], s["days_since_last_clean_restore"]) == (2, 1, 1, 3) + + +def test_unread_locations_or_unfinished_discovery_leave_counts_null_not_zero(): + unread = vb.summarize([location(read=False, restores_read=False)], SETTINGS, NOW) + assert (unread["backups_total"], unread["restores_total"], unread["backup_locations"], unread["backup_locations_read"]) == (None, None, 1, 0) + undiscovered = vb.summarize([], SETTINGS, NOW, discovered=False) + assert (undiscovered["backup_locations"], undiscovered["backups_total"], undiscovered["restores_total"]) == (None, None, None) + empty = vb.summarize([], SETTINGS, NOW) + assert (empty["backup_locations"], empty["backups_total"], empty["schedules_observed"]) == (0, 0, 0) + + +def test_a_schedule_without_any_completed_backup_leaves_the_max_age_null(): + bk = [vb.backup_record("f1", backup("f1", phase="Failed"), NOW, 24)] + s = vb.summarize([location(bk)], SETTINGS, NOW) + assert (s["schedules_with_success_within_window"], s["max_hours_since_scheduled_success"]) == (0, None) + + +def test_summary_matches_the_instructions(assert_summary_documented): + s = vb.summarize([location()], SETTINGS, NOW) + s.update(vb.ResourceExclusions().summary([], ("storage_account",), {"storage_accounts": 1})) + assert_summary_documented("azure/velero_backups", s) + + +def test_blob_reads_refuse_any_endpoint_arm_did_not_return(): + reader = vb.BlobReader(namedtuple("C", "session cred")(None, None), [ENDPOINT]) + for endpoint in ("https://evil.blob.core.windows.net/", "http://backupsa.blob.core.windows.net/"): + with pytest.raises(ValueError, match="untrusted"): + reader._get(endpoint, "velero") + + +# --- full run against synthetic ARM and Blob responses --- + +class _Resp: + def __init__(self, status, body=None, content=None, headers=None): + self.status_code, self._body, self.reason = status, body, "" + self.content = content if content is not None else json.dumps(body or {}).encode() + self.text = self.content.decode(errors="replace") + self.headers = headers or {} + + def json(self): + return self._body + + +def _list_xml(names): + blobs = "".join(f"{n}Fri, 13 Nov 2026 01:00:00 GMT" for n in names) + return f'{blobs}'.encode() + + +class _Azure: + """requests.Session for subscription s1: one GRS account with a 'velero' container, two backups and one restore.""" + + def __init__(self, blob_status=200): + self.blob_status, self.urls, self.headers = blob_status, [], [] + now = datetime.now(timezone.utc) + self.blobs = { + "backups/daily-1/velero-backup.json": json.dumps(backup("daily-1", done_hours_ago=5, now=now)).encode(), + "backups/daily-1/daily-1.tar.gz": b"x", + "backups/old-1/velero-backup.json": json.dumps(backup("old-1", schedule=None, expires_in_hours=-48, now=now)).encode(), + "restores/r1/restore-r1-results.gz": gzip.compress(json.dumps(results()).encode()), + "restores/r1/restore-r1-logs.gz": b"x", + } + + def get(self, url, params=None, headers=None, timeout=None, allow_redirects=True): + self.urls.append(url) + self.headers.append(headers or {}) + host, path = urlparse(url).netloc, urlparse(url).path + if host == "backupsa.blob.core.windows.net": + if self.blob_status != 200: + return _Resp(self.blob_status, content=b"", headers={"x-ms-error-code": "AuthorizationPermissionMismatch"}) + if (params or {}).get("comp") == "list": + return _Resp(200, content=_list_xml(sorted(n for n in self.blobs if n.startswith(params["prefix"])))) + return _Resp(200, content=self.blobs[path.split("/velero/", 1)[1]]) + if path.endswith("/providers/Microsoft.Storage/storageAccounts"): + return _Resp(200, {"value": [{"id": ACCOUNT_ID, "name": "backupsa", "sku": {"name": "Standard_GRS"}, "location": "eastus", + "properties": {"encryption": {"keySource": "Microsoft.Storage"}, "primaryEndpoints": {"blob": ENDPOINT}}}]}) + if path.endswith("/blobServices/default/containers"): + return _Resp(200, {"value": [{"name": "velero"}, {"name": "logs"}]}) + return _Resp(404, {"error": {"code": "NotFound", "message": path}}) + + +class _Cred: + def get_token(self, scope): + return namedtuple("T", "token expires_on")("tok-" + scope, 9_999_999_999) + + +@pytest.fixture +def azure(monkeypatch, tmp_path): + def serve(**kwargs): + session = _Azure(**kwargs) + monkeypatch.setattr("requests.Session", lambda: session) + return session + + monkeypatch.setattr(azure_rest, "pinned_credential", lambda sub: _Cred()) + monkeypatch.setattr(azure_rest, "REPO_ROOT", tmp_path / "repo") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("AZURE_SUBSCRIPTION_ID", "s1") + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.setenv("FETCHER_STATUS_FILE", str(tmp_path / "status.json")) + for env in ("AZURE_AUTHORITY_HOST", "AZURE_TENANT_ID", "AZURE_ENVIRONMENT", "AZURE_RESOURCE_EXCLUSIONS", "AZURE_TARGET_SUBSCRIPTION_ONLY", + vb.CONTAINERS_ENV, vb.LOCATIONS_ENV, vb.WINDOW_ENV, vb.GRACE_ENV, vb.INTERVAL_ENV, + vb.EXPECTED_ENV, vb.RPO_ENV, vb.MISSED_LOOKBACK_ENV, vb.DELIBERATE_ENV): + monkeypatch.delenv(env, raising=False) + return serve + + +def _evidence(tmp_path): + [path] = tmp_path.glob("azure_velero_backups_s1.json") + return json.loads(path.read_text()) + + +def test_main_reads_backups_and_restores_from_the_velero_container(azure, tmp_path): + session = azure() + assert vb.main() == 0 + ev = _evidence(tmp_path) + s = ev["summary"] + assert ev["metadata"]["partial_failure"] is False + assert (s["backup_locations"], s["backup_locations_read"], s["backup_locations_geo_replicated"], s["backup_locations_customer_managed_key"]) == (1, 1, 1, 0) + assert (s["backups_total"], s["backups_completed"], s["expired_backups_not_deleted"]) == (2, 2, 1) + assert (s["schedules_observed"], s["schedules_with_success_within_window"]) == (1, 1) + assert s["restores_total"] == 1 and s["restores_without_errors"] == 1 + blob_calls = [h for u, h in zip(session.urls, session.headers) if "blob.core" in u] + assert blob_calls and all(h["Authorization"] == "Bearer tok-https://storage.azure.com/.default" and h["x-ms-version"] for h in blob_calls) + assert not any("logs" in u and "blob.core" in u for u in session.urls) + + +def test_a_missing_data_role_fails_the_run_and_leaves_counts_null(azure, tmp_path): + azure(blob_status=403) + assert vb.main() != 0 + s = _evidence(tmp_path)["summary"] + assert (s["backup_locations"], s["backup_locations_read"], s["backups_total"], s["restores_total"]) == (1, 0, None, None) + assert "AuthorizationPermissionMismatch" in (tmp_path / "status.json").read_text() + + +def test_an_explicit_location_outside_the_subscription_fails_the_run(azure, tmp_path, monkeypatch): + monkeypatch.setenv(vb.LOCATIONS_ENV, "elsewhere/velero") + azure() + assert vb.main() != 0 + s = _evidence(tmp_path)["summary"] + assert (s["backup_locations"], s["backups_total"]) == (None, None) + + +def test_strict_subscription_mode_still_reads_blobs_from_arm_returned_endpoints(azure, tmp_path, monkeypatch): + monkeypatch.setenv("AZURE_TARGET_SUBSCRIPTION_ONLY", "true") + session = azure() + assert vb.main() == 0 + assert _evidence(tmp_path)["summary"]["backups_total"] == 2 + assert all(urlparse(u).netloc in ("management.azure.com", "backupsa.blob.core.windows.net") for u in session.urls) + + +# --- real Velero 1.16 objects captured from a sandbox (tests/fixtures/velero) --- + +REAL = REPO_ROOT / "tests" / "fixtures" / "velero" +# 1h03m after the 5-minute backup expired, and before Velero's hourly cleanup removed it. +CAPTURED = datetime(2026, 11, 17, 1, 0, tzinfo=timezone.utc) + + +class _RealVelero(_Azure): + """Serves the captured container: every real blob name in the listing, and real bytes for the ones the fetcher reads.""" + + def __init__(self): + super().__init__() + self.blobs = {n: (REAL / n).read_bytes() if (REAL / n).exists() else None + for n in (REAL / "blob-names.txt").read_text().split()} + self.fetched = [] + + def get(self, url, params=None, headers=None, timeout=None, allow_redirects=True): + if "blob.core" in url and (params or {}).get("comp") != "list": + name = urlparse(url).path.split("/velero/", 1)[1] + self.fetched.append(name) + if self.blobs.get(name) is None: + return _Resp(404, content=b"", headers={"x-ms-error-code": "BlobNotFound"}) + return super().get(url, params, headers, timeout, allow_redirects) + + +def test_real_velero_objects_are_judged_as_velero_wrote_them(monkeypatch): + monkeypatch.setenv(vb.GRACE_ENV, "1") + for env in (vb.CONTAINERS_ENV, vb.LOCATIONS_ENV, vb.WINDOW_ENV, "AZURE_TARGET_SUBSCRIPTION_ONLY", "AZURE_RESOURCE_EXCLUSIONS", + vb.EXPECTED_ENV, vb.RPO_ENV, vb.MISSED_LOOKBACK_ENV, vb.INTERVAL_ENV): + monkeypatch.delenv(env, raising=False) + session = _RealVelero() + client = azure_rest.AzureRestClient(_Cred(), session) + collector = sys.modules["azure_common"].Collector(vb.logger) + results, s = vb.collect(client, "s1", collector, now=CAPTURED) + assert collector.failures == [] + # Only the two backup records and the restore results were read; tarballs, logs and other files were not. + assert sorted(session.fetched) == ["backups/manual-1/velero-backup.json", "backups/short-lived/velero-backup.json", + "restores/r1/restore-r1-results.gz"] + assert (s["backups_total"], s["backups_completed"], s["backups_failed"], s["backups_metadata_unreadable"]) == (2, 2, 0, 0) + # Manual backups carry no schedule label; the hourly schedule had not fired when these were captured. + assert (s["schedules_observed"], s["schedules_with_success_within_window"], s["max_hours_since_scheduled_success"]) == (0, 0, None) + # The 5-minute backup is past its expiration by more than the 1-hour grace and is still stored. + assert s["expired_backups_not_deleted"] == 1 + assert [b["name"] for b in results["backup_locations"][0]["backups"] if b["expired_not_deleted"]] == ["short-lived"] + # The real restore lists no errors ("errors": {}) and one namespace warning, so it is clean. + [restore] = results["backup_locations"][0]["restores"] + assert (restore["errors"], restore["warnings"], restore["clean"]) == (0, 1, True) + assert (s["restores_total"], s["restores_without_errors"], s["restores_with_errors"], s["restores_results_missing"]) == (1, 1, 0, 0) + # No volumes in the sandbox workload: Velero omits the snapshot counters, which read as nothing attempted. + assert s["backups_volume_snapshots_incomplete"] == 0 + + +def test_restore_interval_is_judged_only_once_adopted(): + rs = [vb.restore_record("r1", results(), iso(NOW - timedelta(days=40)))] + assert vb.summarize([location([], rs)], SETTINGS, NOW)["clean_restore_within_interval"] is None + adopted = {**SETTINGS, "restore_test_interval_days": 30} + assert vb.summarize([location([], rs)], adopted, NOW)["clean_restore_within_interval"] is False + assert vb.summarize([location([], rs)], {**adopted, "restore_test_interval_days": 60}, NOW)["clean_restore_within_interval"] is True + assert vb.summarize([location([], [])], adopted, NOW)["clean_restore_within_interval"] is False + + +# --- expected schedules, missed runs and the recovery point objective --- + +def _daily(hours_ago_list, schedule="daily", phase="Completed"): + return [vb.backup_record(f"{schedule}-{h}", backup(f"{schedule}-{h}", schedule=schedule, done_hours_ago=h, phase=phase), NOW, 24) + for h in hours_ago_list] + + +def test_expected_schedules_parse_name_and_interval(): + assert vb.parse_expected(None) == (None, []) + assert vb.parse_expected("daily:24, weekly-db:168 , bad, x:0") == ( + [{"name": "daily", "interval_hours": 24}, {"name": "weekly-db", "interval_hours": 168}], ["bad", "x:0"]) + + +def test_missed_daily_runs_are_each_24_hour_period_without_a_completed_backup(): + # Completed 6, 30, 54 and 126 hours ago: periods 0, 1, 2 and 5 are covered; 3, 4 and 6 are missed. + bk = _daily([6, 30, 54, 126]) + _daily([78], phase="Failed") + settings = {**SETTINGS, "expected_schedules": [{"name": "daily", "interval_hours": 24}], "rpo_hours": 168} + [c] = vb.schedule_checks(bk, settings, NOW) + assert (c["periods_checked"], c["missed_runs"], c["hours_since_success"], c["exceeds_rpo"]) == (7, 3, 6.0, False) + + +def test_an_expected_schedule_with_no_backups_is_reported_and_exceeds_the_rpo(): + settings = {**SETTINGS, "expected_schedules": [{"name": "daily", "interval_hours": 24}, {"name": "weekly", "interval_hours": 168}], "rpo_hours": 168} + s = vb.summarize([location(_daily([6]))], settings, NOW) + assert (s["expected_schedules"], s["expected_schedules_without_backups"], s["expected_schedules_without_backups_names"]) == (2, 1, ["weekly"]) + assert (s["missed_scheduled_runs"], s["schedules_with_missed_runs"], s["schedules_exceeding_rpo"]) == (7, 2, 1) + + +def test_rpo_is_judged_against_observed_schedules_when_none_are_listed(): + s = vb.summarize([location(_daily([200], schedule="old") + _daily([5]))], {**SETTINGS, "rpo_hours": 168}, NOW) + assert (s["expected_schedules"], s["missed_scheduled_runs"], s["schedules_exceeding_rpo"]) == (None, None, 1) + + +def test_without_listed_schedules_or_an_rpo_the_new_counts_stay_null(): + s = vb.summarize([location(_daily([5]))], SETTINGS, NOW) + assert (s["expected_schedules"], s["expected_schedules_without_backups"], s["missed_scheduled_runs"], s["schedules_exceeding_rpo"], s["rpo_hours"]) == (None, None, None, None, None) + + +def test_a_restore_without_a_readable_date_is_unknown_and_never_recent(): + rs = [vb.restore_record("r1", results(), None), vb.restore_record("r2", results(), iso(NOW - timedelta(days=90)))] + s = vb.summarize([location([], rs)], {**SETTINGS, "restore_test_interval_days": 30}, NOW) + assert (s["restores_without_errors"], s["restores_date_unknown"], s["days_since_last_clean_restore"], s["clean_restore_within_interval"]) == (2, 1, 90, False) + + +def test_invalid_expected_schedules_or_rpo_fail_the_run(azure, tmp_path, monkeypatch): + monkeypatch.setenv(vb.EXPECTED_ENV, "daily") + monkeypatch.setenv(vb.RPO_ENV, "a week") + azure() + assert vb.main() != 0 + s = _evidence(tmp_path)["summary"] + assert (s["expected_schedules"], s["expected_schedules_without_backups"], s["rpo_hours"]) == (None, None, None) + + +def test_real_sandbox_shows_the_daily_schedule_has_not_produced_a_backup(monkeypatch): + monkeypatch.setenv(vb.EXPECTED_ENV, "daily:24") + monkeypatch.setenv(vb.RPO_ENV, "168") + for env in (vb.CONTAINERS_ENV, vb.LOCATIONS_ENV, vb.WINDOW_ENV, vb.INTERVAL_ENV, vb.MISSED_LOOKBACK_ENV, "AZURE_TARGET_SUBSCRIPTION_ONLY", "AZURE_RESOURCE_EXCLUSIONS"): + monkeypatch.delenv(env, raising=False) + client = azure_rest.AzureRestClient(_Cred(), _RealVelero()) + results, s = vb.collect(client, "s1", sys.modules["azure_common"].Collector(vb.logger), now=CAPTURED) + # The schedule existed in the cluster but had not run yet: storage holds only the two manual backups. + assert (s["expected_schedules"], s["expected_schedules_without_backups"], s["expected_schedules_without_backups_names"]) == (1, 1, ["daily"]) + assert (s["missed_scheduled_runs"], s["schedules_exceeding_rpo"]) == (7, 1) + assert results["schedules"][0]["backups"] == 0 + + +# --- review fixes: retention-bounded, gap-based missed runs; per-schedule windows; history; blob reads --- + +def _runs(schedule, done_times, ttl_hours, phase="Completed", attempted=1, completed=1, name=None): + out = [] + for i, done in enumerate(done_times): + raw = {"metadata": {"labels": {"velero.io/schedule-name": schedule}}, + "status": {"phase": phase, "startTimestamp": iso(done - timedelta(minutes=5)), "completionTimestamp": iso(done), + "expiration": iso(done + timedelta(hours=ttl_hours)), + "volumeSnapshotsAttempted": attempted, "volumeSnapshotsCompleted": completed}} + out.append(vb.backup_record(name or f"{schedule}-{i}", raw, NOW, 24)) + return out + + +def _expected(name, hours, **extra): + return {**SETTINGS, "expected_schedules": [{"name": name, "interval_hours": hours}], **extra} + + +def test_backups_velero_deleted_at_the_end_of_their_retention_are_not_missed_runs(): + # Every run succeeded; only the retained backups are in storage. + hourly = _runs("hourly", [NOW - timedelta(hours=h, minutes=10) for h in range(24)], ttl_hours=24) + [c] = vb.schedule_checks(hourly, _expected("hourly", 1), NOW) + assert (c["missed_runs"], c["hours_checked_for_missed_runs"]) == (0, 24.0) + daily = _runs("daily", [NOW - timedelta(hours=h) for h in (2, 26, 50)], ttl_hours=72) + [c] = vb.schedule_checks(daily, _expected("daily", 24), NOW) + assert (c["missed_runs"], c["periods_checked"]) == (0, 3) + + +def test_a_backup_finishing_a_few_minutes_late_is_not_a_missed_run(): + # Daily at 02:00 for 8 days, except yesterday's finished at 02:05; the fetcher runs at 02:03. + now = datetime(2026, 11, 16, 2, 3, tzinfo=timezone.utc) + times = [datetime(2026, 11, 16 - d, 2, 5 if d == 1 else 0, tzinfo=timezone.utc) for d in range(1, 9)] + bk = [] + for i, done in enumerate(times): + raw = {"metadata": {"labels": {"velero.io/schedule-name": "daily"}}, + "status": {"phase": "Completed", "completionTimestamp": iso(done), "expiration": iso(done + timedelta(days=30))}} + bk.append(vb.backup_record(f"d{i}", raw, now, 24)) + [c] = vb.schedule_checks(bk, _expected("daily", 24), now) + assert c["missed_runs"] == 0 + + +def test_a_real_gap_still_counts_each_missed_interval(): + daily = _runs("daily", [NOW - timedelta(hours=h) for h in (2, 98)], ttl_hours=720) + [c] = vb.schedule_checks(daily, _expected("daily", 24), NOW) + assert c["missed_runs"] == 5 # 3 runs missed between the two backups, 2 before the older one + + +def test_a_listed_weekly_schedule_is_judged_against_its_own_interval(): + weekly = _runs("weekly", [NOW - timedelta(hours=100)], ttl_hours=720) + s = vb.summarize([location(weekly)], _expected("weekly", 168), NOW) + assert (s["schedules_observed"], s["schedules_with_success_within_window"]) == (1, 1) + s = vb.summarize([location(weekly)], SETTINGS, NOW) # not listed: the 26-hour default applies + assert s["schedules_with_success_within_window"] == 0 + assert vb.success_window_hours(24, SETTINGS) == 26 and vb.success_window_hours(168, SETTINGS) == 184 + + +def test_resolved_and_deliberate_failures_stay_in_history_but_are_not_current(): + resolved = _runs("daily", [NOW - timedelta(hours=50)], 720, phase="Failed", name="daily-failed-old") + current = _runs("hourly", [NOW - timedelta(hours=1)], 720, phase="PartiallyFailed", name="hourly-failed-new") + planned = _runs("daily", [NOW - timedelta(hours=1)], 720, phase="Failed", name="failtest-1") + ok = _runs("daily", [NOW - timedelta(hours=26)], 720) + snaps_old = _runs("weekly", [NOW - timedelta(hours=200)], 720, attempted=2, completed=1, name="weekly-old") + snaps_ok = _runs("weekly", [NOW - timedelta(hours=30)], 720, name="weekly-new") + rs = [vb.restore_record("r-bad-old", results(errors=1), iso(NOW - timedelta(days=5))), + vb.restore_record("r-good", results(), iso(NOW - timedelta(days=2))), + vb.restore_record("r-bad-new", results(errors=2), iso(NOW - timedelta(days=1))), + vb.restore_record("failtest-restore", results(errors=1), iso(NOW - timedelta(hours=3)))] + settings = {**SETTINGS, "deliberate_failure_tests": ["failtest-*"]} + s = vb.summarize([location(resolved + current + planned + ok + snaps_old + snaps_ok, rs)], settings, NOW) + assert (s["backups_failed"], s["backups_partially_failed"], s["backups_failed_unresolved"]) == (2, 1, 1) + assert (s["backups_volume_snapshots_incomplete"], s["backups_volume_snapshots_incomplete_unresolved"]) == (1, 0) + assert (s["restores_with_errors"], s["restores_with_errors_unresolved"]) == (3, 1) + assert (s["deliberate_failure_tests"], s["deliberate_failure_test_patterns"]) == (2, ["failtest-*"]) + + +def test_unresolved_counts_stay_null_when_a_location_is_unread(): + s = vb.summarize([location(read=False, restores_read=False)], SETTINGS, NOW) + assert (s["backups_failed_unresolved"], s["restores_with_errors_unresolved"], s["deliberate_failure_tests"]) == (None, None, None) + + +def test_blob_reads_retry_a_dropped_connection(monkeypatch): + import requests + + class Flaky: + calls = 0 + + def get(self, url, **kw): + Flaky.calls += 1 + if Flaky.calls == 1: + raise requests.ConnectionError("reset") + return _Resp(200, content=_list_xml(["backups/b1/velero-backup.json"])) + + monkeypatch.setattr(vb.time, "sleep", lambda s: None) + reader = vb.BlobReader(namedtuple("C", "session cred")(Flaky(), _Cred()), [ENDPOINT]) + assert [b["name"] for b in reader.list(ENDPOINT, "velero", "backups/")] == ["backups/b1/velero-backup.json"] + assert Flaky.calls == 2 + + +def test_blob_list_xml_with_an_entity_declaration_is_refused(): + evil = b']>' + + class Evil: + def get(self, url, **kw): + return _Resp(200, content=evil) + + reader = vb.BlobReader(namedtuple("C", "session cred")(Evil(), _Cred()), [ENDPOINT]) + with pytest.raises(Exception, match="(?i)entit|forbidden"): + reader.list(ENDPOINT, "velero", "backups/") + + +def test_a_gap_that_crosses_the_window_start_counts_only_runs_due_inside_it(): + # Daily backups with the default 30-day retention and a 7-day lookback. + now = datetime(2026, 11, 16, 3, 0, tzinfo=timezone.utc) + window_start = now - timedelta(days=7) + + def day(d): + return now - timedelta(days=d, hours=1) # each backup finished at 02:00, d days ago + + # Backups stopped from day -20 to day -7 and succeeded every day since day -6: nothing due in the window was missed. + recovered = [day(d) for d in range(30, 20, -1)] + [day(d) for d in range(6, 0, -1)] + assert vb.count_missed(sorted(recovered), 24, window_start, now) == 0 + # An outage from day -10 to day -3: only the four runs due inside the window count, not the seven of the whole gap. + outage = [day(d) for d in range(30, 10, -1)] + [day(d) for d in (2, 1)] + assert vb.count_missed(sorted(outage), 24, window_start, now) == 4 + + +def test_a_failed_backup_without_a_schedule_stays_unresolved(): + failed = vb.backup_record("manual-failed", backup("manual-failed", phase="Failed", schedule=None, done_hours_ago=10), NOW, 24) + later = vb.backup_record("manual-ok", backup("manual-ok", schedule=None, done_hours_ago=2), NOW, 24) + s = vb.summarize([location([failed, later])], SETTINGS, NOW) + assert (s["backups_failed"], s["backups_failed_unresolved"]) == (1, 1) + scheduled_failed = vb.backup_record("d-failed", backup("d-failed", phase="Failed", done_hours_ago=10), NOW, 24) + scheduled_ok = vb.backup_record("d-ok", backup("d-ok", done_hours_ago=2), NOW, 24) + assert vb.summarize([location([scheduled_failed, scheduled_ok])], SETTINGS, NOW)["backups_failed_unresolved"] == 0