diff --git a/CHANGELOG.md b/CHANGELOG.md index 89e507dd..7f6bb5b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,10 @@ schemas and the `paramify` CLI — not the internal code. Completed backups, with 10% grace, and only over the part of the lookback the backups are kept for, so a backup Velero deleted at the end of its retention is not a missed run. A listed schedule is judged against its own interval. Failed backups, incomplete snapshots and restores with errors are counted in full and again as - current (unresolved) failures, with planned failure tests matched by name and counted apart. Blob reads go only + current (unresolved) failures. A failed scheduled backup resolves through a later success of its schedule, + and any failure through its own retry (``) when a retry suffix is set; a failed restore resolves + only through its own retry, since storage doesn't record which backup a restore used. No failure is excluded by + name or label, and labelled validation backups are counted, still judged. Blob reads go only to endpoints Resource Manager returned for in-scope accounts in the target subscription, with redirects refused, retry dropped connections, and parse XML with defusedxml. Unread locations leave counts null rather than 0. - `paramify upload --no-waf-check` turns off the check that drops a file holding `../`, `/etc/passwd`, `/bin/sh`, diff --git a/fetchers/azure/velero_backups/fetcher.py b/fetchers/azure/velero_backups/fetcher.py index 7004e402..b8c742d8 100644 --- a/fetchers/azure/velero_backups/fetcher.py +++ b/fetchers/azure/velero_backups/fetcher.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """Velero backup storage locations in Azure Blob Storage: each location's storage settings, every backup's outcome and expiry, and every restore's result.""" -import fnmatch import gzip import json import logging @@ -54,7 +53,8 @@ EXPECTED_ENV = "AZURE_VELERO_EXPECTED_SCHEDULES" RPO_ENV = "AZURE_VELERO_RPO_HOURS" MISSED_LOOKBACK_ENV = "AZURE_VELERO_MISSED_RUN_LOOKBACK_DAYS" -DELIBERATE_ENV = "AZURE_VELERO_DELIBERATE_FAILURE_TESTS" +RETRY_SUFFIX_ENV = "AZURE_VELERO_RETRY_SUFFIX" +VALIDATION_LABEL_ENV = "AZURE_VELERO_VALIDATION_LABEL" DEFAULT_MISSED_LOOKBACK_DAYS = 7 MAX_MISSED_LOOKBACK_DAYS = 90 DEFAULT_CONTAINERS = ("velero",) @@ -203,6 +203,7 @@ def backup_record(name: str, raw: Dict[str, Any], now: datetime, grace_hours: in return { "name": name, "schedule": labels.get(SCHEDULE_LABEL) or None, + "labels": dict(labels), "phase": status.get("phase"), "started": status.get("startTimestamp"), "completed": status.get("completionTimestamp"), @@ -231,19 +232,24 @@ def restore_record(name: str, results: Optional[Dict[str, Any]], written: Option return {"name": name, "results_read": True, "errors": errors, "warnings": warnings, "clean": errors == 0, "results_written": written} -def is_deliberate(name: str, patterns: List[str]) -> bool: - """A backup or restore whose name matches a deliberate_failure_tests pattern is a planned failure test.""" - return any(fnmatch.fnmatchcase(name, p) for p in patterns) +def parse_label(value: Optional[str]) -> Optional[Tuple[str, str]]: + """`key=value`; None when unset or not in that form.""" + key, sep, val = (value or "").strip().partition("=") + return (key.strip(), val.strip()) if sep and key.strip() and val.strip() else None -def classify_history(backups: List[Dict[str, Any]], restores: List[Dict[str, Any]], patterns: List[str]) -> None: - """Mark each failure as a deliberate test, resolved by a later success, or current; records are updated in place. +def classify_history(backups: List[Dict[str, Any]], restores: List[Dict[str, Any]], retry_suffix: Optional[str]) -> None: + """Classify one location's failures as resolved or current; records are updated in place. + + No failure is excluded by its name or labels. When retry_suffix is set, a failure is also resolved by its own retry: + a backup named that completed later (every attempted snapshot, for a snapshot shortfall), or a + restore named that finished later without errors. A failed retry, or none, leaves it current. A Failed or PartiallyFailed backup is resolved once its schedule has a Completed backup that finished after it; a backup with no schedule is never resolved, since a later manual backup may cover other namespaces. A scheduled backup with incomplete volume snapshots is resolved once a later Completed backup of the same schedule completed every snapshot it attempted. A restore with - errors is resolved once a later restore completed without errors. A failure without a readable time is never + errors is resolved only by its own retry, since storage doesn't record which backup a restore used. A failure without a readable time is never resolved. History is kept: the totals still count every record in storage. """ def when(b): @@ -257,19 +263,35 @@ def when(b): newest_done[x["schedule"]] = max(done, newest_done.get(x["schedule"], done)) if not x["volume_snapshots_incomplete"]: newest_full[x["schedule"]] = max(done, newest_full.get(x["schedule"], done)) + retried_done: Dict[str, datetime] = {} + retried_full: Dict[str, datetime] = {} + if retry_suffix: + for x in backups: + done = parse_time(x["completed"]) if x["phase"] == COMPLETED else None + if done and x["name"].endswith(retry_suffix): + retried_done[x["name"][:-len(retry_suffix)]] = done + if not x["volume_snapshots_incomplete"]: + retried_full[x["name"][:-len(retry_suffix)]] = done for b in backups: - b["deliberate_test"] = is_deliberate(b["name"], patterns) t = when(b) later_done, later_full = newest_done.get(b["schedule"]), newest_full.get(b["schedule"]) # A backup with no schedule label is never resolved: a later manual backup may cover other namespaces entirely. scheduled = b["schedule"] is not None - b["failure_resolved"] = bool(scheduled and t and later_done and later_done > t) if b["phase"] in (FAILED, PARTIALLY_FAILED) else None - b["snapshots_resolved"] = bool(scheduled and t and later_full and later_full > t) if b["volume_snapshots_incomplete"] else None - newest_clean = max((c for c in (parse_time(r["results_written"]) for r in restores if r["clean"]) if c), default=None) + by_retry = bool(t and retried_done.get(b["name"]) and retried_done[b["name"]] > t) + by_full_retry = bool(t and retried_full.get(b["name"]) and retried_full[b["name"]] > t) + failed = b["phase"] in (FAILED, PARTIALLY_FAILED) + b["failure_resolved"] = bool(scheduled and t and later_done and later_done > t) or by_retry if failed else None + b["snapshots_resolved"] = (bool(scheduled and t and later_full and later_full > t) or by_full_retry + if b["volume_snapshots_incomplete"] else None) + b["resolved_by_retry"] = (failed and by_retry) or (bool(b["volume_snapshots_incomplete"]) and by_full_retry) + clean_retry = {r["name"][:-len(retry_suffix)]: parse_time(r["results_written"]) for r in restores + if retry_suffix and r["clean"] and r["name"].endswith(retry_suffix) and parse_time(r["results_written"])} for r in restores: - r["deliberate_test"] = is_deliberate(r["name"], patterns) t = parse_time(r["results_written"]) - r["error_resolved"] = bool(t and newest_clean and newest_clean > t) if r["clean"] is False else None + by_retry = bool(t and clean_retry.get(r["name"]) and clean_retry[r["name"]] > t) + # Backup storage doesn't record which backup a restore used, so only the restore's own retry can resolve it. + r["error_resolved"] = by_retry if r["clean"] is False else None + r["resolved_by_retry"] = r["clean"] is False and by_retry def summarize(locations: List[Dict[str, Any]], settings: Dict[str, Any], now: datetime, discovered: bool = True) -> Dict[str, Any]: @@ -279,8 +301,12 @@ def summarize(locations: List[Dict[str, Any]], settings: Dict[str, Any], now: da backups = [b for loc in read for b in loc["backups"]] restores = [r for loc in locations if loc["restores_read"] for r in loc["restores"]] restores_complete = discovered and all(loc["restores_read"] for loc in locations) - patterns = settings.get("deliberate_failure_tests") or [] - classify_history(backups, restores, patterns) + # Names and schedules are local to an account/container/prefix, not globally unique. + for loc in locations: + classify_history(loc["backups"] if loc["backups_read"] else [], + loc["restores"] if loc["restores_read"] else [], settings.get("retry_suffix")) + label = settings.get("validation_label") + validation = [b for b in backups if label and b.get("labels", {}).get(label[0]) == label[1]] latest: Dict[str, datetime] = {} for b in backups: when = parse_time(b["completed"]) @@ -310,9 +336,9 @@ def known(value): "backups_other_phase": known(sum(1 for b in backups if b["phase"] not in (COMPLETED, PARTIALLY_FAILED, FAILED))), "backups_metadata_unreadable": known(sum(loc["backups_unreadable"] for loc in read)), "backups_volume_snapshots_incomplete": known(sum(1 for b in backups if b["volume_snapshots_incomplete"])), - "backups_failed_unresolved": known(sum(1 for b in backups if b["failure_resolved"] is False and not b["deliberate_test"])), + "backups_failed_unresolved": known(sum(1 for b in backups if b["failure_resolved"] is False)), "backups_volume_snapshots_incomplete_unresolved": known(sum(1 for b in backups - if b["snapshots_resolved"] is False and not b["deliberate_test"])), + if b["snapshots_resolved"] is False)), "expired_backups_not_deleted": known(sum(1 for b in backups if b["expired_not_deleted"])), "schedules_observed": known(len(schedules)), "schedules_with_success_within_window": known(len(recent)), @@ -331,12 +357,15 @@ def known(value): "restores_total": len(restores) if restores_complete else None, "restores_without_errors": sum(1 for r in restores if r["clean"]) if restores_complete else None, "restores_with_errors": sum(1 for r in restores if r["clean"] is False) if restores_complete else None, - "restores_with_errors_unresolved": sum(1 for r in restores if r["error_resolved"] is False and not r["deliberate_test"]) + "restores_with_errors_unresolved": sum(1 for r in restores if r["error_resolved"] is False) if restores_complete else None, "restores_results_missing": sum(1 for r in restores if not r["results_read"]) if restores_complete else None, - "deliberate_failure_tests": sum(1 for b in backups if b["deliberate_test"] and (b["failure_resolved"] is not None or b["snapshots_resolved"] is not None)) - + sum(1 for r in restores if r["deliberate_test"] and r["clean"] is False) if complete and restores_complete else None, - "deliberate_failure_test_patterns": patterns, + "failures_resolved_by_retry": (sum(1 for b in backups if b["resolved_by_retry"]) + sum(1 for r in restores if r["resolved_by_retry"]) + if complete and restores_complete and settings.get("retry_suffix") else None), + "retry_suffix": settings.get("retry_suffix"), + "validation_backups": known(len(validation)) if label else None, + "validation_backups_failed": known(sum(1 for b in validation if b["phase"] in (FAILED, PARTIALLY_FAILED))) if label else None, + "validation_label": f"{label[0]}={label[1]}" if label else None, "days_since_last_clean_restore": int((now - max(clean_times)).total_seconds() // 86400) if restores_complete and clean_times else None, "restores_date_unknown": sum(1 for r in restores if r["results_read"] and not parse_time(r["results_written"])) if restores_complete else None, "restore_test_interval_days": settings.get("restore_test_interval_days"), @@ -365,8 +394,12 @@ def settings_from_env(collector: Optional[Collector] = None) -> Dict[str, Any]: lookback = min(int(raw_lb), MAX_MISSED_LOOKBACK_DAYS) if raw_lb.isdigit() and int(raw_lb) > 0 else DEFAULT_MISSED_LOOKBACK_DAYS if collector is not None and raw_lb and not (raw_lb.isdigit() and int(raw_lb) > 0): collector.record(f"config {MISSED_LOOKBACK_ENV}", ValueError(f"{MISSED_LOOKBACK_ENV}={raw_lb!r} is not a whole number above 0")) + raw_label = (os.environ.get(VALIDATION_LABEL_ENV) or "").strip() + if collector is not None and raw_label and parse_label(raw_label) is None: + collector.record(f"config {VALIDATION_LABEL_ENV}", ValueError(f"{VALIDATION_LABEL_ENV}={raw_label!r} must be key=value; left unset")) return { - "deliberate_failure_tests": _csv(os.environ.get(DELIBERATE_ENV)), + "retry_suffix": (os.environ.get(RETRY_SUFFIX_ENV) or "").strip() or None, + "validation_label": parse_label(os.environ.get(VALIDATION_LABEL_ENV)), "expected_schedules": expected, "rpo_hours": rpo, "missed_run_lookback_days": lookback, diff --git a/fetchers/azure/velero_backups/fetcher.yaml b/fetchers/azure/velero_backups/fetcher.yaml index 4f575391..f2e2cbeb 100644 --- a/fetchers/azure/velero_backups/fetcher.yaml +++ b/fetchers/azure/velero_backups/fetcher.yaml @@ -74,14 +74,22 @@ config_schema: description: > How many days of each expected schedule's periods to check for missed runs, up to 90. A value that is not a whole number above 0 fails the run as bad_config. - deliberate_failure_tests: + retry_suffix: type: string required: false - env: AZURE_VELERO_DELIBERATE_FAILURE_TESTS + env: AZURE_VELERO_RETRY_SUFFIX description: > - Comma-separated name patterns (* and ? wildcards) for backups and restores that are planned failure tests. A - failed backup or a restore with errors whose name matches is counted in deliberate_failure_tests and never as a - current failure. It stays in the totals, so history is kept. + The suffix a retry adds to the name of the backup or restore it retries (such as -retry). When set, a failed + backup or a restore with errors is also resolved by its own retry: finishing later without + failing. A failed retry, or none, leaves the failure current. No failure is ever excluded by its name or labels. + validation_label: + type: string + required: false + env: AZURE_VELERO_VALIDATION_LABEL + description: > + A label, as key=value, that marks validation backups (such as validation-test=true). Matching backups are + counted in validation_backups and validation_backups_failed and are still judged like every other backup. A + value not in that form fails the run as bad_config and leaves it unset. restore_test_interval_days: type: integer required: false @@ -159,8 +167,8 @@ evidence_set: - backups_other_phase — Number of backups in any other phase, such as InProgress - backups_metadata_unreadable — Number of backups whose velero-backup.json could not be read or parsed - backups_volume_snapshots_incomplete — Number of backups that completed fewer volume snapshots (native and CSI) than they attempted - - backups_failed_unresolved — Number of Failed or PartiallyFailed backups that are not deliberate failure tests and have no later Completed backup of the same schedule (a current failure); a backup with no schedule label stays unresolved, since a later manual backup may cover other namespaces; resolved failures stay in backups_failed and backups_partially_failed - - backups_volume_snapshots_incomplete_unresolved — Number of backups with incomplete volume snapshots that are not deliberate failure tests and have no later Completed backup of the same schedule that completed every snapshot it attempted; a backup with no schedule label stays unresolved + - backups_failed_unresolved — Number of Failed or PartiallyFailed backups that have no later Completed backup of the same schedule (a current failure); a backup with no schedule label stays unresolved, since a later manual backup may cover other namespaces; resolved failures stay in backups_failed and backups_partially_failed + - backups_volume_snapshots_incomplete_unresolved — Number of backups with incomplete volume snapshots that have no later Completed backup of the same schedule that completed every snapshot it attempted; a backup with no schedule label stays unresolved - schedules_observed — Number of schedules named by a backup's velero.io/schedule-name label - schedules_with_success_within_window — Number of schedules whose newest Completed backup finished within the schedule's window (a success within the window): its interval plus 10% (at least 2 hours) when listed in expected_schedules, otherwise success_window_hours - schedules_without_success_within_window — List of schedule names without a success within the window @@ -185,10 +193,13 @@ evidence_set: - restores_total — Number of restores Velero recorded in the backup storage locations; null unless every location's restores were read - restores_without_errors — Number of clean restores (a results file that lists no errors) - restores_with_errors — Number of restores whose results file lists at least one error - - restores_with_errors_unresolved — Number of restores with errors that are not deliberate failure tests and have no later restore without errors (a current failure); a restore with errors and no readable date is never resolved + - restores_with_errors_unresolved — Number of restores with errors not resolved by their own retry (a later restore without errors); a clean restore of anything else never resolves one, since storage doesn't record which backup a restore used, and a restore with errors and no readable date is never resolved - restores_results_missing — Number of restores whose results file is missing or could not be read - - deliberate_failure_tests — Number of failed backups and restores with errors whose name matches a deliberate_failure_tests pattern; null unless every location's backups and restores were read - - deliberate_failure_test_patterns — List of the deliberate_failure_tests patterns as given + - failures_resolved_by_retry — Number of failed backups and restores with errors resolved by their own retry (a later that did not fail); null while retry_suffix is not set or any location could not be read + - retry_suffix — The retry name suffix, as configured; null when not set + - validation_backups — Number of backups carrying validation_label; they are judged like every other backup; null while validation_label is not set + - validation_backups_failed — Number of validation backups that are Failed or PartiallyFailed; they also stay in every failure count + - validation_label — The validation label as key=value, as configured; null when not set - restores_date_unknown — Number of restores whose results file was read but has no readable date; they never count as recent - days_since_last_clean_restore — Days since the newest clean restore's results file was written; null if there is no clean restore - restore_test_interval_days — The adopted restore-test interval in days, as configured; null when not set diff --git a/tests/test_azure_velero_backups.py b/tests/test_azure_velero_backups.py index aa7d66de..f4aa98f4 100644 --- a/tests/test_azure_velero_backups.py +++ b/tests/test_azure_velero_backups.py @@ -25,7 +25,8 @@ NOW = datetime(2026, 11, 16, 12, 0, tzinfo=timezone.utc) SETTINGS = {"success_window_hours": 26, "expiry_grace_hours": 24, "container_names": ["velero"], "explicit_locations": [], "restore_test_interval_days": None, - "expected_schedules": None, "rpo_hours": None, "missed_run_lookback_days": 7, "deliberate_failure_tests": []} + "expected_schedules": None, "rpo_hours": None, "missed_run_lookback_days": 7, + "retry_suffix": None, "validation_label": None} SUB = "/subscriptions/s1" ACCOUNT_ID = f"{SUB}/resourceGroups/rg1/providers/Microsoft.Storage/storageAccounts/backupsa" ENDPOINT = "https://backupsa.blob.core.windows.net/" @@ -196,7 +197,7 @@ def serve(**kwargs): monkeypatch.setenv("FETCHER_STATUS_FILE", str(tmp_path / "status.json")) for env in ("AZURE_AUTHORITY_HOST", "AZURE_TENANT_ID", "AZURE_ENVIRONMENT", "AZURE_RESOURCE_EXCLUSIONS", "AZURE_TARGET_SUBSCRIPTION_ONLY", vb.CONTAINERS_ENV, vb.LOCATIONS_ENV, vb.WINDOW_ENV, vb.GRACE_ENV, vb.INTERVAL_ENV, - vb.EXPECTED_ENV, vb.RPO_ENV, vb.MISSED_LOOKBACK_ENV, vb.DELIBERATE_ENV): + vb.EXPECTED_ENV, vb.RPO_ENV, vb.MISSED_LOOKBACK_ENV, vb.RETRY_SUFFIX_ENV, vb.VALIDATION_LABEL_ENV): monkeypatch.delenv(env, raising=False) return serve @@ -427,28 +428,97 @@ def test_a_listed_weekly_schedule_is_judged_against_its_own_interval(): assert vb.success_window_hours(24, SETTINGS) == 26 and vb.success_window_hours(168, SETTINGS) == 184 -def test_resolved_and_deliberate_failures_stay_in_history_but_are_not_current(): +def test_resolved_failures_stay_in_history_but_are_not_current(): resolved = _runs("daily", [NOW - timedelta(hours=50)], 720, phase="Failed", name="daily-failed-old") current = _runs("hourly", [NOW - timedelta(hours=1)], 720, phase="PartiallyFailed", name="hourly-failed-new") - planned = _runs("daily", [NOW - timedelta(hours=1)], 720, phase="Failed", name="failtest-1") ok = _runs("daily", [NOW - timedelta(hours=26)], 720) snaps_old = _runs("weekly", [NOW - timedelta(hours=200)], 720, attempted=2, completed=1, name="weekly-old") snaps_ok = _runs("weekly", [NOW - timedelta(hours=30)], 720, name="weekly-new") rs = [vb.restore_record("r-bad-old", results(errors=1), iso(NOW - timedelta(days=5))), vb.restore_record("r-good", results(), iso(NOW - timedelta(days=2))), - vb.restore_record("r-bad-new", results(errors=2), iso(NOW - timedelta(days=1))), - vb.restore_record("failtest-restore", results(errors=1), iso(NOW - timedelta(hours=3)))] - settings = {**SETTINGS, "deliberate_failure_tests": ["failtest-*"]} - s = vb.summarize([location(resolved + current + planned + ok + snaps_old + snaps_ok, rs)], settings, NOW) - assert (s["backups_failed"], s["backups_partially_failed"], s["backups_failed_unresolved"]) == (2, 1, 1) + vb.restore_record("r-bad-new", results(errors=2), iso(NOW - timedelta(days=1)))] + s = vb.summarize([location(resolved + current + ok + snaps_old + snaps_ok, rs)], SETTINGS, NOW) + assert (s["backups_failed"], s["backups_partially_failed"], s["backups_failed_unresolved"]) == (1, 1, 1) assert (s["backups_volume_snapshots_incomplete"], s["backups_volume_snapshots_incomplete_unresolved"]) == (1, 0) - assert (s["restores_with_errors"], s["restores_with_errors_unresolved"]) == (3, 1) - assert (s["deliberate_failure_tests"], s["deliberate_failure_test_patterns"]) == (2, ["failtest-*"]) + # r-good is a different restore, so it resolves neither failure: storage doesn't say which backup each one used. + assert (s["restores_with_errors"], s["restores_with_errors_unresolved"]) == (2, 2) + + +def _validation_backup(name, phase, hours_ago): + raw = {"metadata": {"name": name, "labels": {"validation-test": "true", "ticket": "TICKET-1"}}, + "status": {"phase": phase, "completionTimestamp": iso(NOW - timedelta(hours=hours_ago)), + "expiration": iso(NOW + timedelta(days=30))}} + return vb.backup_record(name, raw, NOW, 24) + + +def test_a_failure_is_resolved_only_by_its_own_successful_retry(): + # Interruption tests named restore-drill-interrupt-; a retry has the same name plus -retry. + bk = [_validation_backup("restore-drill-interrupt-20261104", "Failed", 100), + _validation_backup("restore-drill-interrupt-20261104-retry", "Completed", 99), + _validation_backup("restore-drill-interrupt-20261110", "Failed", 30), + _validation_backup("restore-drill-interrupt-20261110-retry", "Failed", 29)] + rs = [vb.restore_record("restore-drill-interrupt-20261104", results(errors=1), iso(NOW - timedelta(hours=98))), + vb.restore_record("restore-drill-interrupt-20261104-retry", results(), iso(NOW - timedelta(hours=97)))] + settings = {**SETTINGS, "retry_suffix": "-retry", "validation_label": ("validation-test", "true")} + s = vb.summarize([location(bk, rs)], settings, NOW) + # The 20261104 backup and restore are resolved by their retries; the 20261110 test and its failed retry stay current. + assert (s["backups_failed"], s["backups_failed_unresolved"], s["failures_resolved_by_retry"]) == (3, 2, 2) + assert (s["restores_with_errors"], s["restores_with_errors_unresolved"]) == (1, 0) + # Validation backups are counted, never excluded: all four carry the label and three failed. + assert (s["validation_backups"], s["validation_backups_failed"], s["validation_label"]) == (4, 3, "validation-test=true") + # Without a retry suffix nothing is resolved by retry, and nothing is hidden. + plain = vb.summarize([location(bk, rs)], SETTINGS, NOW) + assert (plain["backups_failed_unresolved"], plain["failures_resolved_by_retry"], plain["validation_backups"]) == (3, None, None) + assert plain["restores_with_errors_unresolved"] == 1 + + +@pytest.mark.parametrize("different_field", ["storage_account", "container", "prefix"]) +@pytest.mark.parametrize("failure_kind", ["backup", "snapshots", "restore"]) +def test_only_a_retry_in_the_same_location_resolves_a_failure(different_field, failure_kind): + identity = {"storage_account": "backupsa", "container": "velero", "prefix": "cluster-a"} + original, unrelated = location(), location() + original.update(identity) + unrelated.update({**identity, different_field: "other"}) + if failure_kind == "restore": + failed = vb.restore_record("drill", results(errors=1), iso(NOW - timedelta(hours=10))) + retry = vb.restore_record("drill-retry", results(), iso(NOW - timedelta(hours=2))) + records, unresolved = "restores", "restores_with_errors_unresolved" + resolved_field = "error_resolved" + else: + raw = backup("drill", schedule=None, done_hours_ago=10, + phase="Failed" if failure_kind == "backup" else "Completed", + attempted=2, completed=1) + failed = vb.backup_record("drill", raw, NOW, 24) + retry = vb.backup_record("drill-retry", backup("drill-retry", schedule=None, done_hours_ago=2), NOW, 24) + records = "backups" + unresolved = "backups_failed_unresolved" if failure_kind == "backup" else "backups_volume_snapshots_incomplete_unresolved" + resolved_field = "failure_resolved" if failure_kind == "backup" else "snapshots_resolved" + original[records].append(failed) + unrelated[records].append(retry) + settings = {**SETTINGS, "retry_suffix": "-retry"} + + summary = vb.summarize([original, unrelated], settings, NOW) + assert summary[unresolved] == 1 + assert summary["failures_resolved_by_retry"] == 0 + assert failed[resolved_field] is False + assert failed["resolved_by_retry"] is False + + original[records].append(dict(retry)) + recovered = vb.summarize([original, unrelated], settings, NOW) + assert recovered[unresolved] == 0 + assert recovered["failures_resolved_by_retry"] == 1 + assert failed[resolved_field] is True + assert failed["resolved_by_retry"] is True + + +def test_a_validation_label_not_in_key_value_form_is_bad_config(): + assert vb.parse_label("validation-test=true") == ("validation-test", "true") + assert vb.parse_label("validation-test") is None and vb.parse_label("=true") is None and vb.parse_label(None) is None def test_unresolved_counts_stay_null_when_a_location_is_unread(): s = vb.summarize([location(read=False, restores_read=False)], SETTINGS, NOW) - assert (s["backups_failed_unresolved"], s["restores_with_errors_unresolved"], s["deliberate_failure_tests"]) == (None, None, None) + assert (s["backups_failed_unresolved"], s["restores_with_errors_unresolved"]) == (None, None) def test_blob_reads_retry_a_dropped_connection(monkeypatch): @@ -505,3 +575,14 @@ def test_a_failed_backup_without_a_schedule_stays_unresolved(): scheduled_failed = vb.backup_record("d-failed", backup("d-failed", phase="Failed", done_hours_ago=10), NOW, 24) scheduled_ok = vb.backup_record("d-ok", backup("d-ok", done_hours_ago=2), NOW, 24) assert vb.summarize([location([scheduled_failed, scheduled_ok])], SETTINGS, NOW)["backups_failed_unresolved"] == 0 + + +def test_a_clean_restore_of_something_else_never_resolves_a_failed_restore(): + # A failed restore, then a clean restore with an unrelated name: still a current failure. + rs = [vb.restore_record("restore-app-a", results(errors=3), iso(NOW - timedelta(days=2))), + vb.restore_record("restore-app-b", results(), iso(NOW - timedelta(days=1)))] + s = vb.summarize([location([], rs)], {**SETTINGS, "retry_suffix": "-retry"}, NOW) + assert (s["restores_with_errors"], s["restores_with_errors_unresolved"], s["failures_resolved_by_retry"]) == (1, 1, 0) + retried = rs + [vb.restore_record("restore-app-a-retry", results(), iso(NOW - timedelta(hours=12)))] + s = vb.summarize([location([], retried)], {**SETTINGS, "retry_suffix": "-retry"}, NOW) + assert (s["restores_with_errors_unresolved"], s["failures_resolved_by_retry"]) == (0, 1)