diff --git a/CHANGELOG.md b/CHANGELOG.md index 01dee889..6b0d15a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,15 @@ schemas and the `paramify` CLI — not the internal code. ### Added +- **`azure_endpoint_availability` fetcher.** Reads Application Insights availability tests and their hourly + results, and Standard load balancers' data-path (VipAvailability) and health-probe (DipAvailability) + availability with measurement coverage, each judged against an availability target once one is set (none by + default) and only when every hour was measured; items that cannot be judged are counted apart, so one new or + unread test never leaves the verdict on the others unknown. A load balancer's data path is read only when it has + load-balancing rules and its health probe only when it has probes, so outbound-only load balancers are counted + apart instead of reading as unmeasured. Reads classic ping test URLs from their XML, and honours location: + exclusions. Reports whether tests check the status code and content. Unread results leave verdicts null rather + than 0. - `paramify upload --no-waf-check` turns off the check that drops a file holding `../`, `/etc/passwd`, `/bin/sh`, `/bin/bash` or ` Optional[str]: + """The Application Insights component a web test reports to, from its hidden-link tag.""" + for key, value in (tags or {}).items(): + if key.lower().startswith("hidden-link:") and (value or "").lower() == "resource": + return key.split(":", 1)[1] + return None + + +def validation(props: Dict[str, Any]) -> Dict[str, Optional[bool]]: + """What a test checks beyond reachability: standard tests carry ValidationRules; classic ping tests carry rules in their XML.""" + rules = props.get("ValidationRules") + if isinstance(rules, dict): + content = ((rules.get("ContentValidation") or {}).get("ContentMatch") or "").strip() + return {"checks_status_code": bool(rules.get("ExpectedHttpStatusCode")), "checks_content": bool(content), + "checks_ssl": bool(rules.get("SSLCheck"))} + xml = ((props.get("Configuration") or {}).get("WebTest") or "") + if xml: + return {"checks_status_code": bool(re.search(r'ExpectedHttpStatusCode="[1-9]', xml)), + "checks_content": "ValidationRuleFindText" in xml and bool(re.search(r'Name="FindText" Value="[^"]+"', xml)), + "checks_ssl": False} + return {"checks_status_code": False, "checks_content": False, "checks_ssl": False} + + +def classic_url(props: Dict[str, Any]) -> Optional[str]: + """A classic URL ping test keeps its URL in the Url attribute of the Request element of its WebTest XML.""" + xml = ((props.get("Configuration") or {}).get("WebTest") or "") + found = re.search(r']*\bUrl="([^"]+)"', xml) + return html.unescape(found.group(1)) if found else None + + +def test_record(raw: Dict[str, Any]) -> Dict[str, Any]: + props = raw.get("properties") or {} + request = props.get("Request") or {} + return {**validation(props), + "id": raw.get("id"), + "name": raw.get("name"), + "test_name": props.get("Name") or raw.get("name"), + "resource_group": resource_group_from_id(raw.get("id")), + "location": raw.get("location"), + "kind": (props.get("Kind") or raw.get("kind") or "").lower() or None, + "enabled": bool(props.get("Enabled")), + "url": request.get("RequestUrl") or classic_url(props), + "frequency_seconds": props.get("Frequency"), + "locations": len(props.get("Locations") or []), + "component_id": component_of(raw.get("tags")), + } + + +def load_balancer_record(raw: Dict[str, Any]) -> Dict[str, Any]: + props = raw.get("properties") or {} + frontends = props.get("frontendIPConfigurations") or [] + return { + "id": raw.get("id"), + "name": raw.get("name"), + "resource_group": resource_group_from_id(raw.get("id")), + "location": raw.get("location"), + "sku": (raw.get("sku") or {}).get("name"), + "standard": ((raw.get("sku") or {}).get("name") or "").lower() == "standard", + "public": any((f.get("properties") or {}).get("publicIPAddress") for f in frontends), + "frontends": len(frontends), + # Data-path availability is measured on load-balancing rules and health-probe status on probes: a load balancer + # without them (an outbound-only one, say) reports no data for that metric, which is not a gap in measurement. + "load_balancing_rules": len(props.get("loadBalancingRules") or []), + "inbound_nat_rules": len(props.get("inboundNatRules") or []), + "probes": len(props.get("probes") or []), + } + + +def hourly_points(metrics: List[Dict[str, Any]], name: str) -> List[Tuple[str, float]]: + """(time, average) for every hour with data, across the metric's series.""" + points: Dict[str, List[float]] = {} + for metric in metrics or []: + if ((metric.get("name") or {}).get("value") or "").lower() != name.lower(): + continue + for series in metric.get("timeseries") or []: + for p in series.get("data") or []: + if p.get("average") is not None: + points.setdefault(p.get("timeStamp"), []).append(float(p["average"])) + return [(t, sum(v) / len(v)) for t, v in sorted(points.items())] + + +def judged(points: List[Tuple[str, float]], status: str, target: Optional[float], hours_expected: int = 0) -> Dict[str, Any]: + """Average over the hours with data, lowest and latest hour, and coverage of the window. Below target is null (unknown) + unless every hour of the window was measured and a target is set, so a few good hours never stand for a whole window.""" + has = status == READ and bool(points) + values = [v for _, v in points] + average = round(sum(values) / len(values), 3) if has else None + measured = len(points) if status == READ else None + complete = status == READ and hours_expected > 0 and measured >= hours_expected + return {"metric_status": status, "hours_expected": hours_expected if status == READ else None, "hours_with_data": measured, + "hours_missing": max(hours_expected - measured, 0) if status == READ else None, + "measurement_coverage_percentage": round(100 * min(measured, hours_expected) / hours_expected, 1) if status == READ and hours_expected else None, + "measurement_complete": complete if status == READ else None, + "availability_percentage": average, "lowest_hour_percentage": round(min(values), 3) if has else None, + "latest_hour_percentage": round(values[-1], 3) if has else None, + "below_target": (average < target) if complete and average is not None and target is not None else None} + + +def _lowest(values: List[Optional[float]]) -> Optional[float]: + known = [v for v in values if v is not None] + return min(known) if known else None + + +def summarize(tests: List[Dict[str, Any]], balancers: List[Dict[str, Any]], settings: Dict[str, Any], + tests_listed: bool = True, balancers_listed: bool = True) -> Dict[str, Any]: + enabled = [t for t in tests if t["enabled"]] + tests_complete = tests_listed and all(t["metric_status"] == READ for t in enabled) + standard = [b for b in balancers if b["standard"]] + lb_complete = balancers_listed and all(b[k]["metric_status"] in (READ, NOT_APPLICABLE) for b in standard for k in LB_METRICS) + target = settings["target"] + + def t_known(value): + return value if tests_complete else None + + def applicable(b): + return [b[k] for k in LB_METRICS if b[k]["metric_status"] != NOT_APPLICABLE] + + # Each test or load balancer is judged against the target only once every hour of the lookback was measured. The + # below-target counts are over the items that could be judged, and the rest are counted as not judged, so one new + # or unread item never leaves the verdict on the others unknown. + tests_judged = [t for t in enabled if t["below_target"] is not None] + lbs_judged = [b for b in standard if applicable(b) and all(m["below_target"] is not None for m in applicable(b))] + + def t_target(value): + return value if tests_listed and target is not None else None + + def lb_target(value): + return value if balancers_listed and target is not None else None + + def _coverage(items): + known = [c for c in items if c is not None] + return min(known) if known else None + + def lb_known(value): + return value if lb_complete else None + + return { + "availability_tests_enabled": len(enabled) if tests_listed else None, + "availability_tests_results_unread": sum(1 for t in enabled if t["metric_status"] == ERROR) if tests_listed else None, + "endpoints_tested": len({t["url"] for t in enabled if t["url"]}) if tests_listed else None, + "availability_tests_checking_application": sum(1 for t in enabled if t["checks_status_code"] and t["checks_content"]) if tests_listed else None, + "availability_tests_reachability_only": sum(1 for t in enabled if not (t["checks_status_code"] and t["checks_content"])) if tests_listed else None, + "availability_tests_with_missing_measurements": t_known(sum(1 for t in enabled if not t["measurement_complete"])), + "lowest_measurement_coverage_percentage": t_known(_coverage([t["measurement_coverage_percentage"] for t in enabled])), + "availability_tests_without_data": t_known(sum(1 for t in enabled if t["availability_percentage"] is None)), + "lowest_availability_percentage": t_known(_lowest([t["availability_percentage"] for t in enabled])), + "availability_tests_below_target": t_target(sum(1 for t in tests_judged if t["below_target"])), + "availability_tests_not_judged": t_target(len(enabled) - len(tests_judged)), + "availability_tests_below_target_latest_hour": t_target(sum( + 1 for t in enabled if t["latest_hour_percentage"] is not None and t["latest_hour_percentage"] < (target or 0))), + "load_balancers_standard": len(standard) if balancers_listed else None, + "load_balancers_public": sum(1 for b in balancers if b["public"]) if balancers_listed else None, + "load_balancers_metrics_unread": sum(1 for b in standard if ERROR in (b["data_path"]["metric_status"], b["health_probe"]["metric_status"])) + if balancers_listed else None, + "lowest_data_path_availability": lb_known(_lowest([b["data_path"]["availability_percentage"] for b in standard])), + "lowest_health_probe_availability": lb_known(_lowest([b["health_probe"]["availability_percentage"] for b in standard])), + "load_balancers_below_target": lb_target(sum(1 for b in lbs_judged if any(m["below_target"] for m in applicable(b)))), + "load_balancers_not_judged": lb_target(sum(1 for b in standard if applicable(b)) - len(lbs_judged)), + "load_balancers_without_measured_traffic": sum(1 for b in standard if not applicable(b)) if balancers_listed else None, + "load_balancers_with_missing_measurements": lb_known(sum(1 for b in standard if any(not m["measurement_complete"] for m in applicable(b)))), + "lowest_load_balancer_measurement_coverage_percentage": lb_known(_coverage( + [b[k]["measurement_coverage_percentage"] for b in standard for k in ("data_path", "health_probe")])), + "lookback_days": settings["lookback_days"], + "availability_target_percentage": settings["target"], + } + + +def settings_from_env(collector: Collector) -> Dict[str, Any]: + days, _ = bounded_int_config(collector, LOOKBACK_ENV, DEFAULT_LOOKBACK_DAYS, MAX_LOOKBACK_DAYS) + raw = (os.environ.get(TARGET_ENV) or "").strip() + target: Optional[float] = None # unset until the offering adopts an availability objective + if raw: + try: + value = float(raw) + if not 0 < value <= 100: + raise ValueError + target = value + except ValueError: + collector.record(f"config {TARGET_ENV}", ValueError( + f"invalid setting {TARGET_ENV}={raw!r}: expected a number above 0 and up to 100; left unset")) + return {"lookback_days": days, "target": target} + + +# --- collection --- + +def metric_points(client, collector: Collector, resource_id: str, namespace: str, metric: str, window: Tuple[datetime, datetime], + label: str, flt: Optional[str] = None) -> Tuple[List[Tuple[str, float]], str]: + params = {"metricnames": metric, "metricnamespace": namespace, "aggregation": "Average", "interval": "PT1H", + "timespan": f"{window[0].strftime('%Y-%m-%dT%H:%M:%SZ')}/{window[1].strftime('%Y-%m-%dT%H:%M:%SZ')}"} + if flt: + params["$filter"] = flt + metrics = collector.guard(f"insights.metrics.list({metric}, {label})", + lambda: client.list(f"{resource_id}/providers/Microsoft.Insights/metrics", METRICS_API, params), default=None) + return (hourly_points(metrics or [], metric), ERROR if metrics is None else READ) + + +def collect(client, subscription_id: str, collector: Collector, exclusions: Optional[ResourceExclusions] = None, + now: Optional[datetime] = None) -> tuple: + exclusions = exclusions if exclusions is not None else ResourceExclusions() + # End at the last complete hour: Azure Monitor publishes each hour a few minutes after it closes. + now = (now or datetime.now(timezone.utc)).replace(minute=0, second=0, microsecond=0) - timedelta(hours=1) + settings = settings_from_env(collector) + window = (now - timedelta(days=settings["lookback_days"]), now) + hours = settings["lookback_days"] * 24 + excluded: List[dict] = [] + + raw_tests = collector.guard("insights.webtests.list", lambda: client.list( + f"/subscriptions/{subscription_id}/providers/Microsoft.Insights/webtests", WEBTESTS_API), default=None) + tests_listed = raw_tests is not None + tests = sorted((test_record(t) for t in raw_tests or []), key=lambda t: arm_id_key(t["id"])) + tests, out = exclusions.split(tests, "availability_test") + excluded += out + for t in tests: + if not t["enabled"]: + t.update(judged([], NOT_APPLICABLE, settings["target"], hours)) + elif not t["component_id"]: + collector.record(f"insights.webtests.component ({t['name']})", ValueError("the test has no hidden-link tag naming its Application Insights component")) + t.update(judged([], ERROR, settings["target"], hours)) + else: + name = (t["test_name"] or "").replace("'", "''") + points, status = metric_points(client, collector, t["component_id"], "microsoft.insights/components", AVAILABILITY_METRIC, + window, t["name"], f"availabilityResult/name eq '{name}'") + t.update(judged(points, status, settings["target"], hours)) + + raw_lbs = collector.guard("network.loadBalancers.list", lambda: client.list( + f"/subscriptions/{subscription_id}/providers/Microsoft.Network/loadBalancers", NETWORK_API), default=None) + balancers_listed = raw_lbs is not None + balancers = sorted((load_balancer_record(b) for b in raw_lbs or []), key=lambda b: arm_id_key(b["id"])) + balancers, out = exclusions.split(balancers, "load_balancer") + excluded += out + for b in balancers: + for key, metric in (("data_path", DATA_PATH_METRIC), ("health_probe", HEALTH_PROBE_METRIC)): + measured = b["load_balancing_rules"] if key == "data_path" else b["probes"] + if not b["standard"] or not measured: # Basic SKU emits neither metric; no rules or probes, nothing to measure + b[key] = judged([], NOT_APPLICABLE, settings["target"], hours) + continue + points, status = metric_points(client, collector, b["id"], "Microsoft.Network/loadBalancers", metric, window, b["name"]) + # The metrics report 0-100 per hour; Basic SKU load balancers emit none. + b[key] = judged(points, status, settings["target"], hours) + + exclusions.record_invalid(collector) + if not (tests_listed and balancers_listed): + exclusions.mark_unread() + exclusions.warn_unmatched(logger) + results = {"availability_tests": tests, "load_balancers": balancers, + "window": {"start": window[0].strftime("%Y-%m-%dT%H:%M:%SZ"), "end": window[1].strftime("%Y-%m-%dT%H:%M:%SZ")}, + "excluded": sorted(excluded, key=lambda e: arm_id_key(e.get("id")))} + summary = {**summarize(tests, balancers, settings, tests_listed, balancers_listed), + **exclusions.summary(excluded, ("availability_test", "load_balancer"), { + "availability_tests": len(tests) if tests_listed else None, + "load_balancers": len(balancers) if balancers_listed else None})} + return results, summary + + +def collect_subscription(client, subscription_id: str, collector: Collector) -> tuple: + return collect(client, subscription_id, collector, ResourceExclusions.from_env()) + + +def main() -> int: + return run_subscription(fetcher=NAME, logger=logger, collect=collect_subscription) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/fetchers/azure/endpoint_availability/fetcher.yaml b/fetchers/azure/endpoint_availability/fetcher.yaml new file mode 100644 index 00000000..25caab84 --- /dev/null +++ b/fetchers/azure/endpoint_availability/fetcher.yaml @@ -0,0 +1,122 @@ +name: azure_endpoint_availability +version: 0.1.0 +description: >- + Shows whether customer-facing endpoints are tested for availability from outside and how available they were, and whether Standard load balancers kept their data path and backend health probes up. +category: azure + +supports_targets: true + +runtime: + type: python + entry: fetcher.py + +output: + type: json + path: azure_endpoint_availability.json + aggregation: per_target + +config_schema: + lookback_days: + type: integer + required: false + default: 7 + env: AZURE_ENDPOINT_AVAILABILITY_LOOKBACK_DAYS + description: > + How many days of hourly availability to read, from 1 to 30. Azure Monitor keeps platform metrics for 93 days; a + larger value is read as 30. A value that is not a whole number above 0 fails the run as bad_config. + availability_target_percentage: + type: string + required: false + env: AZURE_AVAILABILITY_TARGET_PERCENTAGE + description: > + The adopted availability objective, as a percentage above 0 and up to 100 (decimals allowed). There is no + default: while it is unset, every below-target count is null rather than judged against an objective nobody + adopted. A test or load balancer whose average over the lookback is lower counts as below target. A value + outside that range fails the run as bad_config and leaves it unset. + resource_exclusions: + type: patterns + prefixes: [rg, location, id] + bare: false + required: false + env: AZURE_RESOURCE_EXCLUSIONS + description: > + Patterns that take availability tests or load balancers out of scope, as for the other Azure fetchers. + +target_schema: + subscription_id: + type: string + required: false + env: AZURE_SUBSCRIPTION_ID + description: > + Azure subscription id to collect from. Optional — omit to use the first + enabled subscription the ambient credential can see; set one target per + subscription to fan out. + environment: + type: string + required: false + env: AZURE_ENVIRONMENT + description: Deployment environment label (e.g. prod) written into the evidence metadata. + +secrets: [] # DefaultAzureCredential only — see _categories/azure.yaml + +ksis: + - KSI-CNA-OFA # measured availability of customer-facing endpoints and the load balancers in front of them + +evidence_set: + reference_id: EVD-AZURE-ENDPOINT-AVAILABILITY + name: Azure Endpoint Availability + instructions: | + Script: azure/endpoint_availability/fetcher.py + + Access: Reader on the subscription, which includes reading Azure Monitor metrics. + + Commands Executed: + - subscriptions.list() — picks a subscription when none is set + - GET Microsoft.Insights/webtests — every Application Insights availability test, its URL (for a classic URL ping test, from its WebTest XML), locations, frequency, component, and the status code, content and certificate checks it validates + - GET components/{component}/providers/Microsoft.Insights/metrics (per enabled test) — hourly availabilityResults/availabilityPercentage for that test + - GET Microsoft.Network/loadBalancers — every load balancer, its SKU, whether it has a public frontend, and how many load-balancing rules, inbound NAT rules and health probes it has + - GET loadBalancers/{lb}/providers/Microsoft.Insights/metrics (per Standard load balancer) — hourly VipAvailability when it has load-balancing rules and DipAvailability when it has health probes; a load balancer without them (outbound-only, say) reports no data for that metric, so it is not read + + Output: azure_endpoint_availability_.json — each availability test (with what it validates) and Standard load balancer with its hourly availability and measurement coverage over the lookback, which ends at the start of the last complete hour (that hour is not included, because Azure Monitor publishes it a few minutes after it closes); each is judged against the availability target when one is set and every hour was measured. + + Summary fields: + + What was checked? + - availability_tests_enabled — Number of availability tests that are enabled; null if the test list could not be read + - availability_tests_results_unread — Number of enabled availability tests whose results could not be read, including a test with no link to its Application Insights component + - load_balancers_standard — Number of Standard load balancers, the only SKU that reports availability metrics + - load_balancers_public — Number of load balancers with a public frontend IP address + - load_balancers_metrics_unread — Number of Standard load balancers whose data-path or health-probe metric could not be read + - load_balancers_without_measured_traffic — Number of Standard load balancers with neither load-balancing rules nor health probes, such as an outbound-only load balancer, which report no availability metrics and are not judged + - lookback_days — Days of hourly availability read + - availability_target_percentage — The adopted availability objective each average is compared with; null when not set + - resource_exclusion_patterns — List of the resource_exclusions patterns as given + - resource_exclusion_patterns_unmatched — List of the patterns that matched nothing, such as a mistyped name; null when the test or load balancer list could not be read + - excluded_availability_tests — Number of availability tests a pattern took out of scope + - excluded_load_balancers — Number of load balancers a pattern took out of scope + - in_scope_availability_tests — Number of availability tests assessed, enabled or not + - in_scope_load_balancers — Number of load balancers assessed + + Are customer endpoints tested from outside, and do the tests check the application works? + - endpoints_tested — Number of distinct URLs that enabled availability tests request + - availability_tests_checking_application — Number of enabled availability tests that check both an expected HTTP status code and the response content, so they show the application works and not only that it answers + - availability_tests_reachability_only — Number of enabled availability tests that do not check both the status code and the content + - availability_tests_without_data — Number of enabled availability tests with no result in any hour of the lookback; null unless every enabled test's results were read + + Is every hour measured? + - availability_tests_with_missing_measurements — Number of enabled availability tests with at least one hour of the lookback that has no result; null unless every enabled test's results were read + - lowest_measurement_coverage_percentage — Percentage of the lookback's hours with a result, for the enabled test with the fewest (measurement coverage) + - load_balancers_with_missing_measurements — Number of Standard load balancers missing at least one hour of data-path data (when it has load-balancing rules) or health-probe data (when it has health probes) + - lowest_load_balancer_measurement_coverage_percentage — Measurement coverage of the Standard load balancer metric with the fewest hours of data + + Were the endpoints available? + - lowest_availability_percentage — Percentage of successful test runs for the enabled test with the lowest average over the hours that have data (the mean of its hourly averages; read it with the measurement coverage); null if no test has data or any test's results could not be read + - availability_tests_below_target — Number of enabled availability tests with every hour of the lookback measured whose average is lower than availability_target_percentage (below target); a test missing an hour or unread is not judged and is counted in availability_tests_not_judged, so a few good hours never stand for the whole lookback; null while no target is set or the test list could not be read + - availability_tests_not_judged — Number of enabled availability tests not judged against the target because an hour of the lookback has no result or the results could not be read; null while no target is set or the test list could not be read + - availability_tests_below_target_latest_hour — Number of enabled availability tests whose most recent hour with data is lower than availability_target_percentage; a test with no data is not counted; null while no target is set or the test list could not be read + + Did the load balancers stay up? + - lowest_data_path_availability — Percentage for the Standard load balancer with the lowest average data-path availability (VipAvailability: whether traffic reaches the load balancer's frontend) over the lookback; null if none has data or any could not be read + - lowest_health_probe_availability — Percentage for the Standard load balancer with the lowest average health-probe status (DipAvailability: whether its backend instances answer the health probe) over the lookback; null if none has data or any could not be read + - load_balancers_below_target — Number of Standard load balancers, with every hour of each metric that applies measured, whose average data-path availability or health-probe status is lower than availability_target_percentage; one missing an hour or unread is counted in load_balancers_not_judged instead; null while no target is set or the load balancer list could not be read + - load_balancers_not_judged — Number of Standard load balancers with load-balancing rules or health probes that are not judged against the target because an hour of a metric is missing or could not be read; null while no target is set or the load balancer list could not be read diff --git a/tests/test_azure_endpoint_availability.py b/tests/test_azure_endpoint_availability.py new file mode 100644 index 00000000..ac683f9a --- /dev/null +++ b/tests/test_azure_endpoint_availability.py @@ -0,0 +1,303 @@ +"""azure_endpoint_availability: web test and load balancer projection, hourly averages against the target, null-not-zero summaries, and a full run against synthetic ARM and metrics responses. No SDK or network.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from collections import namedtuple +from pathlib import Path +from urllib.parse import urlparse + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +AZURE = REPO_ROOT / "fetchers" / "azure" +sys.path.insert(0, str(REPO_ROOT / "fetchers" / "_lib")) +sys.path.insert(0, str(AZURE / "_shared")) + +_spec = importlib.util.spec_from_file_location("azure_endpoint_availability_under_test", AZURE / "endpoint_availability" / "fetcher.py") +ea = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(ea) +azure_rest = sys.modules["azure_rest"] + +SUB = "/subscriptions/s1" +COMPONENT = f"{SUB}/resourceGroups/rg1/providers/microsoft.insights/components/appi1" +SETTINGS = {"lookback_days": 7, "target": 99.9} + + +def raw_test(name="home", enabled=True, component=COMPONENT, url="https://app.example.com/health", location="eastus"): + tags = {f"hidden-link:{component}": "Resource"} if component else {} + return {"id": f"{SUB}/resourceGroups/rg1/providers/Microsoft.Insights/webtests/{name}-appi1", "name": f"{name}-appi1", "tags": tags, + "location": location, + "properties": {"Name": name, "Kind": "standard", "Enabled": enabled, "Frequency": 300, + "Locations": [{"Id": "us-va-ash-azr"}, {"Id": "us-tx-sn1-azr"}, {"Id": "us-il-ch1-azr"}], + "Request": {"RequestUrl": url}}} + + +def raw_lb(name="lb1", sku="Standard", public=True, rules=1, probes=1, location="eastus"): + fe = {"name": "fe", "properties": {"publicIPAddress": {"id": "pip"}} if public else {"privateIPAddress": "10.0.0.4"}} + return {"id": f"{SUB}/resourceGroups/rg-aks/providers/Microsoft.Network/loadBalancers/{name}", "name": name, "sku": {"name": sku}, + "location": location, "properties": {"frontendIPConfigurations": [fe], "loadBalancingRules": [{"name": f"r{i}"} for i in range(rules)], + "probes": [{"name": f"p{i}"} for i in range(probes)]}} + + +def metric(name, *hourly): + return [{"name": {"value": name}, "timeseries": [{"data": [{"timeStamp": f"2026-11-{16 + h // 24:02d}T{h % 24:02d}:00:00Z", "average": v} + for h, v in enumerate(hourly)]}]}] + + +def test_component_comes_from_the_hidden_link_tag(): + assert ea.component_of({f"hidden-link:{COMPONENT}": "Resource", "env": "x"}) == COMPONENT + assert ea.component_of({"hidden-link:/x": "Other"}) is None and ea.component_of(None) is None + + +def test_records_read_test_and_load_balancer_settings(): + t = ea.test_record(raw_test()) + assert (t["test_name"], t["kind"], t["enabled"], t["locations"], t["url"], t["component_id"]) == ( + "home", "standard", True, 3, "https://app.example.com/health", COMPONENT) + b = ea.load_balancer_record(raw_lb(public=False, sku="Basic")) + assert (b["standard"], b["public"], b["frontends"]) == (False, False, 1) + + +def test_hourly_points_average_series_and_skip_empty_hours(): + m = metric(ea.AVAILABILITY_METRIC, 100, 99) + [{"name": {"value": ea.AVAILABILITY_METRIC}, "timeseries": [ + {"data": [{"timeStamp": "2026-11-16T00:00:00Z", "average": 90}, {"timeStamp": "2026-11-16T02:00:00Z", "average": None}]}]}] + assert ea.hourly_points(m, ea.AVAILABILITY_METRIC) == [("2026-11-16T00:00:00Z", 95.0), ("2026-11-16T01:00:00Z", 99.0)] + + +def test_judgement_is_null_when_unread_or_without_data(): + ok = ea.judged([("a", 100.0), ("b", 99.0)], ea.READ, 99.9, 2) + assert (ok["availability_percentage"], ok["lowest_hour_percentage"], ok["latest_hour_percentage"], ok["below_target"]) == (99.5, 99.0, 99.0, True) + assert ea.judged([], ea.READ, 99.9)["availability_percentage"] is None and ea.judged([], ea.READ, 99.9)["hours_with_data"] == 0 + assert ea.judged([("a", 100.0)], ea.ERROR, 99.9)["hours_with_data"] is None + + +def _test(enabled=True, status=ea.READ, points=(("a", 100.0),), url="https://app.example.com/health"): + t = ea.test_record(raw_test(enabled=enabled, url=url)) + t.update(ea.judged(list(points), status if enabled else ea.NOT_APPLICABLE, 99.9, len(points))) + return t + + +def _lb(sku="Standard", vip=(("a", 100.0),), dip=(("a", 100.0),), status=ea.READ, rules=1, probes=1): + b = ea.load_balancer_record(raw_lb(sku=sku, rules=rules, probes=probes)) + standard = sku == "Standard" + b["data_path"] = ea.judged(list(vip), status if standard and rules else ea.NOT_APPLICABLE, 99.9, len(vip)) + b["health_probe"] = ea.judged(list(dip), status if standard and probes else ea.NOT_APPLICABLE, 99.9, len(dip)) + return b + + +def test_summary_counts_tests_and_load_balancers_against_the_target(): + tests = [_test(points=(("a", 100.0), ("b", 98.0))), _test(url="https://api.example.com/", points=(("a", 100.0),)), _test(enabled=False)] + lbs = [_lb(dip=(("a", 100.0), ("b", 99.0))), _lb(sku="Basic")] + s = ea.summarize(tests, lbs, SETTINGS) + assert (s["availability_tests_enabled"], s["endpoints_tested"], s["lowest_availability_percentage"]) == (2, 2, 99.0) + assert (s["availability_tests_below_target"], s["availability_tests_below_target_latest_hour"]) == (1, 1) + assert (s["load_balancers_standard"], s["lowest_data_path_availability"], s["lowest_health_probe_availability"], s["load_balancers_below_target"]) == (1, 100.0, 99.5, 1) + + +def test_unread_results_leave_verdicts_null_not_zero(): + s = ea.summarize([_test(status=ea.ERROR, points=())], [_lb(status=ea.ERROR, vip=(), dip=())], SETTINGS) + assert (s["availability_tests_results_unread"], s["lowest_availability_percentage"]) == (1, None) + # An unread item is not judged; it never counts as meeting the target. + assert (s["availability_tests_below_target"], s["availability_tests_not_judged"]) == (0, 1) + assert (s["load_balancers_metrics_unread"], s["lowest_data_path_availability"]) == (1, None) + assert (s["load_balancers_below_target"], s["load_balancers_not_judged"]) == (0, 1) + unlisted = ea.summarize([], [], SETTINGS, tests_listed=False, balancers_listed=False) + assert (unlisted["availability_tests_enabled"], unlisted["endpoints_tested"], unlisted["load_balancers_standard"]) == (None, None, None) + + +def test_no_tests_reads_as_zero_tested_endpoints_not_unknown(): + s = ea.summarize([], [], SETTINGS) + assert (s["availability_tests_enabled"], s["endpoints_tested"], s["lowest_availability_percentage"]) == (0, 0, None) + + +def test_summary_matches_the_instructions(assert_summary_documented): + s = ea.summarize([_test()], [_lb()], SETTINGS) + s.update(ea.ResourceExclusions().summary([], ("availability_test", "load_balancer"), {"availability_tests": 1, "load_balancers": 1})) + assert_summary_documented("azure/endpoint_availability", s) + + +# --- full run against synthetic ARM and metrics responses --- + +class _Resp: + def __init__(self, status, body): + self.status_code, self._body, self.reason, self.text, self.headers = status, body, "", json.dumps(body), {} + + def json(self): + return self._body + + +class _Arm: + def __init__(self, tests=None, metrics_status=200): + self.tests = tests if tests is not None else [raw_test(), raw_test("off", enabled=False)] + self.metrics_status, self.calls = metrics_status, [] + self.lbs = [raw_lb(), raw_lb("basic", sku="Basic", public=False)] + + def get(self, url, params=None, headers=None, timeout=None, allow_redirects=True): + path = urlparse(url).path + self.calls.append((path, dict(params or {}))) + if path.endswith("/providers/Microsoft.Insights/webtests"): + return _Resp(200, {"value": self.tests}) + if path.endswith("/providers/Microsoft.Network/loadBalancers"): + return _Resp(200, {"value": self.lbs}) + if path.endswith("/providers/Microsoft.Insights/metrics"): + if self.metrics_status != 200: + return _Resp(self.metrics_status, {"error": {"code": "AuthorizationFailed", "message": "no"}}) + name = params["metricnames"] + # A full day: 24 hourly points (23 at 100 and one at 88 averages 99.5 for the test). + return _Resp(200, {"value": metric(name, *([100.0] * 23 + [88.0])) if name == ea.AVAILABILITY_METRIC else metric(name, *([100.0] * 24))}) + return _Resp(404, {"error": {"code": "NotFound", "message": path}}) + + +class _Cred: + def get_token(self, scope): + return namedtuple("T", "token expires_on")("tok", 9_999_999_999) + + +@pytest.fixture +def arm(monkeypatch, tmp_path): + def serve(**kwargs): + session = _Arm(**kwargs) + monkeypatch.setattr("requests.Session", lambda: session) + return session + + monkeypatch.setattr(azure_rest, "pinned_credential", lambda sub: _Cred()) + monkeypatch.setattr(azure_rest, "REPO_ROOT", tmp_path / "repo") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("AZURE_SUBSCRIPTION_ID", "s1") + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.setenv("FETCHER_STATUS_FILE", str(tmp_path / "status.json")) + for env in ("AZURE_AUTHORITY_HOST", "AZURE_TENANT_ID", "AZURE_ENVIRONMENT", "AZURE_RESOURCE_EXCLUSIONS", "AZURE_TARGET_SUBSCRIPTION_ONLY", + ea.LOOKBACK_ENV, ea.TARGET_ENV): + monkeypatch.delenv(env, raising=False) + return serve + + +def _evidence(tmp_path): + [path] = tmp_path.glob("azure_endpoint_availability_s1.json") + return json.loads(path.read_text()) + + +def test_main_reads_each_enabled_test_and_standard_load_balancer(arm, tmp_path, monkeypatch): + monkeypatch.setenv(ea.TARGET_ENV, "99.9") + monkeypatch.setenv(ea.LOOKBACK_ENV, "1") + session = arm() + assert ea.main() == 0 + s = _evidence(tmp_path)["summary"] + assert (s["in_scope_availability_tests"], s["availability_tests_enabled"], s["endpoints_tested"], s["lowest_availability_percentage"]) == (2, 1, 1, 99.5) + assert (s["availability_tests_below_target"], s["load_balancers_standard"], s["lowest_data_path_availability"], s["load_balancers_below_target"]) == (1, 1, 100.0, 0) + metric_calls = [p for path, p in session.calls if path.endswith("/metrics")] + # One call for the enabled test (filtered to its name, on its component) and two for the Standard load balancer only. + assert [c["metricnames"] for c in metric_calls] == [ea.AVAILABILITY_METRIC, ea.DATA_PATH_METRIC, ea.HEALTH_PROBE_METRIC] + assert metric_calls[0]["$filter"] == "availabilityResult/name eq 'home'" and metric_calls[0]["interval"] == "PT1H" + assert any(path.startswith(COMPONENT) for path, _ in session.calls) + + +def test_unreadable_metrics_fail_the_run_and_leave_verdicts_null(arm, tmp_path): + arm(metrics_status=403) + assert ea.main() != 0 + s = _evidence(tmp_path)["summary"] + assert (s["availability_tests_results_unread"], s["lowest_availability_percentage"], s["load_balancers_metrics_unread"], s["lowest_data_path_availability"]) == (1, None, 1, None) + + +def test_a_test_without_its_component_link_fails_the_run(arm, tmp_path): + arm(tests=[raw_test(component=None)]) + assert ea.main() != 0 + assert _evidence(tmp_path)["summary"]["availability_tests_results_unread"] == 1 + + +def test_an_invalid_target_fails_the_run_and_stays_unset(arm, tmp_path, monkeypatch): + monkeypatch.setenv(ea.TARGET_ENV, "101") + arm() + assert ea.main() != 0 + s = _evidence(tmp_path)["summary"] + assert (s["availability_target_percentage"], s["availability_tests_below_target"]) == (None, None) + + +def test_without_an_adopted_target_below_target_counts_stay_null(): + s = ea.summarize([_test(points=(("a", 50.0),))], [_lb(dip=(("a", 10.0),))], {"lookback_days": 7, "target": None}) + assert (s["availability_target_percentage"], s["availability_tests_below_target"], s["load_balancers_below_target"]) == (None, None, None) + assert (s["lowest_availability_percentage"], s["lowest_health_probe_availability"]) == (50.0, 10.0) + + +# --- measurement coverage and what tests validate --- + +def test_a_partial_window_is_unknown_never_healthy(): + few_good_hours = ea.judged([("a", 100.0), ("b", 100.0), ("c", 100.0)], ea.READ, 99.9, 168) + assert (few_good_hours["availability_percentage"], few_good_hours["hours_missing"], few_good_hours["measurement_coverage_percentage"]) == (100.0, 165, 1.8) + assert (few_good_hours["measurement_complete"], few_good_hours["below_target"]) == (False, None) + + +def test_one_incomplete_test_is_not_judged_and_the_others_still_are(): + complete = _test(points=tuple((str(h), 100.0) for h in range(23)) + (("23", 50.0),)) + partial = ea.test_record(raw_test("api", url="https://api.example.com/")) + partial.update(ea.judged([("a", 100.0)], ea.READ, 99.9, 24)) + s = ea.summarize([complete, partial], [], SETTINGS) + assert (s["availability_tests_with_missing_measurements"], s["lowest_measurement_coverage_percentage"]) == (1, 4.2) + assert (s["availability_tests_below_target"], s["availability_tests_not_judged"]) == (1, 1) + + +def test_status_code_and_content_checks_are_read_from_standard_and_classic_tests(): + standard = {"ValidationRules": {"ExpectedHttpStatusCode": 200, "SSLCheck": True, "ContentValidation": {"ContentMatch": "healthy"}}} + assert ea.validation(standard) == {"checks_status_code": True, "checks_content": True, "checks_ssl": True} + assert ea.validation({"ValidationRules": {"ExpectedHttpStatusCode": 200}})["checks_content"] is False + ping = {"Configuration": {"WebTest": '' + ''}} + assert ea.validation(ping) == {"checks_status_code": True, "checks_content": True, "checks_ssl": False} + assert ea.validation({}) == {"checks_status_code": False, "checks_content": False, "checks_ssl": False} + + +def test_application_checks_are_counted_across_enabled_tests(): + works = ea.test_record({**raw_test(), "properties": {**raw_test()["properties"], "ValidationRules": {"ExpectedHttpStatusCode": 200, "ContentValidation": {"ContentMatch": "ok"}}}}) + works.update(ea.judged([("a", 100.0)], ea.READ, 99.9, 1)) + s = ea.summarize([works, _test(url="https://api.example.com/")], [], SETTINGS) + assert (s["availability_tests_checking_application"], s["availability_tests_reachability_only"]) == (1, 1) + + +# --- review fixes: location exclusions, load balancers without rules or probes, classic test URLs --- + +def test_location_patterns_exclude_tests_and_load_balancers(arm, tmp_path, monkeypatch): + monkeypatch.setenv("AZURE_RESOURCE_EXCLUSIONS", "location:westus2") + session = arm(tests=[raw_test(), raw_test("west", location="westus2")]) + session.lbs = [raw_lb(), raw_lb("west-lb", location="westus2")] + assert ea.main() == 0 + s = _evidence(tmp_path)["summary"] + assert (s["excluded_availability_tests"], s["excluded_load_balancers"], s["resource_exclusion_patterns_unmatched"]) == (1, 1, []) + assert (s["in_scope_availability_tests"], s["in_scope_load_balancers"]) == (1, 1) + + +def test_an_outbound_only_load_balancer_does_not_leave_the_verdict_unknown(arm, tmp_path, monkeypatch): + monkeypatch.setenv(ea.TARGET_ENV, "99.9") + monkeypatch.setenv(ea.LOOKBACK_ENV, "1") + session = arm() + session.lbs = [raw_lb(), raw_lb("outbound", public=True, rules=0, probes=0)] + assert ea.main() == 0 + s = _evidence(tmp_path)["summary"] + assert (s["load_balancers_standard"], s["load_balancers_without_measured_traffic"]) == (2, 1) + assert (s["load_balancers_below_target"], s["load_balancers_not_judged"], s["load_balancers_with_missing_measurements"]) == (0, 0, 0) + lb_calls = [path for path, _ in session.calls if "/loadBalancers/" in path] + assert all("/loadBalancers/lb1/" in path for path in lb_calls) # nothing is asked of the outbound-only load balancer + + +def test_a_load_balancer_without_probes_is_judged_on_its_data_path_only(): + b = _lb(vip=(("a", 98.0),), dip=(), probes=0) + assert (b["data_path"]["metric_status"], b["health_probe"]["metric_status"]) == (ea.READ, ea.NOT_APPLICABLE) + s = ea.summarize([], [b], SETTINGS) + assert (s["load_balancers_below_target"], s["load_balancers_not_judged"], s["load_balancers_with_missing_measurements"]) == (1, 0, 0) + + +def test_classic_ping_tests_read_their_url_from_the_webtest_xml(): + xml = ('') + raw = {**raw_test(), "properties": {**raw_test()["properties"], "Kind": "ping", "Request": None, "Configuration": {"WebTest": xml}}} + t = ea.test_record(raw) + assert t["url"] == "https://app.example.com/?a=1&b=2" + t.update(ea.judged([("a", 100.0)], ea.READ, 99.9, 1)) + assert ea.summarize([t], [], SETTINGS)["endpoints_tested"] == 1 + + +def test_without_a_target_the_not_judged_counts_stay_null(): + s = ea.summarize([_test()], [_lb()], {"lookback_days": 7, "target": None}) + assert (s["availability_tests_not_judged"], s["load_balancers_not_judged"]) == (None, None)