From 209a2537cd5d59e099822f792d2a90be1398dbb0 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 10 Sep 2026 09:36:57 +0200 Subject: [PATCH 01/25] chore: ignore local energy-accounting store --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5eb10711..51e45749 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,6 @@ GTAGS # Things specific to this project # ################################### + +# Local energy-accounting store ($ACCOUNTING_PTH), runtime data +.local-accounting/ From 95821346dfbfc28bba992cf2ad37d7e6455e3f16 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 10 Sep 2026 09:37:16 +0200 Subject: [PATCH 02/25] refactor(wattnet): align with Wattnet API and tidy naming - adopt the renamed `environmental-score` endpoint (was `green-score`) - rename WattNet -> Wattnet throughout, `scores` -> `green_scores` - read WATTNET_EMAIL from the environment instead of hardcoding it - add `GreenDirector.footprint_series()` to fetch a datacenter's carbon/water intensity over an arbitrary window (used by the energy accounting backfill) --- ai4papi/wattnet.py | 99 ++++++++++++++++++++++++++++++++-------------- template.env | 1 + 2 files changed, 71 insertions(+), 29 deletions(-) diff --git a/ai4papi/wattnet.py b/ai4papi/wattnet.py index 9b307b61..ef2e3ee0 100644 --- a/ai4papi/wattnet.py +++ b/ai4papi/wattnet.py @@ -1,25 +1,27 @@ """ -Utilities for the integration with WattNet. +Utilities for the integration with Wattnet. API reference: https://api.wattnet.eu/v1/docs """ -from collections.abc import Iterable import datetime import json -import requests import os import statistics import warnings +from collections.abc import Iterable -from cachetools import cached, TTLCache -from ai4papi import utils -import ai4papi.conf as papiconf +import requests +from cachetools import TTLCache, cached +import ai4papi.conf as papiconf +from ai4papi import utils session = requests.Session() WATTNET_URL = "https://api.wattnet.eu" -WATTNET_EMAIL = "bot@ai4eosc.eu" +WATTNET_EMAIL = os.environ.get("WATTNET_EMAIL") +if not WATTNET_EMAIL: + print("You should define a WATTNET_EMAIL") WATTNET_PASS = os.environ.get("WATTNET_PASSWORD") if not WATTNET_PASS: print("You should define a WATTNET_PASSWORD") @@ -32,11 +34,11 @@ def algorithm(func): class GreenDirector: - # Define sensible default footprint values for datacenter outside WattNet scope (Europe) + # Define sensible default footprint values for datacenter outside Wattnet scope (Europe) DEFAULTS = { "carbon": 301, # default energy quality in gCO2/kWh "water": 12, # default water usage in L/kWh - "green-score": 50, # default green score (combining carbon and water). + "environmental-score": 50, # default environmental score (combining carbon and water). } def __init__(self, algorithm: str = "linear_rank"): @@ -68,7 +70,7 @@ def __init__(self, algorithm: str = "linear_rank"): @cached(cache=TTLCache(maxsize=1024, ttl=20 * 60 * 60)) def _retrieve_token(self): """ - WattNet tokens last only one day, so we cache the response for 20 hours. + Wattnet tokens last only one day, so we cache the response for 20 hours. """ url = f"{WATTNET_URL}/token-request/get_token" headers = {"Content-Type": "application/json"} @@ -80,8 +82,8 @@ def _retrieve_token(self): @cached(cache=TTLCache(maxsize=1024, ttl=15 * 60)) def _fetch_footprint_data(self, lat, lon): """ - Fetch footprint data and green score data from WattNet for a specific - lat-lon location. WattNet has a temporal resolution of 15 minutes, so we + Fetch footprint data and environmental score data from Wattnet for a specific + lat-lon location. Wattnet has a temporal resolution of 15 minutes, so we cache for that amount of time. """ end = datetime.datetime.now(datetime.timezone.utc) @@ -98,11 +100,11 @@ def _fetch_footprint_data(self, lat, lon): r1 = session.get(f"{WATTNET_URL}/v1/footprints", headers=headers, params=params) r2 = session.get( - f"{WATTNET_URL}/v1/green-score", headers=headers, params=params + f"{WATTNET_URL}/v1/environmental-score", headers=headers, params=params ) if not (r1.ok and r2.ok): warnings.warn( - f"[wattnet] Failed to retrieve WattNet data for coordinates ({lat}, {lon})" + f"[wattnet] Failed to retrieve Wattnet data for coordinates ({lat}, {lon})" ) return [] @@ -110,10 +112,49 @@ def _fetch_footprint_data(self, lat, lon): score = r2.json() if (not footprints) or (not score): raise Exception("Error retrieving information from Wattnet.") - score[0].update({"footprint_type": "green-score"}) + score[0].update({"footprint_type": "environmental-score"}) return footprints + score + def footprint_series(self, datacenter: str, start, end): + """ + Fetch the carbon/water intensity series for a datacenter over an + arbitrary [start, end] window (Wattnet serves historical data well + beyond the 7-day window kept in `self.metrics`). Used by the + energy-accounting sweep when backfilling a deployment. + + Returns `{"carbon": [[iso_ts, gCO2/kWh], ...], "water": [[iso_ts, L/kWh], ...]}` + or `{}` if Wattnet has no data for that location (caller falls back to + DEFAULTS). + """ + dc = papiconf.datacenters.get(datacenter, {}) + lat, lon = dc.get("lat"), dc.get("lon") + if lat is None or lon is None: + return {} + + params = { + "lat": lat, + "lon": lon, + "start": start.isoformat().replace("+00:00", "Z"), + "end": end.isoformat().replace("+00:00", "Z"), + "aggregate": "false", + } + headers = {"Authorization": f"Bearer {self._retrieve_token()}"} + r = session.get(f"{WATTNET_URL}/v1/footprints", headers=headers, params=params) + if not r.ok: + warnings.warn( + f"[wattnet] Failed to retrieve footprint series for {datacenter}" + ) + return {} + + out = {} + for footprint in r.json(): + series = [] + for sublist in footprint.get("series", []): + series += sublist["values"] + out[footprint["footprint_type"]] = sorted(series, key=lambda x: x[0]) + return out + def retrieve_footprints(self): """ Retrieve the footprints for all datacenters. @@ -143,7 +184,7 @@ def retrieve_footprints(self): series = sorted(series, key=lambda data: data[0]) self.metrics[k][fp_type] = series - # For datacenters outside Europe (e.g. Tubitak), WattNet offers no data + # For datacenters outside Europe, Wattnet offers no data # Therefore we return timeseries with default values if not data: round_end = end.replace( @@ -184,13 +225,13 @@ def normalize_list( return [target_min + ((x - val_min) * target_span) / range_span for x in values] @algorithm - def _linear_rank(self, stats: dict, metric: str = "green-score"): + def _linear_rank(self, stats: dict, metric: str = "environmental-score"): """ - Computes a DIRAC-like score for each node: + Computes a DIRAC-like green_score for each node: - score = processor_efficiency / (datacenter_PUE * datacenter_footprint) + green_score = processor_efficiency / (datacenter_PUE * datacenter_footprint) """ - metrics = ["carbon", "water", "green-score"] + metrics = ["carbon", "water", "environmental-score"] if metric not in metrics: raise ValueError(f"Invalid metric: {metric}. Must be one of: {metrics}") @@ -229,21 +270,21 @@ def _linear_rank(self, stats: dict, metric: str = "green-score"): scaled_values = self.normalize_list(gpu_eff.values(), target_range=(1, 2)) gpu_eff = dict(zip(gpu_eff.keys(), scaled_values)) - # Compute the individual node scores (the higher the better) - scores = {} + # Compute the individual node green_scores (the higher the better) + green_scores = {} for dc_name, dc in stats.datacenters.items(): # Compute the overall datacenter footprint mean = statistics.mean([i[1] for i in self.metrics[dc_name][metric]]) - # In the case of green score, high values lower the footprint - mean = 1 / mean if metric in ["green-score"] else mean + # In the case of environmental score, high values lower the footprint + mean = 1 / mean if metric in ["environmental-score"] else mean - # Now compute per node Dirac-style green-score, weighting the datacenter + # Now compute per node Dirac-style environmental-score, weighting the datacenter # footprint with the node specific efficiency for nid, node in dc.nodes.items(): if nid not in cpu_eff.keys(): continue - scores[nid] = {} - scores[nid]["cpu"] = cpu_eff[nid] / (mean * dc.PUE) + green_scores[nid] = {} + green_scores[nid]["cpu"] = cpu_eff[nid] / (mean * dc.PUE) if nid not in gpu_eff.keys(): continue @@ -251,9 +292,9 @@ def _linear_rank(self, stats: dict, metric: str = "green-score"): # We *assume* that a typical GPU workload consumes 80% of its power on # the GPU and 20% on a CPU combined_eff = 0.2 * cpu_eff[nid] + 0.8 * gpu_eff[nid] - scores[nid]["gpu"] = combined_eff / (mean * dc.PUE) + green_scores[nid]["gpu"] = combined_eff / (mean * dc.PUE) - return scores + return green_scores def rank(self, stats: dict): """ diff --git a/template.env b/template.env index 2df68a70..f8f6c1db 100644 --- a/template.env +++ b/template.env @@ -11,4 +11,5 @@ HARBOR_ROBOT_PASSWORD="*****************************" JENKINS_TOKEN="*************************************" PROVENANCE_TOKEN="**********************************" LITELLM_API_KEY="***********************************" +WATTNET_EMAIL="bot@example.org" WATTNET_PASSWORD="**********************************" From 5979dc7d95e6941cd4f5c990d1200b850fc9900d Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 10 Sep 2026 09:37:21 +0200 Subject: [PATCH 03/25] feat(datacenters): add optional TUE column Add a `TUE` column to var/datacenters.csv (grid-to-compute ratio, known so far only for the ifca sites) and let conf.py tolerate blank numeric cells so the other datacenters can leave it empty. --- ai4papi/conf.py | 3 ++- var/datacenters.csv | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ai4papi/conf.py b/ai4papi/conf.py index 6dea1a53..51e4b96f 100644 --- a/ai4papi/conf.py +++ b/ai4papi/conf.py @@ -172,7 +172,8 @@ class AssetConfig(TypedDict): elif k == "country": datacenters[name][k] = v else: - datacenters[name][k] = float(v) + # Numeric columns; allow blank cells (eg. TUE only known for some DCs) + datacenters[name][k] = float(v) if v not in (None, "") else None # Retrieve git info from PAPI, to show current version in the docs papi_commit = subprocess.run( diff --git a/var/datacenters.csv b/var/datacenters.csv index e69c060f..20810fa6 100644 --- a/var/datacenters.csv +++ b/var/datacenters.csv @@ -1,7 +1,7 @@ -name,country,lat,lon,PUE -ifca-ai4eosc,ES,43.471668,-3.802318,1.31 -ifca-imagine,ES,43.471706,-3.802240,1.31 -iisas-ai4eosc,SK,48.170095,17.070058,1.57 -incd-imagine,PT,38.759140,-9.142664,1.45 -tubitak-imagine,TR,40.785667,29.448592,1.35 -psnc-ai4eosc,PL,52.406889,16.953201,1.29 \ No newline at end of file +name,country,lat,lon,PUE,TUE +ifca-ai4eosc,ES,43.471668,-3.802318,1.31,2.03 +ifca-imagine,ES,43.471706,-3.802240,1.31,2.03 +iisas-ai4eosc,SK,48.170095,17.070058,1.57, +incd-imagine,PT,38.759140,-9.142664,1.45, +tubitak-imagine,TR,40.785667,29.448592,1.35, +psnc-ai4eosc,PL,52.406889,16.953201,1.29, From 972a645032c3d4ec1e6436379c4428130b276a72 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 10 Sep 2026 09:37:35 +0200 Subject: [PATCH 04/25] feat(energy): per-deployment energy and footprint accounting engine Core of the energy-accounting feature, no wiring into the API yet: - `mimir.py`: minimal Grafana Mimir (Prometheus) HTTP client - `accounting/compute.py`: pure helpers (resolve allocations, build the PromQL, sanitize + integrate power into 15-min energy buckets, apply the Wattnet footprint, downsample) - `accounting/store.py`: file-based store under `$ACCOUNTING_PTH/energy` (one JSON accumulator per deployment + append-only .series.jsonl) - `accounting/sweep.py`: background sweep that incrementally integrates the un-consolidated Mimir window per deployment and per datacenter, keeps a running accumulator plus a self-healing 24h tail, and the live `pointer_ts -> now` top-up used on the read path - `accounting/__init__.py`: read API (`get_accumulated*`, `get_series`, `get_user_energy`, `get_datacenter_power`, `get_cluster_energy`), degrading to None/empty when disabled or data is missing CPU (scaphandre) and GPU (DCGM) power sources are independent, so a site that only monitors its GPUs still accounts the GPU energy. All values are normalized by the per-datacenter TUE. Config lives in the new `energy` section of etc/main.yaml. --- ai4papi/accounting/__init__.py | 460 +++++++++++++++++++++ ai4papi/accounting/compute.py | 408 ++++++++++++++++++ ai4papi/accounting/store.py | 124 ++++++ ai4papi/accounting/sweep.py | 731 +++++++++++++++++++++++++++++++++ ai4papi/mimir.py | 125 ++++++ etc/main.yaml | 34 ++ 6 files changed, 1882 insertions(+) create mode 100644 ai4papi/accounting/__init__.py create mode 100644 ai4papi/accounting/compute.py create mode 100644 ai4papi/accounting/store.py create mode 100644 ai4papi/accounting/sweep.py create mode 100644 ai4papi/mimir.py diff --git a/ai4papi/accounting/__init__.py b/ai4papi/accounting/__init__.py new file mode 100644 index 00000000..af263289 --- /dev/null +++ b/ai4papi/accounting/__init__.py @@ -0,0 +1,460 @@ +""" +Energy-accounting read API used by the routers. + +* `get_accumulated` / `get_accumulated_bulk`: per-deployment accumulated energy, + carbon and water (persisted accumulator + live Mimir top-up). +* `get_series`: per-deployment time series for the detail-view graphs (persisted + 15-min archive + a live tail so the graph keeps growing). +* `get_user_energy`: per-user aggregate (all of the user's deployments summed, + each with its live top-up), optionally with a merged time series. +* `get_datacenter_power` / `get_cluster_energy`: per-datacenter instantaneous + power and accumulated energy for `/stats/cluster` (no platform-wide total). + +Everything degrades to `None` / empty when the feature is disabled or the data +is not available; it never raises to the caller. +""" + +import datetime +import logging + +from cachetools import TTLCache, cached + +import ai4papi.conf as papiconf +from ai4papi import mimir +from ai4papi.accounting import compute, store +from ai4papi.accounting.sweep import ( + CLUSTER_NS, + _tue, + enabled, + live_series, + live_series_cluster, + process_single, + run_sweep, + topup, + topup_cluster, +) + +__all__ = [ + "enabled", + "run_sweep", + "process_single", + "get_accumulated", + "get_accumulated_bulk", + "get_series", + "get_user_energy", + "get_datacenter_power", + "get_cluster_energy", +] + +LOG = logging.getLogger(__name__) + +_KEYS = ("energy_wh", "carbon_g", "water_l") # everything is TUE-normalized + + +@cached(cache=TTLCache(maxsize=8192, ttl=60)) +def _read_doc(namespace: str, uuid: str): + return store.read_accum(namespace, uuid) + + +def _doc_tue(doc: dict) -> float | None: + """`tue_factor` from the doc; for a doc written before that field existed, + the single datacenter's configured TUE (blended docs then read `None`).""" + if doc.get("tue_factor"): + return doc["tue_factor"] + dcs = doc.get("datacenters") or [] + return _tue(dcs[0]) if len(dcs) == 1 else None + + +def _to_stats(doc: dict) -> dict: + acc = doc.get("accumulated") or {} + # fallback power for a running deployment with no live top-up: the last + # persisted bucket's mean. `_apply_topup` replaces it with the actual last + # Mimir sample (current draw) whenever the top-up succeeds. + tail = doc.get("tail") or [] + running = doc.get("status") == "running" + last = tail[-1] if tail else {} + return { + "energy_wh": acc.get("energy_wh", 0.0), + "tue_factor": _doc_tue(doc), + "carbon_g": acc.get("carbon_g", 0.0), + "water_l": acc.get("water_l", 0.0), + "power_w": last.get("power_w_avg") if running else None, + "since": doc.get("window_start") or doc.get("settled_ts"), + "as_of": doc.get("pointer_ts"), + "live_as_of": None, + "complete": doc.get("complete", True), + "coverage_ratio": doc.get("coverage_ratio", 1.0), + "degraded": False, + "datacenters": doc.get("datacenters", []), + } + + +def _apply_topup(stats: dict, doc: dict) -> dict: + t = topup(doc) + if not t: + return stats + if t.get("degraded"): + stats["degraded"] = True + return stats + # the top-up slice uses the same TUE as the accumulator, so `tue_factor` + # does not move; just add the energy / footprint deltas and refresh power. + for k in _KEYS: + stats[k] += t.get(k, 0.0) + if t.get("power_w") is not None: + stats["power_w"] = t["power_w"] + stats["live_as_of"] = t.get("live_as_of") + return stats + + +def get_accumulated(namespace: str, uuid: str, live: bool = True) -> dict | None: + if not enabled(): + return None + doc = _read_doc(namespace, uuid) + if not doc or doc.get("metrics_available") is False or "accumulated" not in doc: + return None + stats = _to_stats(doc) + return _apply_topup(stats, doc) if live else stats + + +def get_accumulated_bulk( + namespace: str, owner: str, live: bool = True +) -> dict[str, dict]: + if not enabled(): + return {} + out: dict[str, dict] = {} + for uuid, doc in store.iter_owner(namespace, owner): + if doc.get("metrics_available") is False or "accumulated" not in doc: + continue + stats = _to_stats(doc) + out[uuid] = _apply_topup(stats, doc) if live else stats + return out + + +def get_user_energy( + namespace: str, owner: str, series: bool = False, target_points: int | None = None +) -> dict | None: + """ + All of a user's deployments in a VO rolled into one `EnergyStats`-shaped + block: each deployment's persisted accumulator plus its live Mimir top-up, + summed. With `series=True`, also a 15-min time series merged across the + deployments (`full_info=true` on `GET /stats/user`). `None` when the user has + no accounted deployment. + """ + if not enabled(): + return None + + if target_points is None: + target_points = int( + papiconf.MAIN_CONF["energy"].get("user_series_points", 1000) + ) + + rows = [] + for uuid, doc in store.iter_owner(namespace, owner): + if doc.get("metrics_available") is False or "accumulated" not in doc: + continue + rows.append((uuid, doc, _apply_topup(_to_stats(doc), doc))) + if not rows: + return None + + agg = dict.fromkeys(_KEYS, 0.0) + dcs: set[str] = set() + power = raw_energy = 0.0 # raw_energy: Σ energy_wh / tue, to blend `tue_factor` + since, as_of, live_as_of, coverage = [], [], [], [] + complete, degraded = True, False + for _uuid, _doc, s in rows: + for k in _KEYS: + agg[k] += s.get(k, 0.0) or 0.0 + if s.get("tue_factor"): + raw_energy += s["energy_wh"] / s["tue_factor"] + dcs.update(s.get("datacenters") or []) + if s.get("power_w"): + power += s["power_w"] + if s.get("since"): + since.append(s["since"]) + if s.get("as_of"): + as_of.append(s["as_of"]) + if s.get("live_as_of"): + live_as_of.append(s["live_as_of"]) + coverage.append(s.get("coverage_ratio", 1.0)) + complete = complete and s.get("complete", True) + degraded = degraded or bool(s.get("degraded")) + + out = { + # not rounded, same as the per-deployment `EnergyStats`: the values can be + # legitimately tiny (idle CPU workloads) and must reconcile with the sum + # of `series`. + "energy_wh": agg["energy_wh"], + "tue_factor": (agg["energy_wh"] / raw_energy) if raw_energy else None, + "carbon_g": agg["carbon_g"], + "water_l": agg["water_l"], + "power_w": power or None, + # time range these numbers cover: [since, as_of], live top-up to live_as_of + "since": min(since) if since else None, + "as_of": max(as_of) if as_of else None, + "live_as_of": max(live_as_of) if live_as_of else None, + "complete": complete, + "coverage_ratio": round(sum(coverage) / len(coverage), 3), + "degraded": degraded, + "datacenters": sorted(dcs), + "deployments": len(rows), + "series": None, + } + if series: + out["series"] = _user_series(namespace, rows, target_points) or None + return out + + +# --------------------------------------------------------------------------- +# cluster-level instantaneous power / footprint (for /stats/cluster) +# --------------------------------------------------------------------------- + + +def _sum_by_datacenter(promql: str, scale: float) -> dict[str, float]: + """Instant Mimir query grouped `by (datacenter)` -> {dc: value * scale}.""" + out: dict[str, float] = {} + for series in mimir.query(promql): + dc = series.get("metric", {}).get("datacenter") + if not dc: + continue + try: + out[dc] = float(series["value"][1]) * scale + except (KeyError, IndexError, ValueError, TypeError): + continue + return out + + +@cached(cache=TTLCache(maxsize=1, ttl=25)) +def get_datacenter_power() -> dict[str, dict]: + """ + Instantaneous power draw per datacenter, normalized by TUE: + `power_w = (sum host power + sum GPU power) * TUE[dc]`. Best-effort: an + unreachable Mimir yields `{}`, and a datacenter with no power series at all + is simply absent, never reported as zero. A datacenter listed in + `energy.metrics_datacenters` counts with whatever it has (host-only, + GPU-only or both); without that allowlist a host-power series is required so + a stray GPU metric does not fabricate a total. `{}` when disabled. + """ + if not enabled(): + return {} + + metrics_dcs = set(papiconf.MAIN_CONF["energy"].get("metrics_datacenters") or []) + try: + host_w = _sum_by_datacenter( + "sum by (datacenter) (scaph_host_power_microwatts)", 1e-6 + ) + gpu_w = _sum_by_datacenter("sum by (datacenter) (DCGM_FI_DEV_POWER_USAGE)", 1.0) + except (mimir.MimirError, OSError): + LOG.warning("mimir cluster-power query failed") + return {} + + as_of = compute._iso(datetime.datetime.now(datetime.timezone.utc).timestamp()) + out: dict[str, dict] = {} + for dc in set(host_w) | set(gpu_w): + if metrics_dcs: + if dc not in metrics_dcs: + continue + # declared site: host-only, GPU-only or both are all valid totals + elif dc not in host_w: + # no allowlist: require host power so a stray GPU metric for an + # unmonitored datacenter does not fabricate a total + continue + tue = _tue(dc) + out[dc] = { + "power_w": (host_w.get(dc, 0.0) + gpu_w.get(dc, 0.0)) * tue, + "tue_factor": tue, + "as_of": as_of, + } + return out + + +@cached(cache=TTLCache(maxsize=16, ttl=300)) +def _cluster_archive_series(dc: str) -> list[dict]: + """ + Consolidated 15-min buckets of a datacenter (the `.series.jsonl`). Cached for + ~5 min: it only changes when a bucket ages out of the 24h tail, which happens + at sweep cadence, so the 30s `get_cluster_stats` thread need not re-read it. + """ + return store.read_series(CLUSTER_NS, dc, None, None) + + +def _cluster_series( + dc: str, doc: dict, bucket_s: int, target_points: int +) -> list[dict]: + return _build_series( + list(_cluster_archive_series(dc)) + list(doc.get("tail") or []), + live_series_cluster(dc, doc, bucket_s=bucket_s), + target_points, + ) + + +def get_cluster_energy( + series: bool = True, target_points: int | None = None +) -> dict[str, dict]: + """ + Per-datacenter energy for `/stats/cluster`: the instantaneous power from + `get_datacenter_power()` merged with the persisted accumulator plus a live + top-up, plus a downsampled 15-min `series`. `{}` when the feature is disabled. + A datacenter shows up if it has instantaneous power or an accumulator doc. + """ + if not enabled(): + return {} + cfg = papiconf.MAIN_CONF["energy"] + if target_points is None: + target_points = int(cfg.get("cluster_series_points", 500)) + bucket_s = int(cfg["series_bucket_seconds"]) + + inst = get_datacenter_power() + metrics_dcs = cfg.get("metrics_datacenters") or [] + out: dict[str, dict] = {} + + for dc in set(inst) | set(metrics_dcs): + entry = dict(inst.get(dc) or {}) + doc = store.read_accum(CLUSTER_NS, dc) + if doc and doc.get("accumulated") and doc.get("metrics_available") is not False: + acc = {k: doc["accumulated"].get(k, 0.0) for k in _KEYS} + live_as_of, degraded = None, False + t = topup_cluster(dc, doc) + if t and t.get("degraded"): + degraded = True + elif t: + for k in _KEYS: + acc[k] += t.get(k, 0.0) + live_as_of = t.get("live_as_of") + # not rounded: consistent with the per-deployment / per-user blocks, + # and the total must reconcile with the sum of `series`. + entry.update( + energy_wh=acc["energy_wh"], + carbon_g=acc["carbon_g"], + water_l=acc["water_l"], + tue_factor=doc.get("tue_factor") or _tue(dc), + complete=doc.get("complete", True), + since=doc.get("window_start") or doc.get("settled_ts"), + as_of=doc.get("pointer_ts"), + live_as_of=live_as_of, + degraded=degraded, + ) + if series: + entry["series"] = _cluster_series(dc, doc, bucket_s, target_points) + if entry: + out[dc] = entry + return out + + +# --------------------------------------------------------------------------- +# time series helpers +# --------------------------------------------------------------------------- + + +def _build_series( + archive_and_tail: list[dict], live_points: list[dict], target_points: int +) -> list[dict]: + """ + One entity's whole-life time series: persisted 15-min buckets (archive + the + still-mutable tail) `compute.downsample`d to `target_points`, then the live + Mimir tail appended un-downsampled. Shared by the per-deployment (`get_series`) + and per-datacenter (`_cluster_series`) paths. The points carry the per-bucket + increments only; a running total is `cumsum(series[].energy_wh)` and matches + `accumulated.energy_wh`. + """ + raw = sorted(archive_and_tail, key=lambda b: b["ts"]) + series = [ + _point(b, live=False) + for b in compute.downsample([_bucket(b) for b in raw], target_points) + ] + series += [ + _point(_bucket(p), live=True) + for p in live_points + if not series or p["ts"] > series[-1]["ts"] + ] + return series + + +def get_series( + namespace: str, uuid: str, target_points: int | None = None +) -> dict | None: + if not enabled(): + return None + doc = _read_doc(namespace, uuid) + if not doc or doc.get("metrics_available") is False or "accumulated" not in doc: + return None + + cfg = papiconf.MAIN_CONF["energy"] + if target_points is None: + target_points = int(cfg.get("deployment_series_points", 1000)) + step_s = int(cfg["series_bucket_seconds"]) + series = _build_series( + list(store.read_series(namespace, uuid, None, None)) + + list(doc.get("tail") or []), + live_series(doc, bucket_s=step_s), + target_points, + ) + stats = _apply_topup(_to_stats(doc), doc) + + return { + "deployment_uuid": uuid, + "start": series[0]["ts"] if series else (stats.get("since") or ""), + "end": series[-1]["ts"] + if series + else (stats.get("live_as_of") or stats.get("as_of") or ""), + "source": "mixed" if any(p["live"] for p in series) else "store", + "series": series, + "accumulated": stats, + } + + +def _bucket(b: dict) -> compute.Bucket: + return compute.Bucket( + ts=b["ts"], + energy_wh=b.get("energy_wh", 0.0), + power_w_avg=b.get("power_w_avg", 0.0), + carbon_g=b.get("carbon_g", 0.0), + water_l=b.get("water_l", 0.0), + ) + + +def _point(b: compute.Bucket, live: bool) -> dict: + return { + "ts": b.ts, + "power_w": b.power_w_avg, + "energy_wh": b.energy_wh, + "carbon_g": b.carbon_g, + "water_l": b.water_l, + "live": live, + } + + +def _user_series(namespace: str, rows: list, target_points: int) -> list[dict]: + """ + One time series for a set of the user's deployments. The raw 15-min buckets + of every deployment (persisted archive + 24h tail) are merged by timestamp + first, then downsampled once: uniform resolution across the whole series and + the same hard `target_points` cap as the other levels, regardless of how long + each deployment individually has been running. The live Mimir tails are then + merged and appended un-downsampled (as `get_series` / `_cluster_series` do). + """ + step_s = int(papiconf.MAIN_CONF["energy"]["series_bucket_seconds"]) + + persisted = [ + [ + _bucket(b) + for b in list(store.read_series(namespace, uuid, None, None)) + + list(doc.get("tail") or []) + ] + for uuid, doc, _ in rows + ] + merged = sorted(compute.merge_buckets(persisted), key=lambda b: b.ts) + series = [_point(b, live=False) for b in compute.downsample(merged, target_points)] + + live = sorted( + compute.merge_buckets( + [ + [_bucket(b) for b in live_series(doc, bucket_s=step_s)] + for _uuid, doc, _ in rows + ] + ), + key=lambda b: b.ts, + ) + series += [ + _point(b, live=True) for b in live if not series or b.ts > series[-1]["ts"] + ] + return series diff --git a/ai4papi/accounting/compute.py b/ai4papi/accounting/compute.py new file mode 100644 index 00000000..49a753f1 --- /dev/null +++ b/ai4papi/accounting/compute.py @@ -0,0 +1,408 @@ +""" +Pure(ish) helpers for the energy-accounting feature: resolve a deployment's +allocations, build the PromQL, integrate power into energy buckets, apply the +footprint, and downsample. + +The only side effect here is reading from Nomad in `resolve_alloc_ids` (and its +cached node->datacenter lookup); everything else is pure and unit-testable. +""" + +import bisect +from collections import namedtuple +import datetime + +from cachetools import cached, TTLCache + +import ai4papi.conf as papiconf +from ai4papi.nomad_utils import Nomad +from ai4papi.wattnet import GreenDirector + + +# Bucket start `ts` is an ISO-8601 UTC string ("...Z"). Energy in Wh (normalized +# by the datacenter TUE), power in W, footprint per bucket. +Bucket = namedtuple("Bucket", ["ts", "energy_wh", "power_w_avg", "carbon_g", "water_l"]) + +DEFAULT_CARBON = GreenDirector.DEFAULTS["carbon"] # gCO2eq/kWh +DEFAULT_WATER = GreenDirector.DEFAULTS["water"] # L/kWh + +_TERMINAL_STATUSES = {"complete", "failed", "lost"} + + +def _iso(ts: float) -> str: + return ( + datetime.datetime.fromtimestamp(ts, datetime.timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + + +def _parse_iso(s: str) -> float: + return datetime.datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() + + +# --------------------------------------------------------------------------- +# Nomad: deployment -> allocations +# --------------------------------------------------------------------------- + +AllocWindow = namedtuple("AllocWindow", ["alloc_id", "start_ts", "end_ts"]) + + +@cached(cache=TTLCache(maxsize=4096, ttl=60 * 60)) +def _node_datacenter(node_id: str) -> str: + return Nomad.node.get_node(node_id)["Datacenter"] + + +def resolve_alloc_ids(job_id: str, namespace: str): + """ + Return `(allocs_by_dc, alloc_windows, running_allocs, partial_dc)` for every + allocation the job ever had, keeping only datacenters that have power metrics + (`energy.metrics_datacenters`). + + * allocs_by_dc: {datacenter: [alloc_id, ...]} (whole lifetime) + * alloc_windows: [AllocWindow(alloc_id, start_ts, end_ts|None)] (all allocs, + to estimate how long the deployment actually ran) + * running_allocs: [{"alloc_id": ..., "datacenter": ...}] (currently running, + for the live top-up) + * partial_dc: True if some allocation ran in a datacenter without metrics + """ + metrics_dcs = set(papiconf.MAIN_CONF["energy"]["metrics_datacenters"]) + + allocs = Nomad.job.get_allocations(id_=job_id, namespace=namespace) + + allocs_by_dc: dict[str, list[str]] = {} + alloc_windows: list[AllocWindow] = [] + running_allocs: list[dict] = [] + partial_dc = False + + for a in allocs: + try: + dc = _node_datacenter(a["NodeID"]) + except Exception: + continue + + start_ts = a["CreateTime"] / 1e9 if a.get("CreateTime") else None + end_ts = None + if a.get("ClientStatus") in _TERMINAL_STATUSES and a.get("ModifyTime"): + end_ts = a["ModifyTime"] / 1e9 + if start_ts is not None: + alloc_windows.append(AllocWindow(a["ID"], start_ts, end_ts)) + + if dc in metrics_dcs: + allocs_by_dc.setdefault(dc, []).append(a["ID"]) + if a.get("ClientStatus") == "running": + running_allocs.append({"alloc_id": a["ID"], "datacenter": dc}) + else: + partial_dc = True + + return allocs_by_dc, alloc_windows, running_allocs, partial_dc + + +def running_seconds(alloc_windows, start_ts: float, end_ts: float) -> float: + """ + Union of the alloc windows clipped to [start_ts, end_ts]. Approximates how + long the deployment was actually running in that interval (to compare + against the time we have power data for). + """ + spans = [] + for w in alloc_windows: + s = max(w.start_ts, start_ts) + e = min(w.end_ts if w.end_ts else end_ts, end_ts) + if e > s: + spans.append((s, e)) + if not spans: + return 0.0 + spans.sort() + total = 0.0 + cur_s, cur_e = spans[0] + for s, e in spans[1:]: + if s <= cur_e: + cur_e = max(cur_e, e) + else: + total += cur_e - cur_s + cur_s, cur_e = s, e + total += cur_e - cur_s + return total + + +# --------------------------------------------------------------------------- +# PromQL +# --------------------------------------------------------------------------- + + +def power_promql(alloc_ids: list[str]) -> tuple[str, str]: + """ + (cpu_query, gpu_query) for a set of allocation ids (all in the same + datacenter). + + Both are grouped, not fully summed, so the data-quality filter runs on the + original per-source series (per allocation for CPU, per GPU for GPU power): + a single bad reading is dropped without discarding the other sources at the + same instant, and the ceiling / spike checks compare against a physically + bounded signal. The integrator then sums the per-series energy. + """ + regex = "|".join(alloc_ids) + cpu_q = ( + "sum by (container_label_com_hashicorp_nomad_alloc_id) (" + "scaph_process_power_consumption_microwatts" + f'{{container_label_com_hashicorp_nomad_alloc_id=~"{regex}"}})' + ) + gpu_q = ( + "DCGM_FI_DEV_POWER_USAGE" + " * on(UUID, instance) group_left(alloc_id)" + f' nomad_gpu_allocation_info{{alloc_id=~"{regex}"}}' + ) + return cpu_q, gpu_q + + +# --------------------------------------------------------------------------- +# Integration: power samples -> energy buckets +# --------------------------------------------------------------------------- + + +def _series_samples( + result: list[dict], scale: float +) -> list[list[tuple[float, float]]]: + """A Mimir matrix result -> one sorted `[(ts, watts)]` list per series.""" + out: list[list[tuple[float, float]]] = [] + for series in result: + s: list[tuple[float, float]] = [] + for ts, val in series.get("values", []): + try: + s.append((float(ts), float(val) * scale)) + except (TypeError, ValueError): + continue + if s: + s.sort() + out.append(s) + return out + + +def _median(xs: list[float]) -> float: + s = sorted(xs) + n = len(s) + if n == 0: + return 0.0 + m = n // 2 + return s[m] if n % 2 else (s[m - 1] + s[m]) / 2 + + +def sanitize( + samples: list[tuple[float, float]], dq: dict | None +) -> list[tuple[float, float]]: + """ + Data-quality filter on one original power series (per allocation for CPU, + per GPU for GPU power), before integration. Anomalous samples are *dropped* + (the integrator then treats it as a gap), never clamped, so a bad reading + never inflates the accumulated energy. Every check is skippable via + `energy.data_quality`: + + * `min_power_w` : drop samples below this (default 0, kills negatives) + * `max_power_w` : drop samples above this hard ceiling (0 = off). Physically + bounded because it applies per source (one alloc / one GPU). + * `spike_factor` : drop a sample above `spike_factor` * rolling median of its + `spike_window` neighbours (0 = off) + """ + if not dq or not dq.get("enabled", True): + return samples + + lo = dq.get("min_power_w", 0.0) + hi = dq.get("max_power_w", 0.0) + out = [(t, p) for t, p in samples if p >= lo and (not hi or p <= hi)] + + factor = dq.get("spike_factor", 0.0) + window = int(dq.get("spike_window", 11)) or 11 + if factor and len(out) > window: + half = window // 2 + vals = [p for _, p in out] + kept = [] + for i, (t, p) in enumerate(out): + lo_i, hi_i = max(0, i - half), min(len(vals), i + half + 1) + neighbours = vals[lo_i:i] + vals[i + 1 : hi_i] + med = _median(neighbours) + if med > 1e-6 and p > factor * med: + continue # spike -> drop + kept.append((t, p)) + out = kept + + return out + + +def _integrate_series(samples, step_s, bucket_s, gap_factor): + """One power series -> {bucket_ts: [energy_wh, covered_s]} (trapezoidal).""" + max_gap = gap_factor * step_s + out: dict[int, list[float]] = {} + for (t0, p0), (t1, p1) in zip(samples, samples[1:]): + dt = t1 - t0 + if dt <= 0 or dt > max_gap: + continue + wh = (p0 + p1) / 2 * dt / 3600 + b = int(t0 // bucket_s) * bucket_s + cur = out.setdefault(b, [0.0, 0.0]) + cur[0] += wh + cur[1] += dt + return out + + +def integrate( + cpu_result: list[dict], + gpu_result: list[dict], + step_s: int = 30, + bucket_s: int = 900, + tue_factor: float = 1.0, + gap_factor: float = 3.0, + data_quality: dict | None = None, +) -> list[Bucket]: + """ + Per-source data-quality filter, then trapezoidal integral of every power + series (CPU per allocation, GPU per GPU) into fixed `bucket_s` energy + buckets, summed. Gaps longer than `gap_factor * step_s` are not integrated + across (deployment stopped / scrape hole / dropped anomaly). `carbon_g` / + `water_l` are left at 0 (see `apply_footprint`). + """ + acc: dict[int, list[float]] = {} # bucket_ts -> [raw_energy_wh, covered_s] + + series_lists = _series_samples(cpu_result, 1e-6) + _series_samples(gpu_result, 1.0) + for samples in series_lists: + clean = sanitize(samples, data_quality) + if len(clean) < 2: + continue + for b, (e, c) in _integrate_series(clean, step_s, bucket_s, gap_factor).items(): + cur = acc.setdefault(b, [0.0, 0.0]) + cur[0] += e + cur[1] = max(cur[1], c) # max, not sum: covered time of the busiest source + + buckets = [] + for b in sorted(acc): + raw_wh, covered = acc[b] + energy_wh = raw_wh * tue_factor + hours = covered / 3600 if covered else 0.0 + buckets.append( + Bucket( + ts=_iso(b), + energy_wh=energy_wh, + # mean power over the covered part of the bucket, normalized like + # energy_wh so `energy_wh ~= power_w_avg * bucket_hours` + power_w_avg=(energy_wh / hours) if hours else 0.0, + carbon_g=0.0, + water_l=0.0, + ) + ) + return buckets + + +def instant_power( + cpu_result: list[dict], + gpu_result: list[dict], + tue_factor: float, + fresh_after_ts: float, + data_quality: dict | None = None, +) -> float | None: + """ + Current draw from a range query: the sum of the most recent (sanitized) + sample of every power series, normalized by TUE. A series whose last good + sample predates `fresh_after_ts` (source gone, scrape hole) is skipped. + `None` when nothing is fresh. + """ + raw_w, fresh = 0.0, False + for scale, result in ((1e-6, cpu_result), (1.0, gpu_result)): + for samples in _series_samples(result, scale): + clean = sanitize(samples, data_quality) + if clean and clean[-1][0] >= fresh_after_ts: + raw_w += clean[-1][1] + fresh = True + return raw_w * tue_factor if fresh else None + + +def merge_buckets(bucket_lists: list[list[Bucket]]) -> list[Bucket]: + """ + Sum buckets from several sources by timestamp: the datacenters a deployment + ran in, or the deployments of one user. Inputs share the 15-min grid. + """ + acc: dict[str, Bucket] = {} + for buckets in bucket_lists: + for bk in buckets: + cur = acc.get(bk.ts) + if cur is None: + acc[bk.ts] = bk + else: + acc[bk.ts] = cur._replace( + energy_wh=cur.energy_wh + bk.energy_wh, + power_w_avg=cur.power_w_avg + bk.power_w_avg, + carbon_g=cur.carbon_g + bk.carbon_g, + water_l=cur.water_l + bk.water_l, + ) + return [acc[k] for k in sorted(acc)] + + +# --------------------------------------------------------------------------- +# Footprint +# --------------------------------------------------------------------------- + + +def _prep_series(series): + """[[iso_ts, value], ...] -> (sorted ts floats, values) for bisect lookup.""" + if not series: + return [], [] + parsed = sorted((_parse_iso(t), float(v)) for t, v in series) + return [p[0] for p in parsed], [p[1] for p in parsed] + + +def intensity_at(ts_arr, val_arr, ts: float, default: float) -> float: + """Value of the last sample with sample_ts <= ts; `default` if none.""" + if not ts_arr: + return default + i = bisect.bisect_right(ts_arr, ts) + return val_arr[i - 1] if i > 0 else default + + +def apply_footprint( + buckets: list[Bucket], + carbon_series=None, + water_series=None, +) -> list[Bucket]: + """ + Fill `carbon_g` / `water_l` per bucket = (energy_wh / 1000) * intensity(ts), + with the Wattnet 15-min series (already normalized energy). Falls back to + GreenDirector.DEFAULTS when a series is missing or predates the bucket. + """ + c_ts, c_val = _prep_series(carbon_series) + w_ts, w_val = _prep_series(water_series) + + out = [] + for bk in buckets: + ts = _parse_iso(bk.ts) + kwh = bk.energy_wh / 1000 + out.append( + bk._replace( + carbon_g=kwh * intensity_at(c_ts, c_val, ts, DEFAULT_CARBON), + water_l=kwh * intensity_at(w_ts, w_val, ts, DEFAULT_WATER), + ) + ) + return out + + +# --------------------------------------------------------------------------- +# Downsampling (for the time-series responses) +# --------------------------------------------------------------------------- + + +def downsample(buckets: list[Bucket], target_points: int) -> list[Bucket]: + """Aggregate consecutive buckets so the result has at most `target_points`.""" + n = len(buckets) + if n <= target_points or target_points < 1: + return buckets + + group = -(-n // target_points) # ceil + out = [] + for i in range(0, n, group): + chunk = buckets[i : i + group] + out.append( + Bucket( + ts=chunk[0].ts, + energy_wh=sum(b.energy_wh for b in chunk), + power_w_avg=sum(b.power_w_avg for b in chunk) / len(chunk), + carbon_g=sum(b.carbon_g for b in chunk), + water_l=sum(b.water_l for b in chunk), + ) + ) + return out diff --git a/ai4papi/accounting/store.py b/ai4papi/accounting/store.py new file mode 100644 index 00000000..b9d2f2ee --- /dev/null +++ b/ai4papi/accounting/store.py @@ -0,0 +1,124 @@ +""" +File-based store for the energy-accounting feature. + +One JSON document per deployment under `$ACCOUNTING_PTH/energy//`, +plus an append-only `.series.jsonl` with the consolidated 15-min buckets +(`energy.series_bucket_seconds`). The per-datacenter accumulators live the same +way under the `_cluster` pseudo-namespace. Same convention as +`ai4papi.utils.retrieve_from_snapshots`. The interface is deliberately small so +it can be swapped for a database later. +""" + +import json +import os +from pathlib import Path + + +def base_dir() -> Path | None: + """`$ACCOUNTING_PTH/energy`, or None if accounting storage is not configured.""" + root = os.environ.get("ACCOUNTING_PTH") + if not root: + return None + return Path(root) / "energy" + + +def _ns_dir(ns: str) -> Path | None: + base = base_dir() + return (base / ns) if base else None + + +def _accum_path(ns: str, uuid: str) -> Path | None: + d = _ns_dir(ns) + return (d / f"{uuid}.accum.json") if d else None + + +def _series_path(ns: str, uuid: str) -> Path | None: + d = _ns_dir(ns) + return (d / f"{uuid}.series.jsonl") if d else None + + +def _atomic_write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, path) + + +# --------------------------------------------------------------------------- +# accumulator document +# --------------------------------------------------------------------------- + + +def read_accum(ns: str, uuid: str) -> dict | None: + path = _accum_path(ns, uuid) + if not path or not path.is_file(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + + +def write_accum(ns: str, uuid: str, doc: dict) -> None: + path = _accum_path(ns, uuid) + if not path: + return + _atomic_write(path, json.dumps(doc, separators=(",", ":"))) + + +def iter_namespace(ns: str): + """Yield (uuid, doc) for every accumulator document in the namespace.""" + d = _ns_dir(ns) + if not d or not d.is_dir(): + return + for path in d.glob("*.accum.json"): + uuid = path.name[: -len(".accum.json")] + try: + yield uuid, json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + + +def iter_owner(ns: str, owner: str): + """Yield (uuid, doc) for the accumulator documents owned by `owner`.""" + for uuid, doc in iter_namespace(ns): + if doc.get("owner") == owner: + yield uuid, doc + + +# --------------------------------------------------------------------------- +# consolidated time series (append-only .series.jsonl) +# --------------------------------------------------------------------------- + + +def append_series(ns: str, uuid: str, buckets: list[dict]) -> None: + path = _series_path(ns, uuid) + if not path or not buckets: + return + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "a", encoding="utf-8") as f: + for b in buckets: + f.write(json.dumps(b, separators=(",", ":")) + "\n") + + +def read_series(ns: str, uuid: str, start_ts: str | None, end_ts: str | None): + path = _series_path(ns, uuid) + if not path or not path.is_file(): + return [] + out = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + b = json.loads(line) + except json.JSONDecodeError: + continue + if start_ts and b["ts"] < start_ts: + continue + if end_ts and b["ts"] > end_ts: + continue + out.append(b) + out.sort(key=lambda b: b["ts"]) + return out diff --git a/ai4papi/accounting/sweep.py b/ai4papi/accounting/sweep.py new file mode 100644 index 00000000..09a9cb68 --- /dev/null +++ b/ai4papi/accounting/sweep.py @@ -0,0 +1,731 @@ +""" +Background sweep + live top-up for the energy-accounting feature. + +`run_sweep()` runs on a `@repeat_every` thread in `ai4papi.main`. Per deployment +(and per datacenter for `/stats/cluster`) it integrates the un-consolidated +window of Mimir power into energy buckets (`energy.series_bucket_seconds`, +15 min), applies the per-datacenter TUE and the Wattnet footprint, keeps a +running accumulator plus a ~24h "tail" that is re-computed every sweep (so the +footprint self-heals as Wattnet settles), and persists a small JSON document. + +`topup(doc)` is called on the read path: it adds the `pointer_ts -> now` slice +live from Mimir so the dashboard counter moves every ~30s. +""" + +import datetime +import logging + +from cachetools import TTLCache, cached + +import ai4papi.conf as papiconf +from ai4papi import mimir +from ai4papi.accounting import compute, store +from ai4papi.nomad_utils import Nomad +from ai4papi.wattnet import green_director + +LOG = logging.getLogger(__name__) + +_JOB_PREFIXES = ("module", "tool", "batch", "try") +_KEYS = ("energy_wh", "carbon_g", "water_l") # TUE-normalized totals kept per accum +_EMPTY = dict.fromkeys(_KEYS, 0.0) + + +def _cfg() -> dict: + return papiconf.MAIN_CONF["energy"] + + +def enabled() -> bool: + return ( + bool(_cfg().get("enabled")) + and store.base_dir() is not None + and bool(mimir.MIMIR_USER) + ) + + +def _now() -> datetime.datetime: + return datetime.datetime.now(datetime.timezone.utc) + + +def _dt(ts: float) -> datetime.datetime: + return datetime.datetime.fromtimestamp(ts, datetime.timezone.utc) + + +def _floor(ts: float, step: int) -> float: + return (int(ts) // step) * step + + +def _earliest_bucket_ts(ns: str, uuid: str, doc: dict) -> float | None: + """Timestamp of the oldest 15-min bucket on record: the consolidated + `.series.jsonl` if there is one, else the oldest still in `doc["tail"]`.""" + archive = store.read_series(ns, uuid, None, None) + if archive: + return compute._parse_iso(archive[0]["ts"]) + tail = doc.get("tail") or [] + if tail: + return min(compute._parse_iso(b["ts"]) for b in tail) + return None + + +def _window_start( + ns: str, + uuid: str, + doc: dict, + settled_ts: float, + retention_cut: float, + bucket_s: int, +) -> float: + """ + True start of the accumulation window, exposed to the API as `since`. + + `settled_ts` cannot serve this role: it marches forward every sweep as + buckets age into `doc["settled"]`, while the accumulated totals (and the + persisted series) keep every bucket since the beginning. + + Pinned once and only moved forward if it falls behind Mimir retention: + - fresh doc: the first-sweep `settled_ts` is the window start; + - doc written before this field existed (`settled` / `tail` already there, + `settled_ts` already advanced): recover it from the oldest bucket on record. + """ + retention_floor = _floor(retention_cut, bucket_s) + if doc.get("window_start"): + return max(compute._parse_iso(doc["window_start"]), retention_floor) + if "settled" in doc or doc.get("tail"): + earliest = _earliest_bucket_ts(ns, uuid, doc) + if earliest is not None: + return max(min(earliest, settled_ts), retention_floor) + return max(settled_ts, retention_floor) + + +def _tue(dc: str) -> float: + return (papiconf.datacenters.get(dc) or {}).get("TUE") or _cfg()[ + "default_tue_factor" + ] + + +def _sum(buckets) -> dict: + return {k: sum(b[k] for b in buckets) for k in _KEYS} + + +def _add(a: dict, b: dict) -> dict: + return {k: a.get(k, 0.0) + b.get(k, 0.0) for k in _KEYS} + + +def _footprint_series(dc: str, start_ts: float, end_ts: float): + """ + (carbon, water) intensity series for a datacenter over [start, end]. + In steady state uses the 7-day window that `green_director` already keeps; + only fetches a wider range from Wattnet when backfilling further back. + `apply_footprint` falls back to DEFAULTS when a series is empty/None. + """ + metrics = green_director.metrics.get(dc, {}) + if start_ts >= _now().timestamp() - 7 * 86400: + return metrics.get("carbon"), metrics.get("water") + try: + fetched = green_director.footprint_series(dc, _dt(start_ts), _dt(end_ts)) + except Exception: + LOG.warning("wattnet footprint_series failed for %s", dc) + fetched = {} + return ( + fetched.get("carbon") or metrics.get("carbon"), + fetched.get("water") or metrics.get("water"), + ) + + +# --------------------------------------------------------------------------- +# sweep +# --------------------------------------------------------------------------- + + +def run_sweep() -> None: + if not enabled(): + return + now = _now() + for ns in set(papiconf.MAIN_CONF["nomad"]["namespaces"].values()): + try: + _sweep_namespace(ns, now) + except Exception: + LOG.exception("energy sweep failed for namespace %s", ns) + try: + _sweep_cluster(now) + except Exception: + LOG.exception("cluster energy sweep failed") + + +def _sweep_namespace(ns: str, now: datetime.datetime) -> None: + try: + jobs = Nomad.jobs.get_jobs(namespace=ns, filter_='Status != "dead"') + except Exception: + LOG.exception("could not list jobs for namespace %s", ns) + return + + seen = set() + for stub in jobs: + if not stub["Name"].startswith(_JOB_PREFIXES): + continue + seen.add(stub["ID"]) + # the job list stub does not carry Meta, fetch the full job for `owner` + try: + j = Nomad.job.get_job(id_=stub["ID"], namespace=ns) + except Exception: + LOG.exception("could not fetch job %s", stub["ID"]) + continue + meta = j.get("Meta") or {} + info = { + "uuid": j["ID"], + "job_name": j["Name"], + "namespace": ns, + "owner": meta.get("owner", ""), + "submit_time": (j["SubmitTime"] / 1e9) if j.get("SubmitTime") else None, + "status": "running", + } + try: + process_single(info, now) + except Exception: + LOG.exception("energy sweep failed for deployment %s", stub["ID"]) + + # close deployments whose Nomad job has been garbage-collected + for uuid, doc in store.iter_namespace(ns): + if uuid in seen or doc.get("closed_at"): + continue + info = { + "uuid": uuid, + "job_name": doc.get("job_name"), + "namespace": ns, + "owner": doc.get("owner", ""), + "submit_time": doc.get("submit_time_epoch"), + "status": "closed", + } + try: + process_single(info, now, closing=True) + except Exception: + LOG.exception("energy close failed for deployment %s", uuid) + + +def process_single(info: dict, now: datetime.datetime, closing: bool = False) -> None: + cfg = _cfg() + ns, uuid = info["namespace"], info["uuid"] + bucket_s = int(cfg["series_bucket_seconds"]) + step_s = int(cfg["query_step_seconds"]) + now_ts = now.timestamp() + revision_cut = now_ts - cfg["footprint_revision_hours"] * 3600 + retention_cut = now_ts - cfg["mimir_retention_days"] * 86400 + lookback_ts = now_ts - cfg["initial_lookback_hours"] * 3600 + + doc = store.read_accum(ns, uuid) or {} + doc.pop("partial", None) # dropped in favour of `coverage_ratio` (always a float) + + # integration pointer: on first encounter never look back further than + # `initial_lookback_hours` (pre-existing deployments are not fully backfilled). + if doc.get("settled_ts"): + settled_ts = compute._parse_iso(doc["settled_ts"]) + complete = doc.get("complete", True) + else: + submit = info.get("submit_time") + settled_ts = max(submit or lookback_ts, lookback_ts) + complete = bool(submit) and submit >= lookback_ts + if settled_ts < retention_cut: + settled_ts, complete = retention_cut, False + settled_ts = _floor(settled_ts, bucket_s) + window_start = _window_start(ns, uuid, doc, settled_ts, retention_cut, bucket_s) + extract_to = _floor(now_ts - cfg["query_lag_seconds"], bucket_s) + + try: + allocs_by_dc, alloc_windows, running_allocs, partial_dc = ( + compute.resolve_alloc_ids(uuid, ns) + ) + except Exception: + allocs_by_dc, alloc_windows, running_allocs, partial_dc = {}, [], [], False + + # metadata (always refreshed) + doc.update( + { + "deployment_uuid": uuid, + "job_name": info.get("job_name") or doc.get("job_name"), + "namespace": ns, + "owner": info.get("owner") or doc.get("owner", ""), + "status": "closed" if closing else info.get("status", "running"), + "complete": complete, + "updated_at": compute._iso(now_ts), + "active_allocs": running_allocs, + } + ) + if info.get("submit_time"): + doc["submit_time"] = compute._iso(info["submit_time"]) + doc["submit_time_epoch"] = info["submit_time"] + if allocs_by_dc: + doc["datacenters"] = list(allocs_by_dc) + if closing and not doc.get("closed_at"): + doc["closed_at"] = compute._iso(now_ts) + + run_s = compute.running_seconds(alloc_windows, settled_ts, extract_to) + + # no monitored allocations: queued, or ran only in unmonitored sites + if not allocs_by_dc: + if "settled" not in doc and partial_dc and run_s > 2 * bucket_s: + doc.update({"metrics_available": False, "coverage_ratio": 0.0}) + store.write_accum(ns, uuid, doc) + return + + # integrate [settled_ts, extract_to] per datacenter + dq = cfg.get("data_quality") + new_buckets = [] + if settled_ts < extract_to: + span = extract_to - settled_ts + qfn = ( + mimir.query_range_chunked + if span > mimir.MAX_POINTS_PER_QUERY * step_s + else mimir.query_range + ) + start_dt, end_dt = _dt(settled_ts), _dt(extract_to) + per_dc = [] + tue_norm = tue_raw = 0.0 # energy-weighted mean TUE over the new buckets + for dc, alloc_ids in allocs_by_dc.items(): + cpu_q, gpu_q = compute.power_promql(alloc_ids) + # CPU and GPU are treated symmetrically: a missing metric already + # comes back as an empty result, and a transient query error on one + # source must not discard the other (a GPU-only monitored site has + # no `scaph_*` series at all). + try: + cpu = qfn(cpu_q, start_dt, end_dt, step_s) + except mimir.MimirError: + LOG.warning("mimir CPU query failed for %s/%s", ns, uuid) + cpu = [] + try: + gpu = qfn(gpu_q, start_dt, end_dt, step_s) + except mimir.MimirError: + LOG.warning("mimir GPU query failed for %s/%s", ns, uuid) + gpu = [] + if not cpu and not gpu: + continue + tue = _tue(dc) + buckets = compute.integrate( + cpu, gpu, step_s, bucket_s, tue, data_quality=dq + ) + if buckets: + e = sum(b.energy_wh for b in buckets) + tue_norm += e + tue_raw += e / tue + carbon_s, water_s = _footprint_series(dc, settled_ts, now_ts) + buckets = compute.apply_footprint(buckets, carbon_s, water_s) + per_dc.append(buckets) + if tue_raw: + doc["tue_factor"] = tue_norm / tue_raw + elif "tue_factor" not in doc and allocs_by_dc: + doc["tue_factor"] = _tue(next(iter(allocs_by_dc))) + new_buckets = [ + b + for b in compute.merge_buckets(per_dc) + if compute._parse_iso(b.ts) + bucket_s <= extract_to + ] + # clip to the deployment's actual life: drop buckets with no overlap with + # any allocation window. Guards against metric staleness (the per-alloc + # power series and `nomad_gpu_allocation_info` can linger a few minutes + # after an allocation ends, which would otherwise attribute idle GPU / + # host power to a finished deployment). + if alloc_windows: + new_buckets = [ + b + for b in new_buckets + if compute.running_seconds( + alloc_windows, + compute._parse_iso(b.ts), + compute._parse_iso(b.ts) + bucket_s, + ) + > 0 + ] + + # best-effort coverage: no data while the deployment was clearly running + # means the node has no monitoring stack yet -> null, retry next sweep. + if not new_buckets and "settled" not in doc: + if run_s > 2 * bucket_s: + doc.update({"metrics_available": False, "coverage_ratio": 0.0}) + store.write_accum(ns, uuid, doc) + return + + doc["metrics_available"] = True + doc["window_start"] = compute._iso(window_start) + + aged_n, tail_n = _fold_and_accumulate( + ns, uuid, doc, new_buckets, settled_ts, revision_cut, bucket_s + ) + + # coverage: fraction of the run time (including time on unmonitored sites, so + # `partial_dc` is captured implicitly) that ended up with power data. + covered_s = (tail_n + aged_n) * bucket_s + if run_s > 0: + doc["coverage_ratio"] = round(min(1.0, covered_s / run_s), 3) + else: + doc["coverage_ratio"] = 1.0 + + store.write_accum(ns, uuid, doc) + + +def _fold_and_accumulate( + ns: str, + uuid: str, + doc: dict, + new_buckets: list, + settled_ts: float, + revision_cut: float, + bucket_s: int, +) -> tuple[int, int]: + """ + Merge `new_buckets` (Bucket namedtuples) over `doc["tail"]` keyed by `ts`, + age the buckets older than `revision_cut` into `doc["settled"]` (appending + them to the persisted series), and recompute `settled_ts` / `accumulated` / + `pointer_ts` on `doc`. Returns `(aged_count, tail_count)`. + + Shared by the per-deployment (`process_single`) and per-datacenter + (`process_datacenter`) sweeps. + """ + settled = dict(doc.get("settled") or _EMPTY) + + by_ts = {b["ts"]: b for b in (doc.get("tail") or [])} + for b in new_buckets: + by_ts[b.ts] = b._asdict() + + tail, aged_out = [], [] + for ts in sorted(by_ts): + (aged_out if compute._parse_iso(ts) < revision_cut else tail).append(by_ts[ts]) + + if aged_out: + settled = _add(settled, _sum(aged_out)) + store.append_series(ns, uuid, aged_out) + settled_ts = max(compute._parse_iso(b["ts"]) for b in aged_out) + bucket_s + + doc["settled"] = settled + doc["tail"] = tail + doc["settled_ts"] = compute._iso(settled_ts) + doc["accumulated"] = _add(settled, _sum(tail)) + doc["pointer_ts"] = compute._iso( + (compute._parse_iso(tail[-1]["ts"]) + bucket_s) if tail else settled_ts + ) + return len(aged_out), len(tail) + + +# --------------------------------------------------------------------------- +# per-datacenter sweep (cluster-level energy accumulator) +# --------------------------------------------------------------------------- + +# Pseudo-namespace for the datacenter-level documents. Does not collide with any +# VO namespace nor with the deployment sweep's namespace loop. +CLUSTER_NS = "_cluster" + + +def _cluster_promql(dc: str) -> tuple[str, str]: + """ + (host_query, gpu_query) for a datacenter: one series per host and one per + GPU (grouped, not summed) so `compute.integrate` runs the data-quality + filter on each raw Mimir series before summing the energy. + """ + host_q = f'sum by (node) (scaph_host_power_microwatts{{datacenter="{dc}"}})' + gpu_q = f'sum by (UUID) (DCGM_FI_DEV_POWER_USAGE{{datacenter="{dc}"}})' + return host_q, gpu_q + + +def _cluster_dq() -> dict: + """ + Data-quality config for the datacenter sweep: reuse `energy.data_quality` + but drop the absolute `max_power_w` ceiling (it is calibrated per allocation + / per GPU, far below a whole host). Negative-drop and the scale-free spike + check still apply, per host / per GPU series. + """ + return {**(_cfg().get("data_quality") or {}), "max_power_w": 0} + + +def _sweep_cluster(now: datetime.datetime) -> None: + for dc in _cfg()["metrics_datacenters"]: + try: + process_datacenter(dc, now) + except Exception: + LOG.exception("cluster energy sweep failed for datacenter %s", dc) + + +def process_datacenter(dc: str, now: datetime.datetime) -> None: + cfg = _cfg() + bucket_s = int(cfg["series_bucket_seconds"]) + step_s = int(cfg["query_step_seconds"]) + now_ts = now.timestamp() + revision_cut = now_ts - cfg["footprint_revision_hours"] * 3600 + retention_cut = now_ts - cfg["mimir_retention_days"] * 86400 + lookback_ts = now_ts - cfg["initial_lookback_hours"] * 3600 + + doc = store.read_accum(CLUSTER_NS, dc) or {} + + if doc.get("settled_ts"): + settled_ts = compute._parse_iso(doc["settled_ts"]) + complete = doc.get("complete", True) + else: + settled_ts, complete = lookback_ts, False + if settled_ts < retention_cut: + settled_ts, complete = retention_cut, False + settled_ts = _floor(settled_ts, bucket_s) + window_start = _window_start( + CLUSTER_NS, dc, doc, settled_ts, retention_cut, bucket_s + ) + extract_to = _floor(now_ts - cfg["query_lag_seconds"], bucket_s) + + doc.update( + { + "datacenter": dc, + "deployment_uuid": dc, # reuse the same key name as deployment docs + "namespace": CLUSTER_NS, + "complete": complete, + "updated_at": compute._iso(now_ts), + "tue_factor": _tue(dc), + } + ) + + new_buckets = [] + if settled_ts < extract_to: + span = extract_to - settled_ts + qfn = ( + mimir.query_range_chunked + if span > mimir.MAX_POINTS_PER_QUERY * step_s + else mimir.query_range + ) + host_q, gpu_q = _cluster_promql(dc) + start_dt, end_dt = _dt(settled_ts), _dt(extract_to) + try: + host = qfn(host_q, start_dt, end_dt, step_s) + except mimir.MimirError: + LOG.warning("mimir host-power query failed for %s", dc) + host = [] + try: + gpu = qfn(gpu_q, start_dt, end_dt, step_s) + except mimir.MimirError: + gpu = [] + buckets = compute.integrate( + host, gpu, step_s, bucket_s, _tue(dc), data_quality=_cluster_dq() + ) + if buckets: + carbon_s, water_s = _footprint_series(dc, settled_ts, now_ts) + buckets = compute.apply_footprint(buckets, carbon_s, water_s) + new_buckets = [ + b for b in buckets if compute._parse_iso(b.ts) + bucket_s <= extract_to + ] + + if not new_buckets and "settled" not in doc: + # datacenter has no host-power series yet: retry next sweep, no zeros + doc["metrics_available"] = False + store.write_accum(CLUSTER_NS, dc, doc) + return + + doc["metrics_available"] = True + doc["window_start"] = compute._iso(window_start) + _fold_and_accumulate( + CLUSTER_NS, dc, doc, new_buckets, settled_ts, revision_cut, bucket_s + ) + store.write_accum(CLUSTER_NS, dc, doc) + + +# --------------------------------------------------------------------------- +# live path: `pointer_ts -> now` slice from Mimir (read side) +# --------------------------------------------------------------------------- + +_live_cache: TTLCache = TTLCache( + maxsize=8192, ttl=papiconf.MAIN_CONF["energy"].get("topup_cache_seconds", 25) +) + + +def _live_window(pointer_iso: str) -> tuple[float, float]: + cfg = _cfg() + end_ts = _now().timestamp() - cfg["topup_lag_seconds"] + start_ts = max( + compute._parse_iso(pointer_iso), + end_ts - (cfg["sweep_seconds"] + 4 * cfg["series_bucket_seconds"]), + ) + return start_ts, end_ts + + +def _running_key(doc: dict) -> tuple | None: + pointer = doc.get("pointer_ts") or doc.get("settled_ts") + running = doc.get("active_allocs") or [] + if not doc.get("metrics_available", False) or not pointer or not running: + return None + return ( + doc["namespace"], + doc["deployment_uuid"], + pointer, + tuple(sorted((a["alloc_id"], a["datacenter"]) for a in running)), + ) + + +def _live_buckets(alloc_pairs, start_ts: float, end_ts: float, bucket_s: int): + """ + Integrate the live window from Mimir into `bucket_s` buckets (per DC, merged), + plus the current draw from the last sample of that same query. Returns + `(buckets, power_w)`; `power_w` is `None` when nothing is fresh. + """ + cfg = _cfg() + step_s = int(cfg["query_step_seconds"]) + if end_ts - start_ts < step_s: + return [], None + + by_dc: dict[str, list[str]] = {} + for alloc_id, dc in alloc_pairs: + by_dc.setdefault(dc, []).append(alloc_id) + + start_dt, end_dt = _dt(start_ts), _dt(end_ts) + fresh_after = end_ts - 2 * step_s + per_dc = [] + inst_w = 0.0 + any_inst = False + for dc, alloc_ids in by_dc.items(): + cpu_q, gpu_q = compute.power_promql(alloc_ids) + try: + cpu = mimir.query_range(cpu_q, start_dt, end_dt, step_s) + except mimir.MimirError: + cpu = [] + try: + gpu = mimir.query_range(gpu_q, start_dt, end_dt, step_s) + except mimir.MimirError: + gpu = [] + dq = cfg.get("data_quality") + buckets = compute.integrate( + cpu, gpu, step_s, bucket_s, _tue(dc), data_quality=dq + ) + metrics = green_director.metrics.get(dc, {}) + per_dc.append( + compute.apply_footprint( + buckets, metrics.get("carbon"), metrics.get("water") + ) + ) + iw = compute.instant_power(cpu, gpu, _tue(dc), fresh_after, dq) + if iw is not None: + inst_w += iw + any_inst = True + + return compute.merge_buckets(per_dc), (inst_w if any_inst else None) + + +def topup(doc: dict | None) -> dict | None: + """ + Live `pointer_ts -> now` slice as a single delta: dict with the summed energy + deltas, `power_w` (the last sample read, i.e. the current draw) and + `live_as_of`; `{"degraded": True}` on failure; None if not applicable. + Cached for `topup_cache_seconds`. + """ + key = _running_key(doc) + if key is None: + return None + try: + return _compute_topup(key) + except Exception: + LOG.warning( + "energy top-up failed for %s", doc.get("deployment_uuid"), exc_info=True + ) + return {"degraded": True} + + +@cached(_live_cache) +def _compute_topup(key) -> dict: + _ns, _uuid, pointer_iso, alloc_pairs = key + start_ts, end_ts = _live_window(pointer_iso) + result = { + **_EMPTY, + "power_w": None, + "live_as_of": compute._iso(end_ts), + "degraded": False, + } + span = int(end_ts - start_ts) + int(_cfg()["query_step_seconds"]) + buckets, inst_w = _live_buckets( + alloc_pairs, start_ts, end_ts, bucket_s=max(span, 1) + ) + for b in buckets: + for k in _KEYS: + result[k] += getattr(b, k) + # `power_w` is the last sample read (current draw), not a window mean + result["power_w"] = inst_w + return result + + +def live_series(doc: dict | None, bucket_s: int) -> list[dict]: + """ + Live tail as time-series points (so the detail-view graph keeps growing). + `[]` if not applicable or on any failure. + """ + key = _running_key(doc) + if key is None: + return [] + _ns, _uuid, pointer_iso, alloc_pairs = key + start_ts, end_ts = _live_window(pointer_iso) + try: + buckets, _ = _live_buckets( + alloc_pairs, start_ts, end_ts, bucket_s=max(bucket_s, 1) + ) + except Exception: + LOG.warning("energy live_series failed for %s", doc.get("deployment_uuid")) + return [] + return [{**b._asdict(), "live": True} for b in buckets] + + +# --------------------------------------------------------------------------- +# live path for the per-datacenter (cluster) accumulator +# --------------------------------------------------------------------------- + + +def _cluster_live_buckets(dc: str, start_ts: float, end_ts: float, bucket_s: int): + """Integrate the live window from Mimir for one datacenter into buckets.""" + cfg = _cfg() + step_s = int(cfg["query_step_seconds"]) + if end_ts - start_ts < step_s: + return [] + host_q, gpu_q = _cluster_promql(dc) + start_dt, end_dt = _dt(start_ts), _dt(end_ts) + host = mimir.query_range(host_q, start_dt, end_dt, step_s) + try: + gpu = mimir.query_range(gpu_q, start_dt, end_dt, step_s) + except mimir.MimirError: + gpu = [] + buckets = compute.integrate( + host, gpu, step_s, bucket_s, _tue(dc), data_quality=_cluster_dq() + ) + metrics = green_director.metrics.get(dc, {}) + return compute.apply_footprint(buckets, metrics.get("carbon"), metrics.get("water")) + + +def _cluster_pointer(doc: dict | None) -> str | None: + if not doc or not doc.get("metrics_available", False): + return None + return doc.get("pointer_ts") or doc.get("settled_ts") + + +def topup_cluster(dc: str, doc: dict | None) -> dict | None: + """Live `pointer_ts -> now` slice for a datacenter (see `topup`).""" + pointer = _cluster_pointer(doc) + if pointer is None: + return None + try: + return _compute_cluster_topup((CLUSTER_NS, dc, pointer)) + except Exception: + LOG.warning("cluster energy top-up failed for %s", dc, exc_info=True) + return {"degraded": True} + + +@cached(_live_cache) +def _compute_cluster_topup(key) -> dict: + # Only the energy deltas + `live_as_of` are consumed (the datacenter `power_w` + # comes from `get_datacenter_power`'s instant query, not from here). + _ns, dc, pointer_iso = key + start_ts, end_ts = _live_window(pointer_iso) + result = {**_EMPTY, "live_as_of": compute._iso(end_ts), "degraded": False} + span = int(end_ts - start_ts) + int(_cfg()["query_step_seconds"]) + for b in _cluster_live_buckets(dc, start_ts, end_ts, bucket_s=max(span, 1)): + for k in _KEYS: + result[k] += getattr(b, k) + return result + + +def live_series_cluster(dc: str, doc: dict | None, bucket_s: int) -> list[dict]: + """Live tail for a datacenter as time-series points (see `live_series`).""" + pointer = _cluster_pointer(doc) + if pointer is None: + return [] + start_ts, end_ts = _live_window(pointer) + try: + buckets = _cluster_live_buckets(dc, start_ts, end_ts, bucket_s=max(bucket_s, 1)) + except Exception: + LOG.warning("cluster live_series failed for %s", dc) + return [] + return [{**b._asdict(), "live": True} for b in buckets] diff --git a/ai4papi/mimir.py b/ai4papi/mimir.py new file mode 100644 index 00000000..390e006b --- /dev/null +++ b/ai4papi/mimir.py @@ -0,0 +1,125 @@ +""" +Minimal client for the Grafana Mimir (Prometheus-compatible) HTTP API. + +Used by the energy-accounting feature to read per-allocation power time series. +API reference: https://prometheus.io/docs/prometheus/latest/querying/api/ +""" + +import datetime +import os + +import requests + + +MIMIR_URL = "https://mimir.k8s.cloud.ai4eosc.eu/prometheus" +# Optional: the energy-accounting feature is disabled if these are not set. +MIMIR_USER = os.environ.get("MIMIR_USER") +MIMIR_PASSWORD = os.environ.get("MIMIR_PASSWORD") +if not (MIMIR_USER and MIMIR_PASSWORD): + print("You should define MIMIR_USER / MIMIR_PASSWORD to enable energy stats") + +# Mimir keeps each series under a point cap per query; stay well below it. +MAX_POINTS_PER_QUERY = 11_000 + +session = requests.Session() + + +class MimirError(RuntimeError): + """Raised when Mimir returns a non-successful response.""" + + +def _auth(): + return (MIMIR_USER, MIMIR_PASSWORD) if MIMIR_USER else None + + +def query(promql: str, at: datetime.datetime | None = None, timeout: int = 30): + """ + Instantaneous query (`/api/v1/query`). Returns the raw `data.result` list + (vector: one entry per series, each with `metric` and `value` = [ts, str]). + """ + params = {"query": promql} + if at is not None: + params["time"] = at.timestamp() + + r = session.get( + f"{MIMIR_URL}/api/v1/query", + params=params, + auth=_auth(), + timeout=timeout, + ) + r.raise_for_status() + body = r.json() + if body.get("status") != "success": + raise MimirError(f"Mimir query failed: {body}") + return body.get("data", {}).get("result", []) + + +def query_range( + promql: str, + start: datetime.datetime, + end: datetime.datetime, + step_s: int = 30, + timeout: int = 60, +): + """ + Range query (`/api/v1/query_range`). Returns the raw `data.result` list + (matrix: one entry per series, each with `metric` and `values` = + [[ts, str], ...]). + """ + r = session.get( + f"{MIMIR_URL}/api/v1/query_range", + params={ + "query": promql, + "start": start.timestamp(), + "end": end.timestamp(), + "step": f"{step_s}s", + }, + auth=_auth(), + timeout=timeout, + ) + r.raise_for_status() + body = r.json() + if body.get("status") != "success": + raise MimirError(f"Mimir query failed: {body}") + return body.get("data", {}).get("result", []) + + +def _split_chunks(start, end, step_s, max_points): + """ + Yield non-overlapping (chunk_start, chunk_end) tuples so each range query + stays under `max_points` samples per series (relevant when backfilling a + deployment whose pointer is far behind). + """ + while start < end: + remaining_steps = int((end - start).total_seconds() / step_s) + chunk_steps = min(remaining_steps, max_points) + if chunk_steps <= 0: + break + chunk_end = start + datetime.timedelta(seconds=chunk_steps * step_s) + yield start, min(chunk_end, end) + start = chunk_end + datetime.timedelta(seconds=step_s) + + +def query_range_chunked( + promql: str, + start: datetime.datetime, + end: datetime.datetime, + step_s: int = 30, + max_points: int = MAX_POINTS_PER_QUERY, +): + """ + Like `query_range` but splits long windows into consecutive chunks and + merges the results, concatenating each series' `values` sorted by timestamp. + """ + merged: dict = {} + for chunk_start, chunk_end in _split_chunks(start, end, step_s, max_points): + for series in query_range(promql, chunk_start, chunk_end, step_s): + key = frozenset(series.get("metric", {}).items()) + merged.setdefault(key, {"metric": series.get("metric", {}), "values": []}) + merged[key]["values"].extend(series.get("values", [])) + + result = [] + for entry in merged.values(): + entry["values"].sort(key=lambda v: float(v[0])) + result.append(entry) + return result diff --git a/etc/main.yaml b/etc/main.yaml index c4f5ea5b..9af2d466 100644 --- a/etc/main.yaml +++ b/etc/main.yaml @@ -85,3 +85,37 @@ mlflow: vo.ai4life.eu: https://mlflow.cloud.ai4eosc.eu/ kmd4eosc: https://mlflow.cloud.ai4eosc.eu/ tutorials: https://mlflow.cloud.ai4eosc.eu/ + +energy: # per-deployment energy consumption and environmental footprint stats + + enabled: true + sweep_seconds: 900 # background thread cadence + series_bucket_seconds: 900 # persisted time-series bucket size (matches Wattnet's 15-min cadence) + query_step_seconds: 30 # Mimir scrape interval + query_lag_seconds: 120 # ignore the most recent data (ingestion lag) + initial_lookback_hours: 24 # window covered on a deployment's first sweep + mimir_retention_days: 90 # tune to Mimir's actual retention + footprint_revision_hours: 24 # re-compute footprint for buckets newer than this + topup_cache_seconds: 25 # live top-up cache (< dashboard poll so the counter moves) + topup_lag_seconds: 25 # live top-up query stops this far before "now" + deployment_series_points: 1000 # downsample target for the per-deployment series in /deployments/* + user_series_points: 1000 # downsample target for the merged per-user series in /stats/user (full_info=true) + cluster_series_points: 500 # downsample target for the per-datacenter series in /stats/cluster (full_info=true) + + metrics_datacenters: # datacenters where power metrics exist + + - ifca-ai4eosc + - ifca-imagine + + # TUE (grid -> compute ratio, already covers CPU+GPU) is read per-datacenter + # from the `TUE` column of var/datacenters.csv; this is the fallback for + # datacenters that do not have it yet. + default_tue_factor: 2.03 + + data_quality: # drop anomalous samples from each source series before integrating + + enabled: true + min_power_w: 0 # drop samples below this (kills negatives) + max_power_w: 1000 # per-source ceiling (one allocation's CPU, or one GPU); 0 = off + spike_factor: 12 # drop a sample above spike_factor * rolling median (0 = off) + spike_window: 11 # neighbour samples used for the rolling median From c9f2ec455e2c343550f7b1ff6e5e032bd58537d2 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 10 Sep 2026 09:37:44 +0200 Subject: [PATCH 05/25] feat(energy): run the accounting sweep as a background thread Wire `accounting.run_sweep()` into a `@repeat_every` task started on app startup, next to `get_cluster_stats_bg`. No-op unless the `energy` feature is enabled and the MIMIR_USER / MIMIR_PASSWORD credentials (added to template.env) are set. --- ai4papi/main.py | 16 ++++++++++++++++ template.env | 2 ++ 2 files changed, 18 insertions(+) diff --git a/ai4papi/main.py b/ai4papi/main.py index 50ccd7e9..606da822 100644 --- a/ai4papi/main.py +++ b/ai4papi/main.py @@ -11,6 +11,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi_utils.tasks import repeat_every +from ai4papi import accounting from ai4papi.conf import MAIN_CONF, paths, papi_branch, papi_commit from ai4papi.routers import v1 from ai4papi.routers.v1.stats.deployments import get_cluster_stats_bg @@ -48,6 +49,7 @@ async def lifespan(app: fastapi.FastAPI): # on startup await get_cluster_stats_thread() + await energy_accounting_thread() yield # on shutdown # (nothing to do) @@ -152,5 +154,19 @@ def get_cluster_stats_thread(): logging.exception("Error in background task get_cluster_stats_bg") +# Accumulate per-deployment energy / footprint stats in background task. +# No-op unless the `energy` feature is enabled and Mimir credentials are set. +@repeat_every(seconds=MAIN_CONF.get("energy", {}).get("sweep_seconds", 900)) +def energy_accounting_thread(): + """ + Incrementally integrate Mimir power into per-deployment energy/carbon/water. + Not async, same reasoning as get_cluster_stats_thread. + """ + try: + accounting.run_sweep() + except Exception: + logging.exception("Error in background task energy_accounting") + + if __name__ == "__main__": run() diff --git a/template.env b/template.env index f8f6c1db..b7d1e90a 100644 --- a/template.env +++ b/template.env @@ -13,3 +13,5 @@ PROVENANCE_TOKEN="**********************************" LITELLM_API_KEY="***********************************" WATTNET_EMAIL="bot@example.org" WATTNET_PASSWORD="**********************************" +MIMIR_USER="platform-reader" +MIMIR_PASSWORD="**********************************" From bd95edf50d0ac57c32cd1798b9909c970ab36abe Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 10 Sep 2026 09:37:52 +0200 Subject: [PATCH 06/25] feat(energy): expose the energy block on the deployment and stats endpoints Additive, nothing in the existing responses changes: - `EnergyStats` / `EnergyTimeSeries` / `EnergyTimeSeriesPoint` schemas - deployment endpoints (modules, tools, batch, try_me) gain an `energy` key and an `energy` query param (on by default): accumulated `EnergyStats` on the list view, extended `EnergyTimeSeries` with `full_info=true` - `GET /stats/user` gains an `energy` block (the user's deployments summed) and still serves it if the CSV usage stats are unavailable - `GET /stats/cluster` gains a per-datacenter `energy` block (site power draw + accumulated energy/footprint + optional 15-min series); no platform-wide total, and instrumented datacenters are kept even when the VO filter leaves them without nodes - internal callers (create_deployment, green-affinity, try_me capacity check) pass `energy=False` to skip the extra work --- ai4papi/routers/v1/batch.py | 38 ++++++++- ai4papi/routers/v1/deployments/modules.py | 42 +++++++++- ai4papi/routers/v1/deployments/tools.py | 43 +++++++++-- ai4papi/routers/v1/stats/deployments.py | 94 +++++++++++++++++++---- ai4papi/routers/v1/try_me/nomad.py | 28 ++++++- ai4papi/schemas.py | 60 ++++++++++++++- 6 files changed, 274 insertions(+), 31 deletions(-) diff --git a/ai4papi/routers/v1/batch.py b/ai4papi/routers/v1/batch.py index 82f3d293..04280103 100644 --- a/ai4papi/routers/v1/batch.py +++ b/ai4papi/routers/v1/batch.py @@ -10,7 +10,7 @@ from fastapi import APIRouter, Depends, Form, HTTPException, UploadFile from fastapi.security import HTTPBearer -from ai4papi import auth, module_patches, quotas, schemas, utils +from ai4papi import accounting, auth, module_patches, quotas, schemas, utils import ai4papi.conf as papiconf import ai4papi.nomad_utils as nomad_utils from ai4papi.routers import v1 @@ -29,6 +29,7 @@ def get_deployments( vos: schemas.VoList = None, full_info: bool = False, + energy: bool = True, authorization=Depends(security), ): """ @@ -40,6 +41,9 @@ def get_deployments( * **full_info**: retrieve the full information of each deployment. Disabled by default, as it will increase latency too much if there are many deployments. + * **energy**: attach energy/footprint stats to each deployment (`energy` + key). Enabled by default. `full_info=false` -> accumulated `EnergyStats`; + `full_info=true` -> extended `EnergyTimeSeries`. """ # Retrieve authenticated user info auth_info = auth.get_user_info(token=authorization.credentials) @@ -58,6 +62,8 @@ def get_deployments( user_jobs = [] for vo in user_vos: + namespace = papiconf.MAIN_CONF["nomad"]["namespaces"][vo] + # Retrieve all jobs in namespace (including dead jobs) job_filter = ( 'Name matches "^batch" and ' @@ -65,17 +71,19 @@ def get_deployments( + f'Meta.owner == "{auth_info["id"]}"' ) jobs = nomad_utils.Nomad.jobs.get_jobs( - namespace=papiconf.MAIN_CONF["nomad"]["namespaces"][vo], + namespace=namespace, filter_=job_filter, ) # Retrieve info for jobs in namespace + vo_jobs = [] for j in jobs: try: job_info = get_deployment( vo=vo, deployment_uuid=j["ID"], full_info=full_info, + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), @@ -83,7 +91,20 @@ def get_deployments( except HTTPException: # not a module continue - user_jobs.append(job_info) + vo_jobs.append(job_info) + + # `full_info=False` (default): one cheap batch. `full_info=True`: the + # extended time series per deployment (heavier). + if energy and vo_jobs: + if full_info: + for job in vo_jobs: + job["energy"] = accounting.get_series(namespace, job["job_ID"]) + else: + env = accounting.get_accumulated_bulk(namespace, auth_info["id"]) + for job in vo_jobs: + job["energy"] = env.get(job["job_ID"]) + + user_jobs.extend(vo_jobs) # Sort deployments by submission time in descending order sorted_jobs = sorted(user_jobs, key=lambda x: x["submit_time"], reverse=True) @@ -96,6 +117,7 @@ def get_deployment( vo: str, deployment_uuid: str, full_info: bool = True, + energy: bool = True, authorization=Depends(security), ): """ @@ -107,6 +129,9 @@ def get_deployment( * **deployment_uuid**: uuid of deployment to gather info about * **full_info**: retrieve the full information of that deployment (may increase latency) + * **energy**: attach energy/footprint stats (`energy` key). Enabled by + default. `full_info=true` (default here) -> extended `EnergyTimeSeries`; + `full_info=false` -> accumulated `EnergyStats`. Returns a dict with info """ @@ -132,6 +157,12 @@ def get_deployment( detail="This deployment is not a batch job.", ) + if energy: + if full_info: + job["energy"] = accounting.get_series(namespace, deployment_uuid) + else: + job["energy"] = accounting.get_accumulated(namespace, deployment_uuid) + return job @@ -202,6 +233,7 @@ async def create_deployment( # jobs) batch_deps = get_deployments( vos=[vo], + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), diff --git a/ai4papi/routers/v1/deployments/modules.py b/ai4papi/routers/v1/deployments/modules.py index 65ab1d70..6482313e 100644 --- a/ai4papi/routers/v1/deployments/modules.py +++ b/ai4papi/routers/v1/deployments/modules.py @@ -8,7 +8,7 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.security import HTTPBearer -from ai4papi import auth, module_patches, quotas, schemas, utils +from ai4papi import accounting, auth, module_patches, quotas, schemas, utils from ai4papi.wattnet import green_director import ai4papi.conf as papiconf import ai4papi.nomad_utils as nomad_utils @@ -33,6 +33,7 @@ def get_deployments( vos: schemas.VoList = None, full_info: bool = False, + energy: bool = True, authorization=Depends(security), ): """ @@ -44,6 +45,9 @@ def get_deployments( * **full_info**: retrieve the full information of each deployment. Disabled by default, as it will increase latency too much if there are many deployments. + * **energy**: attach energy/footprint stats to each deployment (`energy` + key). Enabled by default. With `full_info=false` it is the accumulated + `EnergyStats`; with `full_info=true` it is the extended `EnergyTimeSeries`. """ # Retrieve authenticated user info auth_info = auth.get_user_info(token=authorization.credentials) @@ -62,20 +66,24 @@ def get_deployments( user_jobs = [] for vo in user_vos: + namespace = papiconf.MAIN_CONF["nomad"]["namespaces"][vo] + # Retrieve all jobs in namespace jobs = nomad_utils.get_deployments( - namespace=papiconf.MAIN_CONF["nomad"]["namespaces"][vo], + namespace=namespace, owner=auth_info["id"], prefix="module", ) # Retrieve info for jobs in namespace + vo_jobs = [] for j in jobs: try: job_info = get_deployment( vo=vo, deployment_uuid=j["ID"], full_info=full_info, + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), @@ -83,7 +91,21 @@ def get_deployments( except HTTPException: # not a module continue - user_jobs.append(job_info) + vo_jobs.append(job_info) + + # Attach energy/footprint stats. `full_info=False` (the default) uses one + # batch of cheap file reads + a single cached Mimir top-up; `full_info=True` + # attaches the extended time series per deployment (heavier). + if energy and vo_jobs: + if full_info: + for job in vo_jobs: + job["energy"] = accounting.get_series(namespace, job["job_ID"]) + else: + env = accounting.get_accumulated_bulk(namespace, auth_info["id"]) + for job in vo_jobs: + job["energy"] = env.get(job["job_ID"]) + + user_jobs.extend(vo_jobs) # Sort deployments by submission time in descending order sorted_jobs = sorted(user_jobs, key=lambda x: x["submit_time"], reverse=True) @@ -96,6 +118,7 @@ def get_deployment( vo: str, deployment_uuid: str, full_info: bool = True, + energy: bool = True, authorization=Depends(security), ): """ @@ -107,6 +130,9 @@ def get_deployment( * **deployment_uuid**: uuid of deployment to gather info about * **full_info**: retrieve the full information of that deployment (may increase latency) + * **energy**: attach energy/footprint stats (`energy` key). Enabled by + default. With `full_info=true` (the default here) it is the extended + `EnergyTimeSeries`; with `full_info=false` the accumulated `EnergyStats`. Returns a dict with info """ @@ -135,6 +161,12 @@ def get_deployment( detail="This deployment is not a module.", ) + if energy: + if full_info: + job["energy"] = accounting.get_series(namespace, deployment_uuid) + else: + job["energy"] = accounting.get_accumulated(namespace, deployment_uuid) + return job @@ -195,12 +227,14 @@ async def create_deployment( # tools) modules_deps = get_deployments( vos=[vo], + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), ) tools_deps = ai4_deployments.tools.get_deployments( vos=[vo], + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), @@ -289,7 +323,7 @@ async def create_deployment( # Add affinity for greener nodes nomad_conf = green_director.add_green_affinities( nomad_conf=nomad_conf, - stats=ai4_stats.get_cluster_stats(vo), + stats=ai4_stats.get_cluster_stats(vo, energy=False), workload_type="cpu" if user_conf["hardware"]["gpu_num"] == 0 else "gpu", ) diff --git a/ai4papi/routers/v1/deployments/tools.py b/ai4papi/routers/v1/deployments/tools.py index 0bbcd762..e16bccb5 100644 --- a/ai4papi/routers/v1/deployments/tools.py +++ b/ai4papi/routers/v1/deployments/tools.py @@ -11,7 +11,7 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.security import HTTPBearer -from ai4papi import auth, quotas, schemas, utils +from ai4papi import accounting, auth, quotas, schemas, utils from ai4papi.wattnet import green_director import ai4papi.conf as papiconf import ai4papi.nomad_utils as nomad_utils @@ -34,6 +34,7 @@ def get_deployments( vos: schemas.VoList = None, full_info: bool = False, + energy: bool = True, authorization=Depends(security), ): """ @@ -45,6 +46,9 @@ def get_deployments( * **full_info**: retrieve the full information of each deployment. Disabled by default, as it will increase latency too much if there are many deployments. + * **energy**: attach energy/footprint stats to each deployment (`energy` + key). Enabled by default. `full_info=false` -> accumulated `EnergyStats`; + `full_info=true` -> extended `EnergyTimeSeries`. """ # Retrieve authenticated user info auth_info = auth.get_user_info(token=authorization.credentials) @@ -63,20 +67,24 @@ def get_deployments( user_jobs = [] for vo in user_vos: + namespace = papiconf.MAIN_CONF["nomad"]["namespaces"][vo] + # Retrieve all jobs in namespace jobs = nomad_utils.get_deployments( - namespace=papiconf.MAIN_CONF["nomad"]["namespaces"][vo], + namespace=namespace, owner=auth_info["id"], prefix="tool", ) # Retrieve info for jobs in namespace + vo_jobs = [] for j in jobs: try: job_info = get_deployment( vo=vo, deployment_uuid=j["ID"], full_info=full_info, + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), @@ -84,7 +92,20 @@ def get_deployments( except HTTPException: # not a tool continue - user_jobs.append(job_info) + vo_jobs.append(job_info) + + # `full_info=False` (default): one cheap batch. `full_info=True`: the + # extended time series per deployment (heavier). + if energy and vo_jobs: + if full_info: + for job in vo_jobs: + job["energy"] = accounting.get_series(namespace, job["job_ID"]) + else: + env = accounting.get_accumulated_bulk(namespace, auth_info["id"]) + for job in vo_jobs: + job["energy"] = env.get(job["job_ID"]) + + user_jobs.extend(vo_jobs) # Sort deployments by submission time in descending order sorted_jobs = sorted(user_jobs, key=lambda x: x["submit_time"], reverse=True) @@ -97,6 +118,7 @@ def get_deployment( vo: str, deployment_uuid: str, full_info: bool = True, + energy: bool = True, authorization=Depends(security), ): """ @@ -108,6 +130,9 @@ def get_deployment( * **deployment_uuid**: uuid of deployment to gather info about * **full_info**: retrieve the full information of that deployment (may increase latency) + * **energy**: attach energy/footprint stats (`energy` key). Enabled by + default. `full_info=true` (default here) -> extended `EnergyTimeSeries`; + `full_info=false` -> accumulated `EnergyStats`. Returns a dict with info """ @@ -165,6 +190,12 @@ def get_deployment( k for k in job["active_endpoints"] if k not in ignore ] + if energy: + if full_info: + job["energy"] = accounting.get_series(namespace, deployment_uuid) + else: + job["energy"] = accounting.get_accumulated(namespace, deployment_uuid) + return job @@ -246,9 +277,9 @@ def create_deployment( # Check if requested hardware is within the user total quota (summing modules and # tools) - tools_deps = get_deployments(vos=[vo], authorization=auth_arg) + tools_deps = get_deployments(vos=[vo], energy=False, authorization=auth_arg) modules_deps = ai4_deployments.modules.get_deployments( - vos=[vo], authorization=auth_arg + vos=[vo], energy=False, authorization=auth_arg ) quotas.check_userwise( conf=user_conf, @@ -755,7 +786,7 @@ def create_deployment( # Add affinity for greener nodes nomad_conf = green_director.add_green_affinities( nomad_conf=nomad_conf, - stats=ai4_stats.get_cluster_stats(vo), + stats=ai4_stats.get_cluster_stats(vo, energy=False), workload_type="cpu" if user_conf.get("hardware", {}).get("gpu_num", 0) == 0 else "gpu", diff --git a/ai4papi/routers/v1/stats/deployments.py b/ai4papi/routers/v1/stats/deployments.py index d14a31d8..78b4e43b 100644 --- a/ai4papi/routers/v1/stats/deployments.py +++ b/ai4papi/routers/v1/stats/deployments.py @@ -4,6 +4,7 @@ import copy import csv +import logging import time from datetime import datetime, timedelta import os @@ -13,7 +14,7 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.security import HTTPBearer -from ai4papi import auth, schemas +from ai4papi import accounting, auth, schemas from ai4papi.wattnet import green_director import ai4papi.conf as papiconf from ai4papi.nomad_utils import Nomad @@ -90,6 +91,8 @@ def load_stats( @router.get("/user") def get_user_stats( vo: str, + full_info: bool = False, + energy: bool = True, authorization=Depends(security), ): """ @@ -100,6 +103,11 @@ def get_user_stats( Parameters: * **vo**: Virtual Organization where you want the stats from. + * **energy**: attach the `energy` accounting block (the user's accumulated + consumption and footprint, each deployment's live Mimir top-up included). + Enabled by default. + * **full_info**: when `true`, the `energy` block also carries a merged 15-min + time series of the user's deployments. Off by default (accumulated only). """ # Retrieve authenticated user info @@ -109,18 +117,31 @@ def get_user_stats( # Retrieve the associated namespace to that VO namespace = papiconf.MAIN_CONF["nomad"]["namespaces"][vo] - # Load proper namespace stats - full_stats = load_stats(namespace=namespace) - - # Keep only stats from the current user - user_stats = copy.deepcopy(full_stats) + # Load proper namespace stats (CSV usage stats from the `ai4-accounting` repo). + # These are independent of the energy accounting below: if they are missing, + # still serve the energy block instead of failing the whole endpoint. try: - idx = full_stats["users-agg"]["owner"].index(auth_info["id"]) - user_stats["users-agg"] = { - k: v[idx] for k, v in full_stats["users-agg"].items() - } - except ValueError: # user has still no recorded stats - user_stats["users-agg"] = None + full_stats = load_stats(namespace=namespace) + user_stats = copy.deepcopy(full_stats) + try: + idx = full_stats["users-agg"]["owner"].index(auth_info["id"]) + user_stats["users-agg"] = { + k: v[idx] for k, v in full_stats["users-agg"].items() + } + except ValueError: # user has still no recorded stats + user_stats["users-agg"] = None + except HTTPException: + if not (energy and accounting.enabled()): + raise + user_stats = {} + + # Accumulated energy consumption + footprint of the user in that VO (each + # deployment's accumulator + its live top-up, summed). `full_info` adds the + # merged time series. + if energy and accounting.enabled(): + user_stats["energy"] = accounting.get_user_energy( + namespace, auth_info["id"], series=full_info + ) return user_stats @@ -160,6 +181,8 @@ def get_proper_allocation(allocs): @cached(cache=TTLCache(maxsize=1024, ttl=30)) def get_cluster_stats( vo: str | None = None, + full_info: bool = False, + energy: bool = True, ) -> schemas.ClusterStats: """ Returns the following stats of the nodes and the cluster (per resource type): @@ -170,6 +193,14 @@ def get_cluster_stats( ---------- vo: string Keep only the nodes supporting a specific VO. If not provided, returns all nodes. + energy: bool + Attach the per-datacenter `energy` block (site infrastructure power draw plus + the accumulated energy/footprint). Enabled by default. + full_info: bool + When `true`, each `datacenters[].energy` block also carries its 15-min + time series. Off by default (accumulated numbers only); the live top-up is + always applied either way. Energy is reported per datacenter, there is no + platform-wide total. """ global cluster_stats, cluster_stats_updated_at @@ -180,7 +211,7 @@ def get_cluster_stats( # So if None, we need to initialize it cluster_stats = get_cluster_stats_bg() - # If the background task fails for some reason (failed Nomad calls, failed WattNet + # If the background task fails for some reason (failed Nomad calls, failed Wattnet # calls, etc), the stats won't be updated and this endpoint will keep serving the # same (old) stats, which can be misleading because it gives the impression that # everything works normally. So we give a 1 hour grace time and then raise an Error. @@ -192,6 +223,12 @@ def get_cluster_stats( stats = copy.deepcopy(cluster_stats) + # Drop the energy blocks when not requested (the background thread computes + # them regardless; this just keeps them out of the response). + if not energy: + for dc_stats in stats.datacenters.values(): + dc_stats.energy = None + namespace = papiconf.MAIN_CONF["nomad"]["namespaces"][vo] if vo else "all" for k, v in list(stats.datacenters.items()): # we make an object copy with list() @@ -201,8 +238,9 @@ def get_cluster_stats( if namespace == "all" or namespace in n_stats.namespaces: nodes[n_id] = n_stats - # Ignore datacenters with no nodes - if not nodes: + # Ignore datacenters with no nodes, unless they carry an energy block + # (site infrastructure energy is reported regardless of the VO filter). + if not nodes and stats.datacenters[k].energy is None: del stats.datacenters[k] else: stats.datacenters[k].nodes = nodes @@ -232,6 +270,14 @@ def get_cluster_stats( stats.cluster.gpu_models[model_name].gpu_total += g_stats.gpu_total stats.cluster.gpu_models[model_name].gpu_used += g_stats.gpu_used + # Per-datacenter energy time series only with `full_info=true` (it is large: + # ~500 points per datacenter). The accumulated numbers and the live top-up + # are always present. + if not full_info: + for dc_stats in stats.datacenters.values(): + if dc_stats.energy is not None: + dc_stats.energy.series = None + # Add update time stats.updated_at = ( datetime.fromtimestamp(cluster_stats_updated_at).isoformat() + "Z" # ty: ignore[invalid-argument-type] @@ -419,6 +465,24 @@ def get_cluster_stats_bg() -> schemas.ClusterStats: for g_stats in n_stats.gpu_models.values(): g_stats.gpu_total = n_stats.gpu_used + # Per-datacenter energy (energy feature): instantaneous power draw + the + # accumulated energy/footprint since accounting started + a 15-min series. + # Reported per datacenter only, there is no platform-wide total (clients sum + # the datacenters they care about). Best-effort: a failure here must not + # break the cluster stats. + if accounting.enabled(): + try: + for dc_name, dc_energy in accounting.get_cluster_energy( + series=True + ).items(): + if dc_name not in stats.datacenters: + stats.datacenters[dc_name] = schemas.DatacenterStats( + lat=0.0, lon=0.0, PUE=0.0, nodes={} + ) + stats.datacenters[dc_name].energy = schemas.EnergyStats(**dc_energy) + except Exception: + logging.exception("Error computing cluster energy stats") + # Set the new shared variable global cluster_stats, cluster_stats_updated_at cluster_stats = stats diff --git a/ai4papi/routers/v1/try_me/nomad.py b/ai4papi/routers/v1/try_me/nomad.py index 448d887d..bca9adb4 100644 --- a/ai4papi/routers/v1/try_me/nomad.py +++ b/ai4papi/routers/v1/try_me/nomad.py @@ -10,7 +10,7 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.security import HTTPBearer -from ai4papi import auth +from ai4papi import accounting, auth import ai4papi.conf as papiconf from ai4papi.routers.v1.catalog.modules import Modules from ai4papi.routers.v1.stats.deployments import get_cluster_stats @@ -39,6 +39,7 @@ @router.get("") def get_deployments( full_info: bool = False, + energy: bool = True, authorization=Depends(security), ): """ @@ -50,6 +51,9 @@ def get_deployments( * **full_info**: retrieve the full information of each deployment. Disabled by default, as it will increase latency too much if there are many deployments. + * **energy**: attach energy/footprint stats (`energy` key). Enabled by + default. `full_info=false` -> accumulated `EnergyStats`; `full_info=true` + -> extended `EnergyTimeSeries`. """ # Retrieve authenticated user info auth_info = auth.get_user_info(token=authorization.credentials) @@ -67,6 +71,7 @@ def get_deployments( job_info = get_deployment( deployment_uuid=j["ID"], full_info=full_info, + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), @@ -76,6 +81,15 @@ def get_deployments( user_jobs.append(job_info) + if energy and user_jobs: + if full_info: + for job in user_jobs: + job["energy"] = accounting.get_series(NAMESPACE, job["job_ID"]) + else: + env = accounting.get_accumulated_bulk(NAMESPACE, auth_info["id"]) + for job in user_jobs: + job["energy"] = env.get(job["job_ID"]) + # Sort deployments by submission time in descending order sorted_jobs = sorted(user_jobs, key=lambda x: x["submit_time"], reverse=True) @@ -86,6 +100,7 @@ def get_deployments( def get_deployment( deployment_uuid: str, full_info: bool = True, + energy: bool = True, authorization=Depends(security), ): """ @@ -95,6 +110,9 @@ def get_deployment( Parameters: * **deployment_uuid**: uuid of deployment to gather info about + * **energy**: attach energy/footprint stats (`energy` key). Enabled by + default. `full_info=true` (default here) -> extended `EnergyTimeSeries`; + `full_info=false` -> accumulated `EnergyStats`. Returns a dict with info """ @@ -112,6 +130,12 @@ def get_deployment( # Rewrite main endpoint, otherwise it automatically selects DEEPaaS API job["main_endpoint"] = "ui" + if energy: + if full_info: + job["energy"] = accounting.get_series(NAMESPACE, deployment_uuid) + else: + job["energy"] = accounting.get_accumulated(NAMESPACE, deployment_uuid) + return job @@ -188,7 +212,7 @@ def create_deployment( # Check that the target node (ie. tag='tryme') resources are available because # these jobs cannot be left queueing # We check for every resource metric (cpu, disk, ram) - cluster_stats = get_cluster_stats(vo=VO) + cluster_stats = get_cluster_stats(vo=VO, energy=False) resources = ["cpu", "ram", "disk"] keys = [f"{i}_used" for i in resources] + [f"{i}_total" for i in resources] status = dict.fromkeys(keys, 0) diff --git a/ai4papi/schemas.py b/ai4papi/schemas.py index da6b5ce2..577b5c0b 100644 --- a/ai4papi/schemas.py +++ b/ai4papi/schemas.py @@ -38,6 +38,60 @@ class NodeInfo(ResourceStats): cpu_model: str +class EnergyTimeSeriesPoint(BaseModel): + # Per-bucket increments only. A running total is `cumsum(series[].energy_wh)` + # and matches `accumulated.energy_wh` (same for carbon and water). + ts: str + power_w: Optional[float] = None # mean over the bucket, normalized (x TUE) + energy_wh: Optional[float] = None # normalized, incremental for the bucket + carbon_g: Optional[float] = None + water_l: Optional[float] = None + live: bool = False # point from the live tail (not yet persisted) + + +class EnergyStats(BaseModel): + """ + Energy consumption and footprint (carbon and water, scope 2) of one entity, + always with the live Mimir top-up applied. The same shape at every level: + `energy` per deployment (list endpoints), `accumulated` in the per-deployment + time series, `energy` per datacenter in `GET /stats/cluster`, and `energy` + per user in `GET /stats/user`. + + Every value is normalized by the datacenter TUE; `tue_factor` is reported so a + client that wants the raw meter figure divides by it (the raw is not sent). + `coverage_ratio` / `datacenters` are `null` on the datacenter block; + `deployments` is set only on the per-user aggregate. + """ + + energy_wh: Optional[float] = None # accumulated, normalized (x TUE) + tue_factor: Optional[float] = None # normalized / raw (per-datacenter or blended) + carbon_g: Optional[float] = None # gCO2eq + water_l: Optional[float] = None # L + power_w: Optional[float] = ( + None # current draw (last Mimir sample, ~30-60s old); None if stopped + ) + since: Optional[str] = None # start of the accumulation window + as_of: Optional[str] = None # store pointer timestamp + live_as_of: Optional[str] = None # instant of the live top-up + complete: bool = True # False if it started late / clamped by Mimir retention + degraded: bool = False # Mimir/Wattnet unavailable for the top-up + coverage_ratio: Optional[float] = None # per deployment / user: fraction with data + datacenters: Optional[list[str]] = None # per deployment / user + deployments: Optional[int] = None # per-user aggregate only + series: Optional[list[EnergyTimeSeriesPoint]] = ( + None # 15-min buckets; only with full_info=true + ) + + +class EnergyTimeSeries(BaseModel): + deployment_uuid: str + start: str # first point (equivalently `accumulated.since`) + end: str # last point (the live edge while running) + source: str # "store" (archive only) | "mixed" (archive + live tail) + series: list[EnergyTimeSeriesPoint] + accumulated: EnergyStats + + class DatacenterStats(BaseModel): # CSV metadata lat: float @@ -48,7 +102,8 @@ class DatacenterStats(BaseModel): nodes: Dict[str, NodeInfo] # Optional fields (added dynamically) - footprints: Optional[Dict[str, Any]] = None # fromwattnet.GreenDirector + footprints: Optional[Dict[str, Any]] = None # from wattnet.GreenDirector + energy: Optional[EnergyStats] = None # current power + accumulated energy/footprint class ClusterStats(BaseModel): @@ -60,3 +115,6 @@ class ClusterStats(BaseModel): # Added in get_cluster_stats() updated_at: Optional[str] = None + + # Energy is reported per datacenter (`datacenters[].energy`); there is no + # platform-wide total, clients sum the datacenters they care about. From a32cecbe63abb884b350953e3119b26ee0a0aeed Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 10 Sep 2026 09:37:59 +0200 Subject: [PATCH 07/25] test(energy): pure-function, sweep-pipeline and read-API checks Runs without the monitoring stack: pure helpers always, the sweep and read API against a faked Mimir/Nomad, and an optional live Mimir probe behind PAPI_TESTS_MIMIR. Covers the GPU-only site case (no scaphandre series) at both the deployment and datacenter level. --- tests/test_energy.py | 510 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 510 insertions(+) create mode 100644 tests/test_energy.py diff --git a/tests/test_energy.py b/tests/test_energy.py new file mode 100644 index 00000000..1ad446dd --- /dev/null +++ b/tests/test_energy.py @@ -0,0 +1,510 @@ +""" +Tests for the energy-accounting feature. + +Pure-function checks run everywhere. The infra checks (Mimir / a real +deployment) are skipped unless MIMIR_USER/MIMIR_PASSWORD and PAPI_TESTS_ENERGY_VO +are set, so the suite stays runnable without the monitoring stack. +""" + +import datetime + +from ai4papi import mimir +from ai4papi.accounting import compute + + +def _series(values, scale_str=str): + """Build a Mimir matrix result with one series from [(ts, val)].""" + return [{"metric": {}, "values": [[t, scale_str(v)] for t, v in values]}] + + +# --- compute.integrate -------------------------------------------------------- + +# 1 W constant for 1h at 30s step -> 1 Wh at TUE 1.0, 2.03 Wh at TUE 2.03 +step = 30 +n = 3600 // step + 1 +cpu = _series([(1_000_000 + i * step, 1_000_000) for i in range(n)]) # 1e6 uW = 1 W +buckets = compute.integrate(cpu, [], step_s=step, bucket_s=3600, tue_factor=1.0) +total = sum(b.energy_wh for b in buckets) +assert abs(total - 1.0) < 1e-3, total + +buckets_tue = compute.integrate(cpu, [], step_s=step, bucket_s=3600, tue_factor=2.03) +assert abs(sum(b.energy_wh for b in buckets_tue) - 2.03) < 1e-2 + +# gap longer than gap_factor*step is not integrated across +gapped = _series([(0, 1_000_000), (30, 1_000_000), (600, 1_000_000), (630, 1_000_000)]) +gb = compute.integrate(gapped, [], step_s=step, bucket_s=3600, tue_factor=1.0) +# only the two 30s intervals contribute: 2 * (1W * 30s) = 60 Ws = 60/3600 Wh +assert abs(sum(b.energy_wh for b in gb) - 60 / 3600) < 1e-4 + +# a single sample -> no energy +assert compute.integrate(_series([(0, 1_000_000)]), [], 30, 3600) == [] + +# --- compute.sanitize (data quality) --------------------------------------- + +dq = {"enabled": True, "min_power_w": 0, "max_power_w": 1000, "spike_factor": 5} +pw = [(0, 100.0), (30, -5.0), (60, 100.0), (90, 99999.0), (120, 100.0)] +clean = compute.sanitize(pw, dq) +assert (30, -5.0) not in clean # negative dropped +assert (90, 99999.0) not in clean # over ceiling dropped +assert len(clean) == 3 + +# spike detection against the rolling median +spiky = [(i * 30, 100.0) for i in range(20)] +spiky[10] = (300, 5000.0) +clean = compute.sanitize(spiky, {"spike_factor": 10, "spike_window": 7}) +assert (300, 5000.0) not in clean and len(clean) == 19 + +# disabled -> passthrough +assert compute.sanitize(pw, {"enabled": False}) == pw +assert compute.sanitize(pw, None) == pw + +# integrate drops the anomaly so it never inflates the accumulated energy +anom = _series( + [(t, 1_000_000) for t in range(0, 300, 30)] + + [(300, 900_000_000)] + + [(t, 1_000_000) for t in range(330, 600, 30)] +) +raw = sum( + b.energy_wh + for b in compute.integrate(anom, [], step_s=30, bucket_s=3600, tue_factor=1.0) +) +clean = sum( + b.energy_wh + for b in compute.integrate( + anom, + [], + step_s=30, + bucket_s=3600, + tue_factor=1.0, + data_quality={"max_power_w": 100}, + ) +) +assert raw > 5 # the 900 W spike dominates the ~1 W baseline +assert clean < 0.3 # spike dropped -> only the ~1 W baseline remains + +# CPU + GPU add up +cpu2 = _series([(0, 2_000_000), (30, 2_000_000)]) # 2 W +gpu2 = _series([(0, 8), (30, 8)]) # 8 W +mb = compute.integrate(cpu2, gpu2, step_s=30, bucket_s=3600, tue_factor=1.0) +assert abs(sum(b.energy_wh for b in mb) - 10 * 30 / 3600) < 1e-4 + +# per-source data quality: a spike in one series does not discard the others +two_gpu = [ + {"metric": {"UUID": "g0"}, "values": [[0, "10"], [30, "10"], [60, "10"]]}, + {"metric": {"UUID": "g1"}, "values": [[0, "10"], [30, "9000"], [60, "10"]]}, +] +mb = compute.integrate( + [], + two_gpu, + step_s=30, + bucket_s=3600, + tue_factor=1.0, + data_quality={"max_power_w": 100}, +) +got = sum(b.energy_wh for b in mb) +# g0: 3 samples of 10 W over 60 s = 10*60/3600. g1: middle sample dropped, gap +# 0->60 s at 10 W = 10*60/3600. Total ~ 2 * 10 * 60/3600. +assert abs(got - 2 * 10 * 60 / 3600) < 1e-4, got + +# --- compute.apply_footprint ------------------------------------------------- + +b = compute.Bucket("2026-01-01T00:00:00Z", 2000.0, 40.0, 0.0, 0.0) # 2 kWh +out = compute.apply_footprint([b]) # no series -> DEFAULTS +assert abs(out[0].carbon_g - 2.0 * compute.DEFAULT_CARBON) < 1e-6 +assert abs(out[0].water_l - 2.0 * compute.DEFAULT_WATER) < 1e-6 + +series = [["2026-01-01T00:00:00Z", 100.0], ["2026-01-01T00:15:00Z", 200.0]] +out = compute.apply_footprint([b], carbon_series=series, water_series=series) +assert abs(out[0].carbon_g - 2.0 * 100.0) < 1e-6 + +# bucket before the series -> DEFAULTS +early = compute.Bucket("2025-12-31T00:00:00Z", 1000.0, 20.0, 0.0, 0.0) +out = compute.apply_footprint([early], carbon_series=series) +assert abs(out[0].carbon_g - 1.0 * compute.DEFAULT_CARBON) < 1e-6 + +# --- compute.intensity_at --------------------------------------------------- + +ts_arr = [10.0, 20.0, 30.0] +val_arr = [1.0, 2.0, 3.0] +assert compute.intensity_at(ts_arr, val_arr, 5.0, -1) == -1 # before first +assert compute.intensity_at(ts_arr, val_arr, 20.0, -1) == 2.0 # exact +assert compute.intensity_at(ts_arr, val_arr, 25.0, -1) == 2.0 # between +assert compute.intensity_at(ts_arr, val_arr, 999.0, -1) == 3.0 # after last +assert compute.intensity_at([], [], 1.0, 7) == 7 # empty + +# --- compute.downsample --------------------------------------------------- + +many = [compute.Bucket(compute._iso(i * 300), 1.0, 10.0, 0.1, 0.01) for i in range(100)] +ds = compute.downsample(many, 10) +assert len(ds) == 10 +assert abs(sum(b.energy_wh for b in ds) - sum(b.energy_wh for b in many)) < 1e-6 +assert compute.downsample(many, 500) is many # nothing to do + +# --- compute.merge_buckets ------------------------------------------------ + +b1 = compute.Bucket("t0", 2.0, 20.0, 0.5, 0.02) +b2 = compute.Bucket("t0", 3.0, 30.0, 0.7, 0.03) +merged = compute.merge_buckets([[b1], [b2]]) +assert len(merged) == 1 +assert merged[0].energy_wh == 5.0 and merged[0].power_w_avg == 50.0 + +# --- compute.running_seconds -------------------------------------------- + +W = compute.AllocWindow +assert compute.running_seconds([W("a", 0, 100)], 0, 200) == 100 +assert compute.running_seconds([W("a", 0, 100), W("b", 50, 150)], 0, 200) == 150 +assert compute.running_seconds([W("a", 100, None)], 0, 200) == 100 # still running + + +print("🟢 energy: pure-function checks passed!") + + +# --- live Mimir check (optional, before any monkeypatching) ------------------ + +import os # noqa: E402 + +if os.environ.get("PAPI_TESTS_MIMIR") and mimir.MIMIR_USER: + _e = datetime.datetime.now(datetime.timezone.utc) + _s = _e - datetime.timedelta(hours=1) + assert isinstance(mimir.query_range("up", _s, _e, step_s=60), list) + print("🟢 energy: live Mimir query_range reachable!") +else: + print("🟡 energy: skipping live Mimir check (set PAPI_TESTS_MIMIR + MIMIR_* creds)") + + +# --- sweep pipeline (fake Mimir + Nomad, no infra) --------------------------- + +import json # noqa: E402 +import tempfile # noqa: E402 + +os.environ["ACCOUNTING_PTH"] = tempfile.mkdtemp() +# make the feature "enabled" for the read API even without real Mimir creds +mimir.MIMIR_USER = mimir.MIMIR_USER or "test" +mimir.MIMIR_PASSWORD = mimir.MIMIR_PASSWORD or "test" + +from ai4papi.accounting import store, sweep # noqa: E402 +import ai4papi.accounting as accounting # noqa: E402 + +_NOW = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0) +_NS, _UUID, _DC = "ai4eosc", "sweep-test-uuid", "ifca-ai4eosc" + + +def _fake_query_range(promql, start, end, step_s=30): + # 100 W constant (100e6 uW), one sample per step + t, out = start.timestamp(), [] + while t <= end.timestamp(): + out.append([t, "100000000" if "scaph" in promql else "0"]) + t += step_s + return [{"metric": {}, "values": out}] + + +mimir.query_range = _fake_query_range +mimir.query_range_chunked = _fake_query_range +compute.resolve_alloc_ids = lambda job_id, ns: ( + {_DC: ["alloc-1"]}, + [compute.AllocWindow("alloc-1", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-1", "datacenter": _DC}], + False, +) +sweep.green_director.metrics = {_DC: {"carbon": [], "water": []}} + +_info = { + "uuid": _UUID, + "job_name": f"module-{_UUID}", + "namespace": _NS, + "owner": "user@egi.eu", + "submit_time": _NOW.timestamp() - 3 * 3600, + "status": "running", +} + +sweep.process_single(_info, _NOW) +doc = store.read_accum(_NS, _UUID) +acc = doc["accumulated"] +assert doc["metrics_available"] is True +assert doc["tue_factor"] == 2.03 +# ~3h of 100 W -> ~300 Wh raw, x 2.03 TUE (11 or 12 complete 15-min buckets +# depending on where "now" falls in the current bucket) +assert 270 * 2.03 < acc["energy_wh"] < 305 * 2.03, acc +assert abs(acc["carbon_g"] - acc["energy_wh"] / 1000 * compute.DEFAULT_CARBON) < 1.0 + +# idempotent: same "now" -> byte-identical document +before = json.dumps(doc, sort_keys=True) +accounting._read_doc.cache_clear() +sweep.process_single(_info, _NOW) +assert json.dumps(store.read_accum(_NS, _UUID), sort_keys=True) == before + +# monotonic on a later sweep +accounting._read_doc.cache_clear() +sweep.process_single(_info, _NOW + datetime.timedelta(hours=1)) +assert store.read_accum(_NS, _UUID)["accumulated"]["energy_wh"] >= acc["energy_wh"] + +# buckets age out of the 24h tail into the persisted series +accounting._read_doc.cache_clear() +sweep.process_single(_info, _NOW + datetime.timedelta(hours=30)) +assert len(store.read_series(_NS, _UUID, None, None)) > 0 + +# read API + pydantic validation +from ai4papi import schemas # noqa: E402 + +accounting._read_doc.cache_clear() +stats = accounting.get_accumulated(_NS, _UUID, live=False) +schemas.EnergyStats(**stats) +series = accounting.get_series(_NS, _UUID) +schemas.EnergyTimeSeries(**series) +assert series["series"] and all(p["energy_wh"] is not None for p in series["series"]) +# per-bucket increments only; running total is cumsum(series[].energy_wh) +_pt = set(series["series"][-1]) +assert _pt == { + "ts", + "power_w", + "energy_wh", + "carbon_g", + "water_l", + "live", +}, _pt +_ps = accounting.get_accumulated(_NS, _UUID, live=True) +assert _ps["power_w"] and _ps["tue_factor"] == 2.03 + +# per-user aggregate: sums the owner's deployments, EnergyStats-shaped +accounting._read_doc.cache_clear() +_ue = accounting.get_user_energy(_NS, "user@egi.eu", series=False) +schemas.EnergyStats(**_ue) +assert _ue["deployments"] >= 1 and _ue["series"] is None +assert _ue["energy_wh"] >= acc["energy_wh"] and _ue["since"] +_ues = accounting.get_user_energy(_NS, "user@egi.eu", series=True) +schemas.EnergyStats(**_ues) +assert _ues["series"] and all(p["energy_wh"] is not None for p in _ues["series"]) +# the aggregate total must reconcile with the sum of the series (no rounding) +assert abs(_ues["energy_wh"] - sum(p["energy_wh"] for p in _ues["series"])) < 1e-9, _ues +assert abs(_ues["carbon_g"] - sum(p["carbon_g"] for p in _ues["series"])) < 1e-9 +# merge-then-downsample: uniform step across the whole persisted series +_uts = [ + datetime.datetime.fromisoformat(p["ts"].replace("Z", "+00:00")) + for p in _ues["series"] + if not p["live"] +] +_steps = {round((_uts[i + 1] - _uts[i]).total_seconds()) for i in range(len(_uts) - 1)} +assert len(_steps) == 1, _steps +# hard cap on the persisted point count, same as the other levels +_capped = accounting.get_user_energy(_NS, "user@egi.eu", series=True, target_points=5) +assert len([p for p in _capped["series"] if not p["live"]]) <= 5 +assert accounting.get_user_energy(_NS, "nobody@egi.eu") is None + +# live tail: a running deployment's graph keeps growing past the stored series +_live_doc = { + "namespace": _NS, + "deployment_uuid": _UUID, + "metrics_available": True, + "pointer_ts": compute._iso( + datetime.datetime.now(datetime.timezone.utc).timestamp() - 600 + ), + "active_allocs": [{"alloc_id": "alloc-1", "datacenter": _DC}], +} +_lp = sweep.live_series(_live_doc, bucket_s=60) +assert _lp and all(p["live"] for p in _lp), _lp +assert all("energy_wh" in p and "carbon_g" in p for p in _lp) + +# the live top-up also carries the current power (100 W raw, x TUE) +_tu = sweep.topup(_live_doc) +_tue = sweep._tue(_DC) +assert _tu["power_w"] is not None +assert abs(_tu["power_w"] - 100.0 * _tue) < 5.0 * _tue, _tu + +# a running deployment's accumulated stats expose a power figure +accounting._read_doc.cache_clear() +_st = accounting.get_accumulated(_NS, _UUID, live=True) +assert _st["power_w"] is not None + +# the accumulated stats always carry a float coverage_ratio (1.0 = full) +assert isinstance(_st["coverage_ratio"], float) + +# GPU/CPU energy is clipped to the job's life: a finished allocation whose power +# metrics linger for a few minutes must not keep accumulating energy. +_ENDED_UUID = "sweep-ended-uuid" +_ended_end = _NOW.timestamp() - 2 * 3600 # allocation stopped 2h ago +compute.resolve_alloc_ids = lambda job_id, ns: ( + {_DC: ["alloc-e"]}, + [compute.AllocWindow("alloc-e", _NOW.timestamp() - 5 * 3600, _ended_end)], + [], # nothing running + False, +) +_ended_info = { + "uuid": _ENDED_UUID, + "job_name": f"module-{_ENDED_UUID}", + "namespace": _NS, + "owner": "user@egi.eu", + "submit_time": _NOW.timestamp() - 5 * 3600, + "status": "running", +} +accounting._read_doc.cache_clear() +sweep.process_single(_ended_info, _NOW) +_edoc = store.read_accum(_NS, _ENDED_UUID) +_last_ts = max( + (b["ts"] for b in _edoc.get("tail", [])), + default=_edoc.get("settled_ts", ""), +) +# fake Mimir returns 100 W for the whole [settled, now] window, but no bucket +# should land after the allocation ended (+ one bucket of slack). +assert compute._parse_iso(_last_ts) <= _ended_end + 300, _last_ts + +# a GPU-only monitored site (no scaphandre series at all) still accounts energy +# from the GPU power alone: a missing `scaph_*` metric comes back as an empty +# result, not an error, so the integrator just sums the GPU series. +_GPU_UUID = "sweep-gpu-only-uuid" + + +def _fake_qr_gpu_only(promql, start, end, step_s=30): + if "scaph" in promql: + return [] + t, out = start.timestamp(), [] + while t <= end.timestamp(): + out.append([t, "150"]) # 150 W per GPU + t += step_s + return [{"metric": {}, "values": out}] + + +mimir.query_range = _fake_qr_gpu_only +mimir.query_range_chunked = _fake_qr_gpu_only +compute.resolve_alloc_ids = lambda job_id, ns: ( + {_DC: ["alloc-g"]}, + [compute.AllocWindow("alloc-g", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-g", "datacenter": _DC}], + False, +) +_gpu_info = { + "uuid": _GPU_UUID, + "job_name": f"module-{_GPU_UUID}", + "namespace": _NS, + "owner": "gpu@egi.eu", + "submit_time": _NOW.timestamp() - 3 * 3600, + "status": "running", +} +accounting._read_doc.cache_clear() +sweep.process_single(_gpu_info, _NOW) +_gdoc = store.read_accum(_NS, _GPU_UUID) +assert _gdoc["metrics_available"] is True, _gdoc +assert _gdoc["accumulated"]["energy_wh"] > 0, _gdoc + +print("🟢 energy: sweep pipeline checks passed!") + + +# --- cluster energy: instantaneous power + accumulator + series -------------- + +_CNS = sweep.CLUSTER_NS +_ISO0 = "2020-01-01T00:00:00Z" +sweep.green_director.metrics = { + _DC: {"carbon": [[_ISO0, 300.0]], "water": [[_ISO0, 10.0]]} +} +_tue_dc = sweep._tue(_DC) + + +def _fake_query(promql, at=None, timeout=30): + if "scaph_host_power" in promql: + # 500 W reported in microwatts + return [ + {"metric": {"datacenter": _DC}, "value": [_NOW.timestamp(), "500000000"]} + ] + if "DCGM" in promql: + return [ + {"metric": {"datacenter": _DC}, "value": [_NOW.timestamp(), "200"]} + ] # W + return [] + + +mimir.query = _fake_query + +# 1. instantaneous power: power_w = (host + gpu) * TUE, normalized only +accounting.get_datacenter_power.cache_clear() +_cp = accounting.get_datacenter_power() +assert _DC in _cp, _cp +assert abs(_cp[_DC]["power_w"] - (500.0 + 200.0) * _tue_dc) < 1.0, _cp +assert set(_cp[_DC]) == {"power_w", "tue_factor", "as_of"}, _cp[_DC] + + +# 2. per-datacenter sweep: several raw host series -> sanitize each -> sum +def _fake_cluster_qr(promql, start, end, step_s=30): + def _rows(pairs): + out = [] + for label, val in pairs: + vals, t = [], start.timestamp() + while t <= end.timestamp(): + vals.append([t, val]) + t += step_s + out.append({"metric": {"node": label, "UUID": label}, "values": vals}) + return out + + if "scaph_host_power" in promql: # two hosts: 400 W + 600 W (in microwatts) + return _rows([("nodeA", "400000000"), ("nodeB", "600000000")]) + if "DCGM" in promql: # one GPU: 250 W + return _rows([("gpu0", "250")]) + return [] + + +mimir.query_range = _fake_cluster_qr +mimir.query_range_chunked = _fake_cluster_qr +sweep._live_cache.clear() + +sweep.process_datacenter(_DC, _NOW) +cdoc = store.read_accum(_CNS, _DC) +assert cdoc["metrics_available"] is True +cacc = cdoc["accumulated"] +assert cdoc["tue_factor"] == _tue_dc +# window ~= initial_lookback_hours of (400 + 600 + 250) = 1250 W raw, x TUE +_expected = sweep._cfg()["initial_lookback_hours"] * 1250.0 * _tue_dc +assert 0.75 * _expected < cacc["energy_wh"] <= 1.05 * _expected, cacc +assert abs(cacc["carbon_g"] - cacc["energy_wh"] / 1000 * 300.0) < 1.0 + +# monotonic on a later sweep +sweep.process_datacenter(_DC, _NOW + datetime.timedelta(hours=1)) +assert store.read_accum(_CNS, _DC)["accumulated"]["energy_wh"] >= cacc["energy_wh"] + +# buckets age out of the 24h tail into the persisted series +sweep.process_datacenter(_DC, _NOW + datetime.timedelta(hours=30)) +assert len(store.read_series(_CNS, _DC, None, None)) > 0 + +# 3. read API: get_cluster_energy merges power + accumulator + series +accounting.get_datacenter_power.cache_clear() +accounting._cluster_archive_series.cache_clear() +sweep._live_cache.clear() +_ce = accounting.get_cluster_energy(series=True) +assert _DC in _ce, _ce +_row = _ce[_DC] +assert _row["power_w"] is not None and _row["energy_wh"] is not None +assert _row["since"] and _row["as_of"] +assert _row["series"] and all(p["energy_wh"] is not None for p in _row["series"]) +for _p in _row["series"]: + schemas.EnergyTimeSeriesPoint(**_p) +schemas.EnergyStats(**_row) +assert _row["tue_factor"] == _tue_dc # only conversion factor exposed +# the accumulated total reconciles with the sum of the series (no rounding) +assert abs(_row["energy_wh"] - sum(p["energy_wh"] for p in _row["series"])) < 1e-9, _row + +# 4. datacenter live top-up carries the (TUE-normalized) energy deltas +sweep._live_cache.clear() +_ct = sweep.topup_cluster(_DC, cdoc) +assert _ct and _ct["energy_wh"] > 0 and _ct["live_as_of"] +assert abs(_ct["carbon_g"] - _ct["energy_wh"] / 1000 * 300.0) < 1.0, _ct + + +# 5. a GPU-only monitored datacenter (in metrics_datacenters, no host power) is +# still reported: for a declared site, GPU power alone is a valid total. +def _fake_query_gpu_only(promql, at=None, timeout=30): + if "DCGM" in promql: + return [{"metric": {"datacenter": _DC}, "value": [_NOW.timestamp(), "200"]}] + return [] + + +mimir.query = _fake_query_gpu_only +accounting.get_datacenter_power.cache_clear() +_gp = accounting.get_datacenter_power() +assert _DC in _gp, _gp +assert abs(_gp[_DC]["power_w"] - 200.0 * _tue_dc) < 1.0, _gp + +# 6. a datacenter with no power series at all is absent from get_datacenter_power +mimir.query = lambda promql, at=None, timeout=30: [] +mimir.query_range = lambda *a, **k: [] +mimir.query_range_chunked = lambda *a, **k: [] +accounting.get_datacenter_power.cache_clear() +assert accounting.get_datacenter_power() == {} + +print("🟢 energy: cluster energy checks passed!") From 520fb9ce629860a21da60123c71ef0f692836588 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 10 Sep 2026 09:38:03 +0200 Subject: [PATCH 08/25] docs(energy): architecture and dashboard-integration guides - docs/energy-accounting.md: how the sweep, store, footprint and read paths work, plus the config reference and failure modes - docs/energy-api.md: the `energy` block shape for dashboard integration - README: short pointer to both, under the stats section --- README.md | 2 + docs/energy-accounting.md | 322 ++++++++++++++++++++++++++++++++++++++ docs/energy-api.md | 265 +++++++++++++++++++++++++++++++ 3 files changed, 589 insertions(+) create mode 100644 docs/energy-accounting.md create mode 100644 docs/energy-api.md diff --git a/README.md b/README.md index fca63093..a084f706 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,8 @@ More details can be found in the [API docs](https://api.cloud.ai4eosc.eu/docs). It will serve the contents of the `ai4-accounting/summaries` folder. + Energy consumption and footprint (carbon and water) is attached additively to the existing responses: an `energy` key per deployment, per user in `stats/user` and per datacenter in `stats/cluster`. It needs the `MIMIR_USER` / `MIMIR_PASSWORD` variables and the `energy` section of `etc/main.yaml`. Dashboard integration reference: [docs/energy-api.md](./docs/energy-api.md); architecture and internals: [docs/energy-accounting.md](./docs/energy-accounting.md). +
The API methods can also be accessed by interacting directly with diff --git a/docs/energy-accounting.md b/docs/energy-accounting.md new file mode 100644 index 00000000..183bab24 --- /dev/null +++ b/docs/energy-accounting.md @@ -0,0 +1,322 @@ +# Per-deployment energy consumption and footprint + +This feature exposes, for every PAPI deployment, its accumulated energy consumption and footprint (carbon and water, scope 2), plus time series for the dashboard graphs. + +- Accumulated (indefinite retention): energy (Wh), carbon (gCO2eq), water (L). +- Time series of the same three variables, for the deployment detail view. + +It is fully additive: nothing in the existing API changes behaviour and no new endpoints are added. Deployments get a new `energy` key, `/v1/deployments/stats/user` gains an `energy` block (the user's deployments summed), `/v1/deployments/stats/cluster` gains an `energy` block per datacenter (no platform-wide total, clients sum), and there is one extra background thread. If the feature is disabled or its inputs are missing, `energy` is `null` (or absent) and the rest of the API is identical. + +## Inputs + +| Input | Native resolution | Availability | Notes | +|---|---|---|---| +| CPU power: `scaph_process_power_consumption_microwatts` (uW) | 30 s | Mimir retention (tunable, will drop to ~3 months) | label `container_label_com_hashicorp_nomad_alloc_id`; aggregates every container of the alloc (task `main` + sidecars). The Nomad nodes are VMs, so scaphandre's per-process figures are small: real CPU energy only shows up under real load. | +| GPU power: `DCGM_FI_DEV_POWER_USAGE` (W) | 30 s | Mimir retention | board-level power, labels `UUID`, `instance` | +| GPU to alloc mapping: `nomad_gpu_allocation_info` | 30 s | only while a GPU deployment runs | info gauge, label `alloc_id`; kept as the mapping mechanism on purpose (other accounting software relies on it to avoid depending on the Nomad API) | +| Carbon and water intensity (Wattnet) | 15 min | full history on demand; `green_director` keeps a rolling 7-day window | already integrated in `ai4papi/wattnet.py`. Recent intervals are provisional and get revised every ~15 min until settled. `DEFAULTS` (301 gCO2/kWh, 12 L/kWh) outside coverage. | +| Allocations of a deployment | event | while the Nomad job is not garbage-collected (hours) | `Nomad.job.get_allocations`; several allocations over a job's life due to reallocations | + +Power metrics currently exist only in the `ifca-*` datacenters, and not on every node. Availability is effectively per allocation: it is decided by whether Mimir has data for that `alloc_id`, not by the datacenter alone. The CPU (scaphandre) and GPU (DCGM) sources are independent: whichever one has data contributes, so a site that only monitors its GPUs still accounts the GPU energy (a missing metric comes back as an empty Mimir result, not an error). The feature is best-effort: if there is data it is reported; if there is none, `energy` is `null` (never zeros); if there is data for only part of the deployment's life, that part is accounted and `coverage_ratio` drops below `1.0`. + +Mimir endpoint: `https://mimir.k8s.cloud.ai4eosc.eu/prometheus/api/v1/query_range`, HTTP Basic auth, no `X-Scope-OrgID` header. + +## Normalization + +`energy_wh = integrated_Wh * TUE[datacenter]`, where `integrated_Wh` is the raw trapezoidal integral of `CPU + GPU` power. + +The TUE (Total Usage Effectiveness, grid to compute ratio) is per datacenter and already covers CPU plus GPU, so it is applied once, before the footprint (which is scope 2, so it must reflect the whole facility draw). It is a single empirical factor per datacenter (default 2.03), consistent with the platform's existing energy accounting. Every energy, power, carbon and water value in an API response is already TUE-normalized; the applied factor is reported as `tue_factor` (per datacenter, or blended `energy_wh / raw` on multi-datacenter and per-user aggregates) so a client that wants the raw meter figure divides by it. The accumulator stores `tue_factor` per document. + +The TUE is read from the `TUE` column of `var/datacenters.csv`, only populated for the datacenters that have a value (currently `ifca-ai4eosc` and `ifca-imagine`). Blank cells load as `None` and `energy.default_tue_factor` is used as fallback. + +## Architecture + +Two layers: a durable background sweep that owns the persisted record, and a live read-time top-up that makes the dashboard counter move. + +``` + background thread in ai4papi.main: @repeat_every(energy.sweep_seconds) + accounting.sweep.run_sweep() + per deployment: Nomad (jobs + allocations + node -> datacenter) + Mimir query_range (per-alloc power, un-consolidated window) + per datacenter: Mimir query_range (per-host + per-GPU power) + green_director.metrics (Wattnet 15-min intensity) + integrate 30 s samples -> 15-min buckets, x TUE, x intensity + | + v atomic write, one JSON per deployment / per datacenter + $ACCOUNTING_PTH/energy//.accum.json meta + accumulator + 24h tail + $ACCOUNTING_PTH/energy//.series.jsonl consolidated 15-min buckets, append-only + $ACCOUNTING_PTH/energy/_cluster/.accum.json per-datacenter accumulator + 24h tail + $ACCOUNTING_PTH/energy/_cluster/.series.jsonl + | + v cached read + PAPI API (stateless) + energy block = stored accumulator + live top-up (Mimir, short window, cache 25 s) + GET /v1/deployments/{modules,tools,batch,try_me/nomad} -> + energy (EnergyStats, list view) + GET /v1/deployments/{modules,tools,batch,try_me/nomad}/{uuid} -> + energy (EnergyTimeSeries, detail view) + GET /v1/deployments/stats/user -> + energy (per-user total) + GET /v1/deployments/stats/cluster -> + energy (per-datacenter: power + accumulated + series) +``` + +The shape of the deployment `energy` block follows the existing `full_info` query param: `full_info=false` (the list default) attaches the accumulated `EnergyStats`; `full_info=true` (the individual-endpoint default) attaches the extended `EnergyTimeSeries`. + +### Layer 1: durable sweep + +`accounting.sweep.run_sweep()` runs on a `@repeat_every(energy.sweep_seconds)` thread started in `ai4papi.main`'s `lifespan`, next to `get_cluster_stats_thread`. Same posture: not async, one `try/except` per deployment, no-op if the feature is disabled. + +Per namespace it lists the active Nomad jobs (`Status != "dead"`, names matching `^(module|tool|batch|try)`) and also every existing `.accum.json` whose job has vanished, to close it out. Per deployment it calls `process_single(info, now)`: + +1. Load `.accum.json`. The integration pointer `settled_ts` comes from the doc; on first encounter it is clamped to `now - initial_lookback_hours` (pre-existing deployments are not fully backfilled) and to `now - mimir_retention_days` (older than that, `complete` is `false`). `window_start` (exposed as `since`) is pinned to that first-encounter value and never moves afterwards unless it falls behind `now - mimir_retention_days`; unlike `settled_ts` it is not a moving pointer, so it always marks the true start of the span the totals cover. A doc written before this field existed has it backfilled once, from the oldest bucket on record (`.series.jsonl`, else the oldest still in `tail`). +2. Resolve allocations via `compute.resolve_alloc_ids` into `{datacenter: [alloc_id]}` (whole lifetime), the currently-running allocs (for the top-up), the alloc time windows (to estimate `run_seconds`, which spans the whole life including time on datacenters without metrics), and `partial_dc` (an alloc ran in a datacenter without metrics). +3. Per datacenter, query Mimir for `[settled_ts, now - query_lag_seconds]` (chunked only if the span is huge), `sanitize` the samples, `integrate` trapezoidally into 15-min buckets applying that datacenter's TUE, then `apply_footprint` with the Wattnet intensity of that datacenter. Merge the per-datacenter buckets by timestamp. +4. Best-effort coverage: if the deployment was clearly running (`run_seconds` significant) but Mimir returned nothing, mark `metrics_available: false` and retry next sweep. Otherwise `metrics_available: true` and `coverage_ratio = covered_time / run_seconds` (`1.0` = full; lower means part of the life ran unmonitored). +5. Merge the new buckets over the existing 24h tail. Buckets older than `now - footprint_revision_hours` are consolidated: their totals are added to the running `settled` accumulator, `settled_ts` advances past them, and they are appended to `.series.jsonl`. The rest stay in `tail` and are recomputed in full every sweep, so the footprint self-heals as Wattnet settles. `settled_ts` therefore trails `now` by `footprint_revision_hours` in steady state and must not be read as the window start; that is `window_start` / `since`. +6. `accumulated = settled + sum(tail)`. `pointer_ts` is the end of the last tail bucket. Write the doc atomically. + +The sweep is idempotent: re-running with the same `now` produces the same document (buckets keyed by timestamp, `settled` only advances over buckets that leave the 24h window). + +The consolidation step (fold new buckets over the tail, age the old ones into `settled` + `.series.jsonl`, recompute `accumulated` / `pointer_ts`) is `sweep._fold_and_accumulate`, shared with the datacenter sweep below. + +### Layer 1b: per-datacenter sweep + +After the deployment loop, `run_sweep()` calls `_sweep_cluster(now)`, which runs `process_datacenter(dc, now)` for every `energy.metrics_datacenters`. Documents live under the pseudo-namespace `_cluster` (`$ACCOUNTING_PTH/energy/_cluster/.accum.json` + `.series.jsonl`), reusing `accounting.store` unchanged. + +`process_datacenter` is `process_single` without the Nomad parts: no allocation resolution, no `coverage_ratio`, no owner/close handling. + +1. Load the doc; `settled_ts` from it, else clamped to `initial_lookback_hours` / `mimir_retention_days` (retention clamp sets `complete: false`). `window_start` / `since` is pinned on the first sweep, same as the per-deployment path. +2. Query Mimir over `[settled_ts, now - query_lag_seconds]` with `_cluster_promql(dc)`: `sum by (node) (scaph_host_power_microwatts{datacenter=""})` (one series per host) and `sum by (UUID) (DCGM_FI_DEV_POWER_USAGE{datacenter=""})` (one series per GPU). Grouped, not summed, so `compute.integrate` runs the data-quality filter on each raw host / GPU series before summing, exactly as the per-deployment path does per allocation. +3. Data quality is `_cluster_dq()`: `energy.data_quality` with `max_power_w` forced to `0` (the per-alloc / per-GPU ceiling of ~1000 W is meaningless for a whole host). Negative-drop and the scale-free rolling-median spike check still apply. +4. `integrate` with `TUE[dc]`, `apply_footprint` with the datacenter's Wattnet intensity, then `_fold_and_accumulate`. If the datacenter has no host-power series at all: `metrics_available: false`, retry next sweep, no zeros. + +Note on double counting: scaphandre host power is RAPL (CPU package + DRAM), it does not include the GPU board, so `host + GPU` is correct. + +### Layer 2: read-time top-up + +The stored accumulator only reaches `pointer_ts` (up to one sweep interval behind). On each API read, `accounting.sweep.topup(doc)` queries Mimir live for `[pointer_ts, now - topup_lag_seconds]` (a single `query_range`, no chunking), applies the TUE and the current Wattnet intensity, and returns the energy deltas plus `live_as_of` and `power_w`. `power_w` is the **current draw**: the TUE-normalized sum of the last (sanitized, still fresh) sample of every series in that same query, not a window mean. The result is cached for `topup_cache_seconds` (25 s, below the dashboard's 30 s poll, so the number visibly moves). Any failure returns `{"degraded": true}` and the API serves the stored value. + +`topup_cluster(dc, doc)` / `live_series_cluster(dc, doc, bucket_s)` are the datacenter equivalents (same window logic, `_cluster_promql` instead of the per-alloc query). They are computed inside `get_cluster_stats_bg()` (which runs every 30 s anyway), so the datacenter counter moves at the same cadence as the per-deployment one. + +`accumulated (stored, frozen for up to a sweep) + top-up (growing)` stays continuous: when the next sweep advances `pointer_ts` and folds that slice into `accumulated`, the top-up window shrinks by the same amount. + +## Persistence + +One small JSON per deployment under `$ACCOUNTING_PTH/energy//`, atomic writes (temp file + `os.replace`), the sweep thread is the only writer. Same convention as `ai4papi.utils.retrieve_from_snapshots`. No database. The `accounting.store` interface is deliberately small so it can move to PostgreSQL later without touching the routers. + +`.accum.json` (rewritten every sweep, ~25 KB with a full 24h tail): + +```json +{ + "deployment_uuid": "...", "job_name": "module-...", "namespace": "vo-...", + "owner": "...@egi.eu", "submit_time": "2026-01-05T10:00:00Z", + "submit_time_epoch": 1736071200.0, + "status": "running", "closed_at": null, + "metrics_available": true, "complete": true, "coverage_ratio": 1.0, + "datacenters": ["ifca-imagine"], + "active_allocs": [{"alloc_id": "...", "datacenter": "ifca-imagine"}], + "window_start": "2026-09-02T12:00:00Z", + "settled_ts": "2026-09-02T12:00:00Z", "pointer_ts": "2026-09-03T12:45:00Z", + "updated_at": "2026-09-03T12:50:00Z", "tue_factor": 2.03, + "settled": {"energy_wh": 41200.0, "carbon_g": 12100.0, "water_l": 480.0}, + "tail": [{"ts": "2026-09-02T12:00:00Z", "energy_wh": 4.1, "power_w_avg": 24.3, "carbon_g": 1.2, "water_l": 0.05}], + "accumulated": {"energy_wh": 42873.4, "carbon_g": 12550.2, "water_l": 498.1} +} +``` + +`.series.jsonl`: one line per consolidated bucket (`energy.series_bucket_seconds`, 15 min), append-only (a bucket is written once, when it leaves the 24h tail, never rewritten). About 35k lines (~3 MB) per year for an always-on deployment, a few KB appended per sweep. + +There is no summary file. Per-user and per-datacenter aggregates are computed on read from the `.accum.json` documents (`get_user_energy`, `get_cluster_energy`), because they need the live top-up anyway. If the per-user read (a namespace scan filtered by owner) ever gets heavy, cache a namespace snapshot rather than reviving a stale summary. + +## Resolution strategy + +| Layer | Resolution | Why | +|---|---|---| +| Integration for the accumulator | 30 s (native) | the accumulator is as precise as the source, independent of bucket size | +| Persisted archive series | 15 min (`energy.series_bucket_seconds`) | matches Wattnet's 15-min cadence, ~3 MB/year for a long deployment, append-only file | +| Recent-window graph | 30 s from Mimir, downsampled to ~1000 points | full resolution and always current while the data is in Mimir | +| Old-window graph (past Mimir retention) | 15 min from the store | the only source left, enough to see the shape | +| Live counter tail (`pointer_ts -> now`) | 30 s | the number moves at scrape cadence | + +The store is the permanent low-resolution archive; Mimir is the high-resolution source for anything recent. + +What the `series` in an API response actually is: the persisted 15-min archive (plus the still-mutable tail) `compute.downsample`d to a `target_points` cap, then the live 15-min tail appended un-downsampled. Caps: deployment detail `energy.deployment_series_points` (1000), datacenter `energy.cluster_series_points` (500), per-user `energy.user_series_points` (1000, merged across the user's deployments first, then one downsample). Each function falls back to its hardcoded default when the key is absent from the config. Below the cap you get the raw 15-min buckets; above it, `downsample` groups `ceil(n / cap)` consecutive buckets, so the effective step is `ceil(n / cap) * 15 min`, uniform across the series. `get_series(namespace, uuid)` takes no time-range or resolution arguments (the routers always call it with the two positional args): the response is always the whole life. + +## Computation details + +Module `ai4papi/accounting/compute.py`, all pure and unit-testable except the cached Nomad lookup: + +- `resolve_alloc_ids(job_id, namespace)`: `Nomad.job.get_allocations` plus `_node_datacenter` (TTLCache), filtered to `energy.metrics_datacenters`. +- `power_promql(alloc_ids)`: the CPU and GPU queries for one datacenter's alloc subset. Grouped, not fully summed, so the data-quality filter runs on the original per-source series. + - CPU: `sum by (container_label_com_hashicorp_nomad_alloc_id) (scaph_process_power_consumption_microwatts{container_label_com_hashicorp_nomad_alloc_id=~"id1|id2"})` (one series per allocation) + - GPU: `DCGM_FI_DEV_POWER_USAGE * on(UUID, instance) group_left(alloc_id) nomad_gpu_allocation_info{alloc_id=~"id1|id2"}` (one series per GPU) +- `sanitize(samples, dq)`: data-quality filter on one power series (see below). +- `integrate(cpu_result, gpu_result, step_s, bucket_s, tue_factor, gap_factor, data_quality)`: CPU uW/1e6 plus GPU W, `sanitize` per source series, trapezoidal integral of each into `bucket_s` energy buckets, summed, then `energy_wh = raw_integral * tue_factor`. Gaps longer than `gap_factor * step_s` are not integrated across (deployment stopped, scrape hole, dropped anomaly). +- `apply_footprint(buckets, carbon_series, water_series)`: `carbon_g = (energy_wh / 1000) * intensity_at(ts)`, same for water. `intensity_at` is a bisect for the last Wattnet sample at or before the bucket, `DEFAULTS` otherwise. +- `merge_buckets`, `downsample`, `running_seconds`. + +`Bucket` is a namedtuple `(ts, energy_wh, power_w_avg, carbon_g, water_l)` with `ts` an ISO-8601 UTC string and every value already TUE-normalized. + +### Footprint consolidation + +The sweep uses `green_director.metrics[dc]` (Wattnet, already refreshed every 30 s for the cluster stats) with `DEFAULTS` as fallback. For a backfill older than the 7-day window it calls `green_director.footprint_series(dc, start, end)`, which reuses the Wattnet token and hits `/v1/footprints` for the requested range. + +Every sweep recomputes carbon and water for all buckets newer than `now - footprint_revision_hours` (24 h) with the current intensity, so a bucket's footprint firms up as Wattnet settles it. Older buckets are frozen. The accumulated footprint is always `sum` over buckets, so it self-corrects. + +The live top-up multiplies its `delta_wh` by the current provisional intensity; the next sweep reconciles it (typical correction under 1-2%). + +### Data quality + +`compute.sanitize` runs on each original power series (per allocation for CPU, per GPU for GPU power), before integration and before the per-series energies are summed. Anomalous samples are dropped, never clamped, so a bad reading never inflates the accumulated energy (the integrator sees a gap in that one source) and the other sources at the same instant are untouched. Config `energy.data_quality`, every check individually skippable: + +- `min_power_w`: drop samples below this (kills negatives). +- `max_power_w`: drop samples above this hard ceiling (`0` disables). Applies per source, so it is a physically bounded signal (one allocation, one GPU), not the deployment total. +- `spike_factor` plus `spike_window`: drop a sample above `spike_factor` times the rolling median of its `spike_window` neighbours (`0` disables). + +## API surface + +The consumer-facing reference (response formats field by field, examples, polling and UX notes) is in [energy-api.md](./energy-api.md). This section is the maintainer view. + +### `energy` embedded in the deployment info + +The four deployment routers (`modules`, `tools`, `batch`, `try_me/nomad`), in both `get_deployment` and `get_deployments`, attach an `energy` object built from the store plus the live top-up. Its shape follows the existing `full_info` param: + +- `full_info=false` (the `get_deployments` default): `energy` = `accounting.get_accumulated(...)` for the individual endpoint, or one batch `accounting.get_accumulated_bulk(namespace, owner)` mapped over the list. `EnergyStats`. One or two Mimir queries total for a whole list. +- `full_info=true` (the `get_deployment` default): `energy` = `accounting.get_series(namespace, uuid)` (whole life). `EnergyTimeSeries` with `accumulated` nested. On a list this is one `get_series` per deployment, matching the "may increase latency" note that `full_info` already carries. + +A `energy: bool = True` query param lets a client opt out; the internal quota-check calls pass `False`. `nomad_utils` is not touched. + +`energy` is `null` when there is no rollup or `metrics_available` is `false` (deployment in a site or node without monitoring). When `coverage_ratio < 1.0` the available data is still returned, flagged. + +`schemas.EnergyStats`: + +``` +energy_wh accumulated energy, normalized (x TUE); Wh so the dashboard counter visibly moves +tue_factor the normalization applied (energy_wh / raw); divide by it for the raw figure +carbon_g gCO2eq, scope 2 +water_l L, scope 2 +power_w current draw: last Mimir sample (~30-60s old); 0 if running but idle, null if not running +since start of the accumulation window (`window_start`): pinned on the first sweep, only advances if it falls behind Mimir retention +as_of store pointer timestamp +live_as_of instant of the live top-up +complete False if it started late or was clamped by Mimir retention +coverage_ratio fraction of run time that has power data (float, 1.0 = full); null on the per-datacenter block +degraded Mimir or Wattnet unavailable for the top-up +datacenters list of datacenters the deployment or user has run in; null on the per-datacenter block +deployments set only on the per-user aggregate: number of deployments summed +series 15-min EnergyTimeSeriesPoint list; null unless full_info=true (per-user aggregate only; the per-deployment full_info uses EnergyTimeSeries instead) +``` + +`power_w` is a rate, not an accumulator, so it is never summed. The `power_w` in the block (deployment, user, datacenter) is the **current draw**: the last power sample read from Mimir. Deployment and user take it from the last point of the top-up range query (`compute.instant_power`, which drops stale or spiky last samples); the datacenter takes it from a dedicated instant query (`get_datacenter_power`). `series[].power_w` in the time series is different: a **mean** over its 15-min bucket (so `energy_wh ~= power_w * bucket_hours`). + +### Time series, for the detail view + +The detail view calls the deployment's own individual endpoint (`GET /v1/deployments/{type}/{uuid}`, which defaults to `full_info=true`) and reads `energy` from the response. That `energy` is `schemas.EnergyTimeSeries` from `accounting.get_series(namespace, uuid)`: `{deployment_uuid, start, end, source, series, accumulated}`, where `series` is a list of points (`energy` / `carbon` / `water` per-bucket increment, `power_w` per point; a running total is `cumsum` client-side and matches `accumulated`) and `accumulated` is the `EnergyStats`. It serves the persisted 15-min archive downsampled to ~1000 points plus the live tail (`pointer_ts -> now` from Mimir, `live: true`) appended un-downsampled, so the graph keeps growing on every refresh; `source` is then `"mixed"`. Ownership and 404 handling are the individual endpoint's usual behaviour; `energy` is simply `null` when there is no rollup. + +### Per-datacenter energy, in the cluster stats + +`get_cluster_stats_bg()` (the 30 s background thread, which already refreshes `green_director`) calls `accounting.get_cluster_energy(series=True)`, which merges three things per datacenter into `schemas.EnergyStats`: + +1. **Instantaneous power** from `accounting.get_datacenter_power()`: two instant Mimir queries, `power_w = (sum by (datacenter) scaph_host_power_microwatts + sum by (datacenter) DCGM_FI_DEV_POWER_USAGE) * TUE[dc]`. The host / GPU split is not exposed (on the ifca VMs `scaph_host_power` reads near zero, so it would be misleading; in practice `power_w` is GPU-dominated). A datacenter listed in `energy.metrics_datacenters` is reported with whatever it has (host-only, GPU-only or both); without that allowlist a host-power series is required so a stray GPU metric for an unmonitored site does not fabricate a total. +2. **Accumulated energy** from the `_cluster/.accum.json` document (persisted by the datacenter sweep) plus a `topup_cluster` live slice: `energy_wh`, `carbon_g`, `water_l`, `since`, `complete`, `live_as_of`. +3. **Time series**: the consolidated `.series.jsonl` (`_cluster_archive_series`, cached ~5 min so the 30 s thread does not re-parse it) plus the 24h tail, downsampled to `energy.cluster_series_points`, with the live tail (`live_series_cluster`) appended. Points carry per-bucket increments only. + +Attached as `stats.datacenters[].energy`. There is **no** response-root total: `schemas.ClusterStats` has no `energy` field, and clients sum the datacenter blocks they want. `get_cluster_stats()` (the VO-filtered endpoint) prunes each datacenter's node list but keeps the `energy` block, so an instrumented datacenter with no nodes for that VO is still returned (empty `nodes`). The background thread always computes the series; `get_cluster_stats(full_info=false)` (the default) strips it from every datacenter block before returning, so the series only ships when the caller asks for it. Any failure in this block is caught and leaves the datacenters without `energy`. A datacenter that is not in `energy.metrics_datacenters` and has no accumulator doc simply has no `energy` block; a monitored GPU-only site does get one (accumulator plus series, and an instantaneous `power_w` from its GPU draw). + +### Per-user aggregate + +`stats.deployments.get_user_stats` gains an `energy` key from `accounting.get_user_energy(namespace, owner, series=full_info)`: it iterates the owner's `.accum.json` documents, applies each one's live top-up, and sums them into one `EnergyStats`-shaped block (`since` = earliest, `as_of` / `live_as_of` = latest, `complete` = AND, `degraded` = OR, `power_w` = sum of the running ones, `coverage_ratio` = mean, `tue_factor` = blended `energy_wh / raw`, `datacenters` = union, plus a `deployments` count). With `full_info=true` it also returns a merged time series (`_user_series`): the raw 15-min buckets of every deployment (archive + tail) are summed by timestamp first, then `compute.downsample`d once to `target_points` (1000). Merging before downsampling keeps the resolution uniform across the whole series and gives the same hard cap as the other levels, regardless of how long each deployment individually has been running; the live tails are merged and appended un-downsampled, as elsewhere. The energy block is independent of the pre-existing CSV usage stats: if those are missing, the endpoint still returns the `energy` block instead of failing. + +## Configuration + +`etc/main.yaml`, `energy` section: + +```yaml +energy: + enabled: true + sweep_seconds: 900 # background thread cadence + series_bucket_seconds: 900 # persisted time-series bucket size (matches Wattnet's 15-min cadence) + query_step_seconds: 30 # Mimir scrape interval + query_lag_seconds: 120 # ignore the most recent data (ingestion lag) + initial_lookback_hours: 24 # window covered on a deployment's first sweep + mimir_retention_days: 90 # tune to Mimir's actual retention + footprint_revision_hours: 24 # re-compute footprint for buckets newer than this + topup_cache_seconds: 25 # live top-up cache (< dashboard poll) + topup_lag_seconds: 25 # live top-up query stops this far before "now" + deployment_series_points: 1000 # downsample target for the per-deployment series in /deployments/* + user_series_points: 1000 # downsample target for the merged per-user series in /stats/user + cluster_series_points: 500 # downsample target for the per-datacenter series in /stats/cluster + metrics_datacenters: [ifca-ai4eosc, ifca-imagine] + default_tue_factor: 2.03 # fallback when a datacenter has no TUE in datacenters.csv + data_quality: {enabled: true, min_power_w: 0, max_power_w: 1000, spike_factor: 12, spike_window: 11} +``` + +`max_power_w` is a per-source ceiling (one allocation's CPU, or one GPU), not the deployment total, so `1000` W is safe for any single GPU and any CPU allocation on a VM. A dropped sample becomes a gap in that source only. The datacenter sweep reuses this `data_quality` block but forces `max_power_w` to `0` (a whole host draws far more than any single alloc), keeping only the negative-drop and the rolling-median spike check. + +- `ai4papi/mimir.py`: `MIMIR_URL` is a module constant. `MIMIR_USER` and `MIMIR_PASSWORD` come from the environment (`os.environ.get`, like `wattnet.py`, not `load_env`, so a missing value disables the feature instead of breaking PAPI startup). Add both to `template.env` and to the PAPI deployment. +- `var/datacenters.csv`: `TUE` column, populated only for the datacenters that have a value. `ai4papi/conf.py` tolerates a blank numeric cell (loads it as `None`). +- The Prometheus `datacenter` label values must match the `name` column of `datacenters.csv`. + +## Cadences + +| What | Value | Reason | +|---|---|---| +| Mimir scrape | 30 s | given | +| Sweep thread | `sweep_seconds` (900) | keeps the accumulator current, consolidates the footprint at Wattnet's revision cadence, short enough to catch short jobs before Nomad GC | +| Top-up cache | `topup_cache_seconds` (25) | below the dashboard's 30 s poll, so the counter moves | +| Top-up lag | `topup_lag_seconds` (25) | Mimir ingestion margin; the displayed value trails real time by ~25-30 s | +| Cluster archive-series cache | 300 s | the `.series.jsonl` only changes at sweep cadence; keeps the 30 s thread from re-parsing it | + +For the user to perceive the increment every 30 s the frontend must show Wh (or kWh with 3 decimals) or animate the counter: at 50 W, 30 s is ~0.4 Wh. + +## Degradation + +Same posture as `get_cluster_stats`. + +- Mimir down: the sweep logs and skips (store frozen); the top-up returns `degraded: true`; the API serves the stored accumulator with its `as_of`. +- Wattnet down: energy is still computed; the footprint falls back to `DEFAULTS`; `degraded: true`. +- Unreadable file: `get_accumulated` returns `None`, so `energy` is `null`. +- `energy.enabled: false` or no Mimir credentials: `get_accumulated` / `get_series` / `get_datacenter_power` return `None` / `{}`, so `energy` is `null` everywhere and the rest of the API is unchanged. +- Deployment on a site or node without power monitoring: Mimir returns nothing, `metrics_available` is `false`, `energy` is `null` (best-effort, no misleading zeros); retried every sweep in case monitoring appears or the alloc moves. +- Partial coverage: the available data is returned with `coverage_ratio < 1.0`. +- Cluster stats: a Mimir failure leaves `DatacenterStats.energy` `null` without affecting the rest of the cluster stats. The datacenter sweep logs and skips per datacenter; the persisted accumulator is served with its `as_of`. +- Datacenter with no host-power series: `metrics_available: false`, retried every sweep. +- Job garbage-collected before the first sweep: `complete: false`; mitigated by the short sweep interval. A deleted running deployment is deregistered without purge, so it stays queryable for ~4 h and the next sweep's close pass captures the final window. + +## Files + +New: + +- `ai4papi/mimir.py` +- `ai4papi/accounting/__init__.py` (read API: `get_accumulated*`, `get_series`, `get_user_energy`, `get_datacenter_power`, `get_cluster_energy`) +- `ai4papi/accounting/store.py` (file store) +- `ai4papi/accounting/compute.py` (pure functions) +- `ai4papi/accounting/sweep.py` (deployment + datacenter sweep, top-up) +- `tests/test_energy.py` + +Modified (all additive): + +- `ai4papi/main.py`: the `energy_accounting_thread` `@repeat_every` and its call in `lifespan` +- `ai4papi/routers/v1/deployments/{modules,tools}.py`, `ai4papi/routers/v1/batch.py`, `ai4papi/routers/v1/try_me/nomad.py`: the `energy` key and param (`EnergyStats` or `EnergyTimeSeries` per `full_info`) +- `ai4papi/routers/v1/stats/deployments.py`: the per-user `energy` aggregate in `get_user_stats` (`full_info` for the series) and the per-datacenter `energy` block in `get_cluster_stats` (`full_info` for the series); no response-root total +- `ai4papi/schemas.py`: `EnergyStats` (per deployment and per user), `EnergyTimeSeriesPoint`, `EnergyTimeSeries` (its point list is `series`); `energy` on `DatacenterStats` (also `EnergyStats`) +- `ai4papi/wattnet.py`: `GreenDirector.footprint_series` +- `ai4papi/conf.py`: blank numeric CSV cell tolerated +- `var/datacenters.csv`: `TUE` column +- `etc/main.yaml` (`energy` section incl. `deployment_series_points` / `user_series_points` / `cluster_series_points`), `template.env`, `tests/test_routes.py` + +## Testing + +`tests/test_energy.py` (plain-script style, picked up by `tests/main.py`): + +- Pure-function checks, no infrastructure: `integrate` (trapezoidal, TUE, gaps, single sample), `sanitize` (negatives, ceiling, spike detection, disabled passthrough), `apply_footprint` (Wattnet series and `DEFAULTS` fallback), `intensity_at`, `downsample`, `merge_buckets`, `running_seconds`. +- Sweep pipeline, Mimir and Nomad monkeypatched: `process_single` writes a correct `.accum.json`, idempotency (same `now` gives a byte-identical document), monotonicity across sweeps, buckets aging into `.series.jsonl`, and `get_accumulated` / `get_series` validate against the pydantic models. +- Cluster energy, `mimir.query` / `mimir.query_range` monkeypatched (several raw host series): `get_datacenter_power()` gives `power_w = (host + gpu) * TUE`; `process_datacenter` writes a correct `_cluster/.accum.json` with per-series sanitize then sum, monotonic across sweeps, buckets aging into `.series.jsonl`; `get_cluster_energy(series=True)` merges power + accumulator + a series that validates as `EnergyTimeSeriesPoint` and whose `cumsum` reconciles with the accumulated total. +- Per-user aggregate: `get_user_energy` sums the owner's deployments (`EnergyStats`-shaped), with and without the merged series, and its total reconciles with the merged series. +- An optional live Mimir check, gated on `PAPI_TESTS_MIMIR` plus real credentials. + +Run with the repo venv: + +```bash +IS_PROD=False env/bin/python tests/test_energy.py +cd tests && IS_PROD=False ../env/bin/python test_routes.py && cd .. +``` + +Note: install `fastapi < 0.116`. From 0.116 `include_router` is lazy and `test_routes.py` does not see the routes until the app starts. diff --git a/docs/energy-api.md b/docs/energy-api.md new file mode 100644 index 00000000..f00a5471 --- /dev/null +++ b/docs/energy-api.md @@ -0,0 +1,265 @@ +# Energy stats API (dashboard integration) + +Reference for the energy consumption and footprint (carbon and water, scope 2) that PAPI exposes. Architecture and internals are in [energy-accounting.md](./energy-accounting.md). + +All of this is additive and no new endpoints were created. The data rides on endpoints that already exist, at three levels. The accumulated numbers (and the live Mimir top-up) are **always** present at every level; the time series is only included when `full_info=true`, because it is large (hundreds to a thousand points per entity). + +| Level | Where | Accumulated (+ top-up) | Time series (`full_info=true`) | +|---|---|---|---| +| Datacenter / site | `GET /v1/deployments/stats/cluster` -> `datacenters[].energy` | always | per site, downsampled to `energy.cluster_series_points` | +| User | `GET /v1/deployments/stats/user?vo=` -> `energy` | always | merged across the user's deployments | +| Deployment | `energy` key in the deployment info (list and individual endpoints, all four types) | always | per deployment. On the individual endpoint `full_info` defaults to `true`, so it comes back as `EnergyTimeSeries`; the list form is `EnergyStats` | + +There is **no platform-wide / VO total** in the response: `/stats/cluster` returns one `energy` block per datacenter and the client sums the datacenters it cares about. + +`full_info` defaults to `false` on the list-shaped endpoints (`/stats/cluster`, `/stats/user`, the deployment list) and to `true` on the single-deployment endpoint. Pass it explicitly to override. + +Every endpoint that attaches energy also takes `energy` (default `true`). Pass `energy=false` to drop the `energy` block entirely, for a leaner and slightly faster response when the caller does not need it. PAPI itself passes `energy=false` on its internal quota and resource checks. + +Notes: + +- A datacenter carries an `energy` block only if it is in `energy.metrics_datacenters` (has a power exporter). `/stats/cluster?vo=` still returns every instrumented datacenter's `energy` even when that VO has no nodes there (site infrastructure energy is a property of the site); the VO filter only prunes the node lists. +- Datacenter energy measures **infrastructure** power (host + GPU of the whole site, times TUE). User and deployment energy is **workload** attribution (power of that deployment's allocations). They are not expected to add up. +- There is no VO-level energy block. The `full-agg` / `timeseries` keys already in `GET /stats/user` are the older CPU/GPU usage stats, unrelated to energy. + +If the feature is disabled or has no data, every one of these is `null` (or an absent key) and the API behaves exactly as before. + +## Units and conventions + +- Energy: `energy_wh` in watt-hours. Divide by 1000 for kWh. +- Carbon: `carbon_g` in grams of CO2 equivalent. +- Water: `water_l` in liters. +- Power: `power_w` in watts. +- Timestamps: ISO-8601 UTC, always with the `Z` suffix (`2026-09-07T07:50:00Z`). +- Every numeric field is a JSON number (float), returned at full precision (not rounded): the accumulated total always equals the sum of `series`, and idle CPU workloads can legitimately sit at 1e-5 Wh. Round for display on the client. +- Every energy, power, carbon and water value is normalized by the datacenter TUE (grid to compute ratio). The raw pre-TUE figure is not sent, but `tue_factor` is: divide by it to get the raw value. On a single-datacenter deployment `tue_factor` is that datacenter's configured TUE; on the per-user aggregate and on a deployment that moved between sites it is the blended `energy_wh / raw_energy_wh`. +- `power_w` in the block is the **current draw**: the last sample read from Mimir (~30 to 60 s old) at every level. `series[].power_w` in the time series is different: the **mean** over that 15-min bucket (so `energy_wh ~= power_w * bucket_hours`). Either way it is a rate, not a total, and is never summed. + +## The `EnergyStats` object + +One object, the same shape at every level, always with the live top-up applied: `energy` per deployment (list endpoints), `accumulated` inside the per-deployment time series, `energy` per datacenter in `GET /stats/cluster`, and `energy` per user in `GET /stats/user`. `coverage_ratio` and `datacenters` are `null` on the datacenter block; `deployments` is set only on the per-user aggregate. + +```json +{ + "energy_wh": 3088.98, + "tue_factor": 2.03, + "carbon_g": 929.78, + "water_l": 37.07, + "power_w": 220.0, + "since": "2026-09-01T10:00:00Z", + "as_of": "2026-09-07T07:50:00Z", + "live_as_of": "2026-09-07T07:54:58Z", + "complete": false, + "coverage_ratio": 1.0, + "degraded": false, + "datacenters": ["ifca-imagine"] +} +``` + +| Field | Type | Meaning | +|---|---|---| +| `energy_wh` | number | Accumulated energy, normalized by the datacenter TUE. This is the headline number. Divide by `tue_factor` for the raw figure at the meter. | +| `tue_factor` | number or null | The normalization applied: `energy_wh / raw_energy_wh`. On a single-datacenter deployment it is that datacenter's configured TUE. `null` only before there is any energy. | +| `carbon_g` | number | Accumulated carbon footprint (gCO2eq). | +| `water_l` | number | Accumulated water footprint (L). | +| `power_w` | number or null | Current draw: the last power sample read from Mimir (~30 to 60 s old), normalized by TUE. `0` for a running deployment drawing nothing measurable; `null` if it is not running. | +| `since` | string or null | Start of the accumulation window: the point `energy_wh` and the footprint are counted from. Pinned when accounting first picks up the deployment (its submit time, or as far back as `initial_lookback_hours` / Mimir retention allow) and stable afterwards, so it is not the same as `as_of`. Pair it with `complete: false` for a "since \" note. | +| `as_of` | string or null | Timestamp of the last persisted point. | +| `live_as_of` | string or null | Instant the numbers actually reflect, once the live top-up is applied. Use this for "updated N ago". Null when the values come straight from the store with no live top-up. | +| `complete` | boolean | `false` if accounting started after the deployment was created, or the window was clamped by Mimir retention. When `false`, treat `energy_wh` and the footprint as a lower bound. | +| `degraded` | boolean | `true` if Mimir or Wattnet was unreachable for the live part. The stored value is still served, it may just be slightly stale. | +| `coverage_ratio` | number or null | Deployment and per-user only: fraction of the run time that has power data (`1.0` = fully covered). `null` on the datacenter block. | +| `datacenters` | string[] or null | Deployment and per-user only: datacenters the deployment(s) ran in. `null` on the datacenter block. | +| `deployments` | number or null | Per-user aggregate only: how many deployments went into the sum. | +| `series` | array or null | Per-user aggregate only, and only with `full_info=true`: the merged 15-min time series. `null` otherwise. | + +The values already include the live top-up: `energy_wh` is the persisted accumulator plus the slice from the last persisted point up to `live_as_of`, computed live against Mimir on each request (cached ~25 s). That is why the counter moves between refreshes without a background job. + +## 1. `energy` in the deployment info + +Added to the existing deployment object. Its shape follows the endpoint's existing `full_info` query param: + +- `full_info=false` -> `energy` is an `EnergyStats` (or `null`). +- `full_info=true` -> `energy` is an `EnergyTimeSeries` (or `null`): the accumulated stats plus the full time series for the detail-view graphs. + +The defaults line up with how the dashboard uses each endpoint: + +| Endpoint | `full_info` default | `energy` shape | +|---|---|---| +| `GET /v1/deployments/modules?vos=` (and `tools` / `batch` / `try_me/nomad` list variants) | `false` | `EnergyStats` per deployment | +| `GET /v1/deployments/modules/{uuid}?vo=` (and `tools` / `batch` variants) | `true` | `EnergyTimeSeries` | +| `GET /v1/deployments/try_me/nomad/{uuid}` (no `vo`, fixed VO) | `true` | `EnergyTimeSeries` | + +You can override in either direction: `...?vos=&full_info=true` gives the full series per deployment in the list (heavier, one Mimir top-up per deployment), and `.../{uuid}?vo=&full_info=false` gives just the accumulated `EnergyStats`. + +- `energy` is `null` when the deployment has no data (site or node without monitoring, still queued, or the feature is off). Show "not available" or hide the widget, never a zero. +- Query param `energy` (default `true`). Pass `?energy=false` to skip the block entirely, for example when you only need the deployment list fast. + +### List view (`full_info=false`) + +```json +{ + "job_ID": "aaaaaaaa-...", + "name": "module-aaaaaaaa-...", + "status": "running", + "energy": { + "energy_wh": 3088.98, + "carbon_g": 929.78, + "water_l": 37.07, + "power_w": 220.0, + "live_as_of": "2026-09-07T07:54:58Z", + "complete": false, "coverage_ratio": 1.0, "degraded": false, + "datacenters": ["ifca-imagine"], + "...": "see EnergyStats above" + } +} +``` + +The list endpoint resolves all of the user's deployments with one or two Mimir queries total, regardless of how many deployments there are. Poll it every 30 to 60 s for the list view. + +### Detail view (`full_info=true`) + +```json +{ + "job_ID": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "name": "module-aaaaaaaa-...", + "status": "running", + "energy": { + "deployment_uuid": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "start": "2026-09-07T00:55:00Z", + "end": "2026-09-07T07:54:58Z", + "source": "mixed", + "series": [ + { + "ts": "2026-09-07T00:55:00Z", + "power_w": 220.0, + "energy_wh": 37.22, + "carbon_g": 11.20, + "water_l": 0.45, + "live": false + }, + { "ts": "2026-09-07T01:00:00Z", "power_w": 220.0, "energy_wh": 37.22, "...": "..." } + ], + "accumulated": { "...": "EnergyStats, exactly the list-view object" } + } +} +``` + +| Field | Type | Meaning | +|---|---|---| +| `deployment_uuid` | string | Echo of the deployment. | +| `start` / `end` | string | Time span covered by `series`. `start` equals `accumulated.since`, `end` the live edge (the last point). Convenience only, both are derivable from the series. | +| `series` | array | Ordered by `ts` ascending. Whole deployment life, downsampled to `energy.deployment_series_points` (default 1000). | +| `series[].ts` | string | Bucket start. | +| `series[].power_w` | number or null | Mean power over the part of the bucket that had data, normalized by TUE. `energy_wh` is about `power_w` times the bucket duration. | +| `series[].energy_wh` | number or null | Energy in that bucket (per-bucket increment). A running total is `cumsum(series[].energy_wh)` and matches `accumulated.energy_wh`. | +| `series[].carbon_g` | number or null | Carbon in that bucket. Same, cumsum matches `accumulated.carbon_g`. | +| `series[].water_l` | number or null | Water in that bucket. Same, cumsum matches `accumulated.water_l`. | +| `series[].live` | boolean | `true` for the tail points served live from Mimir (not yet persisted). They can change on the next refresh. | +| `source` | string | `"store"` (only the persisted archive) or `"mixed"` (archive plus a live tail). Equivalent to `series[-1].live`. | +| `accumulated` | EnergyStats | The current totals, same object as the list view. Use it for the headline numbers next to the chart (and its `degraded` flag). | + +- The persisted archive is at 15-minute resolution (`energy.series_bucket_seconds`); the response is downsampled to `energy.deployment_series_points` (default 1000) across the whole life. +- The deployment is running, so `end` reaches "now" and the last points carry `live: true`. Calling the endpoint again a minute later returns a slightly longer series with updated `live` points. That is what makes the chart grow. +- Poll the detail view every 30 to 60 s. + +## 2. `energy` in the per-user stats + +``` +GET /v1/deployments/stats/user?vo= +GET /v1/deployments/stats/user?vo=&full_info=true +``` + +The existing response gains an `energy` key: all of the user's deployments in that VO, each with its live top-up, summed into one `EnergyStats` object (plus a `deployments` count): + +```json +{ + "full-agg": { "...": "existing compute-usage stats" }, + "timeseries": { "...": "existing" }, + "users-agg": { "...": "existing" }, + "energy": { + "energy_wh": 3088.98, + "tue_factor": 2.03, + "carbon_g": 929.78, + "water_l": 37.07, + "power_w": 220.0, + "since": "2026-09-01T10:00:00Z", + "as_of": "2026-09-09T07:00:00Z", + "live_as_of": "2026-09-09T07:04:58Z", + "complete": false, + "coverage_ratio": 0.98, + "degraded": false, + "datacenters": ["ifca-imagine"], + "deployments": 3, + "series": null + } +} +``` + +- Same fields as the per-deployment `EnergyStats` (see above), aggregated: `since` is the earliest across the user's deployments, `as_of` / `live_as_of` the latest, `complete` is the AND, `degraded` the OR, `power_w` the sum over the running ones, `coverage_ratio` the mean, `datacenters` the union, `tue_factor` the blended `energy_wh / raw_energy_wh`. +- `deployments`: how many accounted deployments went into the sum. +- `series`: `null` unless `full_info=true`, then a 15-min time series of the user's deployments summed by timestamp and downsampled once to `energy.user_series_points` (default 1000; `EnergyTimeSeriesPoint` shape, uniform step across the whole series), for a "my usage over time" chart. +- `null` when the user has no accounted deployments. +- Independent of the pre-existing CSV usage stats: if those are missing the endpoint still returns the `energy` block. + +## 3. `energy` in the cluster stats + +``` +GET /v1/deployments/stats/cluster +GET /v1/deployments/stats/cluster?vo= +GET /v1/deployments/stats/cluster?full_info=true +``` + +Each `datacenters[]` object gains an `energy` block: the same `EnergyStats` object as everywhere else (`coverage_ratio`, `datacenters`, `deployments` are `null` here). There is **no** response-root `energy`: to get a platform-wide or per-VO figure the client sums the datacenter blocks it wants. + +- **`power_w`**: the current draw from the last Mimir scrape, refreshed every ~30 s, normalized by TUE. It is `(sum of scaphandre host power + sum of GPU power) * TUE`. +- **accumulated** energy and footprint since accounting started (`energy_wh`, `carbon_g`, `water_l`, all TUE-normalized, divide by `tue_factor` for raw), kept indefinitely and topped up live on every read. +- **`series`**: `null` by default; with `full_info=true`, a 15-min series downsampled to `energy.cluster_series_points` (default 500). + +`?vo=` filters the node lists but not the `energy` blocks: an instrumented datacenter keeps its `energy` even when the VO has no nodes there (its `nodes` map is then empty). + +```json +{ + "datacenters": { + "ifca-imagine": { + "lat": 43.47, "lon": -3.8, "PUE": 1.31, + "nodes": { "...": "..." }, + "footprints": { "...": "existing" }, + "energy": { + "power_w": 1421.0, + "energy_wh": 184203.5, + "carbon_g": 55450.2, + "water_l": 2210.9, + "tue_factor": 2.03, + "complete": false, + "since": "2026-06-10T00:00:00Z", + "as_of": "2026-09-08T09:10:00Z", + "live_as_of": "2026-09-08T09:12:35Z", + "degraded": false, + "coverage_ratio": null, "datacenters": null, "deployments": null, + "series": null + } + } + }, + "cluster": { "...": "existing ResourceStats" }, + "updated_at": "2026-09-08T09:12:05Z" +} +``` + +Field meanings: see the `EnergyStats` table above. + +- `energy` is `null` for datacenters outside `energy.metrics_datacenters` and whenever Mimir is unreachable; never shown as zero. +- A `vo` filter does not touch the `energy` blocks; it only prunes each datacenter's `nodes`. An instrumented datacenter with no nodes for that VO is still returned, with its `energy` and an empty `nodes`. +- Refreshed by the same 30 s background task as the rest of the cluster stats. The persisted accumulator advances every sweep (~15 min); the live top-up keeps `energy_wh` moving between sweeps. The endpoint itself is cached ~30 s. + +## UX notes + +- Values come at full float precision; round on the client. Show `energy_wh` in Wh, or kWh with enough decimals to move, or animate the counter. At 50 W the increment per 30 s poll is about 0.4 Wh, invisible if you round to whole kWh. +- The displayed value trails real time by about 25 to 30 s (Mimir ingestion margin). Use `live_as_of` for the "updated N seconds ago" label. +- Poll cadence: list view every 30 to 60 s against the deployment list endpoint (`full_info=false`); detail view every 30 to 60 s against the individual deployment endpoint (`full_info=true`). +- Flags to surface in the UI: `complete: false` (show the value as a lower bound, for example a "since " note), `coverage_ratio < 1.0` (for example "78% of run time with data"), `degraded: true` (a small "data may be delayed" hint). +- `vo` values: `vo.ai4eosc.eu`, `vo.imagine-ai.eu`, `vo.ai4life.eu`, `kmd4eosc`, `tutorials`. The token must have a role for that VO. + +## Note on the test suite + +`tests/test_routes.py` requires `fastapi < 0.116` in the environment: from 0.116 `include_router` is lazy and the route assertions do not see the routes until the app starts. From be007d1e05a922ac2105abc1d1c9cd7a3c4080dc Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Mon, 14 Sep 2026 10:50:37 +0200 Subject: [PATCH 09/25] fix(energy): correct IFCA TUE to 2.73 and decouple test from real config The empirical TUE for ifca-ai4eosc/ifca-imagine was 2.03; the correct value is 2.73. Update the datacenters.csv value, the config default, and the docs example. Also stop test_energy.py from asserting against the real per-datacenter TUE in var/datacenters.csv (it now pins its own value), so editing that file no longer breaks the test suite. --- docs/energy-accounting.md | 6 +++--- etc/main.yaml | 2 +- tests/test_energy.py | 13 +++++++++---- var/datacenters.csv | 4 ++-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/energy-accounting.md b/docs/energy-accounting.md index 183bab24..4645ed87 100644 --- a/docs/energy-accounting.md +++ b/docs/energy-accounting.md @@ -25,7 +25,7 @@ Mimir endpoint: `https://mimir.k8s.cloud.ai4eosc.eu/prometheus/api/v1/query_rang `energy_wh = integrated_Wh * TUE[datacenter]`, where `integrated_Wh` is the raw trapezoidal integral of `CPU + GPU` power. -The TUE (Total Usage Effectiveness, grid to compute ratio) is per datacenter and already covers CPU plus GPU, so it is applied once, before the footprint (which is scope 2, so it must reflect the whole facility draw). It is a single empirical factor per datacenter (default 2.03), consistent with the platform's existing energy accounting. Every energy, power, carbon and water value in an API response is already TUE-normalized; the applied factor is reported as `tue_factor` (per datacenter, or blended `energy_wh / raw` on multi-datacenter and per-user aggregates) so a client that wants the raw meter figure divides by it. The accumulator stores `tue_factor` per document. +The TUE (Total Usage Effectiveness, grid to compute ratio) is per datacenter and already covers CPU plus GPU, so it is applied once, before the footprint (which is scope 2, so it must reflect the whole facility draw). It is a single empirical factor per datacenter (default 2.73), consistent with the platform's existing energy accounting. Every energy, power, carbon and water value in an API response is already TUE-normalized; the applied factor is reported as `tue_factor` (per datacenter, or blended `energy_wh / raw` on multi-datacenter and per-user aggregates) so a client that wants the raw meter figure divides by it. The accumulator stores `tue_factor` per document. The TUE is read from the `TUE` column of `var/datacenters.csv`, only populated for the datacenters that have a value (currently `ifca-ai4eosc` and `ifca-imagine`). Blank cells load as `None` and `energy.default_tue_factor` is used as fallback. @@ -114,7 +114,7 @@ One small JSON per deployment under `$ACCOUNTING_PTH/energy//`, atomi "active_allocs": [{"alloc_id": "...", "datacenter": "ifca-imagine"}], "window_start": "2026-09-02T12:00:00Z", "settled_ts": "2026-09-02T12:00:00Z", "pointer_ts": "2026-09-03T12:45:00Z", - "updated_at": "2026-09-03T12:50:00Z", "tue_factor": 2.03, + "updated_at": "2026-09-03T12:50:00Z", "tue_factor": 2.73, "settled": {"energy_wh": 41200.0, "carbon_g": 12100.0, "water_l": 480.0}, "tail": [{"ts": "2026-09-02T12:00:00Z", "energy_wh": 4.1, "power_w_avg": 24.3, "carbon_g": 1.2, "water_l": 0.05}], "accumulated": {"energy_wh": 42873.4, "carbon_g": 12550.2, "water_l": 498.1} @@ -244,7 +244,7 @@ energy: user_series_points: 1000 # downsample target for the merged per-user series in /stats/user cluster_series_points: 500 # downsample target for the per-datacenter series in /stats/cluster metrics_datacenters: [ifca-ai4eosc, ifca-imagine] - default_tue_factor: 2.03 # fallback when a datacenter has no TUE in datacenters.csv + default_tue_factor: 2.73 # fallback when a datacenter has no TUE in datacenters.csv data_quality: {enabled: true, min_power_w: 0, max_power_w: 1000, spike_factor: 12, spike_window: 11} ``` diff --git a/etc/main.yaml b/etc/main.yaml index 9af2d466..3fe209fe 100644 --- a/etc/main.yaml +++ b/etc/main.yaml @@ -110,7 +110,7 @@ energy: # per-deployment energy consumption and environmental footprint stats # TUE (grid -> compute ratio, already covers CPU+GPU) is read per-datacenter # from the `TUE` column of var/datacenters.csv; this is the fallback for # datacenters that do not have it yet. - default_tue_factor: 2.03 + default_tue_factor: 2.73 data_quality: # drop anomalous samples from each source series before integrating diff --git a/tests/test_energy.py b/tests/test_energy.py index 1ad446dd..917940e3 100644 --- a/tests/test_energy.py +++ b/tests/test_energy.py @@ -182,11 +182,16 @@ def _series(values, scale_str=str): mimir.MIMIR_USER = mimir.MIMIR_USER or "test" mimir.MIMIR_PASSWORD = mimir.MIMIR_PASSWORD or "test" +from ai4papi import conf as papiconf # noqa: E402 from ai4papi.accounting import store, sweep # noqa: E402 import ai4papi.accounting as accounting # noqa: E402 _NOW = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0) _NS, _UUID, _DC = "ai4eosc", "sweep-test-uuid", "ifca-ai4eosc" +# fixed, test-owned TUE: independent of whatever var/datacenters.csv says for +# `_DC` in production, so editing that file never breaks this test. +_TUE = 2.5 +papiconf.datacenters[_DC] = {**papiconf.datacenters[_DC], "TUE": _TUE} def _fake_query_range(promql, start, end, step_s=30): @@ -221,10 +226,10 @@ def _fake_query_range(promql, start, end, step_s=30): doc = store.read_accum(_NS, _UUID) acc = doc["accumulated"] assert doc["metrics_available"] is True -assert doc["tue_factor"] == 2.03 -# ~3h of 100 W -> ~300 Wh raw, x 2.03 TUE (11 or 12 complete 15-min buckets +assert abs(doc["tue_factor"] - _TUE) < 1e-9 +# ~3h of 100 W -> ~300 Wh raw, x _TUE (11 or 12 complete 15-min buckets # depending on where "now" falls in the current bucket) -assert 270 * 2.03 < acc["energy_wh"] < 305 * 2.03, acc +assert 270 * _TUE < acc["energy_wh"] < 305 * _TUE, acc assert abs(acc["carbon_g"] - acc["energy_wh"] / 1000 * compute.DEFAULT_CARBON) < 1.0 # idempotent: same "now" -> byte-identical document @@ -263,7 +268,7 @@ def _fake_query_range(promql, start, end, step_s=30): "live", }, _pt _ps = accounting.get_accumulated(_NS, _UUID, live=True) -assert _ps["power_w"] and _ps["tue_factor"] == 2.03 +assert _ps["power_w"] and abs(_ps["tue_factor"] - _TUE) < 1e-9 # per-user aggregate: sums the owner's deployments, EnergyStats-shaped accounting._read_doc.cache_clear() diff --git a/var/datacenters.csv b/var/datacenters.csv index 20810fa6..fc3e5503 100644 --- a/var/datacenters.csv +++ b/var/datacenters.csv @@ -1,6 +1,6 @@ name,country,lat,lon,PUE,TUE -ifca-ai4eosc,ES,43.471668,-3.802318,1.31,2.03 -ifca-imagine,ES,43.471706,-3.802240,1.31,2.03 +ifca-ai4eosc,ES,43.471668,-3.802318,1.31,2.73 +ifca-imagine,ES,43.471706,-3.802240,1.31,2.73 iisas-ai4eosc,SK,48.170095,17.070058,1.57, incd-imagine,PT,38.759140,-9.142664,1.45, tubitak-imagine,TR,40.785667,29.448592,1.35, From be235b8ffa7b60964eb01d2582d826a84c844900 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 17 Sep 2026 13:48:54 +0200 Subject: [PATCH 10/25] feat(energy): serve real-time energy before the sweep has a doc The read API used to report energy: null for any deployment the background sweep had not reached yet, which for a fresh deployment could mean waiting up to sweep_seconds (15 min). Close that gap with two fallback layers ahead of the periodic sweep: - On-demand sweep (Layer 3): when a read finds no accumulator doc, it runs one process_single pass synchronously instead of waiting for the background thread. Applies to single-deployment and bulk/per-user reads. - Pure-Mimir fallback (Layer 4): when even that leaves nothing settled (a deployment younger than one series_bucket_seconds), build a transient, never-persisted doc and read Mimir directly, only surfacing a result once Mimir actually has a sample (never a fabricated zero). Also simplify the live top-up: Mimir data is immutable once ingested, so there was no reason to hold back a fresh sample behind topup_lag_seconds or drop it as "stale" in compute.instant_power. Both the persisted and the live-only path now read the same lag-free window and always report the actual last sample, so live_only_topup collapses into the same topup() used everywhere else. Fix live_only_doc always reporting complete: false even though its window start is pinned to the exact, unclamped submit time. --- ai4papi/accounting/__init__.py | 88 ++++++++- ai4papi/accounting/compute.py | 10 +- ai4papi/accounting/sweep.py | 197 ++++++++++++++++++-- docs/energy-accounting.md | 27 ++- etc/main.yaml | 1 - tests/test_energy.py | 317 +++++++++++++++++++++++++++++++++ 6 files changed, 609 insertions(+), 31 deletions(-) diff --git a/ai4papi/accounting/__init__.py b/ai4papi/accounting/__init__.py index af263289..e3876ce4 100644 --- a/ai4papi/accounting/__init__.py +++ b/ai4papi/accounting/__init__.py @@ -26,6 +26,9 @@ CLUSTER_NS, _tue, enabled, + ensure_swept, + ensure_swept_bulk, + live_only_doc, live_series, live_series_cluster, process_single, @@ -106,12 +109,49 @@ def _apply_topup(stats: dict, doc: dict) -> dict: return stats +def _live_only_stats(namespace: str, uuid: str) -> tuple[dict, dict] | None: + """ + `(doc, stats)` for a deployment with no accumulator doc yet, computed purely + from `live_only_doc` + its top-up. `None` whenever there is nothing to show: + no running allocation, Mimir unreachable (`degraded`), or Mimir simply has + not returned a single sample for this allocation yet -- an all-zero top-up + with no instantaneous power is indistinguishable from "no measurement", so + it is treated as not populated rather than shown as a fabricated `0` reading. + """ + doc = live_only_doc(namespace, uuid) + if doc is None: + return None + stats = _apply_topup(_to_stats(doc), doc) + has_data = stats.get("power_w") is not None or any( + stats.get(k, 0.0) for k in _KEYS + ) + if not has_data: + return None + return doc, stats + + def get_accumulated(namespace: str, uuid: str, live: bool = True) -> dict | None: if not enabled(): return None doc = _read_doc(namespace, uuid) - if not doc or doc.get("metrics_available") is False or "accumulated" not in doc: + if doc is None: + # never swept: run one sweep pass for just this deployment instead of + # waiting up to `sweep_seconds` for the background thread to reach it. + doc = ensure_swept(namespace, uuid) + if doc is not None and doc.get("metrics_available") is False: + # confirmed: ran a significant time with no usable data. Do not keep + # retrying live -- wait for the real sweep to reassess it. return None + if doc is None or "accumulated" not in doc: + # nothing settled yet: either too young for a bucket, or an + # inconclusive on-demand stub written before Nomad had even placed the + # allocation (no `metrics_available` verdict either way). Fall back to + # a pure-Mimir number, but only once Mimir actually has a sample for + # it - never a fabricated zero. + if not live: + return None + res = _live_only_stats(namespace, uuid) + return res[1] if res else None stats = _to_stats(doc) return _apply_topup(stats, doc) if live else stats @@ -121,12 +161,26 @@ def get_accumulated_bulk( ) -> dict[str, dict]: if not enabled(): return {} + live_jobs = ensure_swept_bulk(namespace, owner) out: dict[str, dict] = {} + covered: set[str] = set() for uuid, doc in store.iter_owner(namespace, owner): - if doc.get("metrics_available") is False or "accumulated" not in doc: + if doc.get("metrics_available") is False: + covered.add(uuid) continue + if "accumulated" not in doc: + continue # inconclusive stub: leave uncovered, try live-only below + covered.add(uuid) stats = _to_stats(doc) out[uuid] = _apply_topup(stats, doc) if live else stats + if live: + for stub in live_jobs: + uuid = stub["ID"] + if uuid in covered: + continue + res = _live_only_stats(namespace, uuid) + if res is not None: + out[uuid] = res[1] return out @@ -142,6 +196,7 @@ def get_user_energy( """ if not enabled(): return None + live_jobs = ensure_swept_bulk(namespace, owner) if target_points is None: target_points = int( @@ -149,10 +204,22 @@ def get_user_energy( ) rows = [] + covered: set[str] = set() for uuid, doc in store.iter_owner(namespace, owner): - if doc.get("metrics_available") is False or "accumulated" not in doc: + if doc.get("metrics_available") is False: + covered.add(uuid) continue + if "accumulated" not in doc: + continue # inconclusive stub: leave uncovered, try live-only below + covered.add(uuid) rows.append((uuid, doc, _apply_topup(_to_stats(doc), doc))) + for stub in live_jobs: + uuid = stub["ID"] + if uuid in covered: + continue + res = _live_only_stats(namespace, uuid) + if res is not None: + rows.append((uuid, res[0], res[1])) if not rows: return None @@ -375,8 +442,16 @@ def get_series( if not enabled(): return None doc = _read_doc(namespace, uuid) - if not doc or doc.get("metrics_available") is False or "accumulated" not in doc: + if doc is None: + doc = ensure_swept(namespace, uuid) + if doc is not None and doc.get("metrics_available") is False: return None + live_only_stats = None + if doc is None or "accumulated" not in doc: + res = _live_only_stats(namespace, uuid) + if res is None: + return None + doc, live_only_stats = res cfg = papiconf.MAIN_CONF["energy"] if target_points is None: @@ -388,7 +463,10 @@ def get_series( live_series(doc, bucket_s=step_s), target_points, ) - stats = _apply_topup(_to_stats(doc), doc) + # already computed (and confirmed non-empty) by `_live_only_stats` above; + # `_apply_topup` is cached at the Mimir-query level, so this is not a + # second call for the swept-doc branch either. + stats = live_only_stats or _apply_topup(_to_stats(doc), doc) return { "deployment_uuid": uuid, diff --git a/ai4papi/accounting/compute.py b/ai4papi/accounting/compute.py index 49a753f1..5ef7017f 100644 --- a/ai4papi/accounting/compute.py +++ b/ai4papi/accounting/compute.py @@ -294,20 +294,20 @@ def instant_power( cpu_result: list[dict], gpu_result: list[dict], tue_factor: float, - fresh_after_ts: float, data_quality: dict | None = None, ) -> float | None: """ Current draw from a range query: the sum of the most recent (sanitized) - sample of every power series, normalized by TUE. A series whose last good - sample predates `fresh_after_ts` (source gone, scrape hole) is skipped. - `None` when nothing is fresh. + sample of every power series, normalized by TUE. Mimir data is immutable + once ingested, so the actual last sample is always used, however old it + is -- `None` only when a series has no sample at all (nothing ingested + yet, or dropped entirely by `sanitize`). """ raw_w, fresh = 0.0, False for scale, result in ((1e-6, cpu_result), (1.0, gpu_result)): for samples in _series_samples(result, scale): clean = sanitize(samples, data_quality) - if clean and clean[-1][0] >= fresh_after_ts: + if clean: raw_w += clean[-1][1] fresh = True return raw_w * tue_factor if fresh else None diff --git a/ai4papi/accounting/sweep.py b/ai4papi/accounting/sweep.py index 09a9cb68..50ac5fad 100644 --- a/ai4papi/accounting/sweep.py +++ b/ai4papi/accounting/sweep.py @@ -151,6 +151,19 @@ def run_sweep() -> None: LOG.exception("cluster energy sweep failed") +def _info_from_job(j: dict, ns: str) -> dict: + """`info` dict `process_single` expects, from a raw Nomad job.""" + meta = j.get("Meta") or {} + return { + "uuid": j["ID"], + "job_name": j["Name"], + "namespace": ns, + "owner": meta.get("owner", ""), + "submit_time": (j["SubmitTime"] / 1e9) if j.get("SubmitTime") else None, + "status": "running", + } + + def _sweep_namespace(ns: str, now: datetime.datetime) -> None: try: jobs = Nomad.jobs.get_jobs(namespace=ns, filter_='Status != "dead"') @@ -169,17 +182,8 @@ def _sweep_namespace(ns: str, now: datetime.datetime) -> None: except Exception: LOG.exception("could not fetch job %s", stub["ID"]) continue - meta = j.get("Meta") or {} - info = { - "uuid": j["ID"], - "job_name": j["Name"], - "namespace": ns, - "owner": meta.get("owner", ""), - "submit_time": (j["SubmitTime"] / 1e9) if j.get("SubmitTime") else None, - "status": "running", - } try: - process_single(info, now) + process_single(_info_from_job(j, ns), now) except Exception: LOG.exception("energy sweep failed for deployment %s", stub["ID"]) @@ -201,6 +205,159 @@ def _sweep_namespace(ns: str, now: datetime.datetime) -> None: LOG.exception("energy close failed for deployment %s", uuid) +def ensure_swept(ns: str, uuid: str) -> dict | None: + """ + On-demand equivalent of one sweep pass for a single deployment, called from + the read path when no accumulator doc exists yet: a deployment that already + has real samples in Mimir does not have to wait up to `sweep_seconds` for + the background thread to reach it before the API can report anything. + + A no-op (returns the existing doc untouched, no Nomad/Mimir call) once the + background sweep has processed this deployment at least once, so it never + duplicates the periodic sweep's work. `None` if the job cannot be found (not + a deployment, or already gone). + """ + if not enabled(): + return None + doc = store.read_accum(ns, uuid) + if doc is not None: + return doc + try: + j = Nomad.job.get_job(id_=uuid, namespace=ns) + except Exception: + return None + try: + process_single(_info_from_job(j, ns), _now()) + except Exception: + LOG.warning("on-demand energy sweep failed for %s/%s", ns, uuid, exc_info=True) + return None + return store.read_accum(ns, uuid) + + +_bulk_sweep_cache: TTLCache = TTLCache( + maxsize=2048, ttl=papiconf.MAIN_CONF["energy"].get("topup_cache_seconds", 25) +) + + +@cached(_bulk_sweep_cache) +def ensure_swept_bulk(ns: str, owner: str) -> list[dict]: + """ + On-demand equivalent of one sweep pass, restricted to one owner's active + deployments in `ns`: the bulk counterpart of `ensure_swept`, used by the + list (`get_accumulated_bulk`) and per-user (`get_user_energy`) read paths so + a just-deployed job shows up there too, not only on its own detail endpoint. + + Lists the owner's active jobs (cheap, no per-job fetch) and only runs + `process_single` for the ones with no accumulator doc yet; a no-op once the + background sweep (or a prior on-demand call) has reached all of them. + Cached for `topup_cache_seconds`: without it, every list/`/stats/user` read + would list Nomad on every single request, forever, not just during the + bootstrap window -- a few seconds of staleness for a listing that itself + only matters for brand-new deployments is a good trade. + + Returns the filtered job stubs (`[{"ID": ..., "Name": ...}, ...]`) so the + caller can also try `live_only_doc` for whichever of them still have no doc + afterwards (a deployment too young for even one completed bucket), without + listing Nomad a second time. `[]` when disabled or the listing itself fails. + """ + if not enabled(): + return [] + try: + jobs = Nomad.jobs.get_jobs( + namespace=ns, + filter_=f'Meta.owner == "{owner}" and Status != "dead"', + ) + except Exception: + LOG.warning("could not list jobs for on-demand sweep (%s/%s)", ns, owner) + return [] + now = _now() + stubs = [s for s in jobs if s["Name"].startswith(_JOB_PREFIXES)] + for stub in stubs: + if store.read_accum(ns, stub["ID"]) is not None: + continue + try: + j = Nomad.job.get_job(id_=stub["ID"], namespace=ns) + process_single(_info_from_job(j, ns), now) + except Exception: + LOG.warning( + "on-demand energy sweep failed for %s/%s", ns, stub["ID"], exc_info=True + ) + return stubs + + +_live_only_doc_cache: TTLCache = TTLCache( + maxsize=8192, ttl=papiconf.MAIN_CONF["energy"].get("topup_cache_seconds", 25) +) + + +@cached(_live_only_doc_cache) +def live_only_doc(ns: str, uuid: str) -> dict | None: + """ + Transient stand-in for a deployment `process_single` has not written any doc + for at all yet -- typically younger than one `series_bucket_seconds`, so it + has no completed bucket to fold into an accumulator (see `ensure_swept`, + which still leaves such a deployment with no doc by design: there is + nothing settled to persist). Never written to the store. + + Cached for `topup_cache_seconds`: without it, every read of a deployment + still in this bootstrap window would fetch its Nomad job and resolve its + allocations from scratch on every single request for as long as it stays + doc-less (which can be several minutes). A few seconds of staleness on + which allocation is "current" is a good trade against that. + + Shaped so the normal `topup` / `live_series` machinery -- and therefore + `accounting._to_stats` / `_apply_topup` / `_build_series` -- treats it like + a real doc with an all-zero accumulator and `pointer_ts = submit_time`: the + whole reported number then comes from the live top-up's Mimir query, capped + at the same lookback `topup` itself uses. `None` if the job cannot be + resolved to a running, monitored allocation (not found, queued, or no + allocation yet). + """ + if not enabled(): + return None + try: + j = Nomad.job.get_job(id_=uuid, namespace=ns) + except Exception: + return None + info = _info_from_job(j, ns) + try: + allocs_by_dc, _windows, running_allocs, _partial = compute.resolve_alloc_ids( + uuid, ns + ) + except Exception: + return None + if not running_allocs: + return None + submit = info.get("submit_time") + submit_ts = submit or _now().timestamp() + # `window_start` below is pinned to the exact submit time, same as + # `process_single`'s first-encounter `window_start` -- not started late or + # clamped, so `complete` is `True` unless we had no real `submit_time` to + # pin it to at all (job fetched with no `SubmitTime`, an edge case) or it + # somehow predates the lookback window (impossible in practice: this path + # only ever runs for a deployment too young to have a doc yet). + complete = bool(submit) and submit >= _now().timestamp() - _cfg()[ + "initial_lookback_hours" + ] * 3600 + return { + "deployment_uuid": uuid, + "namespace": ns, + "job_name": info.get("job_name"), + "owner": info.get("owner"), + "status": "running", + "metrics_available": True, + "complete": complete, + "window_start": compute._iso(submit_ts), + "settled_ts": compute._iso(submit_ts), + "pointer_ts": compute._iso(submit_ts), + "active_allocs": running_allocs, + "datacenters": sorted(allocs_by_dc), + "tail": [], + "settled": dict(_EMPTY), + "accumulated": dict(_EMPTY), + } + + def process_single(info: dict, now: datetime.datetime, closing: bool = False) -> None: cfg = _cfg() ns, uuid = info["namespace"], info["uuid"] @@ -529,8 +686,18 @@ def process_datacenter(dc: str, now: datetime.datetime) -> None: def _live_window(pointer_iso: str) -> tuple[float, float]: + """ + `[pointer_ts, now]`, clamped on the left so a long-dormant pointer does not + trigger a huge query. No ingestion-lag cushion: Mimir data is immutable + once ingested (never revised), so querying right up to "now" cannot return + a wrong value -- worst case, a sample scraped a moment ago is not visible + *yet* and simply is not the one returned as "last", which self-corrects on + the very next call. Used by `topup` for both a real, persisted accumulator + and `live_only_doc`'s transient one (nothing persisted): both get the + freshest number Mimir can currently give. + """ cfg = _cfg() - end_ts = _now().timestamp() - cfg["topup_lag_seconds"] + end_ts = _now().timestamp() start_ts = max( compute._parse_iso(pointer_iso), end_ts - (cfg["sweep_seconds"] + 4 * cfg["series_bucket_seconds"]), @@ -555,7 +722,10 @@ def _live_buckets(alloc_pairs, start_ts: float, end_ts: float, bucket_s: int): """ Integrate the live window from Mimir into `bucket_s` buckets (per DC, merged), plus the current draw from the last sample of that same query. Returns - `(buckets, power_w)`; `power_w` is `None` when nothing is fresh. + `(buckets, power_w)`; `power_w` is `None` only when a series has no sample + at all in the window (nothing to read yet), never because the last sample + is "too old" -- Mimir data is immutable once ingested, so the actual last + point is always the right one to show, however old it is. """ cfg = _cfg() step_s = int(cfg["query_step_seconds"]) @@ -567,7 +737,6 @@ def _live_buckets(alloc_pairs, start_ts: float, end_ts: float, bucket_s: int): by_dc.setdefault(dc, []).append(alloc_id) start_dt, end_dt = _dt(start_ts), _dt(end_ts) - fresh_after = end_ts - 2 * step_s per_dc = [] inst_w = 0.0 any_inst = False @@ -591,7 +760,7 @@ def _live_buckets(alloc_pairs, start_ts: float, end_ts: float, bucket_s: int): buckets, metrics.get("carbon"), metrics.get("water") ) ) - iw = compute.instant_power(cpu, gpu, _tue(dc), fresh_after, dq) + iw = compute.instant_power(cpu, gpu, _tue(dc), dq) if iw is not None: inst_w += iw any_inst = True diff --git a/docs/energy-accounting.md b/docs/energy-accounting.md index 4645ed87..d4f3f7a8 100644 --- a/docs/energy-accounting.md +++ b/docs/energy-accounting.md @@ -31,7 +31,7 @@ The TUE is read from the `TUE` column of `var/datacenters.csv`, only populated f ## Architecture -Two layers: a durable background sweep that owns the persisted record, and a live read-time top-up that makes the dashboard counter move. +Four layers: a durable background sweep that owns the persisted record, a live read-time top-up on top of it, an on-demand version of the sweep for a deployment the background thread has not reached yet, and a pure-Mimir fallback for one still too young to have anything settled at all. ``` background thread in ai4papi.main: @repeat_every(energy.sweep_seconds) @@ -59,6 +59,8 @@ Two layers: a durable background sweep that owns the persisted record, and a liv The shape of the deployment `energy` block follows the existing `full_info` query param: `full_info=false` (the list default) attaches the accumulated `EnergyStats`; `full_info=true` (the individual-endpoint default) attaches the extended `EnergyTimeSeries`. +The diagram above is the steady state, once a deployment has a persisted doc. Before that, the read path itself does the work the diagram shows the background thread doing (Layer 3), and short of that, falls back to querying Mimir directly with nothing persisted at all (Layer 4) -- see both below. + ### Layer 1: durable sweep `accounting.sweep.run_sweep()` runs on a `@repeat_every(energy.sweep_seconds)` thread started in `ai4papi.main`'s `lifespan`, next to `get_cluster_stats_thread`. Same posture: not async, one `try/except` per deployment, no-op if the feature is disabled. @@ -91,15 +93,30 @@ Note on double counting: scaphandre host power is RAPL (CPU package + DRAM), it ### Layer 2: read-time top-up -The stored accumulator only reaches `pointer_ts` (up to one sweep interval behind). On each API read, `accounting.sweep.topup(doc)` queries Mimir live for `[pointer_ts, now - topup_lag_seconds]` (a single `query_range`, no chunking), applies the TUE and the current Wattnet intensity, and returns the energy deltas plus `live_as_of` and `power_w`. `power_w` is the **current draw**: the TUE-normalized sum of the last (sanitized, still fresh) sample of every series in that same query, not a window mean. The result is cached for `topup_cache_seconds` (25 s, below the dashboard's 30 s poll, so the number visibly moves). Any failure returns `{"degraded": true}` and the API serves the stored value. +The stored accumulator only reaches `pointer_ts` (up to one sweep interval behind). On each API read, `accounting.sweep.topup(doc)` queries Mimir live for `[pointer_ts, now]` (a single `query_range`, no chunking, no ingestion-lag cushion -- Mimir data is immutable once ingested, so querying right up to "now" cannot return a wrong value, only occasionally miss a sample scraped a moment ago that is not visible *yet*, which self-corrects on the very next call), applies the TUE and the current Wattnet intensity, and returns the energy deltas plus `live_as_of` and `power_w`. `power_w` is the **current draw**: the TUE-normalized sum of the actual last sample of every series in that same query, however old that sample is, not a window mean. The result is cached for `topup_cache_seconds` (25 s, below the dashboard's 30 s poll, so the number visibly moves). Any failure returns `{"degraded": true}` and the API serves the stored value. `topup_cluster(dc, doc)` / `live_series_cluster(dc, doc, bucket_s)` are the datacenter equivalents (same window logic, `_cluster_promql` instead of the per-alloc query). They are computed inside `get_cluster_stats_bg()` (which runs every 30 s anyway), so the datacenter counter moves at the same cadence as the per-deployment one. `accumulated (stored, frozen for up to a sweep) + top-up (growing)` stays continuous: when the next sweep advances `pointer_ts` and folds that slice into `accumulated`, the top-up window shrinks by the same amount. +### Layer 3: on-demand sweep + +A deployment the background thread has not reached yet has no `.accum.json`, so the top-up (which reads `pointer_ts` off the stored doc) has nothing to attach to. Rather than making a fresh deployment wait up to `sweep_seconds` for its first numbers, every read path runs one `process_single` pass synchronously (the same function and same write path the background thread uses) for whatever it is about to serve and has no doc yet: + +- Single-deployment reads (`get_accumulated`, `get_series`) call `accounting.sweep.ensure_swept(namespace, uuid)`: fetch that one Nomad job, sweep it, return the resulting doc. +- Bulk/per-user reads (`get_accumulated_bulk`, `get_user_energy`) call `accounting.sweep.ensure_swept_bulk(namespace, owner)` first: list the owner's active jobs (one cheap Nomad call, no per-job fetch) and sweep only the ones still missing a doc, before falling back to the normal `store.iter_owner` scan. + +Once a doc exists, both are a single cheap `store.read_accum` check (or, for the bulk path, one job-list call) and defer entirely to the background sweep; neither re-queries Mimir beyond that first pass. A job Nomad no longer knows about (or a listing failure) degrades to `energy: null` / that deployment simply absent from the aggregate. `/stats/cluster` (`get_cluster_energy`, per-datacenter) is not covered by this: its instantaneous power is already live regardless (`get_datacenter_power`), and its accumulator is seeded by the small, fixed `metrics_datacenters` list, not by individual deployments. + +`process_single` also persists a bare stub (no `accumulated` key, no `metrics_available` verdict either way) the first time it runs for a job with no allocation resolved at all yet (eg. queued, or Nomad has not placed the allocation on a node). That stub is not a negative result, just "too early to tell": the read path treats "doc is `None`" and "doc exists but has no `accumulated` and `metrics_available` is not explicitly `False`" the same way, both falling through to Layer 4. Only an explicit `metrics_available: False` (a confirmed gap, set after the deployment has run long enough that the absence of data is meaningful) stops the fallback and returns `null` outright, deferring to the next real sweep. + +### Layer 4: pure-Mimir fallback + +`process_single` leaves a deployment with **no** doc when it is younger than one `series_bucket_seconds`: there is nothing settled to fold into an accumulator yet, so Layer 3 alone still reports `energy: null` for the first few minutes of a deployment's life. `sweep.live_only_doc(ns, uuid)` closes that gap: a transient, **never persisted** stand-in doc (`accumulated` all zero, `pointer_ts = submit_time`, the currently-running allocations) shaped so the normal `topup` / `live_series` machinery treats it exactly like a real doc -- the whole reported number then comes straight from Mimir, through the same lag-free `_live_window` as Layer 2. `accounting._live_only_stats()` wraps it and only returns something when the top-up actually found data (`power_w` set, or a non-zero energy/carbon/water delta) -- an all-zero top-up is indistinguishable from "no measurement yet", so it is treated as not populated (`null`) rather than shown as a fabricated `0` reading. Used by all four read functions whenever they still have no usable doc after Layer 3. + ## Persistence -One small JSON per deployment under `$ACCOUNTING_PTH/energy//`, atomic writes (temp file + `os.replace`), the sweep thread is the only writer. Same convention as `ai4papi.utils.retrieve_from_snapshots`. No database. The `accounting.store` interface is deliberately small so it can move to PostgreSQL later without touching the routers. +One small JSON per deployment under `$ACCOUNTING_PTH/energy//`, atomic writes (temp file + `os.replace`). `process_single` / `process_datacenter` are the only writers, run either by the periodic sweep thread or, for a single deployment's very first doc, synchronously from the read path (see [Layer 3](#layer-3-on-demand-sweep)). Same convention as `ai4papi.utils.retrieve_from_snapshots`. No database. The `accounting.store` interface is deliberately small so it can move to PostgreSQL later without touching the routers. `.accum.json` (rewritten every sweep, ~25 KB with a full 24h tail): @@ -204,7 +221,7 @@ deployments set only on the per-user aggregate: number of deployments s series 15-min EnergyTimeSeriesPoint list; null unless full_info=true (per-user aggregate only; the per-deployment full_info uses EnergyTimeSeries instead) ``` -`power_w` is a rate, not an accumulator, so it is never summed. The `power_w` in the block (deployment, user, datacenter) is the **current draw**: the last power sample read from Mimir. Deployment and user take it from the last point of the top-up range query (`compute.instant_power`, which drops stale or spiky last samples); the datacenter takes it from a dedicated instant query (`get_datacenter_power`). `series[].power_w` in the time series is different: a **mean** over its 15-min bucket (so `energy_wh ~= power_w * bucket_hours`). +`power_w` is a rate, not an accumulator, so it is never summed. The `power_w` in the block (deployment, user, datacenter) is the **current draw**: the last power sample read from Mimir, however old it is (`compute.instant_power` does not drop it for being stale -- Mimir data is immutable once ingested, so the actual last sample is always the right one to show; it does still drop spiky samples via `sanitize`'s data-quality filter). Deployment and user take it from the last point of the top-up range query; the datacenter takes it from a dedicated instant query (`get_datacenter_power`). `series[].power_w` in the time series is different: a **mean** over its 15-min bucket (so `energy_wh ~= power_w * bucket_hours`). ### Time series, for the detail view @@ -239,7 +256,6 @@ energy: mimir_retention_days: 90 # tune to Mimir's actual retention footprint_revision_hours: 24 # re-compute footprint for buckets newer than this topup_cache_seconds: 25 # live top-up cache (< dashboard poll) - topup_lag_seconds: 25 # live top-up query stops this far before "now" deployment_series_points: 1000 # downsample target for the per-deployment series in /deployments/* user_series_points: 1000 # downsample target for the merged per-user series in /stats/user cluster_series_points: 500 # downsample target for the per-datacenter series in /stats/cluster @@ -261,7 +277,6 @@ energy: | Mimir scrape | 30 s | given | | Sweep thread | `sweep_seconds` (900) | keeps the accumulator current, consolidates the footprint at Wattnet's revision cadence, short enough to catch short jobs before Nomad GC | | Top-up cache | `topup_cache_seconds` (25) | below the dashboard's 30 s poll, so the counter moves | -| Top-up lag | `topup_lag_seconds` (25) | Mimir ingestion margin; the displayed value trails real time by ~25-30 s | | Cluster archive-series cache | 300 s | the `.series.jsonl` only changes at sweep cadence; keeps the 30 s thread from re-parsing it | For the user to perceive the increment every 30 s the frontend must show Wh (or kWh with 3 decimals) or animate the counter: at 50 W, 30 s is ~0.4 Wh. diff --git a/etc/main.yaml b/etc/main.yaml index 3fe209fe..9fbcd2d5 100644 --- a/etc/main.yaml +++ b/etc/main.yaml @@ -97,7 +97,6 @@ energy: # per-deployment energy consumption and environmental footprint stats mimir_retention_days: 90 # tune to Mimir's actual retention footprint_revision_hours: 24 # re-compute footprint for buckets newer than this topup_cache_seconds: 25 # live top-up cache (< dashboard poll so the counter moves) - topup_lag_seconds: 25 # live top-up query stops this far before "now" deployment_series_points: 1000 # downsample target for the per-deployment series in /deployments/* user_series_points: 1000 # downsample target for the merged per-user series in /stats/user (full_info=true) cluster_series_points: 500 # downsample target for the per-datacenter series in /stats/cluster (full_info=true) diff --git a/tests/test_energy.py b/tests/test_energy.py index 917940e3..1fb3436d 100644 --- a/tests/test_energy.py +++ b/tests/test_energy.py @@ -393,6 +393,323 @@ def _fake_qr_gpu_only(promql, start, end, step_s=30): print("🟢 energy: sweep pipeline checks passed!") +# --- on-demand sweep: read API does not wait for the background thread ------- + +_OND_UUID = "sweep-on-demand-uuid" +mimir.query_range = _fake_query_range +mimir.query_range_chunked = _fake_query_range +compute.resolve_alloc_ids = lambda job_id, ns: ( + {_DC: ["alloc-o"]}, + [compute.AllocWindow("alloc-o", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-o", "datacenter": _DC}], + False, +) + + +def _fake_get_job(id_, namespace): + if id_ != _OND_UUID: + raise KeyError(id_) + return { + "ID": _OND_UUID, + "Name": f"module-{_OND_UUID}", + "SubmitTime": int((_NOW.timestamp() - 3 * 3600) * 1e9), + "Meta": {"owner": "ondemand@egi.eu"}, + } + + +_orig_get_job = sweep.Nomad.job.get_job +sweep.Nomad.job.get_job = _fake_get_job + +# never swept: no doc yet, the read API must trigger one sweep pass itself +assert store.read_accum(_NS, _OND_UUID) is None +accounting._read_doc.cache_clear() +_ond_stats = accounting.get_accumulated(_NS, _OND_UUID, live=False) +assert _ond_stats is not None and _ond_stats["energy_wh"] > 0, _ond_stats +assert store.read_accum(_NS, _OND_UUID) is not None + +# already swept once (on-demand or not): no further Nomad call needed +def _boom(id_, namespace): + raise AssertionError("ensure_swept should not touch Nomad once a doc exists") + + +sweep.Nomad.job.get_job = _boom +accounting._read_doc.cache_clear() +assert accounting.get_accumulated(_NS, _OND_UUID, live=False) is not None + +# job unknown to Nomad: on-demand sweep finds nothing, degrades to `None`, no crash +sweep.Nomad.job.get_job = _fake_get_job +accounting._read_doc.cache_clear() +assert accounting.get_accumulated(_NS, "sweep-unknown-uuid", live=False) is None +accounting._read_doc.cache_clear() +assert accounting.get_series(_NS, "sweep-unknown-uuid") is None + +sweep.Nomad.job.get_job = _orig_get_job + +print("🟢 energy: on-demand sweep checks passed!") + + +# --- live-only fallback: deployment younger than one bucket, no doc at all --- +# `ensure_swept` correctly leaves such a deployment with no doc (nothing has +# settled into a full 15-min bucket yet); the read API must instead fall back +# to a pure live Mimir number -- but ONLY once Mimir actually has a sample for +# it, never a fabricated `0`. + +# anchored just after the start of the current 15-min bucket (not "N minutes +# before now") so `floor(submit, bucket_s) == floor(now, bucket_s)` regardless +# of where in the bucket the suite happens to run -- "now - 3 minutes" would +# occasionally land in the *previous* bucket when run near a :00/:15/:30/:45 +# boundary, making `process_single` see a full elapsed bucket to integrate and +# spuriously write a real doc, which is exactly the "too young" case this +# section means to test. +_live_submit = sweep._floor(_NOW.timestamp(), 900) + 60 +compute.resolve_alloc_ids = lambda job_id, ns: ( + {_DC: ["alloc-live"]}, + [compute.AllocWindow("alloc-live", _live_submit, None)], + [{"alloc_id": "alloc-live", "datacenter": _DC}], + False, +) + +# Mimir has real (fake) samples for this young deployment -> populated, not null +_LIVE_UUID = "sweep-live-only-uuid" + + +def _fake_get_job_live(id_, namespace): + if id_ != _LIVE_UUID: + raise KeyError(id_) + return { + "ID": _LIVE_UUID, + "Name": f"module-{_LIVE_UUID}", + "SubmitTime": int(_live_submit * 1e9), + "Meta": {"owner": "live@egi.eu"}, + } + + +mimir.query_range = _fake_query_range +mimir.query_range_chunked = _fake_query_range +sweep.Nomad.job.get_job = _fake_get_job_live + +assert store.read_accum(_NS, _LIVE_UUID) is None +accounting._read_doc.cache_clear() +_live_stats = accounting.get_accumulated(_NS, _LIVE_UUID, live=True) +assert _live_stats is not None and _live_stats["energy_wh"] > 0, _live_stats +assert _live_stats["power_w"] is not None +# `since` is pinned to the exact submit time (not clamped/started late), so +# `complete` must be `True` even though nothing is persisted yet +assert _live_stats["complete"] is True, _live_stats +# too young for a bucket: `process_single` still writes nothing, on purpose +assert store.read_accum(_NS, _LIVE_UUID) is None + +accounting._read_doc.cache_clear() +_live_series = accounting.get_series(_NS, _LIVE_UUID) +assert _live_series is not None and _live_series["accumulated"]["energy_wh"] > 0 + +# `power_w` must reflect the actual last Mimir sample even when it is stale +# (older than the `2 * query_step_seconds` freshness cushion `topup` itself +# enforces) -- there is nothing persisted here for a stale reading to mislead. +_STALE_UUID = "sweep-live-only-stale-uuid" +_stale_submit = (_NOW - datetime.timedelta(minutes=10)).timestamp() + + +def _fake_get_job_stale(id_, namespace): + if id_ != _STALE_UUID: + raise KeyError(id_) + return { + "ID": _STALE_UUID, + "Name": f"module-{_STALE_UUID}", + "SubmitTime": int(_stale_submit * 1e9), + "Meta": {"owner": "live@egi.eu"}, + } + + +def _fake_query_range_stale(promql, start, end, step_s=30): + # no sample within the last 5 minutes -- well past the 60s cushion + stale_end = end.timestamp() - 300 + t, out = start.timestamp(), [] + while t <= stale_end: + out.append([t, "100000000" if "scaph" in promql else "0"]) + t += step_s + return [{"metric": {}, "values": out}] + + +compute.resolve_alloc_ids = lambda job_id, ns: ( + {_DC: ["alloc-stale"]}, + [compute.AllocWindow("alloc-stale", _stale_submit, None)], + [{"alloc_id": "alloc-stale", "datacenter": _DC}], + False, +) +mimir.query_range = _fake_query_range_stale +mimir.query_range_chunked = _fake_query_range_stale +sweep.Nomad.job.get_job = _fake_get_job_stale +sweep._live_only_doc_cache.clear() +sweep._live_cache.clear() + +# exercise `live_only_doc` / `topup` directly (not through `get_accumulated`'s +# `ensure_swept`): whether `process_single` would also consider this +# deployment "too young" depends on where in the current 15-min bucket the +# suite happens to run, which is irrelevant to what this checks. +_stale_doc = sweep.live_only_doc(_NS, _STALE_UUID) +assert _stale_doc is not None, _stale_doc +_stale_topup = sweep.topup(_stale_doc) +assert _stale_topup is not None and _stale_topup["power_w"] is not None, _stale_topup +assert _stale_topup["power_w"] > 0, _stale_topup + +# Mimir has NOTHING for this alloc yet -> must stay `null`, not a fabricated `0` +sweep._live_cache.clear() # fresh top-up cache: don't reuse the populated result + +_EMPTY_UUID = "sweep-live-only-empty-uuid" + + +def _fake_get_job_empty(id_, namespace): + if id_ != _EMPTY_UUID: + raise KeyError(id_) + return { + "ID": _EMPTY_UUID, + "Name": f"module-{_EMPTY_UUID}", + "SubmitTime": int(_live_submit * 1e9), + "Meta": {"owner": "live@egi.eu"}, + } + + +def _fake_query_range_empty(promql, start, end, step_s=30): + return [] + + +mimir.query_range = _fake_query_range_empty +mimir.query_range_chunked = _fake_query_range_empty +sweep.Nomad.job.get_job = _fake_get_job_empty +compute.resolve_alloc_ids = lambda job_id, ns: ( + {_DC: ["alloc-empty"]}, + [compute.AllocWindow("alloc-empty", _live_submit, None)], + [{"alloc_id": "alloc-empty", "datacenter": _DC}], + False, +) + +accounting._read_doc.cache_clear() +assert accounting.get_accumulated(_NS, _EMPTY_UUID, live=True) is None +accounting._read_doc.cache_clear() +assert accounting.get_series(_NS, _EMPTY_UUID) is None +assert store.read_accum(_NS, _EMPTY_UUID) is None + +sweep.Nomad.job.get_job = _orig_get_job +mimir.query_range = _fake_query_range +mimir.query_range_chunked = _fake_query_range + +print("🟢 energy: live-only fallback checks passed!") + + +# --- an inconclusive stub must not permanently block the live-only fallback -- +# `process_single` persists a bare stub (no "accumulated" key, no +# `metrics_available` verdict) the first time it sees a job with no allocation +# resolved at all yet (eg. queued, alloc not placed by Nomad). That stub must +# not stop later reads from trying `live_only_doc` once Mimir has real data. + +_STUB_UUID = "sweep-inconclusive-stub-uuid" +_stub_submit = (_NOW - datetime.timedelta(minutes=1)).timestamp() + + +def _fake_get_job_stub(id_, namespace): + if id_ != _STUB_UUID: + raise KeyError(id_) + return { + "ID": _STUB_UUID, + "Name": f"module-{_STUB_UUID}", + "SubmitTime": int(_stub_submit * 1e9), + "Meta": {"owner": "stub@egi.eu"}, + } + + +sweep.Nomad.job.get_job = _fake_get_job_stub + +# first on-demand attempt: no allocation resolved at all yet +compute.resolve_alloc_ids = lambda job_id, ns: ({}, [], [], False) +mimir.query_range = _fake_query_range +mimir.query_range_chunked = _fake_query_range + +assert store.read_accum(_NS, _STUB_UUID) is None +accounting._read_doc.cache_clear() +assert accounting.get_accumulated(_NS, _STUB_UUID, live=True) is None +_stub_doc = store.read_accum(_NS, _STUB_UUID) +assert _stub_doc is not None and "accumulated" not in _stub_doc, _stub_doc +assert _stub_doc.get("metrics_available") is not False # inconclusive, not a verdict + +# the allocation is now placed and Mimir has real samples: must populate, not +# stay `null` because of the stub written above +compute.resolve_alloc_ids = lambda job_id, ns: ( + {_DC: ["alloc-stub"]}, + [compute.AllocWindow("alloc-stub", _stub_submit, None)], + [{"alloc_id": "alloc-stub", "datacenter": _DC}], + False, +) +sweep._live_cache.clear() +sweep._live_only_doc_cache.clear() # `live_only_doc` itself is cached now too +accounting._read_doc.cache_clear() +_stub_stats = accounting.get_accumulated(_NS, _STUB_UUID, live=True) +assert _stub_stats is not None and _stub_stats["energy_wh"] > 0, _stub_stats + +sweep.Nomad.job.get_job = _orig_get_job + +print("🟢 energy: inconclusive-stub fallback checks passed!") + + +# --- on-demand bulk sweep: list / per-user reads catch up too ---------------- + +_BULK_UUID = "sweep-bulk-uuid" +_BULK_OWNER = "bulk@egi.eu" +mimir.query_range = _fake_query_range +mimir.query_range_chunked = _fake_query_range +compute.resolve_alloc_ids = lambda job_id, ns: ( + {_DC: ["alloc-b"]}, + [compute.AllocWindow("alloc-b", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-b", "datacenter": _DC}], + False, +) + + +def _fake_get_jobs(namespace, filter_=None): + return [{"ID": _BULK_UUID, "Name": f"module-{_BULK_UUID}"}] + + +def _fake_get_job_bulk(id_, namespace): + if id_ != _BULK_UUID: + raise KeyError(id_) + return { + "ID": _BULK_UUID, + "Name": f"module-{_BULK_UUID}", + "SubmitTime": int((_NOW.timestamp() - 3 * 3600) * 1e9), + "Meta": {"owner": _BULK_OWNER}, + } + + +_orig_get_jobs = sweep.Nomad.jobs.get_jobs +_orig_get_job = sweep.Nomad.job.get_job +sweep.Nomad.jobs.get_jobs = _fake_get_jobs +sweep.Nomad.job.get_job = _fake_get_job_bulk + +# never swept: the bulk/list read must sweep it on demand before serving +assert store.read_accum(_NS, _BULK_UUID) is None +_bulk = accounting.get_accumulated_bulk(_NS, _BULK_OWNER, live=False) +assert _BULK_UUID in _bulk and _bulk[_BULK_UUID]["energy_wh"] > 0, _bulk +assert store.read_accum(_NS, _BULK_UUID) is not None + +_ue = accounting.get_user_energy(_NS, _BULK_OWNER, series=False) +assert _ue is not None and _ue["deployments"] == 1, _ue + +# already swept: the listing is still cheap (Nomad.jobs.get_jobs), but no +# per-job fetch / re-sweep for a uuid that already has a doc +def _boom_get_job(id_, namespace): + raise AssertionError("ensure_swept_bulk should not re-fetch a swept job") + + +sweep.Nomad.job.get_job = _boom_get_job +_bulk2 = accounting.get_accumulated_bulk(_NS, _BULK_OWNER, live=False) +assert _BULK_UUID in _bulk2 + +sweep.Nomad.jobs.get_jobs = _orig_get_jobs +sweep.Nomad.job.get_job = _orig_get_job + +print("🟢 energy: on-demand bulk sweep checks passed!") + + # --- cluster energy: instantaneous power + accumulator + series -------------- _CNS = sweep.CLUSTER_NS From 330e8afd952c0c537c9d8447a3107238416ef6bb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:49:23 +0000 Subject: [PATCH 11/25] ci: auto-format from pre-commit.com hooks --- ai4papi/accounting/__init__.py | 4 +--- ai4papi/accounting/sweep.py | 7 ++++--- tests/test_energy.py | 2 ++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ai4papi/accounting/__init__.py b/ai4papi/accounting/__init__.py index e3876ce4..79d16b2d 100644 --- a/ai4papi/accounting/__init__.py +++ b/ai4papi/accounting/__init__.py @@ -122,9 +122,7 @@ def _live_only_stats(namespace: str, uuid: str) -> tuple[dict, dict] | None: if doc is None: return None stats = _apply_topup(_to_stats(doc), doc) - has_data = stats.get("power_w") is not None or any( - stats.get(k, 0.0) for k in _KEYS - ) + has_data = stats.get("power_w") is not None or any(stats.get(k, 0.0) for k in _KEYS) if not has_data: return None return doc, stats diff --git a/ai4papi/accounting/sweep.py b/ai4papi/accounting/sweep.py index 50ac5fad..4188a7d4 100644 --- a/ai4papi/accounting/sweep.py +++ b/ai4papi/accounting/sweep.py @@ -336,9 +336,10 @@ def live_only_doc(ns: str, uuid: str) -> dict | None: # pin it to at all (job fetched with no `SubmitTime`, an edge case) or it # somehow predates the lookback window (impossible in practice: this path # only ever runs for a deployment too young to have a doc yet). - complete = bool(submit) and submit >= _now().timestamp() - _cfg()[ - "initial_lookback_hours" - ] * 3600 + complete = ( + bool(submit) + and submit >= _now().timestamp() - _cfg()["initial_lookback_hours"] * 3600 + ) return { "deployment_uuid": uuid, "namespace": ns, diff --git a/tests/test_energy.py b/tests/test_energy.py index 1fb3436d..b2a040f2 100644 --- a/tests/test_energy.py +++ b/tests/test_energy.py @@ -427,6 +427,7 @@ def _fake_get_job(id_, namespace): assert _ond_stats is not None and _ond_stats["energy_wh"] > 0, _ond_stats assert store.read_accum(_NS, _OND_UUID) is not None + # already swept once (on-demand or not): no further Nomad call needed def _boom(id_, namespace): raise AssertionError("ensure_swept should not touch Nomad once a doc exists") @@ -694,6 +695,7 @@ def _fake_get_job_bulk(id_, namespace): _ue = accounting.get_user_energy(_NS, _BULK_OWNER, series=False) assert _ue is not None and _ue["deployments"] == 1, _ue + # already swept: the listing is still cheap (Nomad.jobs.get_jobs), but no # per-job fetch / re-sweep for a uuid that already has a doc def _boom_get_job(id_, namespace): From a70c7da39caa2d5d1405cc344b112b22071075da Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 17 Sep 2026 13:53:09 +0200 Subject: [PATCH 12/25] fix(energy): avoid log injection via unsanitized deployment/owner ids ns/uuid/owner in these warnings can originate from request path/auth parameters that are not validated as strict identifiers before reaching here; %r escapes embedded newlines instead of writing them raw, so they cannot be used to forge log lines. --- ai4papi/accounting/sweep.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ai4papi/accounting/sweep.py b/ai4papi/accounting/sweep.py index 4188a7d4..0d02175c 100644 --- a/ai4papi/accounting/sweep.py +++ b/ai4papi/accounting/sweep.py @@ -229,7 +229,7 @@ def ensure_swept(ns: str, uuid: str) -> dict | None: try: process_single(_info_from_job(j, ns), _now()) except Exception: - LOG.warning("on-demand energy sweep failed for %s/%s", ns, uuid, exc_info=True) + LOG.warning("on-demand energy sweep failed for %r/%r", ns, uuid, exc_info=True) return None return store.read_accum(ns, uuid) @@ -268,7 +268,7 @@ def ensure_swept_bulk(ns: str, owner: str) -> list[dict]: filter_=f'Meta.owner == "{owner}" and Status != "dead"', ) except Exception: - LOG.warning("could not list jobs for on-demand sweep (%s/%s)", ns, owner) + LOG.warning("could not list jobs for on-demand sweep (%r/%r)", ns, owner) return [] now = _now() stubs = [s for s in jobs if s["Name"].startswith(_JOB_PREFIXES)] @@ -280,7 +280,7 @@ def ensure_swept_bulk(ns: str, owner: str) -> list[dict]: process_single(_info_from_job(j, ns), now) except Exception: LOG.warning( - "on-demand energy sweep failed for %s/%s", ns, stub["ID"], exc_info=True + "on-demand energy sweep failed for %r/%r", ns, stub["ID"], exc_info=True ) return stubs From b965561345bd13a2c02640399fee56eb5fe62b8f Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 17 Sep 2026 13:57:23 +0200 Subject: [PATCH 13/25] fix(energy): sanitize logged identifiers, not just reformat them The previous fix (%r instead of %s) doesn't register as sanitization to SonarCloud's taint tracker, so the log-injection finding stayed open. Strip CR/LF explicitly via a small helper before these values reach any log call. --- ai4papi/accounting/sweep.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/ai4papi/accounting/sweep.py b/ai4papi/accounting/sweep.py index 0d02175c..cf24b86f 100644 --- a/ai4papi/accounting/sweep.py +++ b/ai4papi/accounting/sweep.py @@ -25,6 +25,14 @@ LOG = logging.getLogger(__name__) + +def _log_safe(value: str) -> str: + """Strip CR/LF from an identifier that may come straight from a request + path or auth token before it is interpolated into a log message, so it + cannot be used to forge log lines (CWE-117).""" + return str(value).replace("\r", "").replace("\n", "") + + _JOB_PREFIXES = ("module", "tool", "batch", "try") _KEYS = ("energy_wh", "carbon_g", "water_l") # TUE-normalized totals kept per accum _EMPTY = dict.fromkeys(_KEYS, 0.0) @@ -229,7 +237,12 @@ def ensure_swept(ns: str, uuid: str) -> dict | None: try: process_single(_info_from_job(j, ns), _now()) except Exception: - LOG.warning("on-demand energy sweep failed for %r/%r", ns, uuid, exc_info=True) + LOG.warning( + "on-demand energy sweep failed for %s/%s", + _log_safe(ns), + _log_safe(uuid), + exc_info=True, + ) return None return store.read_accum(ns, uuid) @@ -268,7 +281,11 @@ def ensure_swept_bulk(ns: str, owner: str) -> list[dict]: filter_=f'Meta.owner == "{owner}" and Status != "dead"', ) except Exception: - LOG.warning("could not list jobs for on-demand sweep (%r/%r)", ns, owner) + LOG.warning( + "could not list jobs for on-demand sweep (%s/%s)", + _log_safe(ns), + _log_safe(owner), + ) return [] now = _now() stubs = [s for s in jobs if s["Name"].startswith(_JOB_PREFIXES)] @@ -280,7 +297,10 @@ def ensure_swept_bulk(ns: str, owner: str) -> list[dict]: process_single(_info_from_job(j, ns), now) except Exception: LOG.warning( - "on-demand energy sweep failed for %r/%r", ns, stub["ID"], exc_info=True + "on-demand energy sweep failed for %s/%s", + _log_safe(ns), + _log_safe(stub["ID"]), + exc_info=True, ) return stubs From 2d3050d15a341c59d8a4675f68928d6b4278b61b Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Mon, 19 Oct 2026 20:51:23 +0200 Subject: [PATCH 14/25] refactor(energy): rename accounting to energy and split the sweep by job Rename the accounting package to energy and move the Mimir client to ai4papi/clients. The 900-line sweep module is split by responsibility: - config: the energy config section, enabled(), TUE and shared helpers - query: the Mimir power queries shared by the sweep and the live path - sweep: the background sweep (write path), deployments and datacenters - live: the live top-up and live series, one code path for deployment and datacenter docs - ondemand: sweeping a deployment from the read path, live-only doc - series: building the time-series points of the API responses Duplicated logic is merged: one series builder for deployment, user and datacenter, one owner aggregate, one deployment resolver shared by the accumulated and series reads, and add_to_job / add_to_jobs replace the energy block copied in the four deployment routers. Also in this change: - TTL caches are created with a lock, as endpoints run in a thread pool - the store keeps an in-memory owner index instead of scanning the whole namespace on each list, and reads a repeated series bucket only once - a failing Mimir query now propagates (the sweep skips that deployment, the live top-up reports degraded) and the sweep stops its pass when Mimir is unavailable - the shared datacenter query lives in compute.cluster_promql --- ai4papi/accounting/sweep.py | 921 --------------------- ai4papi/clients/__init__.py | 0 ai4papi/{ => clients}/mimir.py | 0 ai4papi/{accounting => energy}/__init__.py | 367 ++++---- ai4papi/{accounting => energy}/compute.py | 41 +- ai4papi/energy/config.py | 81 ++ ai4papi/energy/live.py | 169 ++++ ai4papi/energy/ondemand.py | 185 +++++ ai4papi/energy/query.py | 22 + ai4papi/energy/series.py | 61 ++ ai4papi/{accounting => energy}/store.py | 42 +- ai4papi/energy/sweep.py | 447 ++++++++++ ai4papi/main.py | 4 +- ai4papi/routers/v1/batch.py | 20 +- ai4papi/routers/v1/deployments/modules.py | 20 +- ai4papi/routers/v1/deployments/tools.py | 20 +- ai4papi/routers/v1/stats/deployments.py | 13 +- ai4papi/routers/v1/try_me/nomad.py | 18 +- 18 files changed, 1196 insertions(+), 1235 deletions(-) delete mode 100644 ai4papi/accounting/sweep.py create mode 100644 ai4papi/clients/__init__.py rename ai4papi/{ => clients}/mimir.py (100%) rename ai4papi/{accounting => energy}/__init__.py (63%) rename ai4papi/{accounting => energy}/compute.py (94%) create mode 100644 ai4papi/energy/config.py create mode 100644 ai4papi/energy/live.py create mode 100644 ai4papi/energy/ondemand.py create mode 100644 ai4papi/energy/query.py create mode 100644 ai4papi/energy/series.py rename ai4papi/{accounting => energy}/store.py (70%) create mode 100644 ai4papi/energy/sweep.py diff --git a/ai4papi/accounting/sweep.py b/ai4papi/accounting/sweep.py deleted file mode 100644 index cf24b86f..00000000 --- a/ai4papi/accounting/sweep.py +++ /dev/null @@ -1,921 +0,0 @@ -""" -Background sweep + live top-up for the energy-accounting feature. - -`run_sweep()` runs on a `@repeat_every` thread in `ai4papi.main`. Per deployment -(and per datacenter for `/stats/cluster`) it integrates the un-consolidated -window of Mimir power into energy buckets (`energy.series_bucket_seconds`, -15 min), applies the per-datacenter TUE and the Wattnet footprint, keeps a -running accumulator plus a ~24h "tail" that is re-computed every sweep (so the -footprint self-heals as Wattnet settles), and persists a small JSON document. - -`topup(doc)` is called on the read path: it adds the `pointer_ts -> now` slice -live from Mimir so the dashboard counter moves every ~30s. -""" - -import datetime -import logging - -from cachetools import TTLCache, cached - -import ai4papi.conf as papiconf -from ai4papi import mimir -from ai4papi.accounting import compute, store -from ai4papi.nomad_utils import Nomad -from ai4papi.wattnet import green_director - -LOG = logging.getLogger(__name__) - - -def _log_safe(value: str) -> str: - """Strip CR/LF from an identifier that may come straight from a request - path or auth token before it is interpolated into a log message, so it - cannot be used to forge log lines (CWE-117).""" - return str(value).replace("\r", "").replace("\n", "") - - -_JOB_PREFIXES = ("module", "tool", "batch", "try") -_KEYS = ("energy_wh", "carbon_g", "water_l") # TUE-normalized totals kept per accum -_EMPTY = dict.fromkeys(_KEYS, 0.0) - - -def _cfg() -> dict: - return papiconf.MAIN_CONF["energy"] - - -def enabled() -> bool: - return ( - bool(_cfg().get("enabled")) - and store.base_dir() is not None - and bool(mimir.MIMIR_USER) - ) - - -def _now() -> datetime.datetime: - return datetime.datetime.now(datetime.timezone.utc) - - -def _dt(ts: float) -> datetime.datetime: - return datetime.datetime.fromtimestamp(ts, datetime.timezone.utc) - - -def _floor(ts: float, step: int) -> float: - return (int(ts) // step) * step - - -def _earliest_bucket_ts(ns: str, uuid: str, doc: dict) -> float | None: - """Timestamp of the oldest 15-min bucket on record: the consolidated - `.series.jsonl` if there is one, else the oldest still in `doc["tail"]`.""" - archive = store.read_series(ns, uuid, None, None) - if archive: - return compute._parse_iso(archive[0]["ts"]) - tail = doc.get("tail") or [] - if tail: - return min(compute._parse_iso(b["ts"]) for b in tail) - return None - - -def _window_start( - ns: str, - uuid: str, - doc: dict, - settled_ts: float, - retention_cut: float, - bucket_s: int, -) -> float: - """ - True start of the accumulation window, exposed to the API as `since`. - - `settled_ts` cannot serve this role: it marches forward every sweep as - buckets age into `doc["settled"]`, while the accumulated totals (and the - persisted series) keep every bucket since the beginning. - - Pinned once and only moved forward if it falls behind Mimir retention: - - fresh doc: the first-sweep `settled_ts` is the window start; - - doc written before this field existed (`settled` / `tail` already there, - `settled_ts` already advanced): recover it from the oldest bucket on record. - """ - retention_floor = _floor(retention_cut, bucket_s) - if doc.get("window_start"): - return max(compute._parse_iso(doc["window_start"]), retention_floor) - if "settled" in doc or doc.get("tail"): - earliest = _earliest_bucket_ts(ns, uuid, doc) - if earliest is not None: - return max(min(earliest, settled_ts), retention_floor) - return max(settled_ts, retention_floor) - - -def _tue(dc: str) -> float: - return (papiconf.datacenters.get(dc) or {}).get("TUE") or _cfg()[ - "default_tue_factor" - ] - - -def _sum(buckets) -> dict: - return {k: sum(b[k] for b in buckets) for k in _KEYS} - - -def _add(a: dict, b: dict) -> dict: - return {k: a.get(k, 0.0) + b.get(k, 0.0) for k in _KEYS} - - -def _footprint_series(dc: str, start_ts: float, end_ts: float): - """ - (carbon, water) intensity series for a datacenter over [start, end]. - In steady state uses the 7-day window that `green_director` already keeps; - only fetches a wider range from Wattnet when backfilling further back. - `apply_footprint` falls back to DEFAULTS when a series is empty/None. - """ - metrics = green_director.metrics.get(dc, {}) - if start_ts >= _now().timestamp() - 7 * 86400: - return metrics.get("carbon"), metrics.get("water") - try: - fetched = green_director.footprint_series(dc, _dt(start_ts), _dt(end_ts)) - except Exception: - LOG.warning("wattnet footprint_series failed for %s", dc) - fetched = {} - return ( - fetched.get("carbon") or metrics.get("carbon"), - fetched.get("water") or metrics.get("water"), - ) - - -# --------------------------------------------------------------------------- -# sweep -# --------------------------------------------------------------------------- - - -def run_sweep() -> None: - if not enabled(): - return - now = _now() - for ns in set(papiconf.MAIN_CONF["nomad"]["namespaces"].values()): - try: - _sweep_namespace(ns, now) - except Exception: - LOG.exception("energy sweep failed for namespace %s", ns) - try: - _sweep_cluster(now) - except Exception: - LOG.exception("cluster energy sweep failed") - - -def _info_from_job(j: dict, ns: str) -> dict: - """`info` dict `process_single` expects, from a raw Nomad job.""" - meta = j.get("Meta") or {} - return { - "uuid": j["ID"], - "job_name": j["Name"], - "namespace": ns, - "owner": meta.get("owner", ""), - "submit_time": (j["SubmitTime"] / 1e9) if j.get("SubmitTime") else None, - "status": "running", - } - - -def _sweep_namespace(ns: str, now: datetime.datetime) -> None: - try: - jobs = Nomad.jobs.get_jobs(namespace=ns, filter_='Status != "dead"') - except Exception: - LOG.exception("could not list jobs for namespace %s", ns) - return - - seen = set() - for stub in jobs: - if not stub["Name"].startswith(_JOB_PREFIXES): - continue - seen.add(stub["ID"]) - # the job list stub does not carry Meta, fetch the full job for `owner` - try: - j = Nomad.job.get_job(id_=stub["ID"], namespace=ns) - except Exception: - LOG.exception("could not fetch job %s", stub["ID"]) - continue - try: - process_single(_info_from_job(j, ns), now) - except Exception: - LOG.exception("energy sweep failed for deployment %s", stub["ID"]) - - # close deployments whose Nomad job has been garbage-collected - for uuid, doc in store.iter_namespace(ns): - if uuid in seen or doc.get("closed_at"): - continue - info = { - "uuid": uuid, - "job_name": doc.get("job_name"), - "namespace": ns, - "owner": doc.get("owner", ""), - "submit_time": doc.get("submit_time_epoch"), - "status": "closed", - } - try: - process_single(info, now, closing=True) - except Exception: - LOG.exception("energy close failed for deployment %s", uuid) - - -def ensure_swept(ns: str, uuid: str) -> dict | None: - """ - On-demand equivalent of one sweep pass for a single deployment, called from - the read path when no accumulator doc exists yet: a deployment that already - has real samples in Mimir does not have to wait up to `sweep_seconds` for - the background thread to reach it before the API can report anything. - - A no-op (returns the existing doc untouched, no Nomad/Mimir call) once the - background sweep has processed this deployment at least once, so it never - duplicates the periodic sweep's work. `None` if the job cannot be found (not - a deployment, or already gone). - """ - if not enabled(): - return None - doc = store.read_accum(ns, uuid) - if doc is not None: - return doc - try: - j = Nomad.job.get_job(id_=uuid, namespace=ns) - except Exception: - return None - try: - process_single(_info_from_job(j, ns), _now()) - except Exception: - LOG.warning( - "on-demand energy sweep failed for %s/%s", - _log_safe(ns), - _log_safe(uuid), - exc_info=True, - ) - return None - return store.read_accum(ns, uuid) - - -_bulk_sweep_cache: TTLCache = TTLCache( - maxsize=2048, ttl=papiconf.MAIN_CONF["energy"].get("topup_cache_seconds", 25) -) - - -@cached(_bulk_sweep_cache) -def ensure_swept_bulk(ns: str, owner: str) -> list[dict]: - """ - On-demand equivalent of one sweep pass, restricted to one owner's active - deployments in `ns`: the bulk counterpart of `ensure_swept`, used by the - list (`get_accumulated_bulk`) and per-user (`get_user_energy`) read paths so - a just-deployed job shows up there too, not only on its own detail endpoint. - - Lists the owner's active jobs (cheap, no per-job fetch) and only runs - `process_single` for the ones with no accumulator doc yet; a no-op once the - background sweep (or a prior on-demand call) has reached all of them. - Cached for `topup_cache_seconds`: without it, every list/`/stats/user` read - would list Nomad on every single request, forever, not just during the - bootstrap window -- a few seconds of staleness for a listing that itself - only matters for brand-new deployments is a good trade. - - Returns the filtered job stubs (`[{"ID": ..., "Name": ...}, ...]`) so the - caller can also try `live_only_doc` for whichever of them still have no doc - afterwards (a deployment too young for even one completed bucket), without - listing Nomad a second time. `[]` when disabled or the listing itself fails. - """ - if not enabled(): - return [] - try: - jobs = Nomad.jobs.get_jobs( - namespace=ns, - filter_=f'Meta.owner == "{owner}" and Status != "dead"', - ) - except Exception: - LOG.warning( - "could not list jobs for on-demand sweep (%s/%s)", - _log_safe(ns), - _log_safe(owner), - ) - return [] - now = _now() - stubs = [s for s in jobs if s["Name"].startswith(_JOB_PREFIXES)] - for stub in stubs: - if store.read_accum(ns, stub["ID"]) is not None: - continue - try: - j = Nomad.job.get_job(id_=stub["ID"], namespace=ns) - process_single(_info_from_job(j, ns), now) - except Exception: - LOG.warning( - "on-demand energy sweep failed for %s/%s", - _log_safe(ns), - _log_safe(stub["ID"]), - exc_info=True, - ) - return stubs - - -_live_only_doc_cache: TTLCache = TTLCache( - maxsize=8192, ttl=papiconf.MAIN_CONF["energy"].get("topup_cache_seconds", 25) -) - - -@cached(_live_only_doc_cache) -def live_only_doc(ns: str, uuid: str) -> dict | None: - """ - Transient stand-in for a deployment `process_single` has not written any doc - for at all yet -- typically younger than one `series_bucket_seconds`, so it - has no completed bucket to fold into an accumulator (see `ensure_swept`, - which still leaves such a deployment with no doc by design: there is - nothing settled to persist). Never written to the store. - - Cached for `topup_cache_seconds`: without it, every read of a deployment - still in this bootstrap window would fetch its Nomad job and resolve its - allocations from scratch on every single request for as long as it stays - doc-less (which can be several minutes). A few seconds of staleness on - which allocation is "current" is a good trade against that. - - Shaped so the normal `topup` / `live_series` machinery -- and therefore - `accounting._to_stats` / `_apply_topup` / `_build_series` -- treats it like - a real doc with an all-zero accumulator and `pointer_ts = submit_time`: the - whole reported number then comes from the live top-up's Mimir query, capped - at the same lookback `topup` itself uses. `None` if the job cannot be - resolved to a running, monitored allocation (not found, queued, or no - allocation yet). - """ - if not enabled(): - return None - try: - j = Nomad.job.get_job(id_=uuid, namespace=ns) - except Exception: - return None - info = _info_from_job(j, ns) - try: - allocs_by_dc, _windows, running_allocs, _partial = compute.resolve_alloc_ids( - uuid, ns - ) - except Exception: - return None - if not running_allocs: - return None - submit = info.get("submit_time") - submit_ts = submit or _now().timestamp() - # `window_start` below is pinned to the exact submit time, same as - # `process_single`'s first-encounter `window_start` -- not started late or - # clamped, so `complete` is `True` unless we had no real `submit_time` to - # pin it to at all (job fetched with no `SubmitTime`, an edge case) or it - # somehow predates the lookback window (impossible in practice: this path - # only ever runs for a deployment too young to have a doc yet). - complete = ( - bool(submit) - and submit >= _now().timestamp() - _cfg()["initial_lookback_hours"] * 3600 - ) - return { - "deployment_uuid": uuid, - "namespace": ns, - "job_name": info.get("job_name"), - "owner": info.get("owner"), - "status": "running", - "metrics_available": True, - "complete": complete, - "window_start": compute._iso(submit_ts), - "settled_ts": compute._iso(submit_ts), - "pointer_ts": compute._iso(submit_ts), - "active_allocs": running_allocs, - "datacenters": sorted(allocs_by_dc), - "tail": [], - "settled": dict(_EMPTY), - "accumulated": dict(_EMPTY), - } - - -def process_single(info: dict, now: datetime.datetime, closing: bool = False) -> None: - cfg = _cfg() - ns, uuid = info["namespace"], info["uuid"] - bucket_s = int(cfg["series_bucket_seconds"]) - step_s = int(cfg["query_step_seconds"]) - now_ts = now.timestamp() - revision_cut = now_ts - cfg["footprint_revision_hours"] * 3600 - retention_cut = now_ts - cfg["mimir_retention_days"] * 86400 - lookback_ts = now_ts - cfg["initial_lookback_hours"] * 3600 - - doc = store.read_accum(ns, uuid) or {} - doc.pop("partial", None) # dropped in favour of `coverage_ratio` (always a float) - - # integration pointer: on first encounter never look back further than - # `initial_lookback_hours` (pre-existing deployments are not fully backfilled). - if doc.get("settled_ts"): - settled_ts = compute._parse_iso(doc["settled_ts"]) - complete = doc.get("complete", True) - else: - submit = info.get("submit_time") - settled_ts = max(submit or lookback_ts, lookback_ts) - complete = bool(submit) and submit >= lookback_ts - if settled_ts < retention_cut: - settled_ts, complete = retention_cut, False - settled_ts = _floor(settled_ts, bucket_s) - window_start = _window_start(ns, uuid, doc, settled_ts, retention_cut, bucket_s) - extract_to = _floor(now_ts - cfg["query_lag_seconds"], bucket_s) - - try: - allocs_by_dc, alloc_windows, running_allocs, partial_dc = ( - compute.resolve_alloc_ids(uuid, ns) - ) - except Exception: - allocs_by_dc, alloc_windows, running_allocs, partial_dc = {}, [], [], False - - # metadata (always refreshed) - doc.update( - { - "deployment_uuid": uuid, - "job_name": info.get("job_name") or doc.get("job_name"), - "namespace": ns, - "owner": info.get("owner") or doc.get("owner", ""), - "status": "closed" if closing else info.get("status", "running"), - "complete": complete, - "updated_at": compute._iso(now_ts), - "active_allocs": running_allocs, - } - ) - if info.get("submit_time"): - doc["submit_time"] = compute._iso(info["submit_time"]) - doc["submit_time_epoch"] = info["submit_time"] - if allocs_by_dc: - doc["datacenters"] = list(allocs_by_dc) - if closing and not doc.get("closed_at"): - doc["closed_at"] = compute._iso(now_ts) - - run_s = compute.running_seconds(alloc_windows, settled_ts, extract_to) - - # no monitored allocations: queued, or ran only in unmonitored sites - if not allocs_by_dc: - if "settled" not in doc and partial_dc and run_s > 2 * bucket_s: - doc.update({"metrics_available": False, "coverage_ratio": 0.0}) - store.write_accum(ns, uuid, doc) - return - - # integrate [settled_ts, extract_to] per datacenter - dq = cfg.get("data_quality") - new_buckets = [] - if settled_ts < extract_to: - span = extract_to - settled_ts - qfn = ( - mimir.query_range_chunked - if span > mimir.MAX_POINTS_PER_QUERY * step_s - else mimir.query_range - ) - start_dt, end_dt = _dt(settled_ts), _dt(extract_to) - per_dc = [] - tue_norm = tue_raw = 0.0 # energy-weighted mean TUE over the new buckets - for dc, alloc_ids in allocs_by_dc.items(): - cpu_q, gpu_q = compute.power_promql(alloc_ids) - # CPU and GPU are treated symmetrically: a missing metric already - # comes back as an empty result, and a transient query error on one - # source must not discard the other (a GPU-only monitored site has - # no `scaph_*` series at all). - try: - cpu = qfn(cpu_q, start_dt, end_dt, step_s) - except mimir.MimirError: - LOG.warning("mimir CPU query failed for %s/%s", ns, uuid) - cpu = [] - try: - gpu = qfn(gpu_q, start_dt, end_dt, step_s) - except mimir.MimirError: - LOG.warning("mimir GPU query failed for %s/%s", ns, uuid) - gpu = [] - if not cpu and not gpu: - continue - tue = _tue(dc) - buckets = compute.integrate( - cpu, gpu, step_s, bucket_s, tue, data_quality=dq - ) - if buckets: - e = sum(b.energy_wh for b in buckets) - tue_norm += e - tue_raw += e / tue - carbon_s, water_s = _footprint_series(dc, settled_ts, now_ts) - buckets = compute.apply_footprint(buckets, carbon_s, water_s) - per_dc.append(buckets) - if tue_raw: - doc["tue_factor"] = tue_norm / tue_raw - elif "tue_factor" not in doc and allocs_by_dc: - doc["tue_factor"] = _tue(next(iter(allocs_by_dc))) - new_buckets = [ - b - for b in compute.merge_buckets(per_dc) - if compute._parse_iso(b.ts) + bucket_s <= extract_to - ] - # clip to the deployment's actual life: drop buckets with no overlap with - # any allocation window. Guards against metric staleness (the per-alloc - # power series and `nomad_gpu_allocation_info` can linger a few minutes - # after an allocation ends, which would otherwise attribute idle GPU / - # host power to a finished deployment). - if alloc_windows: - new_buckets = [ - b - for b in new_buckets - if compute.running_seconds( - alloc_windows, - compute._parse_iso(b.ts), - compute._parse_iso(b.ts) + bucket_s, - ) - > 0 - ] - - # best-effort coverage: no data while the deployment was clearly running - # means the node has no monitoring stack yet -> null, retry next sweep. - if not new_buckets and "settled" not in doc: - if run_s > 2 * bucket_s: - doc.update({"metrics_available": False, "coverage_ratio": 0.0}) - store.write_accum(ns, uuid, doc) - return - - doc["metrics_available"] = True - doc["window_start"] = compute._iso(window_start) - - aged_n, tail_n = _fold_and_accumulate( - ns, uuid, doc, new_buckets, settled_ts, revision_cut, bucket_s - ) - - # coverage: fraction of the run time (including time on unmonitored sites, so - # `partial_dc` is captured implicitly) that ended up with power data. - covered_s = (tail_n + aged_n) * bucket_s - if run_s > 0: - doc["coverage_ratio"] = round(min(1.0, covered_s / run_s), 3) - else: - doc["coverage_ratio"] = 1.0 - - store.write_accum(ns, uuid, doc) - - -def _fold_and_accumulate( - ns: str, - uuid: str, - doc: dict, - new_buckets: list, - settled_ts: float, - revision_cut: float, - bucket_s: int, -) -> tuple[int, int]: - """ - Merge `new_buckets` (Bucket namedtuples) over `doc["tail"]` keyed by `ts`, - age the buckets older than `revision_cut` into `doc["settled"]` (appending - them to the persisted series), and recompute `settled_ts` / `accumulated` / - `pointer_ts` on `doc`. Returns `(aged_count, tail_count)`. - - Shared by the per-deployment (`process_single`) and per-datacenter - (`process_datacenter`) sweeps. - """ - settled = dict(doc.get("settled") or _EMPTY) - - by_ts = {b["ts"]: b for b in (doc.get("tail") or [])} - for b in new_buckets: - by_ts[b.ts] = b._asdict() - - tail, aged_out = [], [] - for ts in sorted(by_ts): - (aged_out if compute._parse_iso(ts) < revision_cut else tail).append(by_ts[ts]) - - if aged_out: - settled = _add(settled, _sum(aged_out)) - store.append_series(ns, uuid, aged_out) - settled_ts = max(compute._parse_iso(b["ts"]) for b in aged_out) + bucket_s - - doc["settled"] = settled - doc["tail"] = tail - doc["settled_ts"] = compute._iso(settled_ts) - doc["accumulated"] = _add(settled, _sum(tail)) - doc["pointer_ts"] = compute._iso( - (compute._parse_iso(tail[-1]["ts"]) + bucket_s) if tail else settled_ts - ) - return len(aged_out), len(tail) - - -# --------------------------------------------------------------------------- -# per-datacenter sweep (cluster-level energy accumulator) -# --------------------------------------------------------------------------- - -# Pseudo-namespace for the datacenter-level documents. Does not collide with any -# VO namespace nor with the deployment sweep's namespace loop. -CLUSTER_NS = "_cluster" - - -def _cluster_promql(dc: str) -> tuple[str, str]: - """ - (host_query, gpu_query) for a datacenter: one series per host and one per - GPU (grouped, not summed) so `compute.integrate` runs the data-quality - filter on each raw Mimir series before summing the energy. - """ - host_q = f'sum by (node) (scaph_host_power_microwatts{{datacenter="{dc}"}})' - gpu_q = f'sum by (UUID) (DCGM_FI_DEV_POWER_USAGE{{datacenter="{dc}"}})' - return host_q, gpu_q - - -def _cluster_dq() -> dict: - """ - Data-quality config for the datacenter sweep: reuse `energy.data_quality` - but drop the absolute `max_power_w` ceiling (it is calibrated per allocation - / per GPU, far below a whole host). Negative-drop and the scale-free spike - check still apply, per host / per GPU series. - """ - return {**(_cfg().get("data_quality") or {}), "max_power_w": 0} - - -def _sweep_cluster(now: datetime.datetime) -> None: - for dc in _cfg()["metrics_datacenters"]: - try: - process_datacenter(dc, now) - except Exception: - LOG.exception("cluster energy sweep failed for datacenter %s", dc) - - -def process_datacenter(dc: str, now: datetime.datetime) -> None: - cfg = _cfg() - bucket_s = int(cfg["series_bucket_seconds"]) - step_s = int(cfg["query_step_seconds"]) - now_ts = now.timestamp() - revision_cut = now_ts - cfg["footprint_revision_hours"] * 3600 - retention_cut = now_ts - cfg["mimir_retention_days"] * 86400 - lookback_ts = now_ts - cfg["initial_lookback_hours"] * 3600 - - doc = store.read_accum(CLUSTER_NS, dc) or {} - - if doc.get("settled_ts"): - settled_ts = compute._parse_iso(doc["settled_ts"]) - complete = doc.get("complete", True) - else: - settled_ts, complete = lookback_ts, False - if settled_ts < retention_cut: - settled_ts, complete = retention_cut, False - settled_ts = _floor(settled_ts, bucket_s) - window_start = _window_start( - CLUSTER_NS, dc, doc, settled_ts, retention_cut, bucket_s - ) - extract_to = _floor(now_ts - cfg["query_lag_seconds"], bucket_s) - - doc.update( - { - "datacenter": dc, - "deployment_uuid": dc, # reuse the same key name as deployment docs - "namespace": CLUSTER_NS, - "complete": complete, - "updated_at": compute._iso(now_ts), - "tue_factor": _tue(dc), - } - ) - - new_buckets = [] - if settled_ts < extract_to: - span = extract_to - settled_ts - qfn = ( - mimir.query_range_chunked - if span > mimir.MAX_POINTS_PER_QUERY * step_s - else mimir.query_range - ) - host_q, gpu_q = _cluster_promql(dc) - start_dt, end_dt = _dt(settled_ts), _dt(extract_to) - try: - host = qfn(host_q, start_dt, end_dt, step_s) - except mimir.MimirError: - LOG.warning("mimir host-power query failed for %s", dc) - host = [] - try: - gpu = qfn(gpu_q, start_dt, end_dt, step_s) - except mimir.MimirError: - gpu = [] - buckets = compute.integrate( - host, gpu, step_s, bucket_s, _tue(dc), data_quality=_cluster_dq() - ) - if buckets: - carbon_s, water_s = _footprint_series(dc, settled_ts, now_ts) - buckets = compute.apply_footprint(buckets, carbon_s, water_s) - new_buckets = [ - b for b in buckets if compute._parse_iso(b.ts) + bucket_s <= extract_to - ] - - if not new_buckets and "settled" not in doc: - # datacenter has no host-power series yet: retry next sweep, no zeros - doc["metrics_available"] = False - store.write_accum(CLUSTER_NS, dc, doc) - return - - doc["metrics_available"] = True - doc["window_start"] = compute._iso(window_start) - _fold_and_accumulate( - CLUSTER_NS, dc, doc, new_buckets, settled_ts, revision_cut, bucket_s - ) - store.write_accum(CLUSTER_NS, dc, doc) - - -# --------------------------------------------------------------------------- -# live path: `pointer_ts -> now` slice from Mimir (read side) -# --------------------------------------------------------------------------- - -_live_cache: TTLCache = TTLCache( - maxsize=8192, ttl=papiconf.MAIN_CONF["energy"].get("topup_cache_seconds", 25) -) - - -def _live_window(pointer_iso: str) -> tuple[float, float]: - """ - `[pointer_ts, now]`, clamped on the left so a long-dormant pointer does not - trigger a huge query. No ingestion-lag cushion: Mimir data is immutable - once ingested (never revised), so querying right up to "now" cannot return - a wrong value -- worst case, a sample scraped a moment ago is not visible - *yet* and simply is not the one returned as "last", which self-corrects on - the very next call. Used by `topup` for both a real, persisted accumulator - and `live_only_doc`'s transient one (nothing persisted): both get the - freshest number Mimir can currently give. - """ - cfg = _cfg() - end_ts = _now().timestamp() - start_ts = max( - compute._parse_iso(pointer_iso), - end_ts - (cfg["sweep_seconds"] + 4 * cfg["series_bucket_seconds"]), - ) - return start_ts, end_ts - - -def _running_key(doc: dict) -> tuple | None: - pointer = doc.get("pointer_ts") or doc.get("settled_ts") - running = doc.get("active_allocs") or [] - if not doc.get("metrics_available", False) or not pointer or not running: - return None - return ( - doc["namespace"], - doc["deployment_uuid"], - pointer, - tuple(sorted((a["alloc_id"], a["datacenter"]) for a in running)), - ) - - -def _live_buckets(alloc_pairs, start_ts: float, end_ts: float, bucket_s: int): - """ - Integrate the live window from Mimir into `bucket_s` buckets (per DC, merged), - plus the current draw from the last sample of that same query. Returns - `(buckets, power_w)`; `power_w` is `None` only when a series has no sample - at all in the window (nothing to read yet), never because the last sample - is "too old" -- Mimir data is immutable once ingested, so the actual last - point is always the right one to show, however old it is. - """ - cfg = _cfg() - step_s = int(cfg["query_step_seconds"]) - if end_ts - start_ts < step_s: - return [], None - - by_dc: dict[str, list[str]] = {} - for alloc_id, dc in alloc_pairs: - by_dc.setdefault(dc, []).append(alloc_id) - - start_dt, end_dt = _dt(start_ts), _dt(end_ts) - per_dc = [] - inst_w = 0.0 - any_inst = False - for dc, alloc_ids in by_dc.items(): - cpu_q, gpu_q = compute.power_promql(alloc_ids) - try: - cpu = mimir.query_range(cpu_q, start_dt, end_dt, step_s) - except mimir.MimirError: - cpu = [] - try: - gpu = mimir.query_range(gpu_q, start_dt, end_dt, step_s) - except mimir.MimirError: - gpu = [] - dq = cfg.get("data_quality") - buckets = compute.integrate( - cpu, gpu, step_s, bucket_s, _tue(dc), data_quality=dq - ) - metrics = green_director.metrics.get(dc, {}) - per_dc.append( - compute.apply_footprint( - buckets, metrics.get("carbon"), metrics.get("water") - ) - ) - iw = compute.instant_power(cpu, gpu, _tue(dc), dq) - if iw is not None: - inst_w += iw - any_inst = True - - return compute.merge_buckets(per_dc), (inst_w if any_inst else None) - - -def topup(doc: dict | None) -> dict | None: - """ - Live `pointer_ts -> now` slice as a single delta: dict with the summed energy - deltas, `power_w` (the last sample read, i.e. the current draw) and - `live_as_of`; `{"degraded": True}` on failure; None if not applicable. - Cached for `topup_cache_seconds`. - """ - key = _running_key(doc) - if key is None: - return None - try: - return _compute_topup(key) - except Exception: - LOG.warning( - "energy top-up failed for %s", doc.get("deployment_uuid"), exc_info=True - ) - return {"degraded": True} - - -@cached(_live_cache) -def _compute_topup(key) -> dict: - _ns, _uuid, pointer_iso, alloc_pairs = key - start_ts, end_ts = _live_window(pointer_iso) - result = { - **_EMPTY, - "power_w": None, - "live_as_of": compute._iso(end_ts), - "degraded": False, - } - span = int(end_ts - start_ts) + int(_cfg()["query_step_seconds"]) - buckets, inst_w = _live_buckets( - alloc_pairs, start_ts, end_ts, bucket_s=max(span, 1) - ) - for b in buckets: - for k in _KEYS: - result[k] += getattr(b, k) - # `power_w` is the last sample read (current draw), not a window mean - result["power_w"] = inst_w - return result - - -def live_series(doc: dict | None, bucket_s: int) -> list[dict]: - """ - Live tail as time-series points (so the detail-view graph keeps growing). - `[]` if not applicable or on any failure. - """ - key = _running_key(doc) - if key is None: - return [] - _ns, _uuid, pointer_iso, alloc_pairs = key - start_ts, end_ts = _live_window(pointer_iso) - try: - buckets, _ = _live_buckets( - alloc_pairs, start_ts, end_ts, bucket_s=max(bucket_s, 1) - ) - except Exception: - LOG.warning("energy live_series failed for %s", doc.get("deployment_uuid")) - return [] - return [{**b._asdict(), "live": True} for b in buckets] - - -# --------------------------------------------------------------------------- -# live path for the per-datacenter (cluster) accumulator -# --------------------------------------------------------------------------- - - -def _cluster_live_buckets(dc: str, start_ts: float, end_ts: float, bucket_s: int): - """Integrate the live window from Mimir for one datacenter into buckets.""" - cfg = _cfg() - step_s = int(cfg["query_step_seconds"]) - if end_ts - start_ts < step_s: - return [] - host_q, gpu_q = _cluster_promql(dc) - start_dt, end_dt = _dt(start_ts), _dt(end_ts) - host = mimir.query_range(host_q, start_dt, end_dt, step_s) - try: - gpu = mimir.query_range(gpu_q, start_dt, end_dt, step_s) - except mimir.MimirError: - gpu = [] - buckets = compute.integrate( - host, gpu, step_s, bucket_s, _tue(dc), data_quality=_cluster_dq() - ) - metrics = green_director.metrics.get(dc, {}) - return compute.apply_footprint(buckets, metrics.get("carbon"), metrics.get("water")) - - -def _cluster_pointer(doc: dict | None) -> str | None: - if not doc or not doc.get("metrics_available", False): - return None - return doc.get("pointer_ts") or doc.get("settled_ts") - - -def topup_cluster(dc: str, doc: dict | None) -> dict | None: - """Live `pointer_ts -> now` slice for a datacenter (see `topup`).""" - pointer = _cluster_pointer(doc) - if pointer is None: - return None - try: - return _compute_cluster_topup((CLUSTER_NS, dc, pointer)) - except Exception: - LOG.warning("cluster energy top-up failed for %s", dc, exc_info=True) - return {"degraded": True} - - -@cached(_live_cache) -def _compute_cluster_topup(key) -> dict: - # Only the energy deltas + `live_as_of` are consumed (the datacenter `power_w` - # comes from `get_datacenter_power`'s instant query, not from here). - _ns, dc, pointer_iso = key - start_ts, end_ts = _live_window(pointer_iso) - result = {**_EMPTY, "live_as_of": compute._iso(end_ts), "degraded": False} - span = int(end_ts - start_ts) + int(_cfg()["query_step_seconds"]) - for b in _cluster_live_buckets(dc, start_ts, end_ts, bucket_s=max(span, 1)): - for k in _KEYS: - result[k] += getattr(b, k) - return result - - -def live_series_cluster(dc: str, doc: dict | None, bucket_s: int) -> list[dict]: - """Live tail for a datacenter as time-series points (see `live_series`).""" - pointer = _cluster_pointer(doc) - if pointer is None: - return [] - start_ts, end_ts = _live_window(pointer) - try: - buckets = _cluster_live_buckets(dc, start_ts, end_ts, bucket_s=max(bucket_s, 1)) - except Exception: - LOG.warning("cluster live_series failed for %s", dc) - return [] - return [{**b._asdict(), "live": True} for b in buckets] diff --git a/ai4papi/clients/__init__.py b/ai4papi/clients/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai4papi/mimir.py b/ai4papi/clients/mimir.py similarity index 100% rename from ai4papi/mimir.py rename to ai4papi/clients/mimir.py diff --git a/ai4papi/accounting/__init__.py b/ai4papi/energy/__init__.py similarity index 63% rename from ai4papi/accounting/__init__.py rename to ai4papi/energy/__init__.py index 79d16b2d..f81f3da2 100644 --- a/ai4papi/accounting/__init__.py +++ b/ai4papi/energy/__init__.py @@ -16,45 +16,36 @@ import datetime import logging +import threading from cachetools import TTLCache, cached -import ai4papi.conf as papiconf -from ai4papi import mimir -from ai4papi.accounting import compute, store -from ai4papi.accounting.sweep import ( - CLUSTER_NS, - _tue, - enabled, - ensure_swept, - ensure_swept_bulk, - live_only_doc, - live_series, - live_series_cluster, - process_single, - run_sweep, - topup, - topup_cluster, -) +from ai4papi.clients import mimir +from ai4papi.energy import compute, config, store +from ai4papi.energy.series import build_series, persisted +from ai4papi.energy.config import CLUSTER_NS, KEYS, enabled +from ai4papi.energy.live import live_series, topup +from ai4papi.energy.ondemand import ensure_swept, ensure_swept_bulk, live_only_doc +from ai4papi.energy.sweep import process_single, run_sweep __all__ = [ + "add_to_job", + "add_to_jobs", "enabled", - "run_sweep", - "process_single", "get_accumulated", "get_accumulated_bulk", + "get_cluster_energy", + "get_datacenter_power", "get_series", "get_user_energy", - "get_datacenter_power", - "get_cluster_energy", + "process_single", + "run_sweep", ] LOG = logging.getLogger(__name__) -_KEYS = ("energy_wh", "carbon_g", "water_l") # everything is TUE-normalized - -@cached(cache=TTLCache(maxsize=8192, ttl=60)) +@cached(cache=TTLCache(maxsize=8192, ttl=60), lock=threading.Lock()) def _read_doc(namespace: str, uuid: str): return store.read_accum(namespace, uuid) @@ -65,7 +56,7 @@ def _doc_tue(doc: dict) -> float | None: if doc.get("tue_factor"): return doc["tue_factor"] dcs = doc.get("datacenters") or [] - return _tue(dcs[0]) if len(dcs) == 1 else None + return config.tue(dcs[0]) if len(dcs) == 1 else None def _to_stats(doc: dict) -> dict: @@ -101,7 +92,7 @@ def _apply_topup(stats: dict, doc: dict) -> dict: return stats # the top-up slice uses the same TUE as the accumulator, so `tue_factor` # does not move; just add the energy / footprint deltas and refresh power. - for k in _KEYS: + for k in KEYS: stats[k] += t.get(k, 0.0) if t.get("power_w") is not None: stats["power_w"] = t["power_w"] @@ -122,15 +113,18 @@ def _live_only_stats(namespace: str, uuid: str) -> tuple[dict, dict] | None: if doc is None: return None stats = _apply_topup(_to_stats(doc), doc) - has_data = stats.get("power_w") is not None or any(stats.get(k, 0.0) for k in _KEYS) + has_data = stats.get("power_w") is not None or any(stats.get(k, 0.0) for k in KEYS) if not has_data: return None return doc, stats -def get_accumulated(namespace: str, uuid: str, live: bool = True) -> dict | None: - if not enabled(): - return None +def _resolve(namespace: str, uuid: str, live: bool) -> tuple[dict, dict] | None: + """ + `(doc, stats)` of one deployment, or `None` when there is nothing to report. + The stored doc is swept on demand when missing; without a usable doc the + number comes from Mimir alone (`live_only_doc`). + """ doc = _read_doc(namespace, uuid) if doc is None: # never swept: run one sweep pass for just this deployment instead of @@ -146,21 +140,16 @@ def get_accumulated(namespace: str, uuid: str, live: bool = True) -> dict | None # allocation (no `metrics_available` verdict either way). Fall back to # a pure-Mimir number, but only once Mimir actually has a sample for # it - never a fabricated zero. - if not live: - return None - res = _live_only_stats(namespace, uuid) - return res[1] if res else None + return _live_only_stats(namespace, uuid) if live else None stats = _to_stats(doc) - return _apply_topup(stats, doc) if live else stats + return doc, (_apply_topup(stats, doc) if live else stats) -def get_accumulated_bulk( - namespace: str, owner: str, live: bool = True -) -> dict[str, dict]: - if not enabled(): - return {} +def _owner_rows(namespace: str, owner: str, live: bool) -> list[tuple[str, dict, dict]]: + """`(uuid, doc, stats)` of every accounted deployment of `owner` in `namespace`, + sweeping the missing ones on demand first.""" live_jobs = ensure_swept_bulk(namespace, owner) - out: dict[str, dict] = {} + rows: list[tuple[str, dict, dict]] = [] covered: set[str] = set() for uuid, doc in store.iter_owner(namespace, owner): if doc.get("metrics_available") is False: @@ -170,7 +159,7 @@ def get_accumulated_bulk( continue # inconclusive stub: leave uncovered, try live-only below covered.add(uuid) stats = _to_stats(doc) - out[uuid] = _apply_topup(stats, doc) if live else stats + rows.append((uuid, doc, _apply_topup(stats, doc) if live else stats)) if live: for stub in live_jobs: uuid = stub["ID"] @@ -178,94 +167,118 @@ def get_accumulated_bulk( continue res = _live_only_stats(namespace, uuid) if res is not None: - out[uuid] = res[1] - return out + rows.append((uuid, *res)) + return rows -def get_user_energy( - namespace: str, owner: str, series: bool = False, target_points: int | None = None -) -> dict | None: - """ - All of a user's deployments in a VO rolled into one `EnergyStats`-shaped - block: each deployment's persisted accumulator plus its live Mimir top-up, - summed. With `series=True`, also a 15-min time series merged across the - deployments (`full_info=true` on `GET /stats/user`). `None` when the user has - no accounted deployment. - """ +def get_accumulated(namespace: str, uuid: str, live: bool = True) -> dict | None: if not enabled(): return None - live_jobs = ensure_swept_bulk(namespace, owner) - - if target_points is None: - target_points = int( - papiconf.MAIN_CONF["energy"].get("user_series_points", 1000) - ) + res = _resolve(namespace, uuid, live) + return res[1] if res else None - rows = [] - covered: set[str] = set() - for uuid, doc in store.iter_owner(namespace, owner): - if doc.get("metrics_available") is False: - covered.add(uuid) - continue - if "accumulated" not in doc: - continue # inconclusive stub: leave uncovered, try live-only below - covered.add(uuid) - rows.append((uuid, doc, _apply_topup(_to_stats(doc), doc))) - for stub in live_jobs: - uuid = stub["ID"] - if uuid in covered: - continue - res = _live_only_stats(namespace, uuid) - if res is not None: - rows.append((uuid, res[0], res[1])) - if not rows: - return None - agg = dict.fromkeys(_KEYS, 0.0) +def get_accumulated_bulk( + namespace: str, owner: str, live: bool = True +) -> dict[str, dict]: + if not enabled(): + return {} + return {uuid: stats for uuid, _doc, stats in _owner_rows(namespace, owner, live)} + + +def add_to_job(job: dict, namespace: str, full_info: bool) -> None: + """Set `job["energy"]`: the extended time series if `full_info`, else just the + accumulated stats.""" + get = get_series if full_info else get_accumulated + job["energy"] = get(namespace, job["job_ID"]) + + +def add_to_jobs(jobs: list[dict], namespace: str, owner: str, full_info: bool) -> None: + """`add_to_job` for all the `owner`'s jobs of a namespace. Without `full_info` + it uses one batch of cheap file reads and a single cached Mimir top-up; + with it, the extended time series per deployment (heavier).""" + if not jobs: + return + if full_info: + for job in jobs: + add_to_job(job, namespace, full_info) + else: + stats = get_accumulated_bulk(namespace, owner) + for job in jobs: + job["energy"] = stats.get(job["job_ID"]) + + +def _aggregate(stats_list: list[dict]) -> dict: + """Sum the `EnergyStats` of several deployments into one block.""" + agg = dict.fromkeys(KEYS, 0.0) dcs: set[str] = set() power = raw_energy = 0.0 # raw_energy: Σ energy_wh / tue, to blend `tue_factor` since, as_of, live_as_of, coverage = [], [], [], [] - complete, degraded = True, False - for _uuid, _doc, s in rows: - for k in _KEYS: + for s in stats_list: + for k in KEYS: agg[k] += s.get(k, 0.0) or 0.0 if s.get("tue_factor"): raw_energy += s["energy_wh"] / s["tue_factor"] dcs.update(s.get("datacenters") or []) - if s.get("power_w"): - power += s["power_w"] - if s.get("since"): - since.append(s["since"]) - if s.get("as_of"): - as_of.append(s["as_of"]) - if s.get("live_as_of"): - live_as_of.append(s["live_as_of"]) + power += s.get("power_w") or 0.0 + since += [s["since"]] if s.get("since") else [] + as_of += [s["as_of"]] if s.get("as_of") else [] + live_as_of += [s["live_as_of"]] if s.get("live_as_of") else [] coverage.append(s.get("coverage_ratio", 1.0)) - complete = complete and s.get("complete", True) - degraded = degraded or bool(s.get("degraded")) - out = { + return { # not rounded, same as the per-deployment `EnergyStats`: the values can be # legitimately tiny (idle CPU workloads) and must reconcile with the sum # of `series`. - "energy_wh": agg["energy_wh"], + **agg, "tue_factor": (agg["energy_wh"] / raw_energy) if raw_energy else None, - "carbon_g": agg["carbon_g"], - "water_l": agg["water_l"], "power_w": power or None, # time range these numbers cover: [since, as_of], live top-up to live_as_of - "since": min(since) if since else None, - "as_of": max(as_of) if as_of else None, - "live_as_of": max(live_as_of) if live_as_of else None, - "complete": complete, + "since": min(since, default=None), + "as_of": max(as_of, default=None), + "live_as_of": max(live_as_of, default=None), + "complete": all(s.get("complete", True) for s in stats_list), "coverage_ratio": round(sum(coverage) / len(coverage), 3), - "degraded": degraded, + "degraded": any(s.get("degraded") for s in stats_list), "datacenters": sorted(dcs), - "deployments": len(rows), - "series": None, + "deployments": len(stats_list), } + + +def get_user_energy( + namespace: str, owner: str, series: bool = False, target_points: int | None = None +) -> dict | None: + """ + All of a user's deployments in a VO rolled into one `EnergyStats`-shaped + block: each deployment's persisted accumulator plus its live Mimir top-up, + summed. With `series=True`, also a 15-min time series merged across the + deployments (`full_info=true` on `GET /stats/user`). `None` when the user has + no accounted deployment. + """ + if not enabled(): + return None + rows = _owner_rows(namespace, owner, live=True) + if not rows: + return None + + out = _aggregate([stats for _uuid, _doc, stats in rows]) + out["series"] = None if series: - out["series"] = _user_series(namespace, rows, target_points) or None + cfg = config.get() + step_s = int(cfg["series_bucket_seconds"]) + if target_points is None: + target_points = int(cfg.get("user_series_points", 1000)) + out["series"] = ( + build_series( + [ + persisted(store.read_series(namespace, uuid, None, None), doc) + for uuid, doc, _stats in rows + ], + [live_series(doc, bucket_s=step_s) for _uuid, doc, _stats in rows], + target_points, + ) + or None + ) return out @@ -288,7 +301,7 @@ def _sum_by_datacenter(promql: str, scale: float) -> dict[str, float]: return out -@cached(cache=TTLCache(maxsize=1, ttl=25)) +@cached(cache=TTLCache(maxsize=1, ttl=25), lock=threading.Lock()) def get_datacenter_power() -> dict[str, dict]: """ Instantaneous power draw per datacenter, normalized by TUE: @@ -302,7 +315,7 @@ def get_datacenter_power() -> dict[str, dict]: if not enabled(): return {} - metrics_dcs = set(papiconf.MAIN_CONF["energy"].get("metrics_datacenters") or []) + metrics_dcs = set(config.get().get("metrics_datacenters") or []) try: host_w = _sum_by_datacenter( "sum by (datacenter) (scaph_host_power_microwatts)", 1e-6 @@ -312,7 +325,7 @@ def get_datacenter_power() -> dict[str, dict]: LOG.warning("mimir cluster-power query failed") return {} - as_of = compute._iso(datetime.datetime.now(datetime.timezone.utc).timestamp()) + as_of = compute.iso(datetime.datetime.now(datetime.timezone.utc).timestamp()) out: dict[str, dict] = {} for dc in set(host_w) | set(gpu_w): if metrics_dcs: @@ -323,7 +336,7 @@ def get_datacenter_power() -> dict[str, dict]: # no allowlist: require host power so a stray GPU metric for an # unmonitored datacenter does not fabricate a total continue - tue = _tue(dc) + tue = config.tue(dc) out[dc] = { "power_w": (host_w.get(dc, 0.0) + gpu_w.get(dc, 0.0)) * tue, "tue_factor": tue, @@ -332,7 +345,7 @@ def get_datacenter_power() -> dict[str, dict]: return out -@cached(cache=TTLCache(maxsize=16, ttl=300)) +@cached(cache=TTLCache(maxsize=16, ttl=300), lock=threading.Lock()) def _cluster_archive_series(dc: str) -> list[dict]: """ Consolidated 15-min buckets of a datacenter (the `.series.jsonl`). Cached for @@ -342,16 +355,6 @@ def _cluster_archive_series(dc: str) -> list[dict]: return store.read_series(CLUSTER_NS, dc, None, None) -def _cluster_series( - dc: str, doc: dict, bucket_s: int, target_points: int -) -> list[dict]: - return _build_series( - list(_cluster_archive_series(dc)) + list(doc.get("tail") or []), - live_series_cluster(dc, doc, bucket_s=bucket_s), - target_points, - ) - - def get_cluster_energy( series: bool = True, target_points: int | None = None ) -> dict[str, dict]: @@ -363,7 +366,7 @@ def get_cluster_energy( """ if not enabled(): return {} - cfg = papiconf.MAIN_CONF["energy"] + cfg = config.get() if target_points is None: target_points = int(cfg.get("cluster_series_points", 500)) bucket_s = int(cfg["series_bucket_seconds"]) @@ -376,13 +379,13 @@ def get_cluster_energy( entry = dict(inst.get(dc) or {}) doc = store.read_accum(CLUSTER_NS, dc) if doc and doc.get("accumulated") and doc.get("metrics_available") is not False: - acc = {k: doc["accumulated"].get(k, 0.0) for k in _KEYS} + acc = {k: doc["accumulated"].get(k, 0.0) for k in KEYS} live_as_of, degraded = None, False - t = topup_cluster(dc, doc) + t = topup(doc) if t and t.get("degraded"): degraded = True elif t: - for k in _KEYS: + for k in KEYS: acc[k] += t.get(k, 0.0) live_as_of = t.get("live_as_of") # not rounded: consistent with the per-deployment / per-user blocks, @@ -391,7 +394,7 @@ def get_cluster_energy( energy_wh=acc["energy_wh"], carbon_g=acc["carbon_g"], water_l=acc["water_l"], - tue_factor=doc.get("tue_factor") or _tue(dc), + tue_factor=doc.get("tue_factor") or config.tue(dc), complete=doc.get("complete", True), since=doc.get("window_start") or doc.get("settled_ts"), as_of=doc.get("pointer_ts"), @@ -399,7 +402,11 @@ def get_cluster_energy( degraded=degraded, ) if series: - entry["series"] = _cluster_series(dc, doc, bucket_s, target_points) + entry["series"] = build_series( + [persisted(_cluster_archive_series(dc), doc)], + [live_series(doc, bucket_s=bucket_s)], + target_points, + ) if entry: out[dc] = entry return out @@ -410,61 +417,25 @@ def get_cluster_energy( # --------------------------------------------------------------------------- -def _build_series( - archive_and_tail: list[dict], live_points: list[dict], target_points: int -) -> list[dict]: - """ - One entity's whole-life time series: persisted 15-min buckets (archive + the - still-mutable tail) `compute.downsample`d to `target_points`, then the live - Mimir tail appended un-downsampled. Shared by the per-deployment (`get_series`) - and per-datacenter (`_cluster_series`) paths. The points carry the per-bucket - increments only; a running total is `cumsum(series[].energy_wh)` and matches - `accumulated.energy_wh`. - """ - raw = sorted(archive_and_tail, key=lambda b: b["ts"]) - series = [ - _point(b, live=False) - for b in compute.downsample([_bucket(b) for b in raw], target_points) - ] - series += [ - _point(_bucket(p), live=True) - for p in live_points - if not series or p["ts"] > series[-1]["ts"] - ] - return series - - def get_series( namespace: str, uuid: str, target_points: int | None = None ) -> dict | None: if not enabled(): return None - doc = _read_doc(namespace, uuid) - if doc is None: - doc = ensure_swept(namespace, uuid) - if doc is not None and doc.get("metrics_available") is False: + res = _resolve(namespace, uuid, live=True) + if res is None: return None - live_only_stats = None - if doc is None or "accumulated" not in doc: - res = _live_only_stats(namespace, uuid) - if res is None: - return None - doc, live_only_stats = res + doc, stats = res - cfg = papiconf.MAIN_CONF["energy"] + cfg = config.get() if target_points is None: target_points = int(cfg.get("deployment_series_points", 1000)) step_s = int(cfg["series_bucket_seconds"]) - series = _build_series( - list(store.read_series(namespace, uuid, None, None)) - + list(doc.get("tail") or []), - live_series(doc, bucket_s=step_s), + series = build_series( + [persisted(store.read_series(namespace, uuid, None, None), doc)], + [live_series(doc, bucket_s=step_s)], target_points, ) - # already computed (and confirmed non-empty) by `_live_only_stats` above; - # `_apply_topup` is cached at the Mimir-query level, so this is not a - # second call for the swept-doc branch either. - stats = live_only_stats or _apply_topup(_to_stats(doc), doc) return { "deployment_uuid": uuid, @@ -476,61 +447,3 @@ def get_series( "series": series, "accumulated": stats, } - - -def _bucket(b: dict) -> compute.Bucket: - return compute.Bucket( - ts=b["ts"], - energy_wh=b.get("energy_wh", 0.0), - power_w_avg=b.get("power_w_avg", 0.0), - carbon_g=b.get("carbon_g", 0.0), - water_l=b.get("water_l", 0.0), - ) - - -def _point(b: compute.Bucket, live: bool) -> dict: - return { - "ts": b.ts, - "power_w": b.power_w_avg, - "energy_wh": b.energy_wh, - "carbon_g": b.carbon_g, - "water_l": b.water_l, - "live": live, - } - - -def _user_series(namespace: str, rows: list, target_points: int) -> list[dict]: - """ - One time series for a set of the user's deployments. The raw 15-min buckets - of every deployment (persisted archive + 24h tail) are merged by timestamp - first, then downsampled once: uniform resolution across the whole series and - the same hard `target_points` cap as the other levels, regardless of how long - each deployment individually has been running. The live Mimir tails are then - merged and appended un-downsampled (as `get_series` / `_cluster_series` do). - """ - step_s = int(papiconf.MAIN_CONF["energy"]["series_bucket_seconds"]) - - persisted = [ - [ - _bucket(b) - for b in list(store.read_series(namespace, uuid, None, None)) - + list(doc.get("tail") or []) - ] - for uuid, doc, _ in rows - ] - merged = sorted(compute.merge_buckets(persisted), key=lambda b: b.ts) - series = [_point(b, live=False) for b in compute.downsample(merged, target_points)] - - live = sorted( - compute.merge_buckets( - [ - [_bucket(b) for b in live_series(doc, bucket_s=step_s)] - for _uuid, doc, _ in rows - ] - ), - key=lambda b: b.ts, - ) - series += [ - _point(b, live=True) for b in live if not series or b.ts > series[-1]["ts"] - ] - return series diff --git a/ai4papi/accounting/compute.py b/ai4papi/energy/compute.py similarity index 94% rename from ai4papi/accounting/compute.py rename to ai4papi/energy/compute.py index 5ef7017f..542b3423 100644 --- a/ai4papi/accounting/compute.py +++ b/ai4papi/energy/compute.py @@ -8,16 +8,17 @@ """ import bisect -from collections import namedtuple import datetime +import threading +import statistics +from collections import namedtuple -from cachetools import cached, TTLCache +from cachetools import TTLCache, cached import ai4papi.conf as papiconf from ai4papi.nomad_utils import Nomad from ai4papi.wattnet import GreenDirector - # Bucket start `ts` is an ISO-8601 UTC string ("...Z"). Energy in Wh (normalized # by the datacenter TUE), power in W, footprint per bucket. Bucket = namedtuple("Bucket", ["ts", "energy_wh", "power_w_avg", "carbon_g", "water_l"]) @@ -28,7 +29,7 @@ _TERMINAL_STATUSES = {"complete", "failed", "lost"} -def _iso(ts: float) -> str: +def iso(ts: float) -> str: return ( datetime.datetime.fromtimestamp(ts, datetime.timezone.utc) .isoformat() @@ -36,7 +37,7 @@ def _iso(ts: float) -> str: ) -def _parse_iso(s: str) -> float: +def parse_iso(s: str) -> float: return datetime.datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() @@ -47,7 +48,7 @@ def _parse_iso(s: str) -> float: AllocWindow = namedtuple("AllocWindow", ["alloc_id", "start_ts", "end_ts"]) -@cached(cache=TTLCache(maxsize=4096, ttl=60 * 60)) +@cached(cache=TTLCache(maxsize=4096, ttl=60 * 60), lock=threading.Lock()) def _node_datacenter(node_id: str) -> str: return Nomad.node.get_node(node_id)["Datacenter"] @@ -154,6 +155,17 @@ def power_promql(alloc_ids: list[str]) -> tuple[str, str]: return cpu_q, gpu_q +def cluster_promql(dc: str) -> tuple[str, str]: + """ + (host_query, gpu_query) for a whole datacenter: one series per host and one + per GPU (grouped, not summed) so `integrate` runs the data-quality filter on + each raw Mimir series before summing the energy. + """ + host_q = f'sum by (node) (scaph_host_power_microwatts{{datacenter="{dc}"}})' + gpu_q = f'sum by (UUID) (DCGM_FI_DEV_POWER_USAGE{{datacenter="{dc}"}})' + return host_q, gpu_q + + # --------------------------------------------------------------------------- # Integration: power samples -> energy buckets # --------------------------------------------------------------------------- @@ -177,15 +189,6 @@ def _series_samples( return out -def _median(xs: list[float]) -> float: - s = sorted(xs) - n = len(s) - if n == 0: - return 0.0 - m = n // 2 - return s[m] if n % 2 else (s[m - 1] + s[m]) / 2 - - def sanitize( samples: list[tuple[float, float]], dq: dict | None ) -> list[tuple[float, float]]: @@ -218,7 +221,7 @@ def sanitize( for i, (t, p) in enumerate(out): lo_i, hi_i = max(0, i - half), min(len(vals), i + half + 1) neighbours = vals[lo_i:i] + vals[i + 1 : hi_i] - med = _median(neighbours) + med = statistics.median(neighbours) if neighbours else 0.0 if med > 1e-6 and p > factor * med: continue # spike -> drop kept.append((t, p)) @@ -278,7 +281,7 @@ def integrate( hours = covered / 3600 if covered else 0.0 buckets.append( Bucket( - ts=_iso(b), + ts=iso(b), energy_wh=energy_wh, # mean power over the covered part of the bucket, normalized like # energy_wh so `energy_wh ~= power_w_avg * bucket_hours` @@ -343,7 +346,7 @@ def _prep_series(series): """[[iso_ts, value], ...] -> (sorted ts floats, values) for bisect lookup.""" if not series: return [], [] - parsed = sorted((_parse_iso(t), float(v)) for t, v in series) + parsed = sorted((parse_iso(t), float(v)) for t, v in series) return [p[0] for p in parsed], [p[1] for p in parsed] @@ -370,7 +373,7 @@ def apply_footprint( out = [] for bk in buckets: - ts = _parse_iso(bk.ts) + ts = parse_iso(bk.ts) kwh = bk.energy_wh / 1000 out.append( bk._replace( diff --git a/ai4papi/energy/config.py b/ai4papi/energy/config.py new file mode 100644 index 00000000..70bcbe3e --- /dev/null +++ b/ai4papi/energy/config.py @@ -0,0 +1,81 @@ +""" +Shared configuration and tiny helpers of the energy-accounting feature. +""" + +import datetime + +from cachetools import TTLCache + +import ai4papi.conf as papiconf +from ai4papi.clients import mimir +from ai4papi.energy import store + +# Nomad job-name prefixes that are deployments (as opposed to e.g. traefik) +JOB_PREFIXES = ("module", "tool", "batch", "try") +KEYS = ("energy_wh", "carbon_g", "water_l") # TUE-normalized totals kept per accum +EMPTY = dict.fromkeys(KEYS, 0.0) + +# Pseudo-namespace for the datacenter-level documents. Does not collide with any +# VO namespace nor with the deployment sweep's namespace loop. +CLUSTER_NS = "_cluster" + + +def get() -> dict: + """The `energy` section of the main config.""" + return papiconf.MAIN_CONF["energy"] + + +def enabled() -> bool: + return ( + bool(get().get("enabled")) + and store.base_dir() is not None + and bool(mimir.MIMIR_USER) + ) + + +def topup_cache(maxsize: int) -> TTLCache: + """Cache for the live path: shorter than the dashboard poll, so the counter moves.""" + return TTLCache(maxsize=maxsize, ttl=get().get("topup_cache_seconds", 25)) + + +def now() -> datetime.datetime: + return datetime.datetime.now(datetime.timezone.utc) + + +def dt(ts: float) -> datetime.datetime: + return datetime.datetime.fromtimestamp(ts, datetime.timezone.utc) + + +def floor(ts: float, step: int) -> float: + return (int(ts) // step) * step + + +def tue(dc: str) -> float: + return (papiconf.datacenters.get(dc) or {}).get("TUE") or get()[ + "default_tue_factor" + ] + + +def cluster_dq() -> dict: + """ + Data-quality config for the datacenter sweep: reuse `energy.data_quality` + but drop the absolute `max_power_w` ceiling (it is calibrated per allocation + / per GPU, far below a whole host). Negative-drop and the scale-free spike + check still apply, per host / per GPU series. + """ + return {**(get().get("data_quality") or {}), "max_power_w": 0} + + +def sum_totals(buckets) -> dict: + return {k: sum(b[k] for b in buckets) for k in KEYS} + + +def add_totals(a: dict, b: dict) -> dict: + return {k: a.get(k, 0.0) + b.get(k, 0.0) for k in KEYS} + + +def log_safe(value: str) -> str: + """Strip CR/LF from an identifier that may come straight from a request + path or auth token before it is interpolated into a log message, so it + cannot be used to forge log lines (CWE-117).""" + return str(value).replace("\r", "").replace("\n", "") diff --git a/ai4papi/energy/live.py b/ai4papi/energy/live.py new file mode 100644 index 00000000..082afd5e --- /dev/null +++ b/ai4papi/energy/live.py @@ -0,0 +1,169 @@ +""" +Live top-up of the energy-accounting feature (read path). + +The sweep only reaches `pointer_ts` (up to one sweep interval behind). On each +API read, `topup(doc)` adds the `pointer_ts -> now` slice live from Mimir so the +dashboard counter moves every ~30s, and `live_series(doc)` returns that same +slice as time-series points. Both work on a deployment doc and on a datacenter +doc (`namespace == CLUSTER_NS`). +""" + +import logging +import threading + +from cachetools import cached + +from ai4papi.energy import compute, config, query +from ai4papi.wattnet import green_director + +LOG = logging.getLogger(__name__) + +_live_cache = config.topup_cache(8192) + + +def _live_window(pointer_iso: str) -> tuple[float, float]: + """ + `[pointer_ts, now]`, clamped on the left so a long-dormant pointer does not + trigger a huge query. No ingestion-lag cushion: Mimir data is immutable + once ingested (never revised), so querying right up to "now" cannot return + a wrong value -- worst case, a sample scraped a moment ago is not visible + *yet* and simply is not the one returned as "last", which self-corrects on + the very next call. Used by `topup` for both a real, persisted accumulator + and `live_only_doc`'s transient one (nothing persisted): both get the + freshest number Mimir can currently give. + """ + cfg = config.get() + end_ts = config.now().timestamp() + start_ts = max( + compute.parse_iso(pointer_iso), + end_ts - (cfg["sweep_seconds"] + 4 * cfg["series_bucket_seconds"]), + ) + return start_ts, end_ts + + +def _running_key(doc: dict | None) -> tuple | None: + """Hashable summary of what a live query depends on; `None` if there is + nothing to query (no data yet, or a deployment with no running allocation).""" + if not doc: + return None + pointer = doc.get("pointer_ts") or doc.get("settled_ts") + if not doc.get("metrics_available", False) or not pointer: + return None + if doc.get("namespace") == config.CLUSTER_NS: + return (config.CLUSTER_NS, doc["deployment_uuid"], pointer, ()) + running = doc.get("active_allocs") or [] + if not running: + return None + return ( + doc["namespace"], + doc["deployment_uuid"], + pointer, + tuple(sorted((a["alloc_id"], a["datacenter"]) for a in running)), + ) + + +def _sources(key: tuple) -> list[tuple]: + """`(dc, cpu_query, gpu_query, data_quality)` per datacenter.""" + ns, uuid, _pointer, alloc_pairs = key + if ns == config.CLUSTER_NS: + return [(uuid, *compute.cluster_promql(uuid), config.cluster_dq())] + + by_dc: dict[str, list[str]] = {} + for alloc_id, dc in alloc_pairs: + by_dc.setdefault(dc, []).append(alloc_id) + dq = config.get().get("data_quality") + return [(dc, *compute.power_promql(ids), dq) for dc, ids in by_dc.items()] + + +def _live_buckets(key: tuple, start_ts: float, end_ts: float, bucket_s: int): + """ + Integrate the live window from Mimir into `bucket_s` buckets (per DC, merged), + plus the current draw from the last sample of that same query. Returns + `(buckets, power_w)`; `power_w` is `None` only when a series has no sample + at all in the window (nothing to read yet), never because the last sample + is "too old" -- Mimir data is immutable once ingested, so the actual last + point is always the right one to show, however old it is. + """ + step_s = int(config.get()["query_step_seconds"]) + if end_ts - start_ts < step_s: + return [], None + + per_dc = [] + inst_w = 0.0 + any_inst = False + for dc, cpu_q, gpu_q, dq in _sources(key): + cpu, gpu = query.fetch_power(cpu_q, gpu_q, start_ts, end_ts, step_s) + tue = config.tue(dc) + buckets = compute.integrate(cpu, gpu, step_s, bucket_s, tue, data_quality=dq) + metrics = green_director.metrics.get(dc, {}) + per_dc.append( + compute.apply_footprint( + buckets, metrics.get("carbon"), metrics.get("water") + ) + ) + iw = compute.instant_power(cpu, gpu, tue, dq) + if iw is not None: + inst_w += iw + any_inst = True + + return compute.merge_buckets(per_dc), (inst_w if any_inst else None) + + +def topup(doc: dict | None) -> dict | None: + """ + Live `pointer_ts -> now` slice as a single delta: dict with the summed energy + deltas, `power_w` (the last sample read, i.e. the current draw) and + `live_as_of`; `{"degraded": True}` on failure; None if not applicable. + Cached for `topup_cache_seconds`. + """ + key = _running_key(doc) + if key is None: + return None + try: + return _compute_topup(key) + except Exception: + LOG.warning( + "energy top-up failed for %s", + config.log_safe(doc.get("deployment_uuid")), + exc_info=True, + ) + return {"degraded": True} + + +@cached(_live_cache, lock=threading.Lock()) +def _compute_topup(key: tuple) -> dict: + start_ts, end_ts = _live_window(key[2]) + result = { + **config.EMPTY, + "power_w": None, + "live_as_of": compute.iso(end_ts), + "degraded": False, + } + span = int(end_ts - start_ts) + int(config.get()["query_step_seconds"]) + buckets, inst_w = _live_buckets(key, start_ts, end_ts, bucket_s=max(span, 1)) + for b in buckets: + for k in config.KEYS: + result[k] += getattr(b, k) + # `power_w` is the last sample read (current draw), not a window mean + result["power_w"] = inst_w + return result + + +def live_series(doc: dict | None, bucket_s: int) -> list[dict]: + """ + Live tail as time-series points (so the detail-view graph keeps growing). + `[]` if not applicable or on any failure. + """ + key = _running_key(doc) + if key is None: + return [] + start_ts, end_ts = _live_window(key[2]) + try: + buckets, _ = _live_buckets(key, start_ts, end_ts, bucket_s=max(bucket_s, 1)) + except Exception: + LOG.warning( + "energy live_series failed for %s", + config.log_safe(doc.get("deployment_uuid")), + ) + return [] + return [{**b._asdict(), "live": True} for b in buckets] diff --git a/ai4papi/energy/ondemand.py b/ai4papi/energy/ondemand.py new file mode 100644 index 00000000..0fd1e838 --- /dev/null +++ b/ai4papi/energy/ondemand.py @@ -0,0 +1,185 @@ +""" +On-demand sweep for the read path of the energy-accounting feature. + +When a deployment has no accumulator doc yet (just deployed, or the background +sweep has not reached it), the read API sweeps it on the spot instead of +waiting up to `sweep_seconds`: `ensure_swept` for a single deployment, +`ensure_swept_bulk` for the list / per-user reads. `live_only_doc` covers the +last gap, a deployment too young for even one completed bucket. +""" + +import logging +import threading + +from cachetools import cached + +from ai4papi.clients import mimir +from ai4papi.energy import compute, config, store, sweep +from ai4papi.nomad_utils import Nomad + +LOG = logging.getLogger(__name__) + + +def _sweep_one(ns: str, uuid: str, now) -> bool: + """Fetch one Nomad job and run `process_single` on it. `False` when the job + cannot be found (not a deployment, or already gone) or the sweep failed.""" + try: + j = Nomad.job.get_job(id_=uuid, namespace=ns) + except Exception: + return False + try: + sweep.process_single(sweep.info_from_job(j, ns), now) + except mimir.MimirError as e: + LOG.warning("on-demand energy sweep skipped, Mimir unavailable: %s", e) + return False + except Exception: + LOG.warning( + "on-demand energy sweep failed for %s/%s", + config.log_safe(ns), + config.log_safe(uuid), + exc_info=True, + ) + return False + return True + + +def ensure_swept(ns: str, uuid: str) -> dict | None: + """ + On-demand equivalent of one sweep pass for a single deployment, called from + the read path when no accumulator doc exists yet: a deployment that already + has real samples in Mimir does not have to wait up to `sweep_seconds` for + the background thread to reach it before the API can report anything. + + A no-op (returns the existing doc untouched, no Nomad/Mimir call) once the + background sweep has processed this deployment at least once, so it never + duplicates the periodic sweep's work. `None` if the job cannot be found (not + a deployment, or already gone). + """ + if not config.enabled(): + return None + doc = store.read_accum(ns, uuid) + if doc is not None: + return doc + if not _sweep_one(ns, uuid, config.now()): + return None + return store.read_accum(ns, uuid) + + +_bulk_sweep_cache = config.topup_cache(2048) + + +@cached(_bulk_sweep_cache, lock=threading.Lock()) +def ensure_swept_bulk(ns: str, owner: str) -> list[dict]: + """ + On-demand equivalent of one sweep pass, restricted to one owner's active + deployments in `ns`: the bulk counterpart of `ensure_swept`, used by the + list (`get_accumulated_bulk`) and per-user (`get_user_energy`) read paths so + a just-deployed job shows up there too, not only on its own detail endpoint. + + Lists the owner's active jobs (cheap, no per-job fetch) and only runs + `process_single` for the ones with no accumulator doc yet; a no-op once the + background sweep (or a prior on-demand call) has reached all of them. + Cached for `topup_cache_seconds`: without it, every list/`/stats/user` read + would list Nomad on every single request, forever, not just during the + bootstrap window -- a few seconds of staleness for a listing that itself + only matters for brand-new deployments is a good trade. + + Returns the filtered job stubs (`[{"ID": ..., "Name": ...}, ...]`) so the + caller can also try `live_only_doc` for whichever of them still have no doc + afterwards (a deployment too young for even one completed bucket), without + listing Nomad a second time. `[]` when disabled or the listing itself fails. + """ + if not config.enabled(): + return [] + try: + jobs = Nomad.jobs.get_jobs( + namespace=ns, + filter_=f'Meta.owner == "{owner}" and Status != "dead"', + ) + except Exception: + LOG.warning( + "could not list jobs for on-demand sweep (%s/%s)", + config.log_safe(ns), + config.log_safe(owner), + ) + return [] + now = config.now() + stubs = [s for s in jobs if s["Name"].startswith(config.JOB_PREFIXES)] + for stub in stubs: + if store.read_accum(ns, stub["ID"]) is not None: + continue + _sweep_one(ns, stub["ID"], now) + return stubs + + +_live_only_doc_cache = config.topup_cache(8192) + + +@cached(_live_only_doc_cache, lock=threading.Lock()) +def live_only_doc(ns: str, uuid: str) -> dict | None: + """ + Transient stand-in for a deployment `process_single` has not written any doc + for at all yet -- typically younger than one `series_bucket_seconds`, so it + has no completed bucket to fold into an accumulator (see `ensure_swept`, + which still leaves such a deployment with no doc by design: there is + nothing settled to persist). Never written to the store. + + Cached for `topup_cache_seconds`: without it, every read of a deployment + still in this bootstrap window would fetch its Nomad job and resolve its + allocations from scratch on every single request for as long as it stays + doc-less (which can be several minutes). A few seconds of staleness on + which allocation is "current" is a good trade against that. + + Shaped so the normal `topup` / `live_series` machinery -- and therefore + `energy._to_stats` / `_apply_topup` / `_build_series` -- treats it like + a real doc with an all-zero accumulator and `pointer_ts = submit_time`: the + whole reported number then comes from the live top-up's Mimir query, capped + at the same lookback `topup` itself uses. `None` if the job cannot be + resolved to a running, monitored allocation (not found, queued, or no + allocation yet). + """ + if not config.enabled(): + return None + try: + j = Nomad.job.get_job(id_=uuid, namespace=ns) + except Exception: + return None + info = sweep.info_from_job(j, ns) + try: + allocs_by_dc, _windows, running_allocs, _partial = compute.resolve_alloc_ids( + uuid, ns + ) + except Exception: + return None + if not running_allocs: + return None + submit = info.get("submit_time") + submit_ts = submit or config.now().timestamp() + # `window_start` below is pinned to the exact submit time, same as + # `process_single`'s first-encounter `window_start` -- not started late or + # clamped, so `complete` is `True` unless we had no real `submit_time` to + # pin it to at all (job fetched with no `SubmitTime`, an edge case) or it + # somehow predates the lookback window (impossible in practice: this path + # only ever runs for a deployment too young to have a doc yet). + complete = ( + bool(submit) + and submit + >= config.now().timestamp() - config.get()["initial_lookback_hours"] * 3600 + ) + return { + "deployment_uuid": uuid, + "namespace": ns, + "job_name": info.get("job_name"), + "owner": info.get("owner"), + "status": "running", + "metrics_available": True, + "complete": complete, + "window_start": compute.iso(submit_ts), + "settled_ts": compute.iso(submit_ts), + "pointer_ts": compute.iso(submit_ts), + "active_allocs": running_allocs, + "datacenters": sorted(allocs_by_dc), + "tail": [], + "settled": dict(config.EMPTY), + "accumulated": dict(config.EMPTY), + } diff --git a/ai4papi/energy/query.py b/ai4papi/energy/query.py new file mode 100644 index 00000000..1aa5da71 --- /dev/null +++ b/ai4papi/energy/query.py @@ -0,0 +1,22 @@ +""" +Mimir power queries shared by the sweep (write path) and the live top-up. +""" + +from ai4papi.clients import mimir +from ai4papi.energy import config + + +def fetch_power( + cpu_q: str, gpu_q: str, start_ts: float, end_ts: float, step_s: int +) -> tuple[list, list]: + """ + `(cpu_result, gpu_result)` range queries over [start, end]. A missing metric + comes back as an empty result, not an error (a GPU-only site has no `scaph_*` + series), so a query that fails is a real failure and propagates: the sweep + skips that deployment and retries on the next pass, the live path reports + `degraded`. Long windows are split in chunks. + """ + start, end = config.dt(start_ts), config.dt(end_ts) + chunked = end_ts - start_ts > mimir.MAX_POINTS_PER_QUERY * step_s + qfn = mimir.query_range_chunked if chunked else mimir.query_range + return qfn(cpu_q, start, end, step_s), qfn(gpu_q, start, end, step_s) diff --git a/ai4papi/energy/series.py b/ai4papi/energy/series.py new file mode 100644 index 00000000..98a30d38 --- /dev/null +++ b/ai4papi/energy/series.py @@ -0,0 +1,61 @@ +""" +Time-series building of the energy-accounting feature: turns persisted buckets +and live Mimir tails into the points served by the API. +""" + +from ai4papi.energy import compute + + +def _bucket(b: dict) -> compute.Bucket: + return compute.Bucket( + ts=b["ts"], + energy_wh=b.get("energy_wh", 0.0), + power_w_avg=b.get("power_w_avg", 0.0), + carbon_g=b.get("carbon_g", 0.0), + water_l=b.get("water_l", 0.0), + ) + + +def _point(b: compute.Bucket, live: bool) -> dict: + return { + "ts": b.ts, + "power_w": b.power_w_avg, + "energy_wh": b.energy_wh, + "carbon_g": b.carbon_g, + "water_l": b.water_l, + "live": live, + } + + +def persisted(archive: list[dict], doc: dict) -> list[dict]: + """Persisted 15-min buckets of one entity: the consolidated archive plus the + still-mutable 24h tail.""" + return list(archive) + list(doc.get("tail") or []) + + +def build_series( + persisted_lists: list[list[dict]], live: list[list[dict]], target_points: int +) -> list[dict]: + """ + Time series of one or several entities (a deployment, a user's deployments, + a datacenter). The persisted buckets are merged by timestamp across entities + and `compute.downsample`d once, so the resolution is uniform and capped at + `target_points`; the live Mimir tails are merged the same way and appended + un-downsampled. The points carry the per-bucket increments only; a running + total is `cumsum(series[].energy_wh)` and matches `accumulated.energy_wh`. + """ + + def merged(lists: list[list[dict]]) -> list[compute.Bucket]: + out = compute.merge_buckets([[_bucket(b) for b in bs] for bs in lists]) + return sorted(out, key=lambda b: b.ts) + + series = [ + _point(b, live=False) + for b in compute.downsample(merged(persisted_lists), target_points) + ] + series += [ + _point(b, live=True) + for b in merged(live) + if not series or b.ts > series[-1]["ts"] + ] + return series diff --git a/ai4papi/accounting/store.py b/ai4papi/energy/store.py similarity index 70% rename from ai4papi/accounting/store.py rename to ai4papi/energy/store.py index b9d2f2ee..5178f341 100644 --- a/ai4papi/accounting/store.py +++ b/ai4papi/energy/store.py @@ -11,6 +11,8 @@ import json import os +import threading +import time from pathlib import Path @@ -64,6 +66,10 @@ def write_accum(ns: str, uuid: str, doc: dict) -> None: if not path: return _atomic_write(path, json.dumps(doc, separators=(",", ":"))) + with _owner_lock: + entry = _owner_idx.get(ns) + if entry: + entry[1].setdefault(doc.get("owner"), set()).add(uuid) def iter_namespace(ns: str): @@ -79,10 +85,32 @@ def iter_namespace(ns: str): continue +# owner -> uuids of a namespace, so listing one user's deployments does not parse +# every document of the namespace. Kept in sync by `write_accum` and rebuilt from +# disk every `_OWNER_IDX_TTL` seconds (heals anything written by another process). +# shortcut: assumes one PAPI process writes the store, upgrade to a database index +# if PAPI runs as several replicas on the same storage. +_OWNER_IDX_TTL = 300 +_owner_idx: dict[str, tuple[float, dict[str | None, set[str]]]] = {} +_owner_lock = threading.Lock() + + +def _owner_uuids(ns: str, owner: str) -> list[str]: + with _owner_lock: + entry = _owner_idx.get(ns) + if entry is None or time.monotonic() - entry[0] > _OWNER_IDX_TTL: + idx: dict[str | None, set[str]] = {} + for uuid, doc in iter_namespace(ns): + idx.setdefault(doc.get("owner"), set()).add(uuid) + entry = _owner_idx[ns] = (time.monotonic(), idx) + return sorted(entry[1].get(owner, ())) + + def iter_owner(ns: str, owner: str): """Yield (uuid, doc) for the accumulator documents owned by `owner`.""" - for uuid, doc in iter_namespace(ns): - if doc.get("owner") == owner: + for uuid in _owner_uuids(ns, owner): + doc = read_accum(ns, uuid) + if doc is not None and doc.get("owner") == owner: yield uuid, doc @@ -97,15 +125,14 @@ def append_series(ns: str, uuid: str, buckets: list[dict]) -> None: return path.parent.mkdir(parents=True, exist_ok=True) with open(path, "a", encoding="utf-8") as f: - for b in buckets: - f.write(json.dumps(b, separators=(",", ":")) + "\n") + f.writelines(json.dumps(b, separators=(",", ":")) + "\n" for b in buckets) def read_series(ns: str, uuid: str, start_ts: str | None, end_ts: str | None): path = _series_path(ns, uuid) if not path or not path.is_file(): return [] - out = [] + out: dict[str, dict] = {} # by ts: a bucket appended twice counts once with open(path, "r", encoding="utf-8") as f: for line in f: line = line.strip() @@ -119,6 +146,5 @@ def read_series(ns: str, uuid: str, start_ts: str | None, end_ts: str | None): continue if end_ts and b["ts"] > end_ts: continue - out.append(b) - out.sort(key=lambda b: b["ts"]) - return out + out[b["ts"]] = b + return [out[ts] for ts in sorted(out)] diff --git a/ai4papi/energy/sweep.py b/ai4papi/energy/sweep.py new file mode 100644 index 00000000..41323bc2 --- /dev/null +++ b/ai4papi/energy/sweep.py @@ -0,0 +1,447 @@ +""" +Background sweep of the energy-accounting feature (write path). + +`run_sweep()` runs on a `@repeat_every` thread in `ai4papi.main`. Per deployment +(and per datacenter for `/stats/cluster`) it integrates the un-consolidated +window of Mimir power into energy buckets (`energy.series_bucket_seconds`, +15 min), applies the per-datacenter TUE and the Wattnet footprint, keeps a +running accumulator plus a ~24h "tail" that is re-computed every sweep (so the +footprint self-heals as Wattnet settles), and persists a small JSON document. + +The read-side counterparts live in `ai4papi.energy.live` (live top-up) and +`ai4papi.energy.ondemand` (sweep a deployment from the read path). +""" + +import logging +from typing import NamedTuple + +import ai4papi.conf as papiconf +from ai4papi.clients import mimir +from ai4papi.energy import compute, config, query, store +from ai4papi.nomad_utils import Nomad +from ai4papi.wattnet import green_director + +LOG = logging.getLogger(__name__) + + +class Clock(NamedTuple): + """Time references of one sweep pass (all epoch seconds unless noted).""" + + now_ts: float + bucket_s: int + step_s: int + extract_to: float # newest instant to integrate (now minus the ingestion lag) + revision_cut: float # buckets older than this are final (footprint settled) + retention_cut: float # oldest instant Mimir still holds + lookback_ts: float # how far back a first-seen deployment is integrated + + +def _clock(now) -> Clock: + cfg = config.get() + now_ts = now.timestamp() + bucket_s = int(cfg["series_bucket_seconds"]) + return Clock( + now_ts=now_ts, + bucket_s=bucket_s, + step_s=int(cfg["query_step_seconds"]), + extract_to=config.floor(now_ts - cfg["query_lag_seconds"], bucket_s), + revision_cut=now_ts - cfg["footprint_revision_hours"] * 3600, + retention_cut=now_ts - cfg["mimir_retention_days"] * 86400, + lookback_ts=now_ts - cfg["initial_lookback_hours"] * 3600, + ) + + +def _resume_point(doc: dict, t: Clock, submit: float | None = None): + """ + `(settled_ts, complete)`: where integration resumes. On first encounter never + look back further than `initial_lookback_hours` (pre-existing deployments are + not fully backfilled); never further than Mimir retention either. + """ + if doc.get("settled_ts"): + settled_ts = compute.parse_iso(doc["settled_ts"]) + complete = doc.get("complete", True) + else: + settled_ts = max(submit or t.lookback_ts, t.lookback_ts) + complete = bool(submit) and submit >= t.lookback_ts + if settled_ts < t.retention_cut: + settled_ts, complete = t.retention_cut, False + return config.floor(settled_ts, t.bucket_s), complete + + +def _earliest_bucket_ts(ns: str, uuid: str, doc: dict) -> float | None: + """Timestamp of the oldest 15-min bucket on record: the consolidated + `.series.jsonl` if there is one, else the oldest still in `doc["tail"]`.""" + archive = store.read_series(ns, uuid, None, None) + if archive: + return compute.parse_iso(archive[0]["ts"]) + tail = doc.get("tail") or [] + if tail: + return min(compute.parse_iso(b["ts"]) for b in tail) + return None + + +def _window_start(ns: str, uuid: str, doc: dict, settled_ts: float, t: Clock) -> float: + """ + True start of the accumulation window, exposed to the API as `since`. + + `settled_ts` cannot serve this role: it marches forward every sweep as + buckets age into `doc["settled"]`, while the accumulated totals (and the + persisted series) keep every bucket since the beginning. + + Pinned once and only moved forward if it falls behind Mimir retention: + - fresh doc: the first-sweep `settled_ts` is the window start; + - doc written before this field existed (`settled` / `tail` already there, + `settled_ts` already advanced): recover it from the oldest bucket on record. + """ + retention_floor = config.floor(t.retention_cut, t.bucket_s) + if doc.get("window_start"): + return max(compute.parse_iso(doc["window_start"]), retention_floor) + if "settled" in doc or doc.get("tail"): + earliest = _earliest_bucket_ts(ns, uuid, doc) + if earliest is not None: + return max(min(earliest, settled_ts), retention_floor) + return max(settled_ts, retention_floor) + + +def _footprint_series(dc: str, start_ts: float, end_ts: float): + """ + (carbon, water) intensity series for a datacenter over [start, end]. + In steady state uses the 7-day window that `green_director` already keeps; + only fetches a wider range from Wattnet when backfilling further back. + `apply_footprint` falls back to DEFAULTS when a series is empty/None. + """ + metrics = green_director.metrics.get(dc, {}) + if start_ts >= config.now().timestamp() - 7 * 86400: + return metrics.get("carbon"), metrics.get("water") + try: + fetched = green_director.footprint_series( + dc, config.dt(start_ts), config.dt(end_ts) + ) + except Exception: + LOG.warning("wattnet footprint_series failed for %s", dc) + fetched = {} + return ( + fetched.get("carbon") or metrics.get("carbon"), + fetched.get("water") or metrics.get("water"), + ) + + +def _with_footprint(dc: str, buckets: list, settled_ts: float, t: Clock) -> list: + if not buckets: + return buckets + carbon, water = _footprint_series(dc, settled_ts, t.now_ts) + return compute.apply_footprint(buckets, carbon, water) + + +def _complete_buckets(buckets, t: Clock) -> list: + """Only buckets that are fully inside the integrated window.""" + return [b for b in buckets if compute.parse_iso(b.ts) + t.bucket_s <= t.extract_to] + + +# --------------------------------------------------------------------------- +# sweep driver +# --------------------------------------------------------------------------- + + +def run_sweep() -> None: + if not config.enabled(): + return + now = config.now() + for ns in set(papiconf.MAIN_CONF["nomad"]["namespaces"].values()): + try: + _sweep_namespace(ns, now) + except Exception: + LOG.exception("energy sweep failed for namespace %s", ns) + try: + _sweep_cluster(now) + except Exception: + LOG.exception("cluster energy sweep failed") + + +def info_from_job(j: dict, ns: str) -> dict: + """`info` dict `process_single` expects, from a raw Nomad job.""" + meta = j.get("Meta") or {} + return { + "uuid": j["ID"], + "job_name": j["Name"], + "namespace": ns, + "owner": meta.get("owner", ""), + "submit_time": (j["SubmitTime"] / 1e9) if j.get("SubmitTime") else None, + "status": "running", + } + + +def _sweep_namespace(ns: str, now) -> None: + try: + jobs = Nomad.jobs.get_jobs(namespace=ns, filter_='Status != "dead"') + except Exception: + LOG.exception("could not list jobs for namespace %s", ns) + return + + seen = set() + for stub in jobs: + if not stub["Name"].startswith(config.JOB_PREFIXES): + continue + seen.add(stub["ID"]) + # the job list stub does not carry Meta, fetch the full job for `owner` + try: + j = Nomad.job.get_job(id_=stub["ID"], namespace=ns) + except Exception: + LOG.exception("could not fetch job %s", stub["ID"]) + continue + try: + process_single(info_from_job(j, ns), now) + except mimir.MimirError as e: + LOG.warning("Mimir unavailable, stopping the sweep of %s: %s", ns, e) + return + except Exception: + LOG.exception("energy sweep failed for deployment %s", stub["ID"]) + + # close deployments whose Nomad job has been garbage-collected + for uuid, doc in store.iter_namespace(ns): + if uuid in seen or doc.get("closed_at"): + continue + info = { + "uuid": uuid, + "job_name": doc.get("job_name"), + "namespace": ns, + "owner": doc.get("owner", ""), + "submit_time": doc.get("submit_time_epoch"), + "status": "closed", + } + try: + process_single(info, now, closing=True) + except mimir.MimirError as e: + LOG.warning("Mimir unavailable, stopping the sweep of %s: %s", ns, e) + return + except Exception: + LOG.exception("energy close failed for deployment %s", uuid) + + +# --------------------------------------------------------------------------- +# per-deployment sweep +# --------------------------------------------------------------------------- + + +def _deployment_buckets( + doc: dict, allocs_by_dc: dict, alloc_windows: list, settled_ts: float, t: Clock +) -> list: + """ + Energy buckets of [settled_ts, extract_to] summed over the deployment's + datacenters, clipped to its actual life. Also records on `doc` the + energy-weighted mean TUE of what was integrated. + """ + if settled_ts >= t.extract_to: + return [] + + dq = config.get().get("data_quality") + per_dc = [] + tue_norm = tue_raw = 0.0 + for dc, alloc_ids in allocs_by_dc.items(): + cpu, gpu = query.fetch_power( + *compute.power_promql(alloc_ids), settled_ts, t.extract_to, t.step_s + ) + if not cpu and not gpu: + continue + tue = config.tue(dc) + buckets = compute.integrate( + cpu, gpu, t.step_s, t.bucket_s, tue, data_quality=dq + ) + e = sum(b.energy_wh for b in buckets) + tue_norm += e + tue_raw += e / tue + per_dc.append(_with_footprint(dc, buckets, settled_ts, t)) + + if tue_raw: + doc["tue_factor"] = tue_norm / tue_raw + elif "tue_factor" not in doc and allocs_by_dc: + doc["tue_factor"] = config.tue(next(iter(allocs_by_dc))) + + new_buckets = _complete_buckets(compute.merge_buckets(per_dc), t) + # clip to the deployment's actual life: drop buckets with no overlap with + # any allocation window. Guards against metric staleness (the per-alloc + # power series and `nomad_gpu_allocation_info` can linger a few minutes + # after an allocation ends, which would otherwise attribute idle GPU / + # host power to a finished deployment). + if alloc_windows: + new_buckets = [ + b + for b in new_buckets + if compute.running_seconds( + alloc_windows, + compute.parse_iso(b.ts), + compute.parse_iso(b.ts) + t.bucket_s, + ) + > 0 + ] + return new_buckets + + +def process_single(info: dict, now, closing: bool = False) -> None: + t = _clock(now) + ns, uuid = info["namespace"], info["uuid"] + + doc = store.read_accum(ns, uuid) or {} + doc.pop("partial", None) # dropped in favour of `coverage_ratio` (always a float) + + settled_ts, complete = _resume_point(doc, t, info.get("submit_time")) + window_start = _window_start(ns, uuid, doc, settled_ts, t) + + try: + allocs_by_dc, alloc_windows, running_allocs, partial_dc = ( + compute.resolve_alloc_ids(uuid, ns) + ) + except Exception: + allocs_by_dc, alloc_windows, running_allocs, partial_dc = {}, [], [], False + + # metadata (always refreshed) + doc.update( + { + "deployment_uuid": uuid, + "job_name": info.get("job_name") or doc.get("job_name"), + "namespace": ns, + "owner": info.get("owner") or doc.get("owner", ""), + "status": "closed" if closing else info.get("status", "running"), + "complete": complete, + "updated_at": compute.iso(t.now_ts), + "active_allocs": running_allocs, + } + ) + if info.get("submit_time"): + doc["submit_time"] = compute.iso(info["submit_time"]) + doc["submit_time_epoch"] = info["submit_time"] + if allocs_by_dc: + doc["datacenters"] = list(allocs_by_dc) + if closing and not doc.get("closed_at"): + doc["closed_at"] = compute.iso(t.now_ts) + + run_s = compute.running_seconds(alloc_windows, settled_ts, t.extract_to) + + # no monitored allocations: queued, or ran only in unmonitored sites + if not allocs_by_dc: + if "settled" not in doc and partial_dc and run_s > 2 * t.bucket_s: + doc.update({"metrics_available": False, "coverage_ratio": 0.0}) + store.write_accum(ns, uuid, doc) + return + + new_buckets = _deployment_buckets(doc, allocs_by_dc, alloc_windows, settled_ts, t) + + # best-effort coverage: no data while the deployment was clearly running + # means the node has no monitoring stack yet -> null, retry next sweep. + if not new_buckets and "settled" not in doc: + if run_s > 2 * t.bucket_s: + doc.update({"metrics_available": False, "coverage_ratio": 0.0}) + store.write_accum(ns, uuid, doc) + return + + doc["metrics_available"] = True + doc["window_start"] = compute.iso(window_start) + + aged_n, tail_n = _fold_and_accumulate(ns, uuid, doc, new_buckets, settled_ts, t) + + # coverage: fraction of the run time (including time on unmonitored sites, so + # `partial_dc` is captured implicitly) that ended up with power data. + covered_s = (tail_n + aged_n) * t.bucket_s + doc["coverage_ratio"] = round(min(1.0, covered_s / run_s), 3) if run_s > 0 else 1.0 + + store.write_accum(ns, uuid, doc) + + +def _fold_and_accumulate( + ns: str, + uuid: str, + doc: dict, + new_buckets: list, + settled_ts: float, + t: Clock, +) -> tuple[int, int]: + """ + Merge `new_buckets` (Bucket namedtuples) over `doc["tail"]` keyed by `ts`, + age the buckets older than `revision_cut` into `doc["settled"]` (appending + them to the persisted series), and recompute `settled_ts` / `accumulated` / + `pointer_ts` on `doc`. Returns `(aged_count, tail_count)`. + + Shared by the per-deployment (`process_single`) and per-datacenter + (`process_datacenter`) sweeps. + """ + settled = dict(doc.get("settled") or config.EMPTY) + + by_ts = {b["ts"]: b for b in (doc.get("tail") or [])} + for b in new_buckets: + by_ts[b.ts] = b._asdict() + + tail, aged_out = [], [] + for ts in sorted(by_ts): + (aged_out if compute.parse_iso(ts) < t.revision_cut else tail).append(by_ts[ts]) + + if aged_out: + settled = config.add_totals(settled, config.sum_totals(aged_out)) + store.append_series(ns, uuid, aged_out) + settled_ts = max(compute.parse_iso(b["ts"]) for b in aged_out) + t.bucket_s + + doc["settled"] = settled + doc["tail"] = tail + doc["settled_ts"] = compute.iso(settled_ts) + doc["accumulated"] = config.add_totals(settled, config.sum_totals(tail)) + doc["pointer_ts"] = compute.iso( + (compute.parse_iso(tail[-1]["ts"]) + t.bucket_s) if tail else settled_ts + ) + return len(aged_out), len(tail) + + +# --------------------------------------------------------------------------- +# per-datacenter sweep (cluster-level energy accumulator) +# --------------------------------------------------------------------------- + + +def _sweep_cluster(now) -> None: + for dc in config.get()["metrics_datacenters"]: + try: + process_datacenter(dc, now) + except Exception: + LOG.exception("cluster energy sweep failed for datacenter %s", dc) + + +def _cluster_buckets(dc: str, settled_ts: float, t: Clock) -> list: + if settled_ts >= t.extract_to: + return [] + cpu, gpu = query.fetch_power( + *compute.cluster_promql(dc), settled_ts, t.extract_to, t.step_s + ) + buckets = compute.integrate( + cpu, gpu, t.step_s, t.bucket_s, config.tue(dc), data_quality=config.cluster_dq() + ) + return _complete_buckets(_with_footprint(dc, buckets, settled_ts, t), t) + + +def process_datacenter(dc: str, now) -> None: + t = _clock(now) + ns = config.CLUSTER_NS + + doc = store.read_accum(ns, dc) or {} + settled_ts, complete = _resume_point(doc, t) + window_start = _window_start(ns, dc, doc, settled_ts, t) + + doc.update( + { + "datacenter": dc, + "deployment_uuid": dc, # reuse the same key name as deployment docs + "namespace": ns, + "complete": complete, + "updated_at": compute.iso(t.now_ts), + "tue_factor": config.tue(dc), + } + ) + + new_buckets = _cluster_buckets(dc, settled_ts, t) + + if not new_buckets and "settled" not in doc: + # datacenter has no host-power series yet: retry next sweep, no zeros + doc["metrics_available"] = False + store.write_accum(ns, dc, doc) + return + + doc["metrics_available"] = True + doc["window_start"] = compute.iso(window_start) + _fold_and_accumulate(ns, dc, doc, new_buckets, settled_ts, t) + store.write_accum(ns, dc, doc) diff --git a/ai4papi/main.py b/ai4papi/main.py index 606da822..06e3495e 100644 --- a/ai4papi/main.py +++ b/ai4papi/main.py @@ -11,7 +11,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi_utils.tasks import repeat_every -from ai4papi import accounting +from ai4papi import energy as energy_accounting from ai4papi.conf import MAIN_CONF, paths, papi_branch, papi_commit from ai4papi.routers import v1 from ai4papi.routers.v1.stats.deployments import get_cluster_stats_bg @@ -163,7 +163,7 @@ def energy_accounting_thread(): Not async, same reasoning as get_cluster_stats_thread. """ try: - accounting.run_sweep() + energy_accounting.run_sweep() except Exception: logging.exception("Error in background task energy_accounting") diff --git a/ai4papi/routers/v1/batch.py b/ai4papi/routers/v1/batch.py index 04280103..d9e18e40 100644 --- a/ai4papi/routers/v1/batch.py +++ b/ai4papi/routers/v1/batch.py @@ -10,7 +10,8 @@ from fastapi import APIRouter, Depends, Form, HTTPException, UploadFile from fastapi.security import HTTPBearer -from ai4papi import accounting, auth, module_patches, quotas, schemas, utils +from ai4papi import auth, module_patches, quotas, schemas, utils +from ai4papi import energy as energy_accounting import ai4papi.conf as papiconf import ai4papi.nomad_utils as nomad_utils from ai4papi.routers import v1 @@ -95,14 +96,10 @@ def get_deployments( # `full_info=False` (default): one cheap batch. `full_info=True`: the # extended time series per deployment (heavier). - if energy and vo_jobs: - if full_info: - for job in vo_jobs: - job["energy"] = accounting.get_series(namespace, job["job_ID"]) - else: - env = accounting.get_accumulated_bulk(namespace, auth_info["id"]) - for job in vo_jobs: - job["energy"] = env.get(job["job_ID"]) + if energy: + energy_accounting.add_to_jobs( + vo_jobs, namespace, auth_info["id"], full_info + ) user_jobs.extend(vo_jobs) @@ -158,10 +155,7 @@ def get_deployment( ) if energy: - if full_info: - job["energy"] = accounting.get_series(namespace, deployment_uuid) - else: - job["energy"] = accounting.get_accumulated(namespace, deployment_uuid) + energy_accounting.add_to_job(job, namespace, full_info) return job diff --git a/ai4papi/routers/v1/deployments/modules.py b/ai4papi/routers/v1/deployments/modules.py index 6482313e..271ef2ca 100644 --- a/ai4papi/routers/v1/deployments/modules.py +++ b/ai4papi/routers/v1/deployments/modules.py @@ -8,7 +8,8 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.security import HTTPBearer -from ai4papi import accounting, auth, module_patches, quotas, schemas, utils +from ai4papi import auth, module_patches, quotas, schemas, utils +from ai4papi import energy as energy_accounting from ai4papi.wattnet import green_director import ai4papi.conf as papiconf import ai4papi.nomad_utils as nomad_utils @@ -96,14 +97,10 @@ def get_deployments( # Attach energy/footprint stats. `full_info=False` (the default) uses one # batch of cheap file reads + a single cached Mimir top-up; `full_info=True` # attaches the extended time series per deployment (heavier). - if energy and vo_jobs: - if full_info: - for job in vo_jobs: - job["energy"] = accounting.get_series(namespace, job["job_ID"]) - else: - env = accounting.get_accumulated_bulk(namespace, auth_info["id"]) - for job in vo_jobs: - job["energy"] = env.get(job["job_ID"]) + if energy: + energy_accounting.add_to_jobs( + vo_jobs, namespace, auth_info["id"], full_info + ) user_jobs.extend(vo_jobs) @@ -162,10 +159,7 @@ def get_deployment( ) if energy: - if full_info: - job["energy"] = accounting.get_series(namespace, deployment_uuid) - else: - job["energy"] = accounting.get_accumulated(namespace, deployment_uuid) + energy_accounting.add_to_job(job, namespace, full_info) return job diff --git a/ai4papi/routers/v1/deployments/tools.py b/ai4papi/routers/v1/deployments/tools.py index e16bccb5..2b49a737 100644 --- a/ai4papi/routers/v1/deployments/tools.py +++ b/ai4papi/routers/v1/deployments/tools.py @@ -11,7 +11,8 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.security import HTTPBearer -from ai4papi import accounting, auth, quotas, schemas, utils +from ai4papi import auth, quotas, schemas, utils +from ai4papi import energy as energy_accounting from ai4papi.wattnet import green_director import ai4papi.conf as papiconf import ai4papi.nomad_utils as nomad_utils @@ -96,14 +97,10 @@ def get_deployments( # `full_info=False` (default): one cheap batch. `full_info=True`: the # extended time series per deployment (heavier). - if energy and vo_jobs: - if full_info: - for job in vo_jobs: - job["energy"] = accounting.get_series(namespace, job["job_ID"]) - else: - env = accounting.get_accumulated_bulk(namespace, auth_info["id"]) - for job in vo_jobs: - job["energy"] = env.get(job["job_ID"]) + if energy: + energy_accounting.add_to_jobs( + vo_jobs, namespace, auth_info["id"], full_info + ) user_jobs.extend(vo_jobs) @@ -191,10 +188,7 @@ def get_deployment( ] if energy: - if full_info: - job["energy"] = accounting.get_series(namespace, deployment_uuid) - else: - job["energy"] = accounting.get_accumulated(namespace, deployment_uuid) + energy_accounting.add_to_job(job, namespace, full_info) return job diff --git a/ai4papi/routers/v1/stats/deployments.py b/ai4papi/routers/v1/stats/deployments.py index 78b4e43b..5100b5b3 100644 --- a/ai4papi/routers/v1/stats/deployments.py +++ b/ai4papi/routers/v1/stats/deployments.py @@ -14,7 +14,8 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.security import HTTPBearer -from ai4papi import accounting, auth, schemas +from ai4papi import auth, schemas +from ai4papi import energy as energy_accounting from ai4papi.wattnet import green_director import ai4papi.conf as papiconf from ai4papi.nomad_utils import Nomad @@ -131,15 +132,15 @@ def get_user_stats( except ValueError: # user has still no recorded stats user_stats["users-agg"] = None except HTTPException: - if not (energy and accounting.enabled()): + if not (energy and energy_accounting.enabled()): raise user_stats = {} # Accumulated energy consumption + footprint of the user in that VO (each # deployment's accumulator + its live top-up, summed). `full_info` adds the # merged time series. - if energy and accounting.enabled(): - user_stats["energy"] = accounting.get_user_energy( + if energy and energy_accounting.enabled(): + user_stats["energy"] = energy_accounting.get_user_energy( namespace, auth_info["id"], series=full_info ) @@ -470,9 +471,9 @@ def get_cluster_stats_bg() -> schemas.ClusterStats: # Reported per datacenter only, there is no platform-wide total (clients sum # the datacenters they care about). Best-effort: a failure here must not # break the cluster stats. - if accounting.enabled(): + if energy_accounting.enabled(): try: - for dc_name, dc_energy in accounting.get_cluster_energy( + for dc_name, dc_energy in energy_accounting.get_cluster_energy( series=True ).items(): if dc_name not in stats.datacenters: diff --git a/ai4papi/routers/v1/try_me/nomad.py b/ai4papi/routers/v1/try_me/nomad.py index bca9adb4..943bdc6a 100644 --- a/ai4papi/routers/v1/try_me/nomad.py +++ b/ai4papi/routers/v1/try_me/nomad.py @@ -10,7 +10,8 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.security import HTTPBearer -from ai4papi import accounting, auth +from ai4papi import auth +from ai4papi import energy as energy_accounting import ai4papi.conf as papiconf from ai4papi.routers.v1.catalog.modules import Modules from ai4papi.routers.v1.stats.deployments import get_cluster_stats @@ -81,14 +82,8 @@ def get_deployments( user_jobs.append(job_info) - if energy and user_jobs: - if full_info: - for job in user_jobs: - job["energy"] = accounting.get_series(NAMESPACE, job["job_ID"]) - else: - env = accounting.get_accumulated_bulk(NAMESPACE, auth_info["id"]) - for job in user_jobs: - job["energy"] = env.get(job["job_ID"]) + if energy: + energy_accounting.add_to_jobs(user_jobs, NAMESPACE, auth_info["id"], full_info) # Sort deployments by submission time in descending order sorted_jobs = sorted(user_jobs, key=lambda x: x["submit_time"], reverse=True) @@ -131,10 +126,7 @@ def get_deployment( job["main_endpoint"] = "ui" if energy: - if full_info: - job["energy"] = accounting.get_series(NAMESPACE, deployment_uuid) - else: - job["energy"] = accounting.get_accumulated(NAMESPACE, deployment_uuid) + energy_accounting.add_to_job(job, NAMESPACE, full_info) return job From 737cb965004f48f633f415d200a44f47e784666b Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Mon, 19 Oct 2026 20:51:33 +0200 Subject: [PATCH 15/25] fix(mimir): shorter request timeouts and a backoff when Mimir is down Measured against the real Mimir, a 24h range query takes under 1 s and a full 11,000-point backfill chunk about 1.5 s, so the timeouts go from 30 s / 60 s to 10 s (instant) / 15 s (range). After a timeout or connection error every call fails fast with MimirError for 60 s, so a hung Mimir costs one timeout per minute instead of one per query: the sweep thread and the deployment list endpoints no longer block for minutes. query and query_range share one request function. --- ai4papi/clients/mimir.py | 70 ++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/ai4papi/clients/mimir.py b/ai4papi/clients/mimir.py index 390e006b..abfc8467 100644 --- a/ai4papi/clients/mimir.py +++ b/ai4papi/clients/mimir.py @@ -7,10 +7,10 @@ import datetime import os +import time import requests - MIMIR_URL = "https://mimir.k8s.cloud.ai4eosc.eu/prometheus" # Optional: the energy-accounting feature is disabled if these are not set. MIMIR_USER = os.environ.get("MIMIR_USER") @@ -20,9 +20,18 @@ # Mimir keeps each series under a point cap per query; stay well below it. MAX_POINTS_PER_QUERY = 11_000 +# Measured against the real Mimir: a 24h range query takes under 1 s and a full +# 11,000-point chunk about 1.5 s, so these leave a wide margin while a hung Mimir +# does not hold a request or the sweep thread for a minute. session = requests.Session() +# After a timeout or connection error Mimir is not asked again for this long (the +# calls fail fast with `MimirError`), so a hung Mimir costs one timeout per minute +# instead of one per query. +BACKOFF_SECONDS = 60 +_down_until = 0.0 + class MimirError(RuntimeError): """Raised when Mimir returns a non-successful response.""" @@ -32,7 +41,26 @@ def _auth(): return (MIMIR_USER, MIMIR_PASSWORD) if MIMIR_USER else None -def query(promql: str, at: datetime.datetime | None = None, timeout: int = 30): +def _get(path: str, params: dict, timeout: int) -> list: + """GET a Prometheus API route and return `data.result`.""" + global _down_until + if time.monotonic() < _down_until: + raise MimirError("Mimir unreachable, retrying shortly") + try: + r = session.get( + f"{MIMIR_URL}{path}", params=params, auth=_auth(), timeout=timeout + ) + except (requests.ConnectionError, requests.Timeout): + _down_until = time.monotonic() + BACKOFF_SECONDS + raise + r.raise_for_status() + body = r.json() + if body.get("status") != "success": + raise MimirError(f"Mimir query failed: {body}") + return body.get("data", {}).get("result", []) + + +def query(promql: str, at: datetime.datetime | None = None, timeout: int = 10): """ Instantaneous query (`/api/v1/query`). Returns the raw `data.result` list (vector: one entry per series, each with `metric` and `value` = [ts, str]). @@ -40,18 +68,7 @@ def query(promql: str, at: datetime.datetime | None = None, timeout: int = 30): params = {"query": promql} if at is not None: params["time"] = at.timestamp() - - r = session.get( - f"{MIMIR_URL}/api/v1/query", - params=params, - auth=_auth(), - timeout=timeout, - ) - r.raise_for_status() - body = r.json() - if body.get("status") != "success": - raise MimirError(f"Mimir query failed: {body}") - return body.get("data", {}).get("result", []) + return _get("/api/v1/query", params, timeout) def query_range( @@ -59,29 +76,20 @@ def query_range( start: datetime.datetime, end: datetime.datetime, step_s: int = 30, - timeout: int = 60, + timeout: int = 15, ): """ Range query (`/api/v1/query_range`). Returns the raw `data.result` list (matrix: one entry per series, each with `metric` and `values` = [[ts, str], ...]). """ - r = session.get( - f"{MIMIR_URL}/api/v1/query_range", - params={ - "query": promql, - "start": start.timestamp(), - "end": end.timestamp(), - "step": f"{step_s}s", - }, - auth=_auth(), - timeout=timeout, - ) - r.raise_for_status() - body = r.json() - if body.get("status") != "success": - raise MimirError(f"Mimir query failed: {body}") - return body.get("data", {}).get("result", []) + params = { + "query": promql, + "start": start.timestamp(), + "end": end.timestamp(), + "step": f"{step_s}s", + } + return _get("/api/v1/query_range", params, timeout) def _split_chunks(start, end, step_s, max_points): From 302b2932e14afbff8117ca7e2f30c440b3e24043 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Mon, 19 Oct 2026 20:51:33 +0200 Subject: [PATCH 16/25] fix(wattnet): time out requests and keep the last good data on failures - Every Wattnet request has a 30 s timeout, so a stalled Wattnet cannot block the background threads. - A failed or empty answer raises instead of returning an empty list, so the 15 min cache no longer keeps a failure; it used to leave a datacenter on default values for a quarter of an hour. - On a failed refresh a datacenter keeps its last good data (default values only if it never had any) and is not asked again for 5 minutes. - One warning when a datacenter starts failing and one info line when it recovers, instead of one per refresh. --- ai4papi/wattnet.py | 65 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/ai4papi/wattnet.py b/ai4papi/wattnet.py index ef2e3ee0..3512a2da 100644 --- a/ai4papi/wattnet.py +++ b/ai4papi/wattnet.py @@ -5,6 +5,7 @@ import datetime import json +import logging import os import statistics import warnings @@ -16,9 +17,13 @@ import ai4papi.conf as papiconf from ai4papi import utils +LOG = logging.getLogger(__name__) + session = requests.Session() WATTNET_URL = "https://api.wattnet.eu" +TIMEOUT = 30 # seconds; without it a stalled Wattnet would block the background threads +FAILURE_BACKOFF = 5 * 60 # seconds a failing datacenter is not asked again WATTNET_EMAIL = os.environ.get("WATTNET_EMAIL") if not WATTNET_EMAIL: print("You should define a WATTNET_EMAIL") @@ -66,6 +71,8 @@ def __init__(self, algorithm: str = "linear_rank"): # Init vars self.algorithm_name = algorithm self.metrics = {} + self._backoff = TTLCache(maxsize=1024, ttl=FAILURE_BACKOFF) + self._failing = set() # datacenters whose last refresh failed (log on change) @cached(cache=TTLCache(maxsize=1024, ttl=20 * 60 * 60)) def _retrieve_token(self): @@ -75,7 +82,9 @@ def _retrieve_token(self): url = f"{WATTNET_URL}/token-request/get_token" headers = {"Content-Type": "application/json"} data = {"email": WATTNET_EMAIL, "password": WATTNET_PASS} - response = requests.post(url, headers=headers, data=json.dumps(data)) + response = requests.post( + url, headers=headers, data=json.dumps(data), timeout=TIMEOUT + ) response.raise_for_status() return response.json()["access_token"] @@ -98,20 +107,30 @@ def _fetch_footprint_data(self, lat, lon): token = self._retrieve_token() headers = {"Authorization": f"Bearer {token}"} - r1 = session.get(f"{WATTNET_URL}/v1/footprints", headers=headers, params=params) + r1 = session.get( + f"{WATTNET_URL}/v1/footprints", + headers=headers, + params=params, + timeout=TIMEOUT, + ) r2 = session.get( - f"{WATTNET_URL}/v1/environmental-score", headers=headers, params=params + f"{WATTNET_URL}/v1/environmental-score", + headers=headers, + params=params, + timeout=TIMEOUT, ) + # Raise instead of returning [] so a failure is not cached for 15 min + # (cachetools only caches return values); the caller falls back to DEFAULTS if not (r1.ok and r2.ok): - warnings.warn( - f"[wattnet] Failed to retrieve Wattnet data for coordinates ({lat}, {lon})" + raise RuntimeError( + f"Wattnet request failed for coordinates ({lat}, {lon}): " + f"{r1.status_code}, {r2.status_code}" ) - return [] footprints = r1.json() score = r2.json() if (not footprints) or (not score): - raise Exception("Error retrieving information from Wattnet.") + raise RuntimeError(f"Empty Wattnet data for coordinates ({lat}, {lon})") score[0].update({"footprint_type": "environmental-score"}) return footprints + score @@ -140,7 +159,12 @@ def footprint_series(self, datacenter: str, start, end): "aggregate": "false", } headers = {"Authorization": f"Bearer {self._retrieve_token()}"} - r = session.get(f"{WATTNET_URL}/v1/footprints", headers=headers, params=params) + r = session.get( + f"{WATTNET_URL}/v1/footprints", + headers=headers, + params=params, + timeout=TIMEOUT, + ) if not r.ok: warnings.warn( f"[wattnet] Failed to retrieve footprint series for {datacenter}" @@ -169,7 +193,30 @@ def retrieve_footprints(self): start = end - datetime.timedelta(days=7) for k, v in datacenters.items(): - data = self._fetch_footprint_data(v["lat"], v["lon"]) + if k in self._backoff: + continue # failed a moment ago: do not wait on a down Wattnet again + try: + data = self._fetch_footprint_data(v["lat"], v["lon"]) + except Exception as e: + self._backoff[k] = True + if k not in self._failing: + self._failing.add(k) + kept = self.metrics.get(k, {}).get("carbon") + LOG.warning( + "Wattnet refresh failed for %s (%s); %s", + k, + e, + f"keeping data up to {kept[-1][0]}" + if kept + else "using default values", + ) + if k in self.metrics: + continue # keep the last good data rather than overwrite it + data = [] # never had data: fall back to DEFAULTS + else: + if k in self._failing: + self._failing.discard(k) + LOG.info("Wattnet refresh recovered for %s", k) # For each footprint type, we concatenate all timeseries, irrespectively of # whether they are [valid=True] (meaning their values are final) or From 2fffd5f15b413d8ebd51e995e25ac0c45397a76e Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Mon, 19 Oct 2026 20:51:33 +0200 Subject: [PATCH 17/25] test(energy): run the suite with pytest and cover the new behaviour test_energy.py becomes a pytest module (a fixture gives each test a fresh store, a fake Mimir and clean caches) and tests/main.py runs it through pytest.main. pytest is an optional "test" extra. New checks: the on-demand and live-only paths, add_to_job(s), the series builder, the store (repeated bucket, owner index), the Mimir backoff and the sweep stopping when Mimir is down, a failing query reported as degraded, and the Wattnet failure handling (nothing cached, backoff, last good data kept, one log line per outage). test_live_only_fallback froze "now" so it no longer fails when run in the first minutes of a quarter of an hour. --- tests/main.py | 8 +- tests/test_energy.py | 1813 +++++++++++++++++++++++++----------------- 2 files changed, 1070 insertions(+), 751 deletions(-) diff --git a/tests/main.py b/tests/main.py index eb226e64..39b50066 100644 --- a/tests/main.py +++ b/tests/main.py @@ -30,6 +30,12 @@ for module in modules: try: - __import__(module) + if module == "test_energy": # pytest-style, unlike the plain-script tests + import pytest + + if pytest.main(["-q", str(Path(__file__).parent / f"{module}.py")]): + raise RuntimeError("pytest reported failures") + else: + __import__(module) except Exception as e: print(f"🔴 Error testing {module}:\n{e}") diff --git a/tests/test_energy.py b/tests/test_energy.py index b2a040f2..1238a6fb 100644 --- a/tests/test_energy.py +++ b/tests/test_energy.py @@ -1,197 +1,36 @@ """ Tests for the energy-accounting feature. -Pure-function checks run everywhere. The infra checks (Mimir / a real -deployment) are skipped unless MIMIR_USER/MIMIR_PASSWORD and PAPI_TESTS_ENERGY_VO -are set, so the suite stays runnable without the monitoring stack. +Pure-function checks run everywhere. The sweep / read-API checks use a fake Mimir +and Nomad (no infra). The live Mimir check is skipped unless PAPI_TESTS_MIMIR and +MIMIR_USER/MIMIR_PASSWORD are set, so the suite stays runnable without the +monitoring stack. """ import datetime +import json +import logging +import os -from ai4papi import mimir -from ai4papi.accounting import compute +import pytest +from ai4papi import conf as papiconf +from ai4papi import energy, schemas +from ai4papi.clients import mimir +from ai4papi.energy import compute, config, live, ondemand, series, store, sweep -def _series(values, scale_str=str): - """Build a Mimir matrix result with one series from [(ts, val)].""" - return [{"metric": {}, "values": [[t, scale_str(v)] for t, v in values]}] - - -# --- compute.integrate -------------------------------------------------------- - -# 1 W constant for 1h at 30s step -> 1 Wh at TUE 1.0, 2.03 Wh at TUE 2.03 -step = 30 -n = 3600 // step + 1 -cpu = _series([(1_000_000 + i * step, 1_000_000) for i in range(n)]) # 1e6 uW = 1 W -buckets = compute.integrate(cpu, [], step_s=step, bucket_s=3600, tue_factor=1.0) -total = sum(b.energy_wh for b in buckets) -assert abs(total - 1.0) < 1e-3, total - -buckets_tue = compute.integrate(cpu, [], step_s=step, bucket_s=3600, tue_factor=2.03) -assert abs(sum(b.energy_wh for b in buckets_tue) - 2.03) < 1e-2 - -# gap longer than gap_factor*step is not integrated across -gapped = _series([(0, 1_000_000), (30, 1_000_000), (600, 1_000_000), (630, 1_000_000)]) -gb = compute.integrate(gapped, [], step_s=step, bucket_s=3600, tue_factor=1.0) -# only the two 30s intervals contribute: 2 * (1W * 30s) = 60 Ws = 60/3600 Wh -assert abs(sum(b.energy_wh for b in gb) - 60 / 3600) < 1e-4 - -# a single sample -> no energy -assert compute.integrate(_series([(0, 1_000_000)]), [], 30, 3600) == [] - -# --- compute.sanitize (data quality) --------------------------------------- - -dq = {"enabled": True, "min_power_w": 0, "max_power_w": 1000, "spike_factor": 5} -pw = [(0, 100.0), (30, -5.0), (60, 100.0), (90, 99999.0), (120, 100.0)] -clean = compute.sanitize(pw, dq) -assert (30, -5.0) not in clean # negative dropped -assert (90, 99999.0) not in clean # over ceiling dropped -assert len(clean) == 3 - -# spike detection against the rolling median -spiky = [(i * 30, 100.0) for i in range(20)] -spiky[10] = (300, 5000.0) -clean = compute.sanitize(spiky, {"spike_factor": 10, "spike_window": 7}) -assert (300, 5000.0) not in clean and len(clean) == 19 - -# disabled -> passthrough -assert compute.sanitize(pw, {"enabled": False}) == pw -assert compute.sanitize(pw, None) == pw - -# integrate drops the anomaly so it never inflates the accumulated energy -anom = _series( - [(t, 1_000_000) for t in range(0, 300, 30)] - + [(300, 900_000_000)] - + [(t, 1_000_000) for t in range(330, 600, 30)] -) -raw = sum( - b.energy_wh - for b in compute.integrate(anom, [], step_s=30, bucket_s=3600, tue_factor=1.0) -) -clean = sum( - b.energy_wh - for b in compute.integrate( - anom, - [], - step_s=30, - bucket_s=3600, - tue_factor=1.0, - data_quality={"max_power_w": 100}, - ) -) -assert raw > 5 # the 900 W spike dominates the ~1 W baseline -assert clean < 0.3 # spike dropped -> only the ~1 W baseline remains - -# CPU + GPU add up -cpu2 = _series([(0, 2_000_000), (30, 2_000_000)]) # 2 W -gpu2 = _series([(0, 8), (30, 8)]) # 8 W -mb = compute.integrate(cpu2, gpu2, step_s=30, bucket_s=3600, tue_factor=1.0) -assert abs(sum(b.energy_wh for b in mb) - 10 * 30 / 3600) < 1e-4 - -# per-source data quality: a spike in one series does not discard the others -two_gpu = [ - {"metric": {"UUID": "g0"}, "values": [[0, "10"], [30, "10"], [60, "10"]]}, - {"metric": {"UUID": "g1"}, "values": [[0, "10"], [30, "9000"], [60, "10"]]}, -] -mb = compute.integrate( - [], - two_gpu, - step_s=30, - bucket_s=3600, - tue_factor=1.0, - data_quality={"max_power_w": 100}, -) -got = sum(b.energy_wh for b in mb) -# g0: 3 samples of 10 W over 60 s = 10*60/3600. g1: middle sample dropped, gap -# 0->60 s at 10 W = 10*60/3600. Total ~ 2 * 10 * 60/3600. -assert abs(got - 2 * 10 * 60 / 3600) < 1e-4, got - -# --- compute.apply_footprint ------------------------------------------------- - -b = compute.Bucket("2026-01-01T00:00:00Z", 2000.0, 40.0, 0.0, 0.0) # 2 kWh -out = compute.apply_footprint([b]) # no series -> DEFAULTS -assert abs(out[0].carbon_g - 2.0 * compute.DEFAULT_CARBON) < 1e-6 -assert abs(out[0].water_l - 2.0 * compute.DEFAULT_WATER) < 1e-6 - -series = [["2026-01-01T00:00:00Z", 100.0], ["2026-01-01T00:15:00Z", 200.0]] -out = compute.apply_footprint([b], carbon_series=series, water_series=series) -assert abs(out[0].carbon_g - 2.0 * 100.0) < 1e-6 - -# bucket before the series -> DEFAULTS -early = compute.Bucket("2025-12-31T00:00:00Z", 1000.0, 20.0, 0.0, 0.0) -out = compute.apply_footprint([early], carbon_series=series) -assert abs(out[0].carbon_g - 1.0 * compute.DEFAULT_CARBON) < 1e-6 - -# --- compute.intensity_at --------------------------------------------------- - -ts_arr = [10.0, 20.0, 30.0] -val_arr = [1.0, 2.0, 3.0] -assert compute.intensity_at(ts_arr, val_arr, 5.0, -1) == -1 # before first -assert compute.intensity_at(ts_arr, val_arr, 20.0, -1) == 2.0 # exact -assert compute.intensity_at(ts_arr, val_arr, 25.0, -1) == 2.0 # between -assert compute.intensity_at(ts_arr, val_arr, 999.0, -1) == 3.0 # after last -assert compute.intensity_at([], [], 1.0, 7) == 7 # empty - -# --- compute.downsample --------------------------------------------------- - -many = [compute.Bucket(compute._iso(i * 300), 1.0, 10.0, 0.1, 0.01) for i in range(100)] -ds = compute.downsample(many, 10) -assert len(ds) == 10 -assert abs(sum(b.energy_wh for b in ds) - sum(b.energy_wh for b in many)) < 1e-6 -assert compute.downsample(many, 500) is many # nothing to do - -# --- compute.merge_buckets ------------------------------------------------ - -b1 = compute.Bucket("t0", 2.0, 20.0, 0.5, 0.02) -b2 = compute.Bucket("t0", 3.0, 30.0, 0.7, 0.03) -merged = compute.merge_buckets([[b1], [b2]]) -assert len(merged) == 1 -assert merged[0].energy_wh == 5.0 and merged[0].power_w_avg == 50.0 - -# --- compute.running_seconds -------------------------------------------- - -W = compute.AllocWindow -assert compute.running_seconds([W("a", 0, 100)], 0, 200) == 100 -assert compute.running_seconds([W("a", 0, 100), W("b", 50, 150)], 0, 200) == 150 -assert compute.running_seconds([W("a", 100, None)], 0, 200) == 100 # still running - - -print("🟢 energy: pure-function checks passed!") - - -# --- live Mimir check (optional, before any monkeypatching) ------------------ - -import os # noqa: E402 - -if os.environ.get("PAPI_TESTS_MIMIR") and mimir.MIMIR_USER: - _e = datetime.datetime.now(datetime.timezone.utc) - _s = _e - datetime.timedelta(hours=1) - assert isinstance(mimir.query_range("up", _s, _e, step_s=60), list) - print("🟢 energy: live Mimir query_range reachable!") -else: - print("🟡 energy: skipping live Mimir check (set PAPI_TESTS_MIMIR + MIMIR_* creds)") - - -# --- sweep pipeline (fake Mimir + Nomad, no infra) --------------------------- - -import json # noqa: E402 -import tempfile # noqa: E402 - -os.environ["ACCOUNTING_PTH"] = tempfile.mkdtemp() -# make the feature "enabled" for the read API even without real Mimir creds -mimir.MIMIR_USER = mimir.MIMIR_USER or "test" -mimir.MIMIR_PASSWORD = mimir.MIMIR_PASSWORD or "test" - -from ai4papi import conf as papiconf # noqa: E402 -from ai4papi.accounting import store, sweep # noqa: E402 -import ai4papi.accounting as accounting # noqa: E402 +_REAL_QUERY_RANGE = mimir.query_range _NOW = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0) _NS, _UUID, _DC = "ai4eosc", "sweep-test-uuid", "ifca-ai4eosc" # fixed, test-owned TUE: independent of whatever var/datacenters.csv says for # `_DC` in production, so editing that file never breaks this test. _TUE = 2.5 -papiconf.datacenters[_DC] = {**papiconf.datacenters[_DC], "TUE": _TUE} + + +def _series(values, scale_str=str): + """Build a Mimir matrix result with one series from [(ts, val)].""" + return [{"metric": {}, "values": [[t, scale_str(v)] for t, v in values]}] def _fake_query_range(promql, start, end, step_s=30): @@ -203,632 +42,1106 @@ def _fake_query_range(promql, start, end, step_s=30): return [{"metric": {}, "values": out}] -mimir.query_range = _fake_query_range -mimir.query_range_chunked = _fake_query_range -compute.resolve_alloc_ids = lambda job_id, ns: ( - {_DC: ["alloc-1"]}, - [compute.AllocWindow("alloc-1", _NOW.timestamp() - 3 * 3600, None)], - [{"alloc_id": "alloc-1", "datacenter": _DC}], - False, -) -sweep.green_director.metrics = {_DC: {"carbon": [], "water": []}} - -_info = { - "uuid": _UUID, - "job_name": f"module-{_UUID}", - "namespace": _NS, - "owner": "user@egi.eu", - "submit_time": _NOW.timestamp() - 3 * 3600, - "status": "running", -} - -sweep.process_single(_info, _NOW) -doc = store.read_accum(_NS, _UUID) -acc = doc["accumulated"] -assert doc["metrics_available"] is True -assert abs(doc["tue_factor"] - _TUE) < 1e-9 -# ~3h of 100 W -> ~300 Wh raw, x _TUE (11 or 12 complete 15-min buckets -# depending on where "now" falls in the current bucket) -assert 270 * _TUE < acc["energy_wh"] < 305 * _TUE, acc -assert abs(acc["carbon_g"] - acc["energy_wh"] / 1000 * compute.DEFAULT_CARBON) < 1.0 - -# idempotent: same "now" -> byte-identical document -before = json.dumps(doc, sort_keys=True) -accounting._read_doc.cache_clear() -sweep.process_single(_info, _NOW) -assert json.dumps(store.read_accum(_NS, _UUID), sort_keys=True) == before - -# monotonic on a later sweep -accounting._read_doc.cache_clear() -sweep.process_single(_info, _NOW + datetime.timedelta(hours=1)) -assert store.read_accum(_NS, _UUID)["accumulated"]["energy_wh"] >= acc["energy_wh"] - -# buckets age out of the 24h tail into the persisted series -accounting._read_doc.cache_clear() -sweep.process_single(_info, _NOW + datetime.timedelta(hours=30)) -assert len(store.read_series(_NS, _UUID, None, None)) > 0 - -# read API + pydantic validation -from ai4papi import schemas # noqa: E402 - -accounting._read_doc.cache_clear() -stats = accounting.get_accumulated(_NS, _UUID, live=False) -schemas.EnergyStats(**stats) -series = accounting.get_series(_NS, _UUID) -schemas.EnergyTimeSeries(**series) -assert series["series"] and all(p["energy_wh"] is not None for p in series["series"]) -# per-bucket increments only; running total is cumsum(series[].energy_wh) -_pt = set(series["series"][-1]) -assert _pt == { - "ts", - "power_w", - "energy_wh", - "carbon_g", - "water_l", - "live", -}, _pt -_ps = accounting.get_accumulated(_NS, _UUID, live=True) -assert _ps["power_w"] and abs(_ps["tue_factor"] - _TUE) < 1e-9 - -# per-user aggregate: sums the owner's deployments, EnergyStats-shaped -accounting._read_doc.cache_clear() -_ue = accounting.get_user_energy(_NS, "user@egi.eu", series=False) -schemas.EnergyStats(**_ue) -assert _ue["deployments"] >= 1 and _ue["series"] is None -assert _ue["energy_wh"] >= acc["energy_wh"] and _ue["since"] -_ues = accounting.get_user_energy(_NS, "user@egi.eu", series=True) -schemas.EnergyStats(**_ues) -assert _ues["series"] and all(p["energy_wh"] is not None for p in _ues["series"]) -# the aggregate total must reconcile with the sum of the series (no rounding) -assert abs(_ues["energy_wh"] - sum(p["energy_wh"] for p in _ues["series"])) < 1e-9, _ues -assert abs(_ues["carbon_g"] - sum(p["carbon_g"] for p in _ues["series"])) < 1e-9 -# merge-then-downsample: uniform step across the whole persisted series -_uts = [ - datetime.datetime.fromisoformat(p["ts"].replace("Z", "+00:00")) - for p in _ues["series"] - if not p["live"] -] -_steps = {round((_uts[i + 1] - _uts[i]).total_seconds()) for i in range(len(_uts) - 1)} -assert len(_steps) == 1, _steps -# hard cap on the persisted point count, same as the other levels -_capped = accounting.get_user_energy(_NS, "user@egi.eu", series=True, target_points=5) -assert len([p for p in _capped["series"] if not p["live"]]) <= 5 -assert accounting.get_user_energy(_NS, "nobody@egi.eu") is None - -# live tail: a running deployment's graph keeps growing past the stored series -_live_doc = { - "namespace": _NS, - "deployment_uuid": _UUID, - "metrics_available": True, - "pointer_ts": compute._iso( - datetime.datetime.now(datetime.timezone.utc).timestamp() - 600 - ), - "active_allocs": [{"alloc_id": "alloc-1", "datacenter": _DC}], -} -_lp = sweep.live_series(_live_doc, bucket_s=60) -assert _lp and all(p["live"] for p in _lp), _lp -assert all("energy_wh" in p and "carbon_g" in p for p in _lp) - -# the live top-up also carries the current power (100 W raw, x TUE) -_tu = sweep.topup(_live_doc) -_tue = sweep._tue(_DC) -assert _tu["power_w"] is not None -assert abs(_tu["power_w"] - 100.0 * _tue) < 5.0 * _tue, _tu - -# a running deployment's accumulated stats expose a power figure -accounting._read_doc.cache_clear() -_st = accounting.get_accumulated(_NS, _UUID, live=True) -assert _st["power_w"] is not None - -# the accumulated stats always carry a float coverage_ratio (1.0 = full) -assert isinstance(_st["coverage_ratio"], float) - -# GPU/CPU energy is clipped to the job's life: a finished allocation whose power -# metrics linger for a few minutes must not keep accumulating energy. -_ENDED_UUID = "sweep-ended-uuid" -_ended_end = _NOW.timestamp() - 2 * 3600 # allocation stopped 2h ago -compute.resolve_alloc_ids = lambda job_id, ns: ( - {_DC: ["alloc-e"]}, - [compute.AllocWindow("alloc-e", _NOW.timestamp() - 5 * 3600, _ended_end)], - [], # nothing running - False, -) -_ended_info = { - "uuid": _ENDED_UUID, - "job_name": f"module-{_ENDED_UUID}", - "namespace": _NS, - "owner": "user@egi.eu", - "submit_time": _NOW.timestamp() - 5 * 3600, - "status": "running", -} -accounting._read_doc.cache_clear() -sweep.process_single(_ended_info, _NOW) -_edoc = store.read_accum(_NS, _ENDED_UUID) -_last_ts = max( - (b["ts"] for b in _edoc.get("tail", [])), - default=_edoc.get("settled_ts", ""), +@pytest.fixture(autouse=True) +def infra(monkeypatch, tmp_path): + """Fresh store + fake Mimir for every test; the feature counts as enabled.""" + monkeypatch.setenv("ACCOUNTING_PTH", str(tmp_path)) + monkeypatch.setattr(mimir, "MIMIR_USER", mimir.MIMIR_USER or "test") + monkeypatch.setattr(mimir, "MIMIR_PASSWORD", mimir.MIMIR_PASSWORD or "test") + monkeypatch.setitem( + papiconf.datacenters, _DC, {**papiconf.datacenters[_DC], "TUE": _TUE} + ) + monkeypatch.setattr(mimir, "_down_until", 0.0) + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + monkeypatch.setattr( + sweep.green_director, "metrics", {_DC: {"carbon": [], "water": []}} + ) + store._owner_idx.clear() + energy._read_doc.cache_clear() + energy.get_datacenter_power.cache_clear() + energy._cluster_archive_series.cache_clear() + for cache in ( + live._live_cache, + ondemand._live_only_doc_cache, + ondemand._bulk_sweep_cache, + ): + cache.clear() + + +@pytest.mark.skipif( + not (os.environ.get("PAPI_TESTS_MIMIR") and mimir.MIMIR_USER), + reason="set PAPI_TESTS_MIMIR + MIMIR_* creds", ) -# fake Mimir returns 100 W for the whole [settled, now] window, but no bucket -# should land after the allocation ended (+ one bucket of slack). -assert compute._parse_iso(_last_ts) <= _ended_end + 300, _last_ts +def test_live_mimir_reachable(): + end = datetime.datetime.now(datetime.timezone.utc) + start = end - datetime.timedelta(hours=1) + assert isinstance(_REAL_QUERY_RANGE("up", start, end, step_s=60), list) -# a GPU-only monitored site (no scaphandre series at all) still accounts energy -# from the GPU power alone: a missing `scaph_*` metric comes back as an empty -# result, not an error, so the integrator just sums the GPU series. -_GPU_UUID = "sweep-gpu-only-uuid" +def test_integrate(): + # --- compute.integrate -------------------------------------------------------- -def _fake_qr_gpu_only(promql, start, end, step_s=30): - if "scaph" in promql: - return [] - t, out = start.timestamp(), [] - while t <= end.timestamp(): - out.append([t, "150"]) # 150 W per GPU - t += step_s - return [{"metric": {}, "values": out}] + # 1 W constant for 1h at 30s step -> 1 Wh at TUE 1.0, 2.03 Wh at TUE 2.03 + step = 30 + n = 3600 // step + 1 + cpu = _series([(1_000_000 + i * step, 1_000_000) for i in range(n)]) # 1e6 uW = 1 W + buckets = compute.integrate(cpu, [], step_s=step, bucket_s=3600, tue_factor=1.0) + total = sum(b.energy_wh for b in buckets) + assert abs(total - 1.0) < 1e-3, total + buckets_tue = compute.integrate( + cpu, [], step_s=step, bucket_s=3600, tue_factor=2.03 + ) + assert abs(sum(b.energy_wh for b in buckets_tue) - 2.03) < 1e-2 -mimir.query_range = _fake_qr_gpu_only -mimir.query_range_chunked = _fake_qr_gpu_only -compute.resolve_alloc_ids = lambda job_id, ns: ( - {_DC: ["alloc-g"]}, - [compute.AllocWindow("alloc-g", _NOW.timestamp() - 3 * 3600, None)], - [{"alloc_id": "alloc-g", "datacenter": _DC}], - False, -) -_gpu_info = { - "uuid": _GPU_UUID, - "job_name": f"module-{_GPU_UUID}", - "namespace": _NS, - "owner": "gpu@egi.eu", - "submit_time": _NOW.timestamp() - 3 * 3600, - "status": "running", -} -accounting._read_doc.cache_clear() -sweep.process_single(_gpu_info, _NOW) -_gdoc = store.read_accum(_NS, _GPU_UUID) -assert _gdoc["metrics_available"] is True, _gdoc -assert _gdoc["accumulated"]["energy_wh"] > 0, _gdoc - -print("🟢 energy: sweep pipeline checks passed!") - - -# --- on-demand sweep: read API does not wait for the background thread ------- - -_OND_UUID = "sweep-on-demand-uuid" -mimir.query_range = _fake_query_range -mimir.query_range_chunked = _fake_query_range -compute.resolve_alloc_ids = lambda job_id, ns: ( - {_DC: ["alloc-o"]}, - [compute.AllocWindow("alloc-o", _NOW.timestamp() - 3 * 3600, None)], - [{"alloc_id": "alloc-o", "datacenter": _DC}], - False, -) + # gap longer than gap_factor*step is not integrated across + gapped = _series( + [(0, 1_000_000), (30, 1_000_000), (600, 1_000_000), (630, 1_000_000)] + ) + gb = compute.integrate(gapped, [], step_s=step, bucket_s=3600, tue_factor=1.0) + # only the two 30s intervals contribute: 2 * (1W * 30s) = 60 Ws = 60/3600 Wh + assert abs(sum(b.energy_wh for b in gb) - 60 / 3600) < 1e-4 + + # a single sample -> no energy + assert compute.integrate(_series([(0, 1_000_000)]), [], 30, 3600) == [] + + +def test_sanitize(): + # --- compute.sanitize (data quality) --------------------------------------- + + dq = {"enabled": True, "min_power_w": 0, "max_power_w": 1000, "spike_factor": 5} + pw = [(0, 100.0), (30, -5.0), (60, 100.0), (90, 99999.0), (120, 100.0)] + clean = compute.sanitize(pw, dq) + assert (30, -5.0) not in clean # negative dropped + assert (90, 99999.0) not in clean # over ceiling dropped + assert len(clean) == 3 + + # spike detection against the rolling median + spiky = [(i * 30, 100.0) for i in range(20)] + spiky[10] = (300, 5000.0) + clean = compute.sanitize(spiky, {"spike_factor": 10, "spike_window": 7}) + assert (300, 5000.0) not in clean and len(clean) == 19 + + # disabled -> passthrough + assert compute.sanitize(pw, {"enabled": False}) == pw + assert compute.sanitize(pw, None) == pw + + # integrate drops the anomaly so it never inflates the accumulated energy + anom = _series( + [(t, 1_000_000) for t in range(0, 300, 30)] + + [(300, 900_000_000)] + + [(t, 1_000_000) for t in range(330, 600, 30)] + ) + raw = sum( + b.energy_wh + for b in compute.integrate(anom, [], step_s=30, bucket_s=3600, tue_factor=1.0) + ) + clean = sum( + b.energy_wh + for b in compute.integrate( + anom, + [], + step_s=30, + bucket_s=3600, + tue_factor=1.0, + data_quality={"max_power_w": 100}, + ) + ) + assert raw > 5 # the 900 W spike dominates the ~1 W baseline + assert clean < 0.3 # spike dropped -> only the ~1 W baseline remains + + # CPU + GPU add up + cpu2 = _series([(0, 2_000_000), (30, 2_000_000)]) # 2 W + gpu2 = _series([(0, 8), (30, 8)]) # 8 W + mb = compute.integrate(cpu2, gpu2, step_s=30, bucket_s=3600, tue_factor=1.0) + assert abs(sum(b.energy_wh for b in mb) - 10 * 30 / 3600) < 1e-4 + + # per-source data quality: a spike in one series does not discard the others + two_gpu = [ + {"metric": {"UUID": "g0"}, "values": [[0, "10"], [30, "10"], [60, "10"]]}, + {"metric": {"UUID": "g1"}, "values": [[0, "10"], [30, "9000"], [60, "10"]]}, + ] + mb = compute.integrate( + [], + two_gpu, + step_s=30, + bucket_s=3600, + tue_factor=1.0, + data_quality={"max_power_w": 100}, + ) + got = sum(b.energy_wh for b in mb) + # g0: 3 samples of 10 W over 60 s = 10*60/3600. g1: middle sample dropped, gap + # 0->60 s at 10 W = 10*60/3600. Total ~ 2 * 10 * 60/3600. + assert abs(got - 2 * 10 * 60 / 3600) < 1e-4, got -def _fake_get_job(id_, namespace): - if id_ != _OND_UUID: - raise KeyError(id_) - return { - "ID": _OND_UUID, - "Name": f"module-{_OND_UUID}", - "SubmitTime": int((_NOW.timestamp() - 3 * 3600) * 1e9), - "Meta": {"owner": "ondemand@egi.eu"}, - } +def test_apply_footprint(): + # --- compute.apply_footprint ------------------------------------------------- + b = compute.Bucket("2026-01-01T00:00:00Z", 2000.0, 40.0, 0.0, 0.0) # 2 kWh + out = compute.apply_footprint([b]) # no series -> DEFAULTS + assert abs(out[0].carbon_g - 2.0 * compute.DEFAULT_CARBON) < 1e-6 + assert abs(out[0].water_l - 2.0 * compute.DEFAULT_WATER) < 1e-6 -_orig_get_job = sweep.Nomad.job.get_job -sweep.Nomad.job.get_job = _fake_get_job + series = [["2026-01-01T00:00:00Z", 100.0], ["2026-01-01T00:15:00Z", 200.0]] + out = compute.apply_footprint([b], carbon_series=series, water_series=series) + assert abs(out[0].carbon_g - 2.0 * 100.0) < 1e-6 -# never swept: no doc yet, the read API must trigger one sweep pass itself -assert store.read_accum(_NS, _OND_UUID) is None -accounting._read_doc.cache_clear() -_ond_stats = accounting.get_accumulated(_NS, _OND_UUID, live=False) -assert _ond_stats is not None and _ond_stats["energy_wh"] > 0, _ond_stats -assert store.read_accum(_NS, _OND_UUID) is not None + # bucket before the series -> DEFAULTS + early = compute.Bucket("2025-12-31T00:00:00Z", 1000.0, 20.0, 0.0, 0.0) + out = compute.apply_footprint([early], carbon_series=series) + assert abs(out[0].carbon_g - 1.0 * compute.DEFAULT_CARBON) < 1e-6 -# already swept once (on-demand or not): no further Nomad call needed -def _boom(id_, namespace): - raise AssertionError("ensure_swept should not touch Nomad once a doc exists") +def test_intensity_at(): + # --- compute.intensity_at --------------------------------------------------- + ts_arr = [10.0, 20.0, 30.0] + val_arr = [1.0, 2.0, 3.0] + assert compute.intensity_at(ts_arr, val_arr, 5.0, -1) == -1 # before first + assert compute.intensity_at(ts_arr, val_arr, 20.0, -1) == 2.0 # exact + assert compute.intensity_at(ts_arr, val_arr, 25.0, -1) == 2.0 # between + assert compute.intensity_at(ts_arr, val_arr, 999.0, -1) == 3.0 # after last + assert compute.intensity_at([], [], 1.0, 7) == 7 # empty -sweep.Nomad.job.get_job = _boom -accounting._read_doc.cache_clear() -assert accounting.get_accumulated(_NS, _OND_UUID, live=False) is not None -# job unknown to Nomad: on-demand sweep finds nothing, degrades to `None`, no crash -sweep.Nomad.job.get_job = _fake_get_job -accounting._read_doc.cache_clear() -assert accounting.get_accumulated(_NS, "sweep-unknown-uuid", live=False) is None -accounting._read_doc.cache_clear() -assert accounting.get_series(_NS, "sweep-unknown-uuid") is None +def test_downsample(): + # --- compute.downsample --------------------------------------------------- -sweep.Nomad.job.get_job = _orig_get_job + many = [ + compute.Bucket(compute.iso(i * 300), 1.0, 10.0, 0.1, 0.01) for i in range(100) + ] + ds = compute.downsample(many, 10) + assert len(ds) == 10 + assert abs(sum(b.energy_wh for b in ds) - sum(b.energy_wh for b in many)) < 1e-6 + assert compute.downsample(many, 500) is many # nothing to do -print("🟢 energy: on-demand sweep checks passed!") +def test_merge_buckets(): + # --- compute.merge_buckets ------------------------------------------------ -# --- live-only fallback: deployment younger than one bucket, no doc at all --- -# `ensure_swept` correctly leaves such a deployment with no doc (nothing has -# settled into a full 15-min bucket yet); the read API must instead fall back -# to a pure live Mimir number -- but ONLY once Mimir actually has a sample for -# it, never a fabricated `0`. + b1 = compute.Bucket("t0", 2.0, 20.0, 0.5, 0.02) + b2 = compute.Bucket("t0", 3.0, 30.0, 0.7, 0.03) + merged = compute.merge_buckets([[b1], [b2]]) + assert len(merged) == 1 + assert merged[0].energy_wh == 5.0 and merged[0].power_w_avg == 50.0 -# anchored just after the start of the current 15-min bucket (not "N minutes -# before now") so `floor(submit, bucket_s) == floor(now, bucket_s)` regardless -# of where in the bucket the suite happens to run -- "now - 3 minutes" would -# occasionally land in the *previous* bucket when run near a :00/:15/:30/:45 -# boundary, making `process_single` see a full elapsed bucket to integrate and -# spuriously write a real doc, which is exactly the "too young" case this -# section means to test. -_live_submit = sweep._floor(_NOW.timestamp(), 900) + 60 -compute.resolve_alloc_ids = lambda job_id, ns: ( - {_DC: ["alloc-live"]}, - [compute.AllocWindow("alloc-live", _live_submit, None)], - [{"alloc_id": "alloc-live", "datacenter": _DC}], - False, -) -# Mimir has real (fake) samples for this young deployment -> populated, not null -_LIVE_UUID = "sweep-live-only-uuid" +def test_running_seconds(): + # --- compute.running_seconds -------------------------------------------- + W = compute.AllocWindow + assert compute.running_seconds([W("a", 0, 100)], 0, 200) == 100 + assert compute.running_seconds([W("a", 0, 100), W("b", 50, 150)], 0, 200) == 150 + assert compute.running_seconds([W("a", 100, None)], 0, 200) == 100 # still running -def _fake_get_job_live(id_, namespace): - if id_ != _LIVE_UUID: - raise KeyError(id_) - return { - "ID": _LIVE_UUID, - "Name": f"module-{_LIVE_UUID}", - "SubmitTime": int(_live_submit * 1e9), - "Meta": {"owner": "live@egi.eu"}, - } +def test_sweep_pipeline(monkeypatch): + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-1"]}, + [compute.AllocWindow("alloc-1", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-1", "datacenter": _DC}], + False, + ), + ) + monkeypatch.setattr( + sweep.green_director, "metrics", {_DC: {"carbon": [], "water": []}} + ) -mimir.query_range = _fake_query_range -mimir.query_range_chunked = _fake_query_range -sweep.Nomad.job.get_job = _fake_get_job_live - -assert store.read_accum(_NS, _LIVE_UUID) is None -accounting._read_doc.cache_clear() -_live_stats = accounting.get_accumulated(_NS, _LIVE_UUID, live=True) -assert _live_stats is not None and _live_stats["energy_wh"] > 0, _live_stats -assert _live_stats["power_w"] is not None -# `since` is pinned to the exact submit time (not clamped/started late), so -# `complete` must be `True` even though nothing is persisted yet -assert _live_stats["complete"] is True, _live_stats -# too young for a bucket: `process_single` still writes nothing, on purpose -assert store.read_accum(_NS, _LIVE_UUID) is None - -accounting._read_doc.cache_clear() -_live_series = accounting.get_series(_NS, _LIVE_UUID) -assert _live_series is not None and _live_series["accumulated"]["energy_wh"] > 0 - -# `power_w` must reflect the actual last Mimir sample even when it is stale -# (older than the `2 * query_step_seconds` freshness cushion `topup` itself -# enforces) -- there is nothing persisted here for a stale reading to mislead. -_STALE_UUID = "sweep-live-only-stale-uuid" -_stale_submit = (_NOW - datetime.timedelta(minutes=10)).timestamp() - - -def _fake_get_job_stale(id_, namespace): - if id_ != _STALE_UUID: - raise KeyError(id_) - return { - "ID": _STALE_UUID, - "Name": f"module-{_STALE_UUID}", - "SubmitTime": int(_stale_submit * 1e9), - "Meta": {"owner": "live@egi.eu"}, + _info = { + "uuid": _UUID, + "job_name": f"module-{_UUID}", + "namespace": _NS, + "owner": "user@egi.eu", + "submit_time": _NOW.timestamp() - 3 * 3600, + "status": "running", } - -def _fake_query_range_stale(promql, start, end, step_s=30): - # no sample within the last 5 minutes -- well past the 60s cushion - stale_end = end.timestamp() - 300 - t, out = start.timestamp(), [] - while t <= stale_end: - out.append([t, "100000000" if "scaph" in promql else "0"]) - t += step_s - return [{"metric": {}, "values": out}] - - -compute.resolve_alloc_ids = lambda job_id, ns: ( - {_DC: ["alloc-stale"]}, - [compute.AllocWindow("alloc-stale", _stale_submit, None)], - [{"alloc_id": "alloc-stale", "datacenter": _DC}], - False, -) -mimir.query_range = _fake_query_range_stale -mimir.query_range_chunked = _fake_query_range_stale -sweep.Nomad.job.get_job = _fake_get_job_stale -sweep._live_only_doc_cache.clear() -sweep._live_cache.clear() - -# exercise `live_only_doc` / `topup` directly (not through `get_accumulated`'s -# `ensure_swept`): whether `process_single` would also consider this -# deployment "too young" depends on where in the current 15-min bucket the -# suite happens to run, which is irrelevant to what this checks. -_stale_doc = sweep.live_only_doc(_NS, _STALE_UUID) -assert _stale_doc is not None, _stale_doc -_stale_topup = sweep.topup(_stale_doc) -assert _stale_topup is not None and _stale_topup["power_w"] is not None, _stale_topup -assert _stale_topup["power_w"] > 0, _stale_topup - -# Mimir has NOTHING for this alloc yet -> must stay `null`, not a fabricated `0` -sweep._live_cache.clear() # fresh top-up cache: don't reuse the populated result - -_EMPTY_UUID = "sweep-live-only-empty-uuid" - - -def _fake_get_job_empty(id_, namespace): - if id_ != _EMPTY_UUID: - raise KeyError(id_) - return { - "ID": _EMPTY_UUID, - "Name": f"module-{_EMPTY_UUID}", - "SubmitTime": int(_live_submit * 1e9), - "Meta": {"owner": "live@egi.eu"}, + sweep.process_single(_info, _NOW) + doc = store.read_accum(_NS, _UUID) + acc = doc["accumulated"] + assert doc["metrics_available"] is True + assert abs(doc["tue_factor"] - _TUE) < 1e-9 + # ~3h of 100 W -> ~300 Wh raw, x _TUE (11 or 12 complete 15-min buckets + # depending on where "now" falls in the current bucket) + assert 270 * _TUE < acc["energy_wh"] < 305 * _TUE, acc + assert abs(acc["carbon_g"] - acc["energy_wh"] / 1000 * compute.DEFAULT_CARBON) < 1.0 + + # idempotent: same "now" -> byte-identical document + before = json.dumps(doc, sort_keys=True) + energy._read_doc.cache_clear() + sweep.process_single(_info, _NOW) + assert json.dumps(store.read_accum(_NS, _UUID), sort_keys=True) == before + + # monotonic on a later sweep + energy._read_doc.cache_clear() + sweep.process_single(_info, _NOW + datetime.timedelta(hours=1)) + assert store.read_accum(_NS, _UUID)["accumulated"]["energy_wh"] >= acc["energy_wh"] + + # buckets age out of the 24h tail into the persisted series + energy._read_doc.cache_clear() + sweep.process_single(_info, _NOW + datetime.timedelta(hours=30)) + assert len(store.read_series(_NS, _UUID, None, None)) > 0 + + # read API + pydantic validation + + energy._read_doc.cache_clear() + stats = energy.get_accumulated(_NS, _UUID, live=False) + schemas.EnergyStats(**stats) + series = energy.get_series(_NS, _UUID) + schemas.EnergyTimeSeries(**series) + assert series["series"] and all( + p["energy_wh"] is not None for p in series["series"] + ) + # per-bucket increments only; running total is cumsum(series[].energy_wh) + _pt = set(series["series"][-1]) + assert _pt == { + "ts", + "power_w", + "energy_wh", + "carbon_g", + "water_l", + "live", + }, _pt + _ps = energy.get_accumulated(_NS, _UUID, live=True) + assert _ps["power_w"] and abs(_ps["tue_factor"] - _TUE) < 1e-9 + + # per-user aggregate: sums the owner's deployments, EnergyStats-shaped + energy._read_doc.cache_clear() + _ue = energy.get_user_energy(_NS, "user@egi.eu", series=False) + schemas.EnergyStats(**_ue) + assert _ue["deployments"] >= 1 and _ue["series"] is None + assert _ue["energy_wh"] >= acc["energy_wh"] and _ue["since"] + _ues = energy.get_user_energy(_NS, "user@egi.eu", series=True) + schemas.EnergyStats(**_ues) + assert _ues["series"] and all(p["energy_wh"] is not None for p in _ues["series"]) + # the aggregate total must reconcile with the sum of the series (no rounding) + assert ( + abs(_ues["energy_wh"] - sum(p["energy_wh"] for p in _ues["series"])) < 1e-9 + ), _ues + assert abs(_ues["carbon_g"] - sum(p["carbon_g"] for p in _ues["series"])) < 1e-9 + # merge-then-downsample: uniform step across the whole persisted series + _uts = [ + datetime.datetime.fromisoformat(p["ts"].replace("Z", "+00:00")) + for p in _ues["series"] + if not p["live"] + ] + _steps = { + round((_uts[i + 1] - _uts[i]).total_seconds()) for i in range(len(_uts) - 1) } - - -def _fake_query_range_empty(promql, start, end, step_s=30): - return [] - - -mimir.query_range = _fake_query_range_empty -mimir.query_range_chunked = _fake_query_range_empty -sweep.Nomad.job.get_job = _fake_get_job_empty -compute.resolve_alloc_ids = lambda job_id, ns: ( - {_DC: ["alloc-empty"]}, - [compute.AllocWindow("alloc-empty", _live_submit, None)], - [{"alloc_id": "alloc-empty", "datacenter": _DC}], - False, -) - -accounting._read_doc.cache_clear() -assert accounting.get_accumulated(_NS, _EMPTY_UUID, live=True) is None -accounting._read_doc.cache_clear() -assert accounting.get_series(_NS, _EMPTY_UUID) is None -assert store.read_accum(_NS, _EMPTY_UUID) is None - -sweep.Nomad.job.get_job = _orig_get_job -mimir.query_range = _fake_query_range -mimir.query_range_chunked = _fake_query_range - -print("🟢 energy: live-only fallback checks passed!") - - -# --- an inconclusive stub must not permanently block the live-only fallback -- -# `process_single` persists a bare stub (no "accumulated" key, no -# `metrics_available` verdict) the first time it sees a job with no allocation -# resolved at all yet (eg. queued, alloc not placed by Nomad). That stub must -# not stop later reads from trying `live_only_doc` once Mimir has real data. - -_STUB_UUID = "sweep-inconclusive-stub-uuid" -_stub_submit = (_NOW - datetime.timedelta(minutes=1)).timestamp() - - -def _fake_get_job_stub(id_, namespace): - if id_ != _STUB_UUID: - raise KeyError(id_) - return { - "ID": _STUB_UUID, - "Name": f"module-{_STUB_UUID}", - "SubmitTime": int(_stub_submit * 1e9), - "Meta": {"owner": "stub@egi.eu"}, + assert len(_steps) == 1, _steps + # hard cap on the persisted point count, same as the other levels + _capped = energy.get_user_energy(_NS, "user@egi.eu", series=True, target_points=5) + assert len([p for p in _capped["series"] if not p["live"]]) <= 5 + assert energy.get_user_energy(_NS, "nobody@egi.eu") is None + + # live tail: a running deployment's graph keeps growing past the stored series + _live_doc = { + "namespace": _NS, + "deployment_uuid": _UUID, + "metrics_available": True, + "pointer_ts": compute.iso( + datetime.datetime.now(datetime.timezone.utc).timestamp() - 600 + ), + "active_allocs": [{"alloc_id": "alloc-1", "datacenter": _DC}], } + _lp = live.live_series(_live_doc, bucket_s=60) + assert _lp and all(p["live"] for p in _lp), _lp + assert all("energy_wh" in p and "carbon_g" in p for p in _lp) + + # the live top-up also carries the current power (100 W raw, x TUE) + _tu = live.topup(_live_doc) + _tue = config.tue(_DC) + assert _tu["power_w"] is not None + assert abs(_tu["power_w"] - 100.0 * _tue) < 5.0 * _tue, _tu + + # a running deployment's accumulated stats expose a power figure + energy._read_doc.cache_clear() + _st = energy.get_accumulated(_NS, _UUID, live=True) + assert _st["power_w"] is not None + + # the accumulated stats always carry a float coverage_ratio (1.0 = full) + assert isinstance(_st["coverage_ratio"], float) + + # GPU/CPU energy is clipped to the job's life: a finished allocation whose power + # metrics linger for a few minutes must not keep accumulating energy. + _ENDED_UUID = "sweep-ended-uuid" + _ended_end = _NOW.timestamp() - 2 * 3600 # allocation stopped 2h ago + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-e"]}, + [compute.AllocWindow("alloc-e", _NOW.timestamp() - 5 * 3600, _ended_end)], + [], # nothing running + False, + ), + ) + _ended_info = { + "uuid": _ENDED_UUID, + "job_name": f"module-{_ENDED_UUID}", + "namespace": _NS, + "owner": "user@egi.eu", + "submit_time": _NOW.timestamp() - 5 * 3600, + "status": "running", + } + energy._read_doc.cache_clear() + sweep.process_single(_ended_info, _NOW) + _edoc = store.read_accum(_NS, _ENDED_UUID) + _last_ts = max( + (b["ts"] for b in _edoc.get("tail", [])), + default=_edoc.get("settled_ts", ""), + ) + # fake Mimir returns 100 W for the whole [settled, now] window, but no bucket + # should land after the allocation ended (+ one bucket of slack). + assert compute.parse_iso(_last_ts) <= _ended_end + 300, _last_ts + + # a GPU-only monitored site (no scaphandre series at all) still accounts energy + # from the GPU power alone: a missing `scaph_*` metric comes back as an empty + # result, not an error, so the integrator just sums the GPU series. + _GPU_UUID = "sweep-gpu-only-uuid" + + def _fake_qr_gpu_only(promql, start, end, step_s=30): + if "scaph" in promql: + return [] + t, out = start.timestamp(), [] + while t <= end.timestamp(): + out.append([t, "150"]) # 150 W per GPU + t += step_s + return [{"metric": {}, "values": out}] + + monkeypatch.setattr(mimir, "query_range", _fake_qr_gpu_only) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_qr_gpu_only) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-g"]}, + [compute.AllocWindow("alloc-g", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-g", "datacenter": _DC}], + False, + ), + ) + _gpu_info = { + "uuid": _GPU_UUID, + "job_name": f"module-{_GPU_UUID}", + "namespace": _NS, + "owner": "gpu@egi.eu", + "submit_time": _NOW.timestamp() - 3 * 3600, + "status": "running", + } + energy._read_doc.cache_clear() + sweep.process_single(_gpu_info, _NOW) + _gdoc = store.read_accum(_NS, _GPU_UUID) + assert _gdoc["metrics_available"] is True, _gdoc + assert _gdoc["accumulated"]["energy_wh"] > 0, _gdoc + + +def test_on_demand_sweep(monkeypatch): + _OND_UUID = "sweep-on-demand-uuid" + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-o"]}, + [compute.AllocWindow("alloc-o", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-o", "datacenter": _DC}], + False, + ), + ) + def _fake_get_job(id_, namespace): + if id_ != _OND_UUID: + raise KeyError(id_) + return { + "ID": _OND_UUID, + "Name": f"module-{_OND_UUID}", + "SubmitTime": int((_NOW.timestamp() - 3 * 3600) * 1e9), + "Meta": {"owner": "ondemand@egi.eu"}, + } + + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job) + + # never swept: no doc yet, the read API must trigger one sweep pass itself + assert store.read_accum(_NS, _OND_UUID) is None + energy._read_doc.cache_clear() + _ond_stats = energy.get_accumulated(_NS, _OND_UUID, live=False) + assert _ond_stats is not None and _ond_stats["energy_wh"] > 0, _ond_stats + assert store.read_accum(_NS, _OND_UUID) is not None + + # already swept once (on-demand or not): no further Nomad call needed + def _boom(id_, namespace): + raise AssertionError("ensure_swept should not touch Nomad once a doc exists") + + monkeypatch.setattr(sweep.Nomad.job, "get_job", _boom) + energy._read_doc.cache_clear() + assert energy.get_accumulated(_NS, _OND_UUID, live=False) is not None + + # job unknown to Nomad: on-demand sweep finds nothing, degrades to `None`, no crash + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job) + energy._read_doc.cache_clear() + assert energy.get_accumulated(_NS, "sweep-unknown-uuid", live=False) is None + energy._read_doc.cache_clear() + assert energy.get_series(_NS, "sweep-unknown-uuid") is None + + +def test_live_only_fallback(monkeypatch): + # `ensure_swept` correctly leaves such a deployment with no doc (nothing has + # settled into a full 15-min bucket yet); the read API must instead fall back + # to a pure live Mimir number -- but ONLY once Mimir actually has a sample for + # it, never a fabricated `0`. + + # "now" is frozen 10 minutes into a 15-min bucket and the deployment is + # submitted one minute into it: always younger than one bucket, whatever the + # wall clock says. A real "now - N minutes" would occasionally land in the + # previous bucket (making `process_single` write a real doc), and in the first + # minute of a bucket the live window would be shorter than one query step. + _frozen = datetime.datetime.fromtimestamp( + config.floor(_NOW.timestamp(), 900) + 600, datetime.timezone.utc + ) + monkeypatch.setattr(config, "now", lambda: _frozen) + _live_submit = config.floor(_frozen.timestamp(), 900) + 60 + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-live"]}, + [compute.AllocWindow("alloc-live", _live_submit, None)], + [{"alloc_id": "alloc-live", "datacenter": _DC}], + False, + ), + ) -sweep.Nomad.job.get_job = _fake_get_job_stub - -# first on-demand attempt: no allocation resolved at all yet -compute.resolve_alloc_ids = lambda job_id, ns: ({}, [], [], False) -mimir.query_range = _fake_query_range -mimir.query_range_chunked = _fake_query_range - -assert store.read_accum(_NS, _STUB_UUID) is None -accounting._read_doc.cache_clear() -assert accounting.get_accumulated(_NS, _STUB_UUID, live=True) is None -_stub_doc = store.read_accum(_NS, _STUB_UUID) -assert _stub_doc is not None and "accumulated" not in _stub_doc, _stub_doc -assert _stub_doc.get("metrics_available") is not False # inconclusive, not a verdict - -# the allocation is now placed and Mimir has real samples: must populate, not -# stay `null` because of the stub written above -compute.resolve_alloc_ids = lambda job_id, ns: ( - {_DC: ["alloc-stub"]}, - [compute.AllocWindow("alloc-stub", _stub_submit, None)], - [{"alloc_id": "alloc-stub", "datacenter": _DC}], - False, -) -sweep._live_cache.clear() -sweep._live_only_doc_cache.clear() # `live_only_doc` itself is cached now too -accounting._read_doc.cache_clear() -_stub_stats = accounting.get_accumulated(_NS, _STUB_UUID, live=True) -assert _stub_stats is not None and _stub_stats["energy_wh"] > 0, _stub_stats - -sweep.Nomad.job.get_job = _orig_get_job - -print("🟢 energy: inconclusive-stub fallback checks passed!") - - -# --- on-demand bulk sweep: list / per-user reads catch up too ---------------- - -_BULK_UUID = "sweep-bulk-uuid" -_BULK_OWNER = "bulk@egi.eu" -mimir.query_range = _fake_query_range -mimir.query_range_chunked = _fake_query_range -compute.resolve_alloc_ids = lambda job_id, ns: ( - {_DC: ["alloc-b"]}, - [compute.AllocWindow("alloc-b", _NOW.timestamp() - 3 * 3600, None)], - [{"alloc_id": "alloc-b", "datacenter": _DC}], - False, -) - + # Mimir has real (fake) samples for this young deployment -> populated, not null + _LIVE_UUID = "sweep-live-only-uuid" + + def _fake_get_job_live(id_, namespace): + if id_ != _LIVE_UUID: + raise KeyError(id_) + return { + "ID": _LIVE_UUID, + "Name": f"module-{_LIVE_UUID}", + "SubmitTime": int(_live_submit * 1e9), + "Meta": {"owner": "live@egi.eu"}, + } + + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job_live) + + assert store.read_accum(_NS, _LIVE_UUID) is None + energy._read_doc.cache_clear() + _live_stats = energy.get_accumulated(_NS, _LIVE_UUID, live=True) + assert _live_stats is not None and _live_stats["energy_wh"] > 0, _live_stats + assert _live_stats["power_w"] is not None + # `since` is pinned to the exact submit time (not clamped/started late), so + # `complete` must be `True` even though nothing is persisted yet + assert _live_stats["complete"] is True, _live_stats + # too young for a bucket: `process_single` still writes nothing, on purpose + assert store.read_accum(_NS, _LIVE_UUID) is None + + energy._read_doc.cache_clear() + _live_series = energy.get_series(_NS, _LIVE_UUID) + assert _live_series is not None and _live_series["accumulated"]["energy_wh"] > 0 + + # `power_w` must reflect the actual last Mimir sample even when it is stale + # (older than the `2 * query_step_seconds` freshness cushion `topup` itself + # enforces) -- there is nothing persisted here for a stale reading to mislead. + _STALE_UUID = "sweep-live-only-stale-uuid" + _stale_submit = (_NOW - datetime.timedelta(minutes=10)).timestamp() + + def _fake_get_job_stale(id_, namespace): + if id_ != _STALE_UUID: + raise KeyError(id_) + return { + "ID": _STALE_UUID, + "Name": f"module-{_STALE_UUID}", + "SubmitTime": int(_stale_submit * 1e9), + "Meta": {"owner": "live@egi.eu"}, + } + + def _fake_query_range_stale(promql, start, end, step_s=30): + # no sample within the last 5 minutes -- well past the 60s cushion + stale_end = end.timestamp() - 300 + t, out = start.timestamp(), [] + while t <= stale_end: + out.append([t, "100000000" if "scaph" in promql else "0"]) + t += step_s + return [{"metric": {}, "values": out}] + + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-stale"]}, + [compute.AllocWindow("alloc-stale", _stale_submit, None)], + [{"alloc_id": "alloc-stale", "datacenter": _DC}], + False, + ), + ) + monkeypatch.setattr(mimir, "query_range", _fake_query_range_stale) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range_stale) + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job_stale) + ondemand._live_only_doc_cache.clear() + live._live_cache.clear() + + # exercise `live_only_doc` / `topup` directly (not through `get_accumulated`'s + # `ensure_swept`): whether `process_single` would also consider this + # deployment "too young" depends on where in the current 15-min bucket the + # suite happens to run, which is irrelevant to what this checks. + _stale_doc = ondemand.live_only_doc(_NS, _STALE_UUID) + assert _stale_doc is not None, _stale_doc + _stale_topup = live.topup(_stale_doc) + assert _stale_topup is not None and _stale_topup["power_w"] is not None, ( + _stale_topup + ) + assert _stale_topup["power_w"] > 0, _stale_topup -def _fake_get_jobs(namespace, filter_=None): - return [{"ID": _BULK_UUID, "Name": f"module-{_BULK_UUID}"}] + # Mimir has NOTHING for this alloc yet -> must stay `null`, not a fabricated `0` + live._live_cache.clear() # fresh top-up cache: don't reuse the populated result + _EMPTY_UUID = "sweep-live-only-empty-uuid" -def _fake_get_job_bulk(id_, namespace): - if id_ != _BULK_UUID: - raise KeyError(id_) - return { - "ID": _BULK_UUID, - "Name": f"module-{_BULK_UUID}", - "SubmitTime": int((_NOW.timestamp() - 3 * 3600) * 1e9), - "Meta": {"owner": _BULK_OWNER}, - } + def _fake_get_job_empty(id_, namespace): + if id_ != _EMPTY_UUID: + raise KeyError(id_) + return { + "ID": _EMPTY_UUID, + "Name": f"module-{_EMPTY_UUID}", + "SubmitTime": int(_live_submit * 1e9), + "Meta": {"owner": "live@egi.eu"}, + } + def _fake_query_range_empty(promql, start, end, step_s=30): + return [] -_orig_get_jobs = sweep.Nomad.jobs.get_jobs -_orig_get_job = sweep.Nomad.job.get_job -sweep.Nomad.jobs.get_jobs = _fake_get_jobs -sweep.Nomad.job.get_job = _fake_get_job_bulk + monkeypatch.setattr(mimir, "query_range", _fake_query_range_empty) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range_empty) + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job_empty) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-empty"]}, + [compute.AllocWindow("alloc-empty", _live_submit, None)], + [{"alloc_id": "alloc-empty", "datacenter": _DC}], + False, + ), + ) -# never swept: the bulk/list read must sweep it on demand before serving -assert store.read_accum(_NS, _BULK_UUID) is None -_bulk = accounting.get_accumulated_bulk(_NS, _BULK_OWNER, live=False) -assert _BULK_UUID in _bulk and _bulk[_BULK_UUID]["energy_wh"] > 0, _bulk -assert store.read_accum(_NS, _BULK_UUID) is not None + energy._read_doc.cache_clear() + assert energy.get_accumulated(_NS, _EMPTY_UUID, live=True) is None + energy._read_doc.cache_clear() + assert energy.get_series(_NS, _EMPTY_UUID) is None + assert store.read_accum(_NS, _EMPTY_UUID) is None -_ue = accounting.get_user_energy(_NS, _BULK_OWNER, series=False) -assert _ue is not None and _ue["deployments"] == 1, _ue + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) -# already swept: the listing is still cheap (Nomad.jobs.get_jobs), but no -# per-job fetch / re-sweep for a uuid that already has a doc -def _boom_get_job(id_, namespace): - raise AssertionError("ensure_swept_bulk should not re-fetch a swept job") +def test_inconclusive_stub(monkeypatch): + # `process_single` persists a bare stub (no "accumulated" key, no + # `metrics_available` verdict) the first time it sees a job with no allocation + # resolved at all yet (eg. queued, alloc not placed by Nomad). That stub must + # not stop later reads from trying `live_only_doc` once Mimir has real data. + _STUB_UUID = "sweep-inconclusive-stub-uuid" + _stub_submit = (_NOW - datetime.timedelta(minutes=1)).timestamp() -sweep.Nomad.job.get_job = _boom_get_job -_bulk2 = accounting.get_accumulated_bulk(_NS, _BULK_OWNER, live=False) -assert _BULK_UUID in _bulk2 + def _fake_get_job_stub(id_, namespace): + if id_ != _STUB_UUID: + raise KeyError(id_) + return { + "ID": _STUB_UUID, + "Name": f"module-{_STUB_UUID}", + "SubmitTime": int(_stub_submit * 1e9), + "Meta": {"owner": "stub@egi.eu"}, + } -sweep.Nomad.jobs.get_jobs = _orig_get_jobs -sweep.Nomad.job.get_job = _orig_get_job + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job_stub) -print("🟢 energy: on-demand bulk sweep checks passed!") + # first on-demand attempt: no allocation resolved at all yet + monkeypatch.setattr( + compute, "resolve_alloc_ids", lambda job_id, ns: ({}, [], [], False) + ) + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + + assert store.read_accum(_NS, _STUB_UUID) is None + energy._read_doc.cache_clear() + assert energy.get_accumulated(_NS, _STUB_UUID, live=True) is None + _stub_doc = store.read_accum(_NS, _STUB_UUID) + assert _stub_doc is not None and "accumulated" not in _stub_doc, _stub_doc + assert ( + _stub_doc.get("metrics_available") is not False + ) # inconclusive, not a verdict + + # the allocation is now placed and Mimir has real samples: must populate, not + # stay `null` because of the stub written above + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-stub"]}, + [compute.AllocWindow("alloc-stub", _stub_submit, None)], + [{"alloc_id": "alloc-stub", "datacenter": _DC}], + False, + ), + ) + live._live_cache.clear() + ondemand._live_only_doc_cache.clear() # `live_only_doc` itself is cached now too + energy._read_doc.cache_clear() + _stub_stats = energy.get_accumulated(_NS, _STUB_UUID, live=True) + assert _stub_stats is not None and _stub_stats["energy_wh"] > 0, _stub_stats + + +def test_on_demand_bulk_sweep(monkeypatch): + _BULK_UUID = "sweep-bulk-uuid" + _BULK_OWNER = "bulk@egi.eu" + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-b"]}, + [compute.AllocWindow("alloc-b", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-b", "datacenter": _DC}], + False, + ), + ) + def _fake_get_jobs(namespace, filter_=None): + return [{"ID": _BULK_UUID, "Name": f"module-{_BULK_UUID}"}] + + def _fake_get_job_bulk(id_, namespace): + if id_ != _BULK_UUID: + raise KeyError(id_) + return { + "ID": _BULK_UUID, + "Name": f"module-{_BULK_UUID}", + "SubmitTime": int((_NOW.timestamp() - 3 * 3600) * 1e9), + "Meta": {"owner": _BULK_OWNER}, + } + + monkeypatch.setattr(sweep.Nomad.jobs, "get_jobs", _fake_get_jobs) + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job_bulk) + + # never swept: the bulk/list read must sweep it on demand before serving + assert store.read_accum(_NS, _BULK_UUID) is None + _bulk = energy.get_accumulated_bulk(_NS, _BULK_OWNER, live=False) + assert _BULK_UUID in _bulk and _bulk[_BULK_UUID]["energy_wh"] > 0, _bulk + assert store.read_accum(_NS, _BULK_UUID) is not None + + _ue = energy.get_user_energy(_NS, _BULK_OWNER, series=False) + assert _ue is not None and _ue["deployments"] == 1, _ue + + # already swept: the listing is still cheap (Nomad.jobs.get_jobs), but no + # per-job fetch / re-sweep for a uuid that already has a doc + def _boom_get_job(id_, namespace): + raise AssertionError("ensure_swept_bulk should not re-fetch a swept job") + + monkeypatch.setattr(sweep.Nomad.job, "get_job", _boom_get_job) + _bulk2 = energy.get_accumulated_bulk(_NS, _BULK_OWNER, live=False) + assert _BULK_UUID in _bulk2 + + +def test_cluster_energy(monkeypatch): + _CNS = config.CLUSTER_NS + _ISO0 = "2020-01-01T00:00:00Z" + monkeypatch.setattr( + sweep.green_director, + "metrics", + {_DC: {"carbon": [[_ISO0, 300.0]], "water": [[_ISO0, 10.0]]}}, + ) + _tue_dc = config.tue(_DC) + + def _fake_query(promql, at=None, timeout=30): + if "scaph_host_power" in promql: + # 500 W reported in microwatts + return [ + { + "metric": {"datacenter": _DC}, + "value": [_NOW.timestamp(), "500000000"], + } + ] + if "DCGM" in promql: + return [ + {"metric": {"datacenter": _DC}, "value": [_NOW.timestamp(), "200"]} + ] # W + return [] -# --- cluster energy: instantaneous power + accumulator + series -------------- + monkeypatch.setattr(mimir, "query", _fake_query) + + # 1. instantaneous power: power_w = (host + gpu) * TUE, normalized only + energy.get_datacenter_power.cache_clear() + _cp = energy.get_datacenter_power() + assert _DC in _cp, _cp + assert abs(_cp[_DC]["power_w"] - (500.0 + 200.0) * _tue_dc) < 1.0, _cp + assert set(_cp[_DC]) == {"power_w", "tue_factor", "as_of"}, _cp[_DC] + + # 2. per-datacenter sweep: several raw host series -> sanitize each -> sum + def _fake_cluster_qr(promql, start, end, step_s=30): + def _rows(pairs): + out = [] + for label, val in pairs: + vals, t = [], start.timestamp() + while t <= end.timestamp(): + vals.append([t, val]) + t += step_s + out.append({"metric": {"node": label, "UUID": label}, "values": vals}) + return out + + if "scaph_host_power" in promql: # two hosts: 400 W + 600 W (in microwatts) + return _rows([("nodeA", "400000000"), ("nodeB", "600000000")]) + if "DCGM" in promql: # one GPU: 250 W + return _rows([("gpu0", "250")]) + return [] -_CNS = sweep.CLUSTER_NS -_ISO0 = "2020-01-01T00:00:00Z" -sweep.green_director.metrics = { - _DC: {"carbon": [[_ISO0, 300.0]], "water": [[_ISO0, 10.0]]} -} -_tue_dc = sweep._tue(_DC) + monkeypatch.setattr(mimir, "query_range", _fake_cluster_qr) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_cluster_qr) + live._live_cache.clear() + + sweep.process_datacenter(_DC, _NOW) + cdoc = store.read_accum(_CNS, _DC) + assert cdoc["metrics_available"] is True + cacc = cdoc["accumulated"] + assert cdoc["tue_factor"] == _tue_dc + # window ~= initial_lookback_hours of (400 + 600 + 250) = 1250 W raw, x TUE + _expected = config.get()["initial_lookback_hours"] * 1250.0 * _tue_dc + assert 0.75 * _expected < cacc["energy_wh"] <= 1.05 * _expected, cacc + assert abs(cacc["carbon_g"] - cacc["energy_wh"] / 1000 * 300.0) < 1.0 + + # monotonic on a later sweep + sweep.process_datacenter(_DC, _NOW + datetime.timedelta(hours=1)) + assert store.read_accum(_CNS, _DC)["accumulated"]["energy_wh"] >= cacc["energy_wh"] + + # buckets age out of the 24h tail into the persisted series + sweep.process_datacenter(_DC, _NOW + datetime.timedelta(hours=30)) + assert len(store.read_series(_CNS, _DC, None, None)) > 0 + + # 3. read API: get_cluster_energy merges power + accumulator + series + energy.get_datacenter_power.cache_clear() + energy._cluster_archive_series.cache_clear() + live._live_cache.clear() + _ce = energy.get_cluster_energy(series=True) + assert _DC in _ce, _ce + _row = _ce[_DC] + assert _row["power_w"] is not None and _row["energy_wh"] is not None + assert _row["since"] and _row["as_of"] + assert _row["series"] and all(p["energy_wh"] is not None for p in _row["series"]) + for _p in _row["series"]: + schemas.EnergyTimeSeriesPoint(**_p) + schemas.EnergyStats(**_row) + assert _row["tue_factor"] == _tue_dc # only conversion factor exposed + # the accumulated total reconciles with the sum of the series (no rounding) + assert ( + abs(_row["energy_wh"] - sum(p["energy_wh"] for p in _row["series"])) < 1e-9 + ), _row + + # 4. datacenter live top-up carries the (TUE-normalized) energy deltas + live._live_cache.clear() + _ct = live.topup(cdoc) + assert _ct and _ct["energy_wh"] > 0 and _ct["live_as_of"] + assert abs(_ct["carbon_g"] - _ct["energy_wh"] / 1000 * 300.0) < 1.0, _ct + + # 5. a GPU-only monitored datacenter (in metrics_datacenters, no host power) is + # still reported: for a declared site, GPU power alone is a valid total. + def _fake_query_gpu_only(promql, at=None, timeout=30): + if "DCGM" in promql: + return [{"metric": {"datacenter": _DC}, "value": [_NOW.timestamp(), "200"]}] + return [] + monkeypatch.setattr(mimir, "query", _fake_query_gpu_only) + energy.get_datacenter_power.cache_clear() + _gp = energy.get_datacenter_power() + assert _DC in _gp, _gp + assert abs(_gp[_DC]["power_w"] - 200.0 * _tue_dc) < 1.0, _gp -def _fake_query(promql, at=None, timeout=30): - if "scaph_host_power" in promql: - # 500 W reported in microwatts - return [ - {"metric": {"datacenter": _DC}, "value": [_NOW.timestamp(), "500000000"]} - ] - if "DCGM" in promql: - return [ - {"metric": {"datacenter": _DC}, "value": [_NOW.timestamp(), "200"]} - ] # W - return [] + # 6. a datacenter with no power series at all is absent from get_datacenter_power + monkeypatch.setattr(mimir, "query", lambda promql, at=None, timeout=30: []) + monkeypatch.setattr(mimir, "query_range", lambda *a, **k: []) + monkeypatch.setattr(mimir, "query_range_chunked", lambda *a, **k: []) + energy.get_datacenter_power.cache_clear() + assert energy.get_datacenter_power() == {} -mimir.query = _fake_query - -# 1. instantaneous power: power_w = (host + gpu) * TUE, normalized only -accounting.get_datacenter_power.cache_clear() -_cp = accounting.get_datacenter_power() -assert _DC in _cp, _cp -assert abs(_cp[_DC]["power_w"] - (500.0 + 200.0) * _tue_dc) < 1.0, _cp -assert set(_cp[_DC]) == {"power_w", "tue_factor", "as_of"}, _cp[_DC] - - -# 2. per-datacenter sweep: several raw host series -> sanitize each -> sum -def _fake_cluster_qr(promql, start, end, step_s=30): - def _rows(pairs): - out = [] - for label, val in pairs: - vals, t = [], start.timestamp() - while t <= end.timestamp(): - vals.append([t, val]) - t += step_s - out.append({"metric": {"node": label, "UUID": label}, "values": vals}) - return out - - if "scaph_host_power" in promql: # two hosts: 400 W + 600 W (in microwatts) - return _rows([("nodeA", "400000000"), ("nodeB", "600000000")]) - if "DCGM" in promql: # one GPU: 250 W - return _rows([("gpu0", "250")]) +def _failing_scaph(promql, start, end, step_s=30): + if "scaph" in promql: + raise mimir.MimirError("down") return [] -mimir.query_range = _fake_cluster_qr -mimir.query_range_chunked = _fake_cluster_qr -sweep._live_cache.clear() - -sweep.process_datacenter(_DC, _NOW) -cdoc = store.read_accum(_CNS, _DC) -assert cdoc["metrics_available"] is True -cacc = cdoc["accumulated"] -assert cdoc["tue_factor"] == _tue_dc -# window ~= initial_lookback_hours of (400 + 600 + 250) = 1250 W raw, x TUE -_expected = sweep._cfg()["initial_lookback_hours"] * 1250.0 * _tue_dc -assert 0.75 * _expected < cacc["energy_wh"] <= 1.05 * _expected, cacc -assert abs(cacc["carbon_g"] - cacc["energy_wh"] / 1000 * 300.0) < 1.0 - -# monotonic on a later sweep -sweep.process_datacenter(_DC, _NOW + datetime.timedelta(hours=1)) -assert store.read_accum(_CNS, _DC)["accumulated"]["energy_wh"] >= cacc["energy_wh"] - -# buckets age out of the 24h tail into the persisted series -sweep.process_datacenter(_DC, _NOW + datetime.timedelta(hours=30)) -assert len(store.read_series(_CNS, _DC, None, None)) > 0 - -# 3. read API: get_cluster_energy merges power + accumulator + series -accounting.get_datacenter_power.cache_clear() -accounting._cluster_archive_series.cache_clear() -sweep._live_cache.clear() -_ce = accounting.get_cluster_energy(series=True) -assert _DC in _ce, _ce -_row = _ce[_DC] -assert _row["power_w"] is not None and _row["energy_wh"] is not None -assert _row["since"] and _row["as_of"] -assert _row["series"] and all(p["energy_wh"] is not None for p in _row["series"]) -for _p in _row["series"]: - schemas.EnergyTimeSeriesPoint(**_p) -schemas.EnergyStats(**_row) -assert _row["tue_factor"] == _tue_dc # only conversion factor exposed -# the accumulated total reconciles with the sum of the series (no rounding) -assert abs(_row["energy_wh"] - sum(p["energy_wh"] for p in _row["series"])) < 1e-9, _row - -# 4. datacenter live top-up carries the (TUE-normalized) energy deltas -sweep._live_cache.clear() -_ct = sweep.topup_cluster(_DC, cdoc) -assert _ct and _ct["energy_wh"] > 0 and _ct["live_as_of"] -assert abs(_ct["carbon_g"] - _ct["energy_wh"] / 1000 * 300.0) < 1.0, _ct - - -# 5. a GPU-only monitored datacenter (in metrics_datacenters, no host power) is -# still reported: for a declared site, GPU power alone is a valid total. -def _fake_query_gpu_only(promql, at=None, timeout=30): - if "DCGM" in promql: - return [{"metric": {"datacenter": _DC}, "value": [_NOW.timestamp(), "200"]}] - return [] - +def test_live_topup_is_degraded_when_mimir_fails(monkeypatch): + # a failing query must show up as `degraded`, never as a silent zero + monkeypatch.setattr(mimir, "query_range", _failing_scaph) + pointer = compute.iso(_NOW.timestamp() - 600) + cluster_doc = { + "namespace": config.CLUSTER_NS, + "deployment_uuid": _DC, + "metrics_available": True, + "pointer_ts": pointer, + } + dep_doc = { + "namespace": _NS, + "deployment_uuid": _UUID, + "metrics_available": True, + "pointer_ts": pointer, + "active_allocs": [{"alloc_id": "alloc-1", "datacenter": _DC}], + } + assert live.topup(cluster_doc) == {"degraded": True} + assert live.topup(dep_doc) == {"degraded": True} + assert live.live_series(dep_doc, bucket_s=60) == [] -mimir.query = _fake_query_gpu_only -accounting.get_datacenter_power.cache_clear() -_gp = accounting.get_datacenter_power() -assert _DC in _gp, _gp -assert abs(_gp[_DC]["power_w"] - 200.0 * _tue_dc) < 1.0, _gp -# 6. a datacenter with no power series at all is absent from get_datacenter_power -mimir.query = lambda promql, at=None, timeout=30: [] -mimir.query_range = lambda *a, **k: [] -mimir.query_range_chunked = lambda *a, **k: [] -accounting.get_datacenter_power.cache_clear() -assert accounting.get_datacenter_power() == {} +def test_no_metrics_verdict_is_not_reported(monkeypatch): + # a deployment confirmed to have no usable data is `None`, never a zero + store.write_accum( + _NS, + _UUID, + {"owner": "u@egi.eu", "metrics_available": False, "accumulated": {}}, + ) + monkeypatch.setattr(sweep.Nomad.jobs, "get_jobs", lambda **kw: []) + assert energy.get_accumulated(_NS, _UUID, live=False) is None + assert energy.get_series(_NS, _UUID) is None + assert energy.get_accumulated_bulk(_NS, "u@egi.eu", live=False) == {} + assert energy.get_user_energy(_NS, "u@egi.eu") is None + + +def test_add_to_jobs(monkeypatch): + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-1"]}, + [compute.AllocWindow("alloc-1", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-1", "datacenter": _DC}], + False, + ), + ) + monkeypatch.setattr(sweep.Nomad.jobs, "get_jobs", lambda **kw: []) + info = { + "uuid": _UUID, + "job_name": f"module-{_UUID}", + "namespace": _NS, + "owner": "u@egi.eu", + "submit_time": _NOW.timestamp() - 3 * 3600, + "status": "running", + } + sweep.process_single(info, _NOW) + + # accumulated stats by default, the extended series with `full_info` + jobs = [{"job_ID": _UUID}, {"job_ID": "not-accounted"}] + energy.add_to_jobs(jobs, _NS, "u@egi.eu", full_info=False) + schemas.EnergyStats(**jobs[0]["energy"]) + assert jobs[1]["energy"] is None + + job = {"job_ID": _UUID} + energy.add_to_job(job, _NS, full_info=True) + schemas.EnergyTimeSeries(**job["energy"]) + energy.add_to_job(job, _NS, full_info=False) + schemas.EnergyStats(**job["energy"]) + + # nothing to attach: no Nomad / store work at all + monkeypatch.setattr(energy, "get_accumulated_bulk", lambda *a, **k: 1 / 0) + energy.add_to_jobs([], _NS, "u@egi.eu", full_info=False) + + +def test_build_series_live_tail_only_after_persisted(): + def b(ts, wh): + return {"ts": ts, "energy_wh": wh, "power_w_avg": 1.0} + + persisted = [[b("2026-01-01T00:00:00Z", 1.0), b("2026-01-01T00:15:00Z", 2.0)]] + live = [[b("2026-01-01T00:15:00Z", 9.0), b("2026-01-01T00:30:00Z", 3.0)]] + points = series.build_series(persisted, live, target_points=100) + assert [(p["ts"], p["energy_wh"], p["live"]) for p in points] == [ + ("2026-01-01T00:00:00Z", 1.0, False), + ("2026-01-01T00:15:00Z", 2.0, False), + ("2026-01-01T00:30:00Z", 3.0, True), + ] + + # several entities (a user's deployments) are summed per timestamp + two = series.build_series([persisted[0], persisted[0]], [], target_points=100) + assert [p["energy_wh"] for p in two] == [2.0, 4.0] + + +def test_wattnet_failure_is_not_cached_nor_overwrites_good_data(monkeypatch, caplog): + from types import SimpleNamespace + + from ai4papi import wattnet + + ts = "2026-01-01T00:00:00Z" + fp = [{"footprint_type": "carbon", "series": [{"values": [[ts, 5.0]]}]}] + score = [{"footprint_type": "x", "series": [{"values": [[ts, 9.0]]}]}] + answers = {"ok": True} + calls = [] + + def fake_get(url, **kwargs): + calls.append(url) + assert kwargs["timeout"] # a stalled Wattnet must not block the threads + if not answers["ok"]: + return SimpleNamespace(ok=False, status_code=503) + body = score if "environmental" in url else fp + return SimpleNamespace(ok=True, status_code=200, json=lambda: body) + + monkeypatch.setattr(wattnet.session, "get", fake_get) + monkeypatch.setattr(papiconf, "datacenters", {"wn-test": {"lat": 1, "lon": 2}}) + g = wattnet.GreenDirector() + monkeypatch.setattr(g, "_retrieve_token", lambda: "t") + caplog.set_level(logging.INFO, logger="ai4papi.wattnet") + default_carbon = wattnet.GreenDirector.DEFAULTS["carbon"] + + # never had data and Wattnet is down: DEFAULTS, and nothing is cached + answers["ok"] = False + g.retrieve_footprints() + assert g.metrics["wn-test"]["carbon"][0][1] == default_carbon + assert len(calls) == 2 + assert "using default values" in caplog.text + + # still within the backoff: Wattnet is not asked again + g.retrieve_footprints() + assert len(calls) == 2 + + # backoff over and recovered: asks again, gets the real data, logs it once + g._backoff.clear() + answers["ok"] = True + g.retrieve_footprints() + assert len(calls) == 4 + assert g.metrics["wn-test"]["carbon"] == [[ts, 5.0]] + assert caplog.text.count("recovered") == 1 + + # blip after good data: the last good data is kept, not overwritten by DEFAULTS + wattnet.GreenDirector._fetch_footprint_data.cache_clear() + g._backoff.clear() + answers["ok"] = False + caplog.clear() + g.retrieve_footprints() + assert g.metrics["wn-test"]["carbon"] == [[ts, 5.0]] + assert f"keeping data up to {ts}" in caplog.text + + # a long outage logs once (on the change), not on every refresh + for _ in range(3): + g._backoff.clear() + g.retrieve_footprints() + assert caplog.text.count("refresh failed") == 1 + + +def test_store_series_dedupes_and_owner_index(monkeypatch): + # a bucket appended twice (crash between the series and the doc write) counts once + b = {"ts": "2026-01-01T00:00:00Z", "energy_wh": 1.0} + store.append_series(_NS, "dup", [b, {**b, "ts": "2026-01-01T00:15:00Z"}]) + store.append_series(_NS, "dup", [b]) + assert [x["ts"] for x in store.read_series(_NS, "dup", None, None)] == [ + "2026-01-01T00:00:00Z", + "2026-01-01T00:15:00Z", + ] + + # listing one owner scans the namespace once, and sees later writes at once + store.write_accum(_NS, "a1", {"owner": "a@egi.eu"}) + store.write_accum(_NS, "b1", {"owner": "b@egi.eu"}) + scans = [] + real_iter = store.iter_namespace + monkeypatch.setattr( + store, "iter_namespace", lambda ns: scans.append(ns) or real_iter(ns) + ) + assert [u for u, _ in store.iter_owner(_NS, "a@egi.eu")] == ["a1"] + store.write_accum(_NS, "a2", {"owner": "a@egi.eu"}) + assert [u for u, _ in store.iter_owner(_NS, "a@egi.eu")] == ["a1", "a2"] + assert [u for u, _ in store.iter_owner(_NS, "nobody@egi.eu")] == [] + assert len(scans) == 1 + + +def test_mimir_backoff_after_a_timeout(monkeypatch): + import requests + + calls = [] + + def hung(url, **kwargs): + calls.append(url) + raise requests.Timeout("no answer") + + monkeypatch.setattr(mimir.session, "get", hung) + start = datetime.datetime.now(datetime.timezone.utc) + + # the first call pays the timeout, the next ones fail fast without a request + with pytest.raises(requests.Timeout): + _REAL_QUERY_RANGE("up", start, start, step_s=30) + for call in ( + lambda: _REAL_QUERY_RANGE("up", start, start, step_s=30), + lambda: mimir.query("up"), + ): + with pytest.raises(mimir.MimirError): + call() + assert len(calls) == 1 + + # once the backoff is over Mimir is asked again + monkeypatch.setattr(mimir, "_down_until", 0.0) + with pytest.raises(requests.Timeout): + mimir.query("up") + assert len(calls) == 2 + + +def test_sweep_stops_the_pass_when_mimir_is_down(monkeypatch): + import requests + + uuids = [f"down-{i}" for i in range(3)] + submit = _NOW.timestamp() - 3 * 3600 + calls = [] + + def hung(url, **kwargs): + calls.append(url) + raise requests.Timeout("no answer") + + monkeypatch.setattr(mimir.session, "get", hung) + monkeypatch.setattr(mimir, "query_range", _REAL_QUERY_RANGE) + monkeypatch.setattr(mimir, "query_range_chunked", _REAL_QUERY_RANGE) + monkeypatch.setattr( + sweep.Nomad.jobs, + "get_jobs", + lambda **kw: [{"ID": u, "Name": f"module-{u}"} for u in uuids], + ) + fetched = [] + + def get_job(id_, namespace): + fetched.append(id_) + return { + "ID": id_, + "Name": f"module-{id_}", + "SubmitTime": int(submit * 1e9), + "Meta": {"owner": "u@egi.eu"}, + } + + monkeypatch.setattr(sweep.Nomad.job, "get_job", get_job) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-1"]}, + [compute.AllocWindow("alloc-1", submit, None)], + [{"alloc_id": "alloc-1", "datacenter": _DC}], + False, + ), + ) -print("🟢 energy: cluster energy checks passed!") + with pytest.raises(requests.Timeout): + sweep.process_single( + sweep.info_from_job(sweep.Nomad.job.get_job(uuids[0], _NS), _NS), _NOW + ) + calls.clear() + fetched.clear() + monkeypatch.setattr(mimir, "_down_until", 0.0) + + # three deployments, Mimir hung: the first pays the timeout, the second fails + # fast and stops the pass, the third is not even fetched from Nomad + sweep._sweep_namespace(_NS, _NOW) + assert len(calls) == 1 + assert fetched == uuids[:2] + assert all(store.read_accum(_NS, u) is None for u in uuids) From 7b53213a6702e661506a9e2e3d33fe6dafca5c81 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Mon, 19 Oct 2026 20:51:33 +0200 Subject: [PATCH 18/25] docs(energy): update the architecture and API docs Describe the new module layout, the owner index, the Mimir and Wattnet timeouts and backoff, and the lock on the TTL caches. `degraded` was documented as set when Mimir or Wattnet fail, but only a failing Mimir top-up sets it; docs and the schema comment now say so. --- ai4papi/schemas.py | 2 +- docs/energy-accounting.md | 93 +++++++++++++++++++++++---------------- docs/energy-api.md | 2 +- 3 files changed, 58 insertions(+), 39 deletions(-) diff --git a/ai4papi/schemas.py b/ai4papi/schemas.py index 577b5c0b..b797a91b 100644 --- a/ai4papi/schemas.py +++ b/ai4papi/schemas.py @@ -74,7 +74,7 @@ class EnergyStats(BaseModel): as_of: Optional[str] = None # store pointer timestamp live_as_of: Optional[str] = None # instant of the live top-up complete: bool = True # False if it started late / clamped by Mimir retention - degraded: bool = False # Mimir/Wattnet unavailable for the top-up + degraded: bool = False # the live Mimir top-up failed coverage_ratio: Optional[float] = None # per deployment / user: fraction with data datacenters: Optional[list[str]] = None # per deployment / user deployments: Optional[int] = None # per-user aggregate only diff --git a/docs/energy-accounting.md b/docs/energy-accounting.md index d4f3f7a8..3b36e2fa 100644 --- a/docs/energy-accounting.md +++ b/docs/energy-accounting.md @@ -35,7 +35,7 @@ Four layers: a durable background sweep that owns the persisted record, a live r ``` background thread in ai4papi.main: @repeat_every(energy.sweep_seconds) - accounting.sweep.run_sweep() + energy.run_sweep() per deployment: Nomad (jobs + allocations + node -> datacenter) Mimir query_range (per-alloc power, un-consolidated window) per datacenter: Mimir query_range (per-host + per-GPU power) @@ -63,7 +63,7 @@ The diagram above is the steady state, once a deployment has a persisted doc. Be ### Layer 1: durable sweep -`accounting.sweep.run_sweep()` runs on a `@repeat_every(energy.sweep_seconds)` thread started in `ai4papi.main`'s `lifespan`, next to `get_cluster_stats_thread`. Same posture: not async, one `try/except` per deployment, no-op if the feature is disabled. +`energy.run_sweep()` runs on a `@repeat_every(energy.sweep_seconds)` thread started in `ai4papi.main`'s `lifespan`, next to `get_cluster_stats_thread`. Same posture: not async, one `try/except` per deployment, no-op if the feature is disabled. Per namespace it lists the active Nomad jobs (`Status != "dead"`, names matching `^(module|tool|batch|try)`) and also every existing `.accum.json` whose job has vanished, to close it out. Per deployment it calls `process_single(info, now)`: @@ -76,26 +76,26 @@ Per namespace it lists the active Nomad jobs (`Status != "dead"`, names matching The sweep is idempotent: re-running with the same `now` produces the same document (buckets keyed by timestamp, `settled` only advances over buckets that leave the 24h window). -The consolidation step (fold new buckets over the tail, age the old ones into `settled` + `.series.jsonl`, recompute `accumulated` / `pointer_ts`) is `sweep._fold_and_accumulate`, shared with the datacenter sweep below. +The consolidation step (fold new buckets over the tail, age the old ones into `settled` + `.series.jsonl`, recompute `accumulated` / `pointer_ts`) is `sweep._fold_and_accumulate` (`ai4papi/energy/sweep.py`), shared with the datacenter sweep below. ### Layer 1b: per-datacenter sweep -After the deployment loop, `run_sweep()` calls `_sweep_cluster(now)`, which runs `process_datacenter(dc, now)` for every `energy.metrics_datacenters`. Documents live under the pseudo-namespace `_cluster` (`$ACCOUNTING_PTH/energy/_cluster/.accum.json` + `.series.jsonl`), reusing `accounting.store` unchanged. +After the deployment loop, `run_sweep()` calls `_sweep_cluster(now)`, which runs `process_datacenter(dc, now)` for every `energy.metrics_datacenters`. Documents live under the pseudo-namespace `_cluster` (`$ACCOUNTING_PTH/energy/_cluster/.accum.json` + `.series.jsonl`), reusing `energy.store` unchanged. `process_datacenter` is `process_single` without the Nomad parts: no allocation resolution, no `coverage_ratio`, no owner/close handling. 1. Load the doc; `settled_ts` from it, else clamped to `initial_lookback_hours` / `mimir_retention_days` (retention clamp sets `complete: false`). `window_start` / `since` is pinned on the first sweep, same as the per-deployment path. -2. Query Mimir over `[settled_ts, now - query_lag_seconds]` with `_cluster_promql(dc)`: `sum by (node) (scaph_host_power_microwatts{datacenter=""})` (one series per host) and `sum by (UUID) (DCGM_FI_DEV_POWER_USAGE{datacenter=""})` (one series per GPU). Grouped, not summed, so `compute.integrate` runs the data-quality filter on each raw host / GPU series before summing, exactly as the per-deployment path does per allocation. -3. Data quality is `_cluster_dq()`: `energy.data_quality` with `max_power_w` forced to `0` (the per-alloc / per-GPU ceiling of ~1000 W is meaningless for a whole host). Negative-drop and the scale-free rolling-median spike check still apply. +2. Query Mimir over `[settled_ts, now - query_lag_seconds]` with `compute.cluster_promql(dc)`: `sum by (node) (scaph_host_power_microwatts{datacenter=""})` (one series per host) and `sum by (UUID) (DCGM_FI_DEV_POWER_USAGE{datacenter=""})` (one series per GPU). Grouped, not summed, so `compute.integrate` runs the data-quality filter on each raw host / GPU series before summing, exactly as the per-deployment path does per allocation. +3. Data quality is `config.cluster_dq()`: `energy.data_quality` with `max_power_w` forced to `0` (the per-alloc / per-GPU ceiling of ~1000 W is meaningless for a whole host). Negative-drop and the scale-free rolling-median spike check still apply. 4. `integrate` with `TUE[dc]`, `apply_footprint` with the datacenter's Wattnet intensity, then `_fold_and_accumulate`. If the datacenter has no host-power series at all: `metrics_available: false`, retry next sweep, no zeros. Note on double counting: scaphandre host power is RAPL (CPU package + DRAM), it does not include the GPU board, so `host + GPU` is correct. ### Layer 2: read-time top-up -The stored accumulator only reaches `pointer_ts` (up to one sweep interval behind). On each API read, `accounting.sweep.topup(doc)` queries Mimir live for `[pointer_ts, now]` (a single `query_range`, no chunking, no ingestion-lag cushion -- Mimir data is immutable once ingested, so querying right up to "now" cannot return a wrong value, only occasionally miss a sample scraped a moment ago that is not visible *yet*, which self-corrects on the very next call), applies the TUE and the current Wattnet intensity, and returns the energy deltas plus `live_as_of` and `power_w`. `power_w` is the **current draw**: the TUE-normalized sum of the actual last sample of every series in that same query, however old that sample is, not a window mean. The result is cached for `topup_cache_seconds` (25 s, below the dashboard's 30 s poll, so the number visibly moves). Any failure returns `{"degraded": true}` and the API serves the stored value. +The stored accumulator only reaches `pointer_ts` (up to one sweep interval behind). On each API read, `energy.live.topup(doc)` queries Mimir live for `[pointer_ts, now]` (a single `query_range`, no chunking, no ingestion-lag cushion -- Mimir data is immutable once ingested, so querying right up to "now" cannot return a wrong value, only occasionally miss a sample scraped a moment ago that is not visible *yet*, which self-corrects on the very next call), applies the TUE and the current Wattnet intensity, and returns the energy deltas plus `live_as_of` and `power_w`. `power_w` is the **current draw**: the TUE-normalized sum of the actual last sample of every series in that same query, however old that sample is, not a window mean. The result is cached for `topup_cache_seconds` (25 s, below the dashboard's 30 s poll, so the number visibly moves). Any failure returns `{"degraded": true}` and the API serves the stored value. -`topup_cluster(dc, doc)` / `live_series_cluster(dc, doc, bucket_s)` are the datacenter equivalents (same window logic, `_cluster_promql` instead of the per-alloc query). They are computed inside `get_cluster_stats_bg()` (which runs every 30 s anyway), so the datacenter counter moves at the same cadence as the per-deployment one. +`topup` / `live_series` also accept a datacenter doc (`namespace == _cluster`), same window logic and `compute.cluster_promql` instead of the per-alloc query). They are computed inside `get_cluster_stats_bg()` (which runs every 30 s anyway), so the datacenter counter moves at the same cadence as the per-deployment one. `accumulated (stored, frozen for up to a sweep) + top-up (growing)` stays continuous: when the next sweep advances `pointer_ts` and folds that slice into `accumulated`, the top-up window shrinks by the same amount. @@ -103,8 +103,8 @@ The stored accumulator only reaches `pointer_ts` (up to one sweep interval behin A deployment the background thread has not reached yet has no `.accum.json`, so the top-up (which reads `pointer_ts` off the stored doc) has nothing to attach to. Rather than making a fresh deployment wait up to `sweep_seconds` for its first numbers, every read path runs one `process_single` pass synchronously (the same function and same write path the background thread uses) for whatever it is about to serve and has no doc yet: -- Single-deployment reads (`get_accumulated`, `get_series`) call `accounting.sweep.ensure_swept(namespace, uuid)`: fetch that one Nomad job, sweep it, return the resulting doc. -- Bulk/per-user reads (`get_accumulated_bulk`, `get_user_energy`) call `accounting.sweep.ensure_swept_bulk(namespace, owner)` first: list the owner's active jobs (one cheap Nomad call, no per-job fetch) and sweep only the ones still missing a doc, before falling back to the normal `store.iter_owner` scan. +- Single-deployment reads (`get_accumulated`, `get_series`) call `energy.ondemand.ensure_swept(namespace, uuid)`: fetch that one Nomad job, sweep it, return the resulting doc. +- Bulk/per-user reads (`get_accumulated_bulk`, `get_user_energy`) call `energy.ondemand.ensure_swept_bulk(namespace, owner)` first: list the owner's active jobs (one cheap Nomad call, no per-job fetch) and sweep only the ones still missing a doc, before reading the owner's documents with `store.iter_owner`, which uses an in-memory owner to uuids index instead of scanning the namespace (see [Persistence](#persistence)). Once a doc exists, both are a single cheap `store.read_accum` check (or, for the bulk path, one job-list call) and defer entirely to the background sweep; neither re-queries Mimir beyond that first pass. A job Nomad no longer knows about (or a listing failure) degrades to `energy: null` / that deployment simply absent from the aggregate. `/stats/cluster` (`get_cluster_energy`, per-datacenter) is not covered by this: its instantaneous power is already live regardless (`get_datacenter_power`), and its accumulator is seeded by the small, fixed `metrics_datacenters` list, not by individual deployments. @@ -112,11 +112,11 @@ Once a doc exists, both are a single cheap `store.read_accum` check (or, for the ### Layer 4: pure-Mimir fallback -`process_single` leaves a deployment with **no** doc when it is younger than one `series_bucket_seconds`: there is nothing settled to fold into an accumulator yet, so Layer 3 alone still reports `energy: null` for the first few minutes of a deployment's life. `sweep.live_only_doc(ns, uuid)` closes that gap: a transient, **never persisted** stand-in doc (`accumulated` all zero, `pointer_ts = submit_time`, the currently-running allocations) shaped so the normal `topup` / `live_series` machinery treats it exactly like a real doc -- the whole reported number then comes straight from Mimir, through the same lag-free `_live_window` as Layer 2. `accounting._live_only_stats()` wraps it and only returns something when the top-up actually found data (`power_w` set, or a non-zero energy/carbon/water delta) -- an all-zero top-up is indistinguishable from "no measurement yet", so it is treated as not populated (`null`) rather than shown as a fabricated `0` reading. Used by all four read functions whenever they still have no usable doc after Layer 3. +`process_single` leaves a deployment with **no** doc when it is younger than one `series_bucket_seconds`: there is nothing settled to fold into an accumulator yet, so Layer 3 alone still reports `energy: null` for the first few minutes of a deployment's life. `ondemand.live_only_doc(ns, uuid)` closes that gap: a transient, **never persisted** stand-in doc (`accumulated` all zero, `pointer_ts = submit_time`, the currently-running allocations) shaped so the normal `topup` / `live_series` machinery treats it exactly like a real doc -- the whole reported number then comes straight from Mimir, through the same lag-free `_live_window` as Layer 2. `energy._live_only_stats()` wraps it and only returns something when the top-up actually found data (`power_w` set, or a non-zero energy/carbon/water delta) -- an all-zero top-up is indistinguishable from "no measurement yet", so it is treated as not populated (`null`) rather than shown as a fabricated `0` reading. Used by all four read functions whenever they still have no usable doc after Layer 3. ## Persistence -One small JSON per deployment under `$ACCOUNTING_PTH/energy//`, atomic writes (temp file + `os.replace`). `process_single` / `process_datacenter` are the only writers, run either by the periodic sweep thread or, for a single deployment's very first doc, synchronously from the read path (see [Layer 3](#layer-3-on-demand-sweep)). Same convention as `ai4papi.utils.retrieve_from_snapshots`. No database. The `accounting.store` interface is deliberately small so it can move to PostgreSQL later without touching the routers. +One small JSON per deployment under `$ACCOUNTING_PTH/energy//`, atomic writes (temp file + `os.replace`). `process_single` / `process_datacenter` are the only writers, run either by the periodic sweep thread or, for a single deployment's very first doc, synchronously from the read path (see [Layer 3](#layer-3-on-demand-sweep)). Same convention as `ai4papi.utils.retrieve_from_snapshots`. No database. The `energy.store` interface is deliberately small so it can move to PostgreSQL later without touching the routers. `.accum.json` (rewritten every sweep, ~25 KB with a full 24h tail): @@ -138,9 +138,9 @@ One small JSON per deployment under `$ACCOUNTING_PTH/energy//`, atomi } ``` -`.series.jsonl`: one line per consolidated bucket (`energy.series_bucket_seconds`, 15 min), append-only (a bucket is written once, when it leaves the 24h tail, never rewritten). About 35k lines (~3 MB) per year for an always-on deployment, a few KB appended per sweep. +`.series.jsonl`: one line per consolidated bucket (`energy.series_bucket_seconds`, 15 min), append-only (a bucket is written once, when it leaves the 24h tail, never rewritten). About 35k lines (~3 MB) per year for an always-on deployment, a few KB appended per sweep. If the process dies between appending the buckets and writing the doc, the next sweep appends them again; `read_series` keeps one line per `ts`, so a repeated bucket counts once. -There is no summary file. Per-user and per-datacenter aggregates are computed on read from the `.accum.json` documents (`get_user_energy`, `get_cluster_energy`), because they need the live top-up anyway. If the per-user read (a namespace scan filtered by owner) ever gets heavy, cache a namespace snapshot rather than reviving a stale summary. +There is no summary file. Per-user and per-datacenter aggregates are computed on read from the `.accum.json` documents (`get_user_energy`, `get_cluster_energy`), because they need the live top-up anyway. Listing one user's deployments does not parse the whole namespace: `store.iter_owner` uses an in-memory `owner -> uuids` index, built with one scan, kept in sync by every `write_accum` and rebuilt from disk every 5 minutes. It assumes a single PAPI process writes the store (marked with a `shortcut:` comment); with several replicas on the same storage a replica would see another one's new documents up to 5 minutes late, and the index should move to a database. ## Resolution strategy @@ -158,12 +158,13 @@ What the `series` in an API response actually is: the persisted 15-min archive ( ## Computation details -Module `ai4papi/accounting/compute.py`, all pure and unit-testable except the cached Nomad lookup: +Module `ai4papi/energy/compute.py`, all pure and unit-testable except the cached Nomad lookup: - `resolve_alloc_ids(job_id, namespace)`: `Nomad.job.get_allocations` plus `_node_datacenter` (TTLCache), filtered to `energy.metrics_datacenters`. - `power_promql(alloc_ids)`: the CPU and GPU queries for one datacenter's alloc subset. Grouped, not fully summed, so the data-quality filter runs on the original per-source series. - CPU: `sum by (container_label_com_hashicorp_nomad_alloc_id) (scaph_process_power_consumption_microwatts{container_label_com_hashicorp_nomad_alloc_id=~"id1|id2"})` (one series per allocation) - GPU: `DCGM_FI_DEV_POWER_USAGE * on(UUID, instance) group_left(alloc_id) nomad_gpu_allocation_info{alloc_id=~"id1|id2"}` (one series per GPU) +- `cluster_promql(dc)`: the host and GPU queries of a whole datacenter (one series per host and per GPU), used by the datacenter sweep and its live top-up. - `sanitize(samples, dq)`: data-quality filter on one power series (see below). - `integrate(cpu_result, gpu_result, step_s, bucket_s, tue_factor, gap_factor, data_quality)`: CPU uW/1e6 plus GPU W, `sanitize` per source series, trapezoidal integral of each into `bucket_s` energy buckets, summed, then `energy_wh = raw_integral * tue_factor`. Gaps longer than `gap_factor * step_s` are not integrated across (deployment stopped, scrape hole, dropped anomaly). - `apply_footprint(buckets, carbon_series, water_series)`: `carbon_g = (energy_wh / 1000) * intensity_at(ts)`, same for water. `intensity_at` is a bisect for the last Wattnet sample at or before the bucket, `DEFAULTS` otherwise. @@ -177,6 +178,8 @@ The sweep uses `green_director.metrics[dc]` (Wattnet, already refreshed every 30 Every sweep recomputes carbon and water for all buckets newer than `now - footprint_revision_hours` (24 h) with the current intensity, so a bucket's footprint firms up as Wattnet settles it. Older buckets are frozen. The accumulated footprint is always `sum` over buckets, so it self-corrects. +Wattnet failures are contained in `GreenDirector.retrieve_footprints` (`ai4papi/wattnet.py`). Every request has a 30 s timeout, so a stalled Wattnet cannot block the background threads. A failed or empty answer raises inside `_fetch_footprint_data`, so the 15-minute cache never stores it. The datacenter keeps its last good data (or gets `DEFAULTS` if it never had any), and is not asked again for 5 minutes (`FAILURE_BACKOFF`), so a Wattnet outage costs one round of timeouts per 5 minutes. The log has one warning when a datacenter starts failing (with the date up to which data is kept) and one info line when it recovers, not one per refresh. + The live top-up multiplies its `delta_wh` by the current provisional intensity; the next sweep reconciles it (typical correction under 1-2%). ### Data quality @@ -193,10 +196,10 @@ The consumer-facing reference (response formats field by field, examples, pollin ### `energy` embedded in the deployment info -The four deployment routers (`modules`, `tools`, `batch`, `try_me/nomad`), in both `get_deployment` and `get_deployments`, attach an `energy` object built from the store plus the live top-up. Its shape follows the existing `full_info` param: +The four deployment routers (`modules`, `tools`, `batch`, `try_me/nomad`), in both `get_deployment` and `get_deployments`, attach an `energy` object built from the store plus the live top-up, through two helpers of the read API: `energy.add_to_job(job, namespace, full_info)` for a single deployment and `energy.add_to_jobs(jobs, namespace, owner, full_info)` for a list (a no-op on an empty list). The router only keeps the `if energy:` check. Its shape follows the existing `full_info` param: -- `full_info=false` (the `get_deployments` default): `energy` = `accounting.get_accumulated(...)` for the individual endpoint, or one batch `accounting.get_accumulated_bulk(namespace, owner)` mapped over the list. `EnergyStats`. One or two Mimir queries total for a whole list. -- `full_info=true` (the `get_deployment` default): `energy` = `accounting.get_series(namespace, uuid)` (whole life). `EnergyTimeSeries` with `accumulated` nested. On a list this is one `get_series` per deployment, matching the "may increase latency" note that `full_info` already carries. +- `full_info=false` (the `get_deployments` default): `energy` = `energy.get_accumulated(...)` for the individual endpoint, or one batch `energy.get_accumulated_bulk(namespace, owner)` mapped over the list. `EnergyStats`. One or two Mimir queries total for a whole list. +- `full_info=true` (the `get_deployment` default): `energy` = `energy.get_series(namespace, uuid)` (whole life). `EnergyTimeSeries` with `accumulated` nested. On a list this is one `get_series` per deployment, matching the "may increase latency" note that `full_info` already carries. A `energy: bool = True` query param lets a client opt out; the internal quota-check calls pass `False`. `nomad_utils` is not touched. @@ -215,7 +218,7 @@ as_of store pointer timestamp live_as_of instant of the live top-up complete False if it started late or was clamped by Mimir retention coverage_ratio fraction of run time that has power data (float, 1.0 = full); null on the per-datacenter block -degraded Mimir or Wattnet unavailable for the top-up +degraded the live Mimir top-up failed (the stored value is served) datacenters list of datacenters the deployment or user has run in; null on the per-datacenter block deployments set only on the per-user aggregate: number of deployments summed series 15-min EnergyTimeSeriesPoint list; null unless full_info=true (per-user aggregate only; the per-deployment full_info uses EnergyTimeSeries instead) @@ -225,21 +228,21 @@ series 15-min EnergyTimeSeriesPoint list; null unless full_info=tr ### Time series, for the detail view -The detail view calls the deployment's own individual endpoint (`GET /v1/deployments/{type}/{uuid}`, which defaults to `full_info=true`) and reads `energy` from the response. That `energy` is `schemas.EnergyTimeSeries` from `accounting.get_series(namespace, uuid)`: `{deployment_uuid, start, end, source, series, accumulated}`, where `series` is a list of points (`energy` / `carbon` / `water` per-bucket increment, `power_w` per point; a running total is `cumsum` client-side and matches `accumulated`) and `accumulated` is the `EnergyStats`. It serves the persisted 15-min archive downsampled to ~1000 points plus the live tail (`pointer_ts -> now` from Mimir, `live: true`) appended un-downsampled, so the graph keeps growing on every refresh; `source` is then `"mixed"`. Ownership and 404 handling are the individual endpoint's usual behaviour; `energy` is simply `null` when there is no rollup. +The detail view calls the deployment's own individual endpoint (`GET /v1/deployments/{type}/{uuid}`, which defaults to `full_info=true`) and reads `energy` from the response. That `energy` is `schemas.EnergyTimeSeries` from `energy.get_series(namespace, uuid)`: `{deployment_uuid, start, end, source, series, accumulated}`, where `series` is a list of points (`energy` / `carbon` / `water` per-bucket increment, `power_w` per point; a running total is `cumsum` client-side and matches `accumulated`) and `accumulated` is the `EnergyStats`. It serves the persisted 15-min archive downsampled to ~1000 points plus the live tail (`pointer_ts -> now` from Mimir, `live: true`) appended un-downsampled, so the graph keeps growing on every refresh; `source` is then `"mixed"`. Ownership and 404 handling are the individual endpoint's usual behaviour; `energy` is simply `null` when there is no rollup. ### Per-datacenter energy, in the cluster stats -`get_cluster_stats_bg()` (the 30 s background thread, which already refreshes `green_director`) calls `accounting.get_cluster_energy(series=True)`, which merges three things per datacenter into `schemas.EnergyStats`: +`get_cluster_stats_bg()` (the 30 s background thread, which already refreshes `green_director`) calls `energy.get_cluster_energy(series=True)`, which merges three things per datacenter into `schemas.EnergyStats`: -1. **Instantaneous power** from `accounting.get_datacenter_power()`: two instant Mimir queries, `power_w = (sum by (datacenter) scaph_host_power_microwatts + sum by (datacenter) DCGM_FI_DEV_POWER_USAGE) * TUE[dc]`. The host / GPU split is not exposed (on the ifca VMs `scaph_host_power` reads near zero, so it would be misleading; in practice `power_w` is GPU-dominated). A datacenter listed in `energy.metrics_datacenters` is reported with whatever it has (host-only, GPU-only or both); without that allowlist a host-power series is required so a stray GPU metric for an unmonitored site does not fabricate a total. -2. **Accumulated energy** from the `_cluster/.accum.json` document (persisted by the datacenter sweep) plus a `topup_cluster` live slice: `energy_wh`, `carbon_g`, `water_l`, `since`, `complete`, `live_as_of`. -3. **Time series**: the consolidated `.series.jsonl` (`_cluster_archive_series`, cached ~5 min so the 30 s thread does not re-parse it) plus the 24h tail, downsampled to `energy.cluster_series_points`, with the live tail (`live_series_cluster`) appended. Points carry per-bucket increments only. +1. **Instantaneous power** from `energy.get_datacenter_power()`: two instant Mimir queries, `power_w = (sum by (datacenter) scaph_host_power_microwatts + sum by (datacenter) DCGM_FI_DEV_POWER_USAGE) * TUE[dc]`. The host / GPU split is not exposed (on the ifca VMs `scaph_host_power` reads near zero, so it would be misleading; in practice `power_w` is GPU-dominated). A datacenter listed in `energy.metrics_datacenters` is reported with whatever it has (host-only, GPU-only or both); without that allowlist a host-power series is required so a stray GPU metric for an unmonitored site does not fabricate a total. +2. **Accumulated energy** from the `_cluster/.accum.json` document (persisted by the datacenter sweep) plus a live top-up slice (`live.topup` on the datacenter doc): `energy_wh`, `carbon_g`, `water_l`, `since`, `complete`, `live_as_of`. +3. **Time series**: the consolidated `.series.jsonl` (`_cluster_archive_series`, cached ~5 min so the 30 s thread does not re-parse it) plus the 24h tail, downsampled to `energy.cluster_series_points`, with the live tail (`live.live_series` on the datacenter doc) appended. Points carry per-bucket increments only. Attached as `stats.datacenters[].energy`. There is **no** response-root total: `schemas.ClusterStats` has no `energy` field, and clients sum the datacenter blocks they want. `get_cluster_stats()` (the VO-filtered endpoint) prunes each datacenter's node list but keeps the `energy` block, so an instrumented datacenter with no nodes for that VO is still returned (empty `nodes`). The background thread always computes the series; `get_cluster_stats(full_info=false)` (the default) strips it from every datacenter block before returning, so the series only ships when the caller asks for it. Any failure in this block is caught and leaves the datacenters without `energy`. A datacenter that is not in `energy.metrics_datacenters` and has no accumulator doc simply has no `energy` block; a monitored GPU-only site does get one (accumulator plus series, and an instantaneous `power_w` from its GPU draw). ### Per-user aggregate -`stats.deployments.get_user_stats` gains an `energy` key from `accounting.get_user_energy(namespace, owner, series=full_info)`: it iterates the owner's `.accum.json` documents, applies each one's live top-up, and sums them into one `EnergyStats`-shaped block (`since` = earliest, `as_of` / `live_as_of` = latest, `complete` = AND, `degraded` = OR, `power_w` = sum of the running ones, `coverage_ratio` = mean, `tue_factor` = blended `energy_wh / raw`, `datacenters` = union, plus a `deployments` count). With `full_info=true` it also returns a merged time series (`_user_series`): the raw 15-min buckets of every deployment (archive + tail) are summed by timestamp first, then `compute.downsample`d once to `target_points` (1000). Merging before downsampling keeps the resolution uniform across the whole series and gives the same hard cap as the other levels, regardless of how long each deployment individually has been running; the live tails are merged and appended un-downsampled, as elsewhere. The energy block is independent of the pre-existing CSV usage stats: if those are missing, the endpoint still returns the `energy` block instead of failing. +`stats.deployments.get_user_stats` gains an `energy` key from `energy.get_user_energy(namespace, owner, series=full_info)`: it reads the owner's `.accum.json` documents (through the owner index), applies each one's live top-up, and sums them into one `EnergyStats`-shaped block (`since` = earliest, `as_of` / `live_as_of` = latest, `complete` = AND, `degraded` = OR, `power_w` = sum of the running ones, `coverage_ratio` = mean, `tue_factor` = blended `energy_wh / raw`, `datacenters` = union, plus a `deployments` count). With `full_info=true` it also returns a merged time series (`series.build_series`): the raw 15-min buckets of every deployment (archive + tail) are summed by timestamp first, then `compute.downsample`d once to `target_points` (1000). Merging before downsampling keeps the resolution uniform across the whole series and gives the same hard cap as the other levels, regardless of how long each deployment individually has been running; the live tails are merged and appended un-downsampled, as elsewhere. The energy block is independent of the pre-existing CSV usage stats: if those are missing, the endpoint still returns the `energy` block instead of failing. ## Configuration @@ -266,7 +269,7 @@ energy: `max_power_w` is a per-source ceiling (one allocation's CPU, or one GPU), not the deployment total, so `1000` W is safe for any single GPU and any CPU allocation on a VM. A dropped sample becomes a gap in that source only. The datacenter sweep reuses this `data_quality` block but forces `max_power_w` to `0` (a whole host draws far more than any single alloc), keeping only the negative-drop and the rolling-median spike check. -- `ai4papi/mimir.py`: `MIMIR_URL` is a module constant. `MIMIR_USER` and `MIMIR_PASSWORD` come from the environment (`os.environ.get`, like `wattnet.py`, not `load_env`, so a missing value disables the feature instead of breaking PAPI startup). Add both to `template.env` and to the PAPI deployment. +- `ai4papi/clients/mimir.py`: `MIMIR_URL` is a module constant; `BACKOFF_SECONDS` too (60 s). Request timeouts are 10 s (instant query) and 15 s (range query): measured against the real Mimir, a 24h range query takes under 1 s and a full 11,000-point backfill chunk about 1.5 s. `MIMIR_USER` and `MIMIR_PASSWORD` come from the environment (`os.environ.get`, like `wattnet.py`, not `load_env`, so a missing value disables the feature instead of breaking PAPI startup). Add both to `template.env` and to the PAPI deployment. - `var/datacenters.csv`: `TUE` column, populated only for the datacenters that have a value. `ai4papi/conf.py` tolerates a blank numeric cell (loads it as `None`). - The Prometheus `datacenter` label values must match the `name` column of `datacenters.csv`. @@ -278,6 +281,11 @@ energy: | Sweep thread | `sweep_seconds` (900) | keeps the accumulator current, consolidates the footprint at Wattnet's revision cadence, short enough to catch short jobs before Nomad GC | | Top-up cache | `topup_cache_seconds` (25) | below the dashboard's 30 s poll, so the counter moves | | Cluster archive-series cache | 300 s | the `.series.jsonl` only changes at sweep cadence; keeps the 30 s thread from re-parsing it | +| Mimir failure backoff | 60 s (`BACKOFF_SECONDS`, constant in `clients/mimir.py`) | after a timeout or connection error every Mimir call fails fast with `MimirError` | +| Wattnet failure backoff | 300 s (`FAILURE_BACKOFF`, constant in `wattnet.py`) | a datacenter whose refresh failed is not asked again meanwhile | +| Owner index rebuild | 300 s | heals anything written outside this process; `write_accum` keeps it current in between | + +Every TTL cache in `energy/` is built with `cachetools.cached(..., lock=threading.Lock())`: endpoints run in a thread pool next to the sweep and stats threads, and an unlocked `cachetools` cache raises `KeyError` / `RuntimeError` under concurrent access. For the user to perceive the increment every 30 s the frontend must show Wh (or kWh with 3 decimals) or animate the counter: at 50 W, 30 s is ~0.4 Wh. @@ -285,8 +293,8 @@ For the user to perceive the increment every 30 s the frontend must show Wh (or Same posture as `get_cluster_stats`. -- Mimir down: the sweep logs and skips (store frozen); the top-up returns `degraded: true`; the API serves the stored accumulator with its `as_of`. -- Wattnet down: energy is still computed; the footprint falls back to `DEFAULTS`; `degraded: true`. +- Mimir down: the sweep logs and skips (store frozen); the top-up returns `degraded: true`; the API serves the stored accumulator with its `as_of`. A Mimir that hangs or refuses connections costs one timeout, then `clients/mimir.py` fails fast for 60 s (`BACKOFF_SECONDS`): the sweep stops the pass of the namespace with one warning instead of waiting two timeouts per deployment, and the on-demand sweep in a list request skips instead of blocking the page. +- Wattnet down: energy is still computed; the footprint keeps using the last good intensity (or `DEFAULTS` for a datacenter that never had data) and `degraded` stays `false`; buckets still in the 24h tail are recomputed once Wattnet is back. See Footprint consolidation for the timeout and backoff. - Unreadable file: `get_accumulated` returns `None`, so `energy` is `null`. - `energy.enabled: false` or no Mimir credentials: `get_accumulated` / `get_series` / `get_datacenter_power` return `None` / `{}`, so `energy` is `null` everywhere and the rest of the API is unchanged. - Deployment on a site or node without power monitoring: Mimir returns nothing, `metrics_available` is `false`, `energy` is `null` (best-effort, no misleading zeros); retried every sweep in case monitoring appears or the alloc moves. @@ -299,11 +307,16 @@ Same posture as `get_cluster_stats`. New: -- `ai4papi/mimir.py` -- `ai4papi/accounting/__init__.py` (read API: `get_accumulated*`, `get_series`, `get_user_energy`, `get_datacenter_power`, `get_cluster_energy`) -- `ai4papi/accounting/store.py` (file store) -- `ai4papi/accounting/compute.py` (pure functions) -- `ai4papi/accounting/sweep.py` (deployment + datacenter sweep, top-up) +- `ai4papi/clients/mimir.py` +- `ai4papi/energy/__init__.py` (read API: `get_accumulated*`, `get_series`, `get_user_energy`, `get_datacenter_power`, `get_cluster_energy`; `add_to_job` / `add_to_jobs` are what the deployment routers call to set `job["energy"]`) +- `ai4papi/energy/store.py` (file store) +- `ai4papi/energy/series.py` (builds the time-series points of the API responses) +- `ai4papi/energy/compute.py` (pure functions) +- `ai4papi/energy/config.py` (config section, `enabled()`, TUE, shared constants and tiny helpers) +- `ai4papi/energy/query.py` (Mimir power queries shared by the sweep and the live top-up) +- `ai4papi/energy/sweep.py` (write path: deployment + datacenter sweep) +- `ai4papi/energy/live.py` (read path: live top-up and live series, for deployments and datacenters) +- `ai4papi/energy/ondemand.py` (read path: on-demand sweep and `live_only_doc`) - `tests/test_energy.py` Modified (all additive): @@ -312,25 +325,31 @@ Modified (all additive): - `ai4papi/routers/v1/deployments/{modules,tools}.py`, `ai4papi/routers/v1/batch.py`, `ai4papi/routers/v1/try_me/nomad.py`: the `energy` key and param (`EnergyStats` or `EnergyTimeSeries` per `full_info`) - `ai4papi/routers/v1/stats/deployments.py`: the per-user `energy` aggregate in `get_user_stats` (`full_info` for the series) and the per-datacenter `energy` block in `get_cluster_stats` (`full_info` for the series); no response-root total - `ai4papi/schemas.py`: `EnergyStats` (per deployment and per user), `EnergyTimeSeriesPoint`, `EnergyTimeSeries` (its point list is `series`); `energy` on `DatacenterStats` (also `EnergyStats`) -- `ai4papi/wattnet.py`: `GreenDirector.footprint_series` +- `ai4papi/wattnet.py`: `GreenDirector.footprint_series`; request timeouts, failures raised instead of cached, last good data kept and failure backoff in `retrieve_footprints` +- `pyproject.toml`: optional `test` extra (`pytest`); `tests/main.py`: runs `test_energy` through `pytest.main` - `ai4papi/conf.py`: blank numeric CSV cell tolerated - `var/datacenters.csv`: `TUE` column - `etc/main.yaml` (`energy` section incl. `deployment_series_points` / `user_series_points` / `cluster_series_points`), `template.env`, `tests/test_routes.py` ## Testing -`tests/test_energy.py` (plain-script style, picked up by `tests/main.py`): +`tests/test_energy.py` (pytest, unlike the plain-script tests; `tests/main.py` runs it through `pytest.main`): - Pure-function checks, no infrastructure: `integrate` (trapezoidal, TUE, gaps, single sample), `sanitize` (negatives, ceiling, spike detection, disabled passthrough), `apply_footprint` (Wattnet series and `DEFAULTS` fallback), `intensity_at`, `downsample`, `merge_buckets`, `running_seconds`. - Sweep pipeline, Mimir and Nomad monkeypatched: `process_single` writes a correct `.accum.json`, idempotency (same `now` gives a byte-identical document), monotonicity across sweeps, buckets aging into `.series.jsonl`, and `get_accumulated` / `get_series` validate against the pydantic models. - Cluster energy, `mimir.query` / `mimir.query_range` monkeypatched (several raw host series): `get_datacenter_power()` gives `power_w = (host + gpu) * TUE`; `process_datacenter` writes a correct `_cluster/.accum.json` with per-series sanitize then sum, monotonic across sweeps, buckets aging into `.series.jsonl`; `get_cluster_energy(series=True)` merges power + accumulator + a series that validates as `EnergyTimeSeriesPoint` and whose `cumsum` reconciles with the accumulated total. - Per-user aggregate: `get_user_energy` sums the owner's deployments (`EnergyStats`-shaped), with and without the merged series, and its total reconciles with the merged series. +- On-demand paths: a never-swept deployment is swept by the read, an existing doc triggers no Nomad call, an unknown job degrades to `None`; the bulk variant for lists and per-user reads; the pure-Mimir fallback for a deployment younger than one bucket (never a fabricated zero); an inconclusive stub that does not block it. +- `add_to_job` / `add_to_jobs` (list and detail shape, empty list), `series.build_series` (live tail only after the persisted points, several entities summed), and the store (a repeated bucket counts once, the owner index scans once and sees later writes). +- Live top-up: a failing host query is `degraded` on a datacenter doc; a confirmed `metrics_available: false` is reported as `None`. +- Mimir backoff: one timeout, then calls fail fast without a request until the backoff is over; with three deployments and a hung Mimir the sweep pays one timeout and stops the pass. +- Wattnet failure handling with a fake session: defaults while it is down and nothing cached, no new request during the backoff, real data after it recovers, last good data kept on a later blip, one log line per outage, timeouts present. - An optional live Mimir check, gated on `PAPI_TESTS_MIMIR` plus real credentials. -Run with the repo venv: +Run with the repo venv (`pip install -e .[test]` installs `pytest`): ```bash -IS_PROD=False env/bin/python tests/test_energy.py +IS_PROD=False env/bin/python -m pytest tests/test_energy.py cd tests && IS_PROD=False ../env/bin/python test_routes.py && cd .. ``` diff --git a/docs/energy-api.md b/docs/energy-api.md index f00a5471..23cd4124 100644 --- a/docs/energy-api.md +++ b/docs/energy-api.md @@ -67,7 +67,7 @@ One object, the same shape at every level, always with the live top-up applied: | `as_of` | string or null | Timestamp of the last persisted point. | | `live_as_of` | string or null | Instant the numbers actually reflect, once the live top-up is applied. Use this for "updated N ago". Null when the values come straight from the store with no live top-up. | | `complete` | boolean | `false` if accounting started after the deployment was created, or the window was clamped by Mimir retention. When `false`, treat `energy_wh` and the footprint as a lower bound. | -| `degraded` | boolean | `true` if Mimir or Wattnet was unreachable for the live part. The stored value is still served, it may just be slightly stale. | +| `degraded` | boolean | `true` if the live Mimir part failed. The stored value is still served, it may just be slightly stale. A Wattnet outage does not set it: the footprint then uses the last good intensity (or the default values) and the numbers stay available. | | `coverage_ratio` | number or null | Deployment and per-user only: fraction of the run time that has power data (`1.0` = fully covered). `null` on the datacenter block. | | `datacenters` | string[] or null | Deployment and per-user only: datacenters the deployment(s) ran in. `null` on the datacenter block. | | `deployments` | number or null | Per-user aggregate only: how many deployments went into the sum. | From 66dd69105f67485f2651580e118101491b7a7fb5 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Mon, 19 Oct 2026 20:54:20 +0200 Subject: [PATCH 19/25] fix(energy): only accept plain names as namespace and deployment ids in the store The namespace and the deployment id were used to build file paths as they came, so an id such as "../../x" could point outside the store (SonarCloud pythonsecurity:S2083, path traversal in the atomic write). Both are now checked to be a plain name (no separators, no leading dot) before building a path; anything else is ignored like a missing document. --- ai4papi/energy/store.py | 22 ++++++++++++++++------ tests/test_energy.py | 24 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/ai4papi/energy/store.py b/ai4papi/energy/store.py index 5178f341..0b107198 100644 --- a/ai4papi/energy/store.py +++ b/ai4papi/energy/store.py @@ -11,6 +11,7 @@ import json import os +import re import threading import time from pathlib import Path @@ -24,19 +25,28 @@ def base_dir() -> Path | None: return Path(root) / "energy" +# A namespace or deployment id is used as a file name: only a plain name is allowed +# (no separators, no leading dot), so an id can never point outside the store. +_SAFE_NAME = re.compile(r"[A-Za-z0-9_][A-Za-z0-9._-]*") + + +def _name(value: str) -> str | None: + return os.path.basename(value) if _SAFE_NAME.fullmatch(value) else None + + def _ns_dir(ns: str) -> Path | None: - base = base_dir() - return (base / ns) if base else None + base, name = base_dir(), _name(ns) + return (base / name) if base and name else None def _accum_path(ns: str, uuid: str) -> Path | None: - d = _ns_dir(ns) - return (d / f"{uuid}.accum.json") if d else None + d, name = _ns_dir(ns), _name(uuid) + return (d / f"{name}.accum.json") if d and name else None def _series_path(ns: str, uuid: str) -> Path | None: - d = _ns_dir(ns) - return (d / f"{uuid}.series.jsonl") if d else None + d, name = _ns_dir(ns), _name(uuid) + return (d / f"{name}.series.jsonl") if d and name else None def _atomic_write(path: Path, text: str) -> None: diff --git a/tests/test_energy.py b/tests/test_energy.py index 1238a6fb..27fd5bb9 100644 --- a/tests/test_energy.py +++ b/tests/test_energy.py @@ -1145,3 +1145,27 @@ def get_job(id_, namespace): assert len(calls) == 1 assert fetched == uuids[:2] assert all(store.read_accum(_NS, u) is None for u in uuids) + + +def test_store_ignores_ids_that_are_not_plain_names(tmp_path): + # an id used as a file name can never point outside the store + for bad in ("../evil", "a/b", "..", ".hidden", "", "x\n", "/etc/passwd"): + store.write_accum(_NS, bad, {"owner": "u@egi.eu"}) + store.append_series(_NS, bad, [{"ts": "2026-01-01T00:00:00Z"}]) + assert store.read_accum(_NS, bad) is None + assert store.read_series(_NS, bad, None, None) == [] + store.write_accum("../escape", "ok", {"owner": "u@egi.eu"}) + assert store.read_accum("../escape", "ok") is None + assert [p for p in tmp_path.rglob("*") if p.is_file()] == [] + assert not (tmp_path.parent / "evil.accum.json").exists() + + # real ids still work + for name in ( + "sweep-test-uuid", + "ifca-ai4eosc", + "a1b2c3d4-0000-1111-2222-333344445555", + ): + store.write_accum(_NS, name, {"owner": "u@egi.eu"}) + assert store.read_accum(_NS, name) == {"owner": "u@egi.eu"} + store.write_accum(config.CLUSTER_NS, "ifca-ai4eosc", {"owner": None}) + assert store.read_accum(config.CLUSTER_NS, "ifca-ai4eosc") is not None From aacef2be64cd624cfff0a390d694fb314f2808d0 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Mon, 19 Oct 2026 21:00:14 +0200 Subject: [PATCH 20/25] chore(energy): mark the store write as a Sonar false positive SonarCloud keeps reporting pythonsecurity:S2083 on the atomic write even though the namespace and deployment id are checked to be plain names before any path is built (see _name and the traversal test). Annotate the line so the finding does not block the quality gate. --- ai4papi/energy/store.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ai4papi/energy/store.py b/ai4papi/energy/store.py index 0b107198..f236ead6 100644 --- a/ai4papi/energy/store.py +++ b/ai4papi/energy/store.py @@ -52,7 +52,9 @@ def _series_path(ns: str, uuid: str) -> Path | None: def _atomic_write(path: Path, text: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix + ".tmp") - tmp.write_text(text, encoding="utf-8") + tmp.write_text( + text, encoding="utf-8" + ) # NOSONAR: path built from names checked by _name os.replace(tmp, path) From 9fc44088ff18450f5185b3ccca621185a7fbb0ff Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Mon, 19 Oct 2026 21:02:24 +0200 Subject: [PATCH 21/25] chore(energy): keep the Sonar annotation on the line of the write --- ai4papi/energy/store.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ai4papi/energy/store.py b/ai4papi/energy/store.py index f236ead6..b3a84e24 100644 --- a/ai4papi/energy/store.py +++ b/ai4papi/energy/store.py @@ -52,9 +52,7 @@ def _series_path(ns: str, uuid: str) -> Path | None: def _atomic_write(path: Path, text: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix + ".tmp") - tmp.write_text( - text, encoding="utf-8" - ) # NOSONAR: path built from names checked by _name + tmp.write_text(text, encoding="utf-8") # NOSONAR: ids are checked by _name os.replace(tmp, path) From f13db0842c8f5ce95aa493efcdc61014d9f51b11 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Mon, 19 Oct 2026 21:05:13 +0200 Subject: [PATCH 22/25] fix(energy): write the store files with open() instead of Path.write_text SonarCloud models Path.write_text as a path sink and treats what was read from a file as untrusted, so the document content flowing back into the atomic write was reported as a path traversal (pythonsecurity:S2083) even though the file name is built only from validated ids. open() plus write() is the same write without that false positive, as in append_series. --- ai4papi/energy/store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ai4papi/energy/store.py b/ai4papi/energy/store.py index b3a84e24..1009bfe9 100644 --- a/ai4papi/energy/store.py +++ b/ai4papi/energy/store.py @@ -52,7 +52,8 @@ def _series_path(ns: str, uuid: str) -> Path | None: def _atomic_write(path: Path, text: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix + ".tmp") - tmp.write_text(text, encoding="utf-8") # NOSONAR: ids are checked by _name + with open(tmp, "w", encoding="utf-8") as f: + f.write(text) os.replace(tmp, path) From 629c31e6b37e312fdf8d2a7082fc497af46f539b Mon Sep 17 00:00:00 2001 From: Ignacio Heredia Date: Wed, 28 Oct 2026 14:49:26 +0100 Subject: [PATCH 23/25] tests: move unit tests to a dedicated folder Integration tests folder will be created after merge with main --- tests/main.py | 8 +------- tests/{ => unit}/test_energy.py | 0 2 files changed, 1 insertion(+), 7 deletions(-) rename tests/{ => unit}/test_energy.py (100%) diff --git a/tests/main.py b/tests/main.py index 39b50066..eb226e64 100644 --- a/tests/main.py +++ b/tests/main.py @@ -30,12 +30,6 @@ for module in modules: try: - if module == "test_energy": # pytest-style, unlike the plain-script tests - import pytest - - if pytest.main(["-q", str(Path(__file__).parent / f"{module}.py")]): - raise RuntimeError("pytest reported failures") - else: - __import__(module) + __import__(module) except Exception as e: print(f"🔴 Error testing {module}:\n{e}") diff --git a/tests/test_energy.py b/tests/unit/test_energy.py similarity index 100% rename from tests/test_energy.py rename to tests/unit/test_energy.py From c814dfedc32437766ef39ce0e04c7f7fb42e68c0 Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Thu, 29 Oct 2026 13:12:20 +0100 Subject: [PATCH 24/25] fix(energy): uniform time windows in the series and continuous per-user series Downsampling grouped a fixed number of buckets by position, so the last persisted chunk was a short remainder and the live tail was appended at 15 min, giving points with different spans. The window boundaries also moved every time a bucket arrived. - downsample into fixed, epoch-aligned time windows and run the live tail through the same downsampling; a point is live if its window holds any live bucket - fill the idle periods of the per-user series with zero points, up to the last complete bucket, but never past the pointer_ts of a running deployment, where the store knows nothing - update the docs and the endpoint docstrings --- ai4papi/energy/__init__.py | 16 ++- ai4papi/energy/compute.py | 42 +++++--- ai4papi/energy/series.py | 56 ++++++++--- ai4papi/routers/v1/stats/deployments.py | 6 +- docs/energy-accounting.md | 8 +- tests/unit/test_energy.py | 124 +++++++++++++++++++++++- 6 files changed, 215 insertions(+), 37 deletions(-) diff --git a/ai4papi/energy/__init__.py b/ai4papi/energy/__init__.py index f81f3da2..8e95df8b 100644 --- a/ai4papi/energy/__init__.py +++ b/ai4papi/energy/__init__.py @@ -268,14 +268,28 @@ def get_user_energy( step_s = int(cfg["series_bucket_seconds"]) if target_points is None: target_points = int(cfg.get("user_series_points", 1000)) + lives = [live_series(doc, bucket_s=step_s) for _uuid, doc, _stats in rows] + # zeros only where nothing was running: past the `pointer_ts` of a running + # deployment the store knows nothing (the live tail covers the recent part), + # so a stale sweep or an empty live tail must not read as idle + until = min( + [config.now().timestamp()] + + [ + compute.parse_iso(doc["pointer_ts"]) + for _uuid, doc, _stats in rows + if doc.get("active_allocs") and doc.get("pointer_ts") + ] + ) out["series"] = ( build_series( [ persisted(store.read_series(namespace, uuid, None, None), doc) for uuid, doc, _stats in rows ], - [live_series(doc, bucket_s=step_s) for _uuid, doc, _stats in rows], + lives, target_points, + fill_s=step_s, # idle periods show as zeros, not as missing points + until=until, ) or None ) diff --git a/ai4papi/energy/compute.py b/ai4papi/energy/compute.py index 542b3423..eca74642 100644 --- a/ai4papi/energy/compute.py +++ b/ai4papi/energy/compute.py @@ -9,6 +9,7 @@ import bisect import datetime +import itertools import threading import statistics from collections import namedtuple @@ -390,22 +391,35 @@ def apply_footprint( def downsample(buckets: list[Bucket], target_points: int) -> list[Bucket]: - """Aggregate consecutive buckets so the result has at most `target_points`.""" + """ + Aggregate buckets into fixed, epoch-aligned time windows so the result has at + most `target_points` and every point spans the same time (only the first and + last window can be partial). Windows are by time, not by position: the + boundaries do not move when a new bucket arrives. Each point is stamped with + its window start. The bucket step is inferred from the smallest gap. + """ n = len(buckets) if n <= target_points or target_points < 1: return buckets - group = -(-n // target_points) # ceil - out = [] - for i in range(0, n, group): - chunk = buckets[i : i + group] - out.append( - Bucket( - ts=chunk[0].ts, - energy_wh=sum(b.energy_wh for b in chunk), - power_w_avg=sum(b.power_w_avg for b in chunk) / len(chunk), - carbon_g=sum(b.carbon_g for b in chunk), - water_l=sum(b.water_l for b in chunk), - ) + ts = [parse_iso(b.ts) for b in buckets] + step = min(b - a for a, b in itertools.pairwise(ts)) + slots = round((ts[-1] - ts[0]) / step) + 1 + # one window less than `target_points`: the first and last may be partial, so + # a span of `slots` can straddle one more window than it holds + # shortcut: target_points == 1 can still yield 2 windows, nobody asks for one + window = -(-slots // max(target_points - 1, 1)) * step + + groups: dict[int, list[Bucket]] = {} + for t, b in zip(ts, buckets): + groups.setdefault(int(t // window), []).append(b) + return [ + Bucket( + ts=iso(k * window), + energy_wh=sum(b.energy_wh for b in chunk), + power_w_avg=sum(b.power_w_avg for b in chunk) / len(chunk), + carbon_g=sum(b.carbon_g for b in chunk), + water_l=sum(b.water_l for b in chunk), ) - return out + for k, chunk in groups.items() + ] diff --git a/ai4papi/energy/series.py b/ai4papi/energy/series.py index 98a30d38..e3acfff9 100644 --- a/ai4papi/energy/series.py +++ b/ai4papi/energy/series.py @@ -33,15 +33,37 @@ def persisted(archive: list[dict], doc: dict) -> list[dict]: return list(archive) + list(doc.get("tail") or []) +def _fill_gaps( + buckets: list[compute.Bucket], step_s: int, end_ts: float +) -> list[compute.Bucket]: + """Zero buckets on the `step_s` grid wherever `buckets` (sorted) has a hole, + and from the last one up to `end_ts` (inclusive).""" + by_ts = {b.ts: b for b in buckets} + out, t = [], compute.parse_iso(buckets[0].ts) + end = max(end_ts, compute.parse_iso(buckets[-1].ts)) + while t <= end: + ts = compute.iso(t) + out.append(by_ts.get(ts) or compute.Bucket(ts, 0.0, 0.0, 0.0, 0.0)) + t += step_s + return out + + def build_series( - persisted_lists: list[list[dict]], live: list[list[dict]], target_points: int + persisted_lists: list[list[dict]], + live: list[list[dict]], + target_points: int, + fill_s: int | None = None, + until: float | None = None, ) -> list[dict]: """ Time series of one or several entities (a deployment, a user's deployments, a datacenter). The persisted buckets are merged by timestamp across entities - and `compute.downsample`d once, so the resolution is uniform and capped at - `target_points`; the live Mimir tails are merged the same way and appended - un-downsampled. The points carry the per-bucket increments only; a running + and `compute.downsample`d once together with the live Mimir tails (merged the + same way, only the part after the persisted buckets), so the resolution is + uniform and capped at `target_points`. With `fill_s`, holes in the persisted buckets (no entity + active) are filled with zero points on that grid, before downsampling, so + the series is continuous; with `until` (a timestamp) it also runs up to the last + complete bucket before it, stopping short of the live tail's first bucket. The points carry the per-bucket increments only; a running total is `cumsum(series[].energy_wh)` and matches `accumulated.energy_wh`. """ @@ -49,13 +71,19 @@ def merged(lists: list[list[dict]]) -> list[compute.Bucket]: out = compute.merge_buckets([[_bucket(b) for b in bs] for bs in lists]) return sorted(out, key=lambda b: b.ts) - series = [ - _point(b, live=False) - for b in compute.downsample(merged(persisted_lists), target_points) - ] - series += [ - _point(b, live=True) - for b in merged(live) - if not series or b.ts > series[-1]["ts"] - ] - return series + buckets = merged(persisted_lists) + live_buckets = merged(live) + if fill_s and buckets: + end = (int(until) // fill_s - 1) * fill_s if until else 0.0 + if live_buckets: # leave the live buckets to the live tail + end = min(end, compute.parse_iso(live_buckets[0].ts) - fill_s) + buckets = _fill_gaps(buckets, fill_s, end) + # the live tail goes through the same downsampling, so every point spans the + # same time; a point is `live` if its window holds any live bucket + last = buckets[-1].ts if buckets else "" + tail = [b for b in live_buckets if b.ts > last] + points = compute.downsample(buckets + tail, target_points) + if not tail: + return [_point(p, live=False) for p in points] + live_from = max(p.ts for p in points if p.ts <= tail[0].ts) + return [_point(p, live=p.ts >= live_from) for p in points] diff --git a/ai4papi/routers/v1/stats/deployments.py b/ai4papi/routers/v1/stats/deployments.py index 5100b5b3..ea16cb0d 100644 --- a/ai4papi/routers/v1/stats/deployments.py +++ b/ai4papi/routers/v1/stats/deployments.py @@ -107,7 +107,7 @@ def get_user_stats( * **energy**: attach the `energy` accounting block (the user's accumulated consumption and footprint, each deployment's live Mimir top-up included). Enabled by default. - * **full_info**: when `true`, the `energy` block also carries a merged 15-min + * **full_info**: when `true`, the `energy` block also carries a merged time series of the user's deployments. Off by default (accumulated only). """ @@ -198,7 +198,7 @@ def get_cluster_stats( Attach the per-datacenter `energy` block (site infrastructure power draw plus the accumulated energy/footprint). Enabled by default. full_info: bool - When `true`, each `datacenters[].energy` block also carries its 15-min + When `true`, each `datacenters[].energy` block also carries its time series. Off by default (accumulated numbers only); the live top-up is always applied either way. Energy is reported per datacenter, there is no platform-wide total. @@ -467,7 +467,7 @@ def get_cluster_stats_bg() -> schemas.ClusterStats: g_stats.gpu_total = n_stats.gpu_used # Per-datacenter energy (energy feature): instantaneous power draw + the - # accumulated energy/footprint since accounting started + a 15-min series. + # accumulated energy/footprint since accounting started + a time series. # Reported per datacenter only, there is no platform-wide total (clients sum # the datacenters they care about). Best-effort: a failure here must not # break the cluster stats. diff --git a/docs/energy-accounting.md b/docs/energy-accounting.md index 3b36e2fa..09d9a83e 100644 --- a/docs/energy-accounting.md +++ b/docs/energy-accounting.md @@ -154,7 +154,7 @@ There is no summary file. Per-user and per-datacenter aggregates are computed on The store is the permanent low-resolution archive; Mimir is the high-resolution source for anything recent. -What the `series` in an API response actually is: the persisted 15-min archive (plus the still-mutable tail) `compute.downsample`d to a `target_points` cap, then the live 15-min tail appended un-downsampled. Caps: deployment detail `energy.deployment_series_points` (1000), datacenter `energy.cluster_series_points` (500), per-user `energy.user_series_points` (1000, merged across the user's deployments first, then one downsample). Each function falls back to its hardcoded default when the key is absent from the config. Below the cap you get the raw 15-min buckets; above it, `downsample` groups `ceil(n / cap)` consecutive buckets, so the effective step is `ceil(n / cap) * 15 min`, uniform across the series. `get_series(namespace, uuid)` takes no time-range or resolution arguments (the routers always call it with the two positional args): the response is always the whole life. +What the `series` in an API response actually is: the persisted 15-min archive (plus the still-mutable tail) merged with the live 15-min tail (only the part after the persisted buckets) and `compute.downsample`d once to a `target_points` cap. Caps: deployment detail `energy.deployment_series_points` (1000), datacenter `energy.cluster_series_points` (500), per-user `energy.user_series_points` (1000, merged across the user's deployments first, then one downsample). Each function falls back to its hardcoded default when the key is absent from the config. Below the cap you get the raw 15-min buckets; above it, `downsample` groups the buckets into fixed, epoch-aligned time windows (a multiple of 15 min, sized so the span fits in `cap - 1` windows), so every point spans the same time and the boundaries do not move when a new bucket arrives. Only the first and the last window can be partial. Each point is stamped with its window start, and a point is `live: true` when its window holds any live bucket. A hole in the data (Mimir outage, an allocation that was not running) stays a hole: it is not filled. The per-user series is the exception: its idle periods are filled with zero points on the 15-min grid, from the first bucket up to the last complete one (`fill_s` and `until` of `build_series`), except that the trailing zeros are skipped while a running deployment has no live tail. `get_series(namespace, uuid)` takes no time-range or resolution arguments (the routers always call it with the two positional args): the response is always the whole life. ## Computation details @@ -221,14 +221,14 @@ coverage_ratio fraction of run time that has power data (float, 1.0 = full degraded the live Mimir top-up failed (the stored value is served) datacenters list of datacenters the deployment or user has run in; null on the per-datacenter block deployments set only on the per-user aggregate: number of deployments summed -series 15-min EnergyTimeSeriesPoint list; null unless full_info=true (per-user aggregate only; the per-deployment full_info uses EnergyTimeSeries instead) +series EnergyTimeSeriesPoint list (15-min buckets, grouped into equal time windows above the point cap); null unless full_info=true (per-user aggregate only; the per-deployment full_info uses EnergyTimeSeries instead) ``` `power_w` is a rate, not an accumulator, so it is never summed. The `power_w` in the block (deployment, user, datacenter) is the **current draw**: the last power sample read from Mimir, however old it is (`compute.instant_power` does not drop it for being stale -- Mimir data is immutable once ingested, so the actual last sample is always the right one to show; it does still drop spiky samples via `sanitize`'s data-quality filter). Deployment and user take it from the last point of the top-up range query; the datacenter takes it from a dedicated instant query (`get_datacenter_power`). `series[].power_w` in the time series is different: a **mean** over its 15-min bucket (so `energy_wh ~= power_w * bucket_hours`). ### Time series, for the detail view -The detail view calls the deployment's own individual endpoint (`GET /v1/deployments/{type}/{uuid}`, which defaults to `full_info=true`) and reads `energy` from the response. That `energy` is `schemas.EnergyTimeSeries` from `energy.get_series(namespace, uuid)`: `{deployment_uuid, start, end, source, series, accumulated}`, where `series` is a list of points (`energy` / `carbon` / `water` per-bucket increment, `power_w` per point; a running total is `cumsum` client-side and matches `accumulated`) and `accumulated` is the `EnergyStats`. It serves the persisted 15-min archive downsampled to ~1000 points plus the live tail (`pointer_ts -> now` from Mimir, `live: true`) appended un-downsampled, so the graph keeps growing on every refresh; `source` is then `"mixed"`. Ownership and 404 handling are the individual endpoint's usual behaviour; `energy` is simply `null` when there is no rollup. +The detail view calls the deployment's own individual endpoint (`GET /v1/deployments/{type}/{uuid}`, which defaults to `full_info=true`) and reads `energy` from the response. That `energy` is `schemas.EnergyTimeSeries` from `energy.get_series(namespace, uuid)`: `{deployment_uuid, start, end, source, series, accumulated}`, where `series` is a list of points (`energy` / `carbon` / `water` per-bucket increment, `power_w` per point; a running total is `cumsum` client-side and matches `accumulated`) and `accumulated` is the `EnergyStats`. It serves the persisted 15-min archive plus the live tail (`pointer_ts -> now` from Mimir), downsampled together to ~1000 points with the points that hold live buckets flagged `live: true`, so the graph keeps growing on every refresh; `source` is then `"mixed"`. Ownership and 404 handling are the individual endpoint's usual behaviour; `energy` is simply `null` when there is no rollup. ### Per-datacenter energy, in the cluster stats @@ -236,7 +236,7 @@ The detail view calls the deployment's own individual endpoint (`GET /v1/deploym 1. **Instantaneous power** from `energy.get_datacenter_power()`: two instant Mimir queries, `power_w = (sum by (datacenter) scaph_host_power_microwatts + sum by (datacenter) DCGM_FI_DEV_POWER_USAGE) * TUE[dc]`. The host / GPU split is not exposed (on the ifca VMs `scaph_host_power` reads near zero, so it would be misleading; in practice `power_w` is GPU-dominated). A datacenter listed in `energy.metrics_datacenters` is reported with whatever it has (host-only, GPU-only or both); without that allowlist a host-power series is required so a stray GPU metric for an unmonitored site does not fabricate a total. 2. **Accumulated energy** from the `_cluster/.accum.json` document (persisted by the datacenter sweep) plus a live top-up slice (`live.topup` on the datacenter doc): `energy_wh`, `carbon_g`, `water_l`, `since`, `complete`, `live_as_of`. -3. **Time series**: the consolidated `.series.jsonl` (`_cluster_archive_series`, cached ~5 min so the 30 s thread does not re-parse it) plus the 24h tail, downsampled to `energy.cluster_series_points`, with the live tail (`live.live_series` on the datacenter doc) appended. Points carry per-bucket increments only. +3. **Time series**: the consolidated `.series.jsonl` (`_cluster_archive_series`, cached ~5 min so the 30 s thread does not re-parse it) plus the 24h tail, merged with the live tail (`live.live_series` on the datacenter doc) and downsampled together to `energy.cluster_series_points`. Points carry per-bucket increments only. Attached as `stats.datacenters[].energy`. There is **no** response-root total: `schemas.ClusterStats` has no `energy` field, and clients sum the datacenter blocks they want. `get_cluster_stats()` (the VO-filtered endpoint) prunes each datacenter's node list but keeps the `energy` block, so an instrumented datacenter with no nodes for that VO is still returned (empty `nodes`). The background thread always computes the series; `get_cluster_stats(full_info=false)` (the default) strips it from every datacenter block before returning, so the series only ships when the caller asks for it. Any failure in this block is caught and leaves the datacenters without `energy`. A datacenter that is not in `energy.metrics_datacenters` and has no accumulator doc simply has no `energy` block; a monitored GPU-only site does get one (accumulator plus series, and an instantaneous `power_w` from its GPU draw). diff --git a/tests/unit/test_energy.py b/tests/unit/test_energy.py index 27fd5bb9..25e9d345 100644 --- a/tests/unit/test_energy.py +++ b/tests/unit/test_energy.py @@ -8,6 +8,7 @@ """ import datetime +import itertools import json import logging import os @@ -213,7 +214,13 @@ def test_downsample(): compute.Bucket(compute.iso(i * 300), 1.0, 10.0, 0.1, 0.01) for i in range(100) ] ds = compute.downsample(many, 10) - assert len(ds) == 10 + assert 2 <= len(ds) <= 10 + # fixed time windows: all the same span except the first and last (partial) + _w = { + round(compute.parse_iso(b.ts) - compute.parse_iso(a.ts)) + for a, b in itertools.pairwise(ds) + } + assert len(_w) == 1, _w assert abs(sum(b.energy_wh for b in ds) - sum(b.energy_wh for b in many)) < 1e-6 assert compute.downsample(many, 500) is many # nothing to do @@ -1169,3 +1176,118 @@ def test_store_ignores_ids_that_are_not_plain_names(tmp_path): assert store.read_accum(_NS, name) == {"owner": "u@egi.eu"} store.write_accum(config.CLUSTER_NS, "ifca-ai4eosc", {"owner": None}) assert store.read_accum(config.CLUSTER_NS, "ifca-ai4eosc") is not None + + +def test_build_series_fill_gaps(): + def b(ts, wh): + return {"ts": ts, "energy_wh": wh, "power_w_avg": 5.0} + + persisted = [[b("2026-01-01T00:00:00Z", 1.0), b("2026-01-01T00:45:00Z", 2.0)]] + points = series.build_series(persisted, [], target_points=100, fill_s=900) + assert [(p["ts"], p["energy_wh"], p["power_w"]) for p in points] == [ + ("2026-01-01T00:00:00Z", 1.0, 5.0), + ("2026-01-01T00:15:00Z", 0.0, 0.0), + ("2026-01-01T00:30:00Z", 0.0, 0.0), + ("2026-01-01T00:45:00Z", 2.0, 5.0), + ] + + +def test_build_series_fill_until_now(): + def b(ts, wh): + return {"ts": ts, "energy_wh": wh, "power_w_avg": 5.0} + + persisted = [[b("2026-01-01T00:00:00Z", 1.0)]] + until = compute.parse_iso("2026-01-01T00:50:00Z") # last complete bucket: 00:30 + points = series.build_series(persisted, [], 100, fill_s=900, until=until) + assert [p["ts"] for p in points] == [ + "2026-01-01T00:00:00Z", + "2026-01-01T00:15:00Z", + "2026-01-01T00:30:00Z", + ] + + # a live tail takes over where it starts: no zero is placed over it + live = [[b("2026-01-01T00:30:00Z", 7.0)]] + points = series.build_series(persisted, live, 100, fill_s=900, until=until) + assert [(p["ts"], p["energy_wh"], p["live"]) for p in points] == [ + ("2026-01-01T00:00:00Z", 1.0, False), + ("2026-01-01T00:15:00Z", 0.0, False), + ("2026-01-01T00:30:00Z", 7.0, True), + ] + + +def test_user_series_is_continuous(monkeypatch): + """Idle periods of the user's series are zero points, up to the last complete + bucket, instead of missing points.""" + step = 900 + base = config.floor(config.now().timestamp(), step) + first, second = base - 5 * step, base - 4 * step + stats = {"energy_wh": 3.0, "carbon_g": 0.0, "water_l": 0.0} + monkeypatch.setattr(energy, "enabled", lambda: True) + monkeypatch.setattr( + energy, "_owner_rows", lambda ns, owner, live: [("u", {}, stats)] + ) + monkeypatch.setattr(energy, "live_series", lambda doc, bucket_s: []) + monkeypatch.setattr( + store, + "read_series", + lambda *a: [ + {"ts": compute.iso(t), "energy_wh": 1.5, "power_w_avg": 6.0} + for t in (first, second) + ], + ) + out = energy.get_user_energy(_NS, "u@egi.eu", series=True) + pts = out["series"] + assert [compute.parse_iso(p["ts"]) for p in pts] == [ + first + i * step + for i in range(5) # up to base - step, the last complete one + ] + assert [p["energy_wh"] for p in pts] == [1.5, 1.5, 0.0, 0.0, 0.0] + assert sum(p["energy_wh"] for p in pts) == out["energy_wh"] + + +def test_build_series_uniform_resolution_with_live_tail(): + def b(i, wh=1.0): + return {"ts": compute.iso(i * 900), "energy_wh": wh, "power_w_avg": 5.0} + + # 100 persisted buckets + 3 live ones, squeezed into 10 points: every point + # spans the same time, the live tail included, and nothing is lost + points = series.build_series( + [[b(i) for i in range(100)]], + [[b(i) for i in range(100, 103)]], + target_points=10, + ) + assert len(points) <= 10 + steps = { + round(compute.parse_iso(q["ts"]) - compute.parse_iso(p["ts"])) + for p, q in itertools.pairwise(points) + } + assert len(steps) == 1, steps + assert sum(p["energy_wh"] for p in points) == 103.0 + assert not points[0]["live"] and points[-1]["live"] + assert [p["live"] for p in points] == sorted(p["live"] for p in points) + + +def test_user_series_no_trailing_zeros_when_live_tail_is_missing(monkeypatch): + """A running deployment whose store is behind (stale sweep, Mimir down): what + is past its `pointer_ts` is unknown, so the series ends there instead of zeros.""" + step = 900 + base = config.floor(config.now().timestamp(), step) + stats = {"energy_wh": 1.0, "carbon_g": 0.0, "water_l": 0.0} + doc = { + "active_allocs": [{"alloc_id": "a", "datacenter": _DC}], + "pointer_ts": compute.iso(base - 4 * step), # persisted up to base - 5 * step + } + monkeypatch.setattr(energy, "enabled", lambda: True) + monkeypatch.setattr( + energy, "_owner_rows", lambda ns, owner, live: [("u", doc, stats)] + ) + monkeypatch.setattr(energy, "live_series", lambda doc, bucket_s: []) + monkeypatch.setattr( + store, + "read_series", + lambda *a: [ + {"ts": compute.iso(base - 5 * step), "energy_wh": 1.0, "power_w_avg": 4.0} + ], + ) + pts = energy.get_user_energy(_NS, "u@egi.eu", series=True)["series"] + assert [p["ts"] for p in pts] == [compute.iso(base - 5 * step)] From b66d15e4abe99ddc42b72bf2dc58465f0f813e0e Mon Sep 17 00:00:00 2001 From: Jaime Iglesias Blanco Date: Fri, 30 Oct 2026 08:57:41 +0100 Subject: [PATCH 25/25] feat(energy): cap the series points with an aligned window ladder The series are aggregated into fixed windows picked from a ladder (15m, 30m, 1h, 2h, 3h, 6h, 12h, 1d, 1w): the finest one that keeps the series within the per-level point cap. Windows are anchored to UTC midnight (weeks to Monday), so daily totals are exact. The response does not carry the resolution, it is the spacing of the timestamps. - keep the per-user series continuous, filling idle periods with zeros without going past the known data of a running deployment - build the cluster series per request instead of in the background thread - take start/end of the deployment series from the accumulated stats - keep off-grid buckets when filling gaps --- ai4papi/energy/__init__.py | 66 +++++----- ai4papi/energy/compute.py | 41 +++---- ai4papi/energy/series.py | 77 +++++++++--- ai4papi/routers/v1/stats/deployments.py | 27 +++-- ai4papi/schemas.py | 10 +- docs/energy-accounting.md | 20 +-- docs/energy-api.md | 26 ++-- etc/main.yaml | 6 +- tests/unit/test_energy.py | 155 ++++++++++++++++++++---- 9 files changed, 288 insertions(+), 140 deletions(-) diff --git a/ai4papi/energy/__init__.py b/ai4papi/energy/__init__.py index 8e95df8b..307c0641 100644 --- a/ai4papi/energy/__init__.py +++ b/ai4papi/energy/__init__.py @@ -189,8 +189,10 @@ def get_accumulated_bulk( def add_to_job(job: dict, namespace: str, full_info: bool) -> None: """Set `job["energy"]`: the extended time series if `full_info`, else just the accumulated stats.""" - get = get_series if full_info else get_accumulated - job["energy"] = get(namespace, job["job_ID"]) + if full_info: + job["energy"] = get_series(namespace, job["job_ID"]) + else: + job["energy"] = get_accumulated(namespace, job["job_ID"]) def add_to_jobs(jobs: list[dict], namespace: str, owner: str, full_info: bool) -> None: @@ -246,12 +248,14 @@ def _aggregate(stats_list: list[dict]) -> dict: def get_user_energy( - namespace: str, owner: str, series: bool = False, target_points: int | None = None + namespace: str, + owner: str, + series: bool = False, ) -> dict | None: """ All of a user's deployments in a VO rolled into one `EnergyStats`-shaped block: each deployment's persisted accumulator plus its live Mimir top-up, - summed. With `series=True`, also a 15-min time series merged across the + summed. With `series=True`, also a time series merged across the deployments (`full_info=true` on `GET /stats/user`). `None` when the user has no accounted deployment. """ @@ -264,10 +268,7 @@ def get_user_energy( out = _aggregate([stats for _uuid, _doc, stats in rows]) out["series"] = None if series: - cfg = config.get() - step_s = int(cfg["series_bucket_seconds"]) - if target_points is None: - target_points = int(cfg.get("user_series_points", 1000)) + step_s = int(config.get()["series_bucket_seconds"]) lives = [live_series(doc, bucket_s=step_s) for _uuid, doc, _stats in rows] # zeros only where nothing was running: past the `pointer_ts` of a running # deployment the store knows nothing (the live tail covers the recent part), @@ -280,19 +281,17 @@ def get_user_energy( if doc.get("active_allocs") and doc.get("pointer_ts") ] ) - out["series"] = ( - build_series( - [ - persisted(store.read_series(namespace, uuid, None, None), doc) - for uuid, doc, _stats in rows - ], - lives, - target_points, - fill_s=step_s, # idle periods show as zeros, not as missing points - until=until, - ) - or None + out["series"] = build_series( + [ + persisted(store.read_series(namespace, uuid, None, None), doc) + for uuid, doc, _stats in rows + ], + lives, + int(config.get().get("user_series_points", 1000)), + fill_s=step_s, # idle periods show as zeros, not as missing points + until=until, ) + out["series"] = out["series"] or None return out @@ -369,20 +368,16 @@ def _cluster_archive_series(dc: str) -> list[dict]: return store.read_series(CLUSTER_NS, dc, None, None) -def get_cluster_energy( - series: bool = True, target_points: int | None = None -) -> dict[str, dict]: +def get_cluster_energy(series: bool = True) -> dict[str, dict]: """ Per-datacenter energy for `/stats/cluster`: the instantaneous power from `get_datacenter_power()` merged with the persisted accumulator plus a live - top-up, plus a downsampled 15-min `series`. `{}` when the feature is disabled. + top-up, plus a `series` (window picked by the point cap). `{}` when the feature is disabled. A datacenter shows up if it has instantaneous power or an accumulator doc. """ if not enabled(): return {} cfg = config.get() - if target_points is None: - target_points = int(cfg.get("cluster_series_points", 500)) bucket_s = int(cfg["series_bucket_seconds"]) inst = get_datacenter_power() @@ -419,7 +414,7 @@ def get_cluster_energy( entry["series"] = build_series( [persisted(_cluster_archive_series(dc), doc)], [live_series(doc, bucket_s=bucket_s)], - target_points, + int(cfg.get("cluster_series_points", 500)), ) if entry: out[dc] = entry @@ -431,9 +426,7 @@ def get_cluster_energy( # --------------------------------------------------------------------------- -def get_series( - namespace: str, uuid: str, target_points: int | None = None -) -> dict | None: +def get_series(namespace: str, uuid: str) -> dict | None: if not enabled(): return None res = _resolve(namespace, uuid, live=True) @@ -441,22 +434,17 @@ def get_series( return None doc, stats = res - cfg = config.get() - if target_points is None: - target_points = int(cfg.get("deployment_series_points", 1000)) - step_s = int(cfg["series_bucket_seconds"]) + step_s = int(config.get()["series_bucket_seconds"]) series = build_series( [persisted(store.read_series(namespace, uuid, None, None), doc)], [live_series(doc, bucket_s=step_s)], - target_points, + int(config.get().get("deployment_series_points", 1000)), ) return { "deployment_uuid": uuid, - "start": series[0]["ts"] if series else (stats.get("since") or ""), - "end": series[-1]["ts"] - if series - else (stats.get("live_as_of") or stats.get("as_of") or ""), + "start": stats.get("since") or "", + "end": stats.get("live_as_of") or stats.get("as_of") or "", "source": "mixed" if any(p["live"] for p in series) else "store", "series": series, "accumulated": stats, diff --git a/ai4papi/energy/compute.py b/ai4papi/energy/compute.py index eca74642..90e0c429 100644 --- a/ai4papi/energy/compute.py +++ b/ai4papi/energy/compute.py @@ -9,7 +9,6 @@ import bisect import datetime -import itertools import threading import statistics from collections import namedtuple @@ -390,32 +389,30 @@ def apply_footprint( # --------------------------------------------------------------------------- -def downsample(buckets: list[Bucket], target_points: int) -> list[Bucket]: +# Windows are anchored to a Monday midnight (the epoch itself is a Thursday): a +# window that divides the day is unaffected, and a week window starts on Monday. +WINDOW_ANCHOR_S = int( + datetime.datetime(1970, 1, 5, tzinfo=datetime.timezone.utc).timestamp() +) + + +def downsample(buckets: list[Bucket], window_s: int) -> list[Bucket]: """ - Aggregate buckets into fixed, epoch-aligned time windows so the result has at - most `target_points` and every point spans the same time (only the first and - last window can be partial). Windows are by time, not by position: the - boundaries do not move when a new bucket arrives. Each point is stamped with - its window start. The bucket step is inferred from the smallest gap. + Aggregate buckets into fixed windows of `window_s` seconds anchored to a Monday + midnight UTC (energy, carbon and water summed, power averaged). Windows are by + time, not by position, so their boundaries never move when a new bucket + arrives; only the first and the last window can be partial. Each point is + stamped with its window start. Pass a window that divides the day or is a whole + number of days, so a client grouping the points by UTC day gets exact totals. """ - n = len(buckets) - if n <= target_points or target_points < 1: - return buckets - - ts = [parse_iso(b.ts) for b in buckets] - step = min(b - a for a, b in itertools.pairwise(ts)) - slots = round((ts[-1] - ts[0]) / step) + 1 - # one window less than `target_points`: the first and last may be partial, so - # a span of `slots` can straddle one more window than it holds - # shortcut: target_points == 1 can still yield 2 windows, nobody asks for one - window = -(-slots // max(target_points - 1, 1)) * step - groups: dict[int, list[Bucket]] = {} - for t, b in zip(ts, buckets): - groups.setdefault(int(t // window), []).append(b) + for b in buckets: + groups.setdefault( + int((parse_iso(b.ts) - WINDOW_ANCHOR_S) // window_s), [] + ).append(b) return [ Bucket( - ts=iso(k * window), + ts=iso(k * window_s + WINDOW_ANCHOR_S), energy_wh=sum(b.energy_wh for b in chunk), power_w_avg=sum(b.power_w_avg for b in chunk) / len(chunk), carbon_g=sum(b.carbon_g for b in chunk), diff --git a/ai4papi/energy/series.py b/ai4papi/energy/series.py index e3acfff9..5f41d43c 100644 --- a/ai4papi/energy/series.py +++ b/ai4papi/energy/series.py @@ -5,6 +5,36 @@ from ai4papi.energy import compute +# Windows the 15-min buckets are aggregated into, finest first. Every one is a +# multiple of 15 min, and from 1 h on of the hour, and they tile the UTC day (or +# are whole weeks), so the buckets always line up and a client grouping the points +# by day / ISO week gets exact totals. +RESOLUTIONS: dict[str, int] = { + "15m": 900, + "30m": 1800, + "1h": 3600, + "2h": 7200, + "3h": 10800, + "6h": 21600, + "12h": 43200, + "1d": 86400, + "1w": 604800, +} + + +def pick_resolution(first_ts: float, last_ts: float, max_points: int) -> str: + """ + The finest window whose number of points over `[first_ts, last_ts]` stays + within `max_points`. The cap is soft: the windows are a fixed ladder, so a + series usually ends up under it, by up to a factor of two between rungs. Beyond + the coarsest window (weeks) the cap no longer applies. + """ + for name, w in RESOLUTIONS.items(): + o = compute.WINDOW_ANCHOR_S + if int((last_ts - o) // w) - int((first_ts - o) // w) + 1 <= max_points: + return name + return "1w" + def _bucket(b: dict) -> compute.Bucket: return compute.Bucket( @@ -36,35 +66,41 @@ def persisted(archive: list[dict], doc: dict) -> list[dict]: def _fill_gaps( buckets: list[compute.Bucket], step_s: int, end_ts: float ) -> list[compute.Bucket]: - """Zero buckets on the `step_s` grid wherever `buckets` (sorted) has a hole, - and from the last one up to `end_ts` (inclusive).""" - by_ts = {b.ts: b for b in buckets} - out, t = [], compute.parse_iso(buckets[0].ts) + """`buckets` (sorted) plus a zero bucket on the `step_s` grid at every missing + slot, from the first bucket up to `end_ts` (inclusive). Buckets off the grid + are kept, so no energy is lost.""" + have = {b.ts for b in buckets} + out = list(buckets) + t = compute.parse_iso(buckets[0].ts) end = max(end_ts, compute.parse_iso(buckets[-1].ts)) while t <= end: ts = compute.iso(t) - out.append(by_ts.get(ts) or compute.Bucket(ts, 0.0, 0.0, 0.0, 0.0)) + if ts not in have: + out.append(compute.Bucket(ts, 0.0, 0.0, 0.0, 0.0)) t += step_s - return out + return sorted(out, key=lambda b: b.ts) def build_series( persisted_lists: list[list[dict]], live: list[list[dict]], - target_points: int, + max_points: int, fill_s: int | None = None, until: float | None = None, ) -> list[dict]: """ Time series of one or several entities (a deployment, a user's deployments, - a datacenter). The persisted buckets are merged by timestamp across entities - and `compute.downsample`d once together with the live Mimir tails (merged the - same way, only the part after the persisted buckets), so the resolution is - uniform and capped at `target_points`. With `fill_s`, holes in the persisted buckets (no entity - active) are filled with zero points on that grid, before downsampling, so - the series is continuous; with `until` (a timestamp) it also runs up to the last - complete bucket before it, stopping short of the live tail's first bucket. The points carry the per-bucket increments only; a running - total is `cumsum(series[].energy_wh)` and matches `accumulated.energy_wh`. + a datacenter). The persisted buckets are merged by + timestamp across entities and `compute.downsample`d together with the live + Mimir tails (merged the same way, only the part after the persisted buckets) + into fixed windows: the finest of `RESOLUTIONS` that keeps the series within + about `max_points` (`pick_resolution`), so every point spans the same time. With `fill_s`, holes in the persisted + buckets (no entity active) are filled with zero points on that grid, before + aggregating, so the series is continuous; with `until` (a timestamp) it also + runs up to the last complete bucket before it, stopping short of the live + tail's first bucket. The points carry the per-window increments only; a + running total is `cumsum(series[].energy_wh)` and matches + `accumulated.energy_wh`. """ def merged(lists: list[list[dict]]) -> list[compute.Bucket]: @@ -78,11 +114,18 @@ def merged(lists: list[list[dict]]) -> list[compute.Bucket]: if live_buckets: # leave the live buckets to the live tail end = min(end, compute.parse_iso(live_buckets[0].ts) - fill_s) buckets = _fill_gaps(buckets, fill_s, end) - # the live tail goes through the same downsampling, so every point spans the + # the live tail goes through the same aggregation, so every point spans the # same time; a point is `live` if its window holds any live bucket last = buckets[-1].ts if buckets else "" tail = [b for b in live_buckets if b.ts > last] - points = compute.downsample(buckets + tail, target_points) + allb = buckets + tail + if not allb: + return [] + name = pick_resolution( + compute.parse_iso(allb[0].ts), compute.parse_iso(allb[-1].ts), max_points + ) + w = RESOLUTIONS[name] + points = compute.downsample(allb, w) if not tail: return [_point(p, live=False) for p in points] live_from = max(p.ts for p in points if p.ts <= tail[0].ts) diff --git a/ai4papi/routers/v1/stats/deployments.py b/ai4papi/routers/v1/stats/deployments.py index ea16cb0d..1be7891d 100644 --- a/ai4papi/routers/v1/stats/deployments.py +++ b/ai4papi/routers/v1/stats/deployments.py @@ -271,13 +271,21 @@ def get_cluster_stats( stats.cluster.gpu_models[model_name].gpu_total += g_stats.gpu_total stats.cluster.gpu_models[model_name].gpu_used += g_stats.gpu_used - # Per-datacenter energy time series only with `full_info=true` (it is large: - # ~500 points per datacenter). The accumulated numbers and the live top-up - # are always present. - if not full_info: - for dc_stats in stats.datacenters.values(): - if dc_stats.energy is not None: - dc_stats.energy.series = None + # Per-datacenter energy time series only with `full_info=true` (it is large). + # The background thread computes the accumulated numbers and the live top-up + # only; the series is built here, per request. + if energy and full_info and energy_accounting.enabled(): + try: + for dc_name, dc_energy in energy_accounting.get_cluster_energy( + series=True + ).items(): + dc_stats = stats.datacenters.get(dc_name) + if dc_stats is not None and dc_stats.energy is not None: + dc_stats.energy.series = schemas.EnergyStats( + series=dc_energy.get("series") + ).series + except Exception: + logging.exception("Error computing cluster energy series") # Add update time stats.updated_at = ( @@ -467,14 +475,15 @@ def get_cluster_stats_bg() -> schemas.ClusterStats: g_stats.gpu_total = n_stats.gpu_used # Per-datacenter energy (energy feature): instantaneous power draw + the - # accumulated energy/footprint since accounting started + a time series. + # accumulated energy/footprint since accounting started (the series is built per request in + # `get_cluster_stats`). # Reported per datacenter only, there is no platform-wide total (clients sum # the datacenters they care about). Best-effort: a failure here must not # break the cluster stats. if energy_accounting.enabled(): try: for dc_name, dc_energy in energy_accounting.get_cluster_energy( - series=True + series=False ).items(): if dc_name not in stats.datacenters: stats.datacenters[dc_name] = schemas.DatacenterStats( diff --git a/ai4papi/schemas.py b/ai4papi/schemas.py index b797a91b..9a3867d0 100644 --- a/ai4papi/schemas.py +++ b/ai4papi/schemas.py @@ -42,8 +42,8 @@ class EnergyTimeSeriesPoint(BaseModel): # Per-bucket increments only. A running total is `cumsum(series[].energy_wh)` # and matches `accumulated.energy_wh` (same for carbon and water). ts: str - power_w: Optional[float] = None # mean over the bucket, normalized (x TUE) - energy_wh: Optional[float] = None # normalized, incremental for the bucket + power_w: Optional[float] = None # mean over the window, normalized (x TUE) + energy_wh: Optional[float] = None # normalized, incremental for the window carbon_g: Optional[float] = None water_l: Optional[float] = None live: bool = False # point from the live tail (not yet persisted) @@ -79,14 +79,14 @@ class EnergyStats(BaseModel): datacenters: Optional[list[str]] = None # per deployment / user deployments: Optional[int] = None # per-user aggregate only series: Optional[list[EnergyTimeSeriesPoint]] = ( - None # 15-min buckets; only with full_info=true + None # equal time windows; only with full_info=true ) class EnergyTimeSeries(BaseModel): deployment_uuid: str - start: str # first point (equivalently `accumulated.since`) - end: str # last point (the live edge while running) + start: str # `accumulated.since` + end: str # live edge while running (`accumulated.live_as_of`) source: str # "store" (archive only) | "mixed" (archive + live tail) series: list[EnergyTimeSeriesPoint] accumulated: EnergyStats diff --git a/docs/energy-accounting.md b/docs/energy-accounting.md index 09d9a83e..06822a22 100644 --- a/docs/energy-accounting.md +++ b/docs/energy-accounting.md @@ -148,13 +148,13 @@ There is no summary file. Per-user and per-datacenter aggregates are computed on |---|---|---| | Integration for the accumulator | 30 s (native) | the accumulator is as precise as the source, independent of bucket size | | Persisted archive series | 15 min (`energy.series_bucket_seconds`) | matches Wattnet's 15-min cadence, ~3 MB/year for a long deployment, append-only file | -| Recent-window graph | 30 s from Mimir, downsampled to ~1000 points | full resolution and always current while the data is in Mimir | +| Recent-window graph | 30 s from Mimir, integrated into the live 15-min tail and aggregated into the series windows | full resolution and always current while the data is in Mimir | | Old-window graph (past Mimir retention) | 15 min from the store | the only source left, enough to see the shape | | Live counter tail (`pointer_ts -> now`) | 30 s | the number moves at scrape cadence | The store is the permanent low-resolution archive; Mimir is the high-resolution source for anything recent. -What the `series` in an API response actually is: the persisted 15-min archive (plus the still-mutable tail) merged with the live 15-min tail (only the part after the persisted buckets) and `compute.downsample`d once to a `target_points` cap. Caps: deployment detail `energy.deployment_series_points` (1000), datacenter `energy.cluster_series_points` (500), per-user `energy.user_series_points` (1000, merged across the user's deployments first, then one downsample). Each function falls back to its hardcoded default when the key is absent from the config. Below the cap you get the raw 15-min buckets; above it, `downsample` groups the buckets into fixed, epoch-aligned time windows (a multiple of 15 min, sized so the span fits in `cap - 1` windows), so every point spans the same time and the boundaries do not move when a new bucket arrives. Only the first and the last window can be partial. Each point is stamped with its window start, and a point is `live: true` when its window holds any live bucket. A hole in the data (Mimir outage, an allocation that was not running) stays a hole: it is not filled. The per-user series is the exception: its idle periods are filled with zero points on the 15-min grid, from the first bucket up to the last complete one (`fill_s` and `until` of `build_series`), except that the trailing zeros are skipped while a running deployment has no live tail. `get_series(namespace, uuid)` takes no time-range or resolution arguments (the routers always call it with the two positional args): the response is always the whole life. +What the `series` in an API response actually is: the persisted 15-min archive (plus the still-mutable tail) merged with the live 15-min tail (only the part after the persisted buckets) and aggregated by `compute.downsample` into fixed windows. The window is picked per series by `series.pick_resolution`: the finest of `series.RESOLUTIONS` (`15m`, `30m`, `1h`, `2h`, `3h`, `6h`, `12h`, `1d`, `1w`) whose number of points over the series span stays within the cap. Caps: deployment detail `energy.deployment_series_points` (1000), datacenter `energy.cluster_series_points` (500), per-user `energy.user_series_points` (1000, merged across the user's deployments first, then one aggregation). Each function falls back to its hardcoded default when the key is absent from the config. The cap is soft: the ladder is discrete, so the series usually ends up under it, and past weeks it no longer applies. The response does not name the window, the client reads it from the spacing of the `ts`. Every window is a multiple of 15 min, from 1 h on a multiple of the hour, and they tile the UTC day (the week windows start on Monday), so every point spans the same time (only the first and the last window can be partial), the boundaries do not move when a new bucket arrives, and a client that groups the points by UTC day (or ISO week) gets exact totals (a window that does not divide the day made one day in three look lower and the other two higher). Each point is stamped with its window start, `power_w` is the mean over the window, and a point is `live: true` when its window holds any live bucket. A hole in the data (Mimir outage, an allocation that was not running) stays a hole: it is not filled. The per-user series is the exception: its idle periods are filled with zero points on the 15-min grid, from the first bucket up to the last complete one (`fill_s` and `until` of `build_series`), but never past the `pointer_ts` of a running deployment, where the store knows nothing. `get_series(namespace, uuid)` takes no time-range argument: the response is always the whole life. ## Computation details @@ -221,14 +221,14 @@ coverage_ratio fraction of run time that has power data (float, 1.0 = full degraded the live Mimir top-up failed (the stored value is served) datacenters list of datacenters the deployment or user has run in; null on the per-datacenter block deployments set only on the per-user aggregate: number of deployments summed -series EnergyTimeSeriesPoint list (15-min buckets, grouped into equal time windows above the point cap); null unless full_info=true (per-user aggregate only; the per-deployment full_info uses EnergyTimeSeries instead) +series EnergyTimeSeriesPoint list (equal time windows chosen to fit the point cap); null unless full_info=true (per-user aggregate only; the per-deployment full_info uses EnergyTimeSeries instead) ``` -`power_w` is a rate, not an accumulator, so it is never summed. The `power_w` in the block (deployment, user, datacenter) is the **current draw**: the last power sample read from Mimir, however old it is (`compute.instant_power` does not drop it for being stale -- Mimir data is immutable once ingested, so the actual last sample is always the right one to show; it does still drop spiky samples via `sanitize`'s data-quality filter). Deployment and user take it from the last point of the top-up range query; the datacenter takes it from a dedicated instant query (`get_datacenter_power`). `series[].power_w` in the time series is different: a **mean** over its 15-min bucket (so `energy_wh ~= power_w * bucket_hours`). +`power_w` is a rate, not an accumulator, so it is never summed. The `power_w` in the block (deployment, user, datacenter) is the **current draw**: the last power sample read from Mimir, however old it is (`compute.instant_power` does not drop it for being stale -- Mimir data is immutable once ingested, so the actual last sample is always the right one to show; it does still drop spiky samples via `sanitize`'s data-quality filter). Deployment and user take it from the last point of the top-up range query; the datacenter takes it from a dedicated instant query (`get_datacenter_power`). `series[].power_w` in the time series is different: a **mean** over its window (so `energy_wh ~= power_w * window_hours`). ### Time series, for the detail view -The detail view calls the deployment's own individual endpoint (`GET /v1/deployments/{type}/{uuid}`, which defaults to `full_info=true`) and reads `energy` from the response. That `energy` is `schemas.EnergyTimeSeries` from `energy.get_series(namespace, uuid)`: `{deployment_uuid, start, end, source, series, accumulated}`, where `series` is a list of points (`energy` / `carbon` / `water` per-bucket increment, `power_w` per point; a running total is `cumsum` client-side and matches `accumulated`) and `accumulated` is the `EnergyStats`. It serves the persisted 15-min archive plus the live tail (`pointer_ts -> now` from Mimir), downsampled together to ~1000 points with the points that hold live buckets flagged `live: true`, so the graph keeps growing on every refresh; `source` is then `"mixed"`. Ownership and 404 handling are the individual endpoint's usual behaviour; `energy` is simply `null` when there is no rollup. +The detail view calls the deployment's own individual endpoint (`GET /v1/deployments/{type}/{uuid}`, which defaults to `full_info=true`) and reads `energy` from the response. That `energy` is `schemas.EnergyTimeSeries` from `energy.get_series(namespace, uuid)`: `{deployment_uuid, start, end, source, series, accumulated}`, where `series` is a list of points (`energy` / `carbon` / `water` per-window increment, `power_w` per point; a running total is `cumsum` client-side and matches `accumulated`) and `accumulated` is the `EnergyStats`. It serves the persisted 15-min archive plus the live tail (`pointer_ts -> now` from Mimir), aggregated together into windows that fit `energy.deployment_series_points`, with the points that hold live buckets flagged `live: true`, so the graph keeps growing on every refresh; `source` is then `"mixed"`. Ownership and 404 handling are the individual endpoint's usual behaviour; `energy` is simply `null` when there is no rollup. ### Per-datacenter energy, in the cluster stats @@ -236,13 +236,13 @@ The detail view calls the deployment's own individual endpoint (`GET /v1/deploym 1. **Instantaneous power** from `energy.get_datacenter_power()`: two instant Mimir queries, `power_w = (sum by (datacenter) scaph_host_power_microwatts + sum by (datacenter) DCGM_FI_DEV_POWER_USAGE) * TUE[dc]`. The host / GPU split is not exposed (on the ifca VMs `scaph_host_power` reads near zero, so it would be misleading; in practice `power_w` is GPU-dominated). A datacenter listed in `energy.metrics_datacenters` is reported with whatever it has (host-only, GPU-only or both); without that allowlist a host-power series is required so a stray GPU metric for an unmonitored site does not fabricate a total. 2. **Accumulated energy** from the `_cluster/.accum.json` document (persisted by the datacenter sweep) plus a live top-up slice (`live.topup` on the datacenter doc): `energy_wh`, `carbon_g`, `water_l`, `since`, `complete`, `live_as_of`. -3. **Time series**: the consolidated `.series.jsonl` (`_cluster_archive_series`, cached ~5 min so the 30 s thread does not re-parse it) plus the 24h tail, merged with the live tail (`live.live_series` on the datacenter doc) and downsampled together to `energy.cluster_series_points`. Points carry per-bucket increments only. +3. **Time series**: the consolidated `.series.jsonl` (`_cluster_archive_series`, cached ~5 min so the 30 s thread does not re-parse it) plus the 24h tail, merged with the live tail (`live.live_series` on the datacenter doc) and aggregated together into windows that fit `energy.cluster_series_points`. The series is built per request in `get_cluster_stats` (the background thread only computes the accumulated numbers and the live top-up). Points carry per-bucket increments only. Attached as `stats.datacenters[].energy`. There is **no** response-root total: `schemas.ClusterStats` has no `energy` field, and clients sum the datacenter blocks they want. `get_cluster_stats()` (the VO-filtered endpoint) prunes each datacenter's node list but keeps the `energy` block, so an instrumented datacenter with no nodes for that VO is still returned (empty `nodes`). The background thread always computes the series; `get_cluster_stats(full_info=false)` (the default) strips it from every datacenter block before returning, so the series only ships when the caller asks for it. Any failure in this block is caught and leaves the datacenters without `energy`. A datacenter that is not in `energy.metrics_datacenters` and has no accumulator doc simply has no `energy` block; a monitored GPU-only site does get one (accumulator plus series, and an instantaneous `power_w` from its GPU draw). ### Per-user aggregate -`stats.deployments.get_user_stats` gains an `energy` key from `energy.get_user_energy(namespace, owner, series=full_info)`: it reads the owner's `.accum.json` documents (through the owner index), applies each one's live top-up, and sums them into one `EnergyStats`-shaped block (`since` = earliest, `as_of` / `live_as_of` = latest, `complete` = AND, `degraded` = OR, `power_w` = sum of the running ones, `coverage_ratio` = mean, `tue_factor` = blended `energy_wh / raw`, `datacenters` = union, plus a `deployments` count). With `full_info=true` it also returns a merged time series (`series.build_series`): the raw 15-min buckets of every deployment (archive + tail) are summed by timestamp first, then `compute.downsample`d once to `target_points` (1000). Merging before downsampling keeps the resolution uniform across the whole series and gives the same hard cap as the other levels, regardless of how long each deployment individually has been running; the live tails are merged and appended un-downsampled, as elsewhere. The energy block is independent of the pre-existing CSV usage stats: if those are missing, the endpoint still returns the `energy` block instead of failing. +`stats.deployments.get_user_stats` gains an `energy` key from `energy.get_user_energy(namespace, owner, series=full_info)`: it reads the owner's `.accum.json` documents (through the owner index), applies each one's live top-up, and sums them into one `EnergyStats`-shaped block (`since` = earliest, `as_of` / `live_as_of` = latest, `complete` = AND, `degraded` = OR, `power_w` = sum of the running ones, `coverage_ratio` = mean, `tue_factor` = blended `energy_wh / raw`, `datacenters` = union, plus a `deployments` count). With `full_info=true` it also returns a merged time series (`series.build_series`): the raw 15-min buckets of every deployment (archive + tail) are summed by timestamp first, then aggregated once into windows that fit `energy.user_series_points`, together with the live tails. Merging before aggregating keeps the resolution uniform across the whole series. The energy block is independent of the pre-existing CSV usage stats: if those are missing, the endpoint still returns the `energy` block instead of failing. ## Configuration @@ -259,9 +259,9 @@ energy: mimir_retention_days: 90 # tune to Mimir's actual retention footprint_revision_hours: 24 # re-compute footprint for buckets newer than this topup_cache_seconds: 25 # live top-up cache (< dashboard poll) - deployment_series_points: 1000 # downsample target for the per-deployment series in /deployments/* - user_series_points: 1000 # downsample target for the merged per-user series in /stats/user - cluster_series_points: 500 # downsample target for the per-datacenter series in /stats/cluster + deployment_series_points: 1000 # point cap of the per-deployment series in /deployments/* + user_series_points: 1000 # point cap of the merged per-user series in /stats/user + cluster_series_points: 500 # point cap of the per-datacenter series in /stats/cluster metrics_datacenters: [ifca-ai4eosc, ifca-imagine] default_tue_factor: 2.73 # fallback when a datacenter has no TUE in datacenters.csv data_quality: {enabled: true, min_power_w: 0, max_power_w: 1000, spike_factor: 12, spike_window: 11} diff --git a/docs/energy-api.md b/docs/energy-api.md index 23cd4124..a3e3d228 100644 --- a/docs/energy-api.md +++ b/docs/energy-api.md @@ -6,7 +6,7 @@ All of this is additive and no new endpoints were created. The data rides on end | Level | Where | Accumulated (+ top-up) | Time series (`full_info=true`) | |---|---|---|---| -| Datacenter / site | `GET /v1/deployments/stats/cluster` -> `datacenters[].energy` | always | per site, downsampled to `energy.cluster_series_points` | +| Datacenter / site | `GET /v1/deployments/stats/cluster` -> `datacenters[].energy` | always | per site, window picked by `energy.cluster_series_points` | | User | `GET /v1/deployments/stats/user?vo=` -> `energy` | always | merged across the user's deployments | | Deployment | `energy` key in the deployment info (list and individual endpoints, all four types) | always | per deployment. On the individual endpoint `full_info` defaults to `true`, so it comes back as `EnergyTimeSeries`; the list form is `EnergyStats` | @@ -14,6 +14,8 @@ There is **no platform-wide / VO total** in the response: `/stats/cluster` retur `full_info` defaults to `false` on the list-shaped endpoints (`/stats/cluster`, `/stats/user`, the deployment list) and to `true` on the single-deployment endpoint. Pass it explicitly to override. +The time series (`series`) have a point cap per level: `energy.deployment_series_points` (1000), `energy.user_series_points` (1000) and `energy.cluster_series_points` (500). The points are the 15-min buckets aggregated into fixed windows, the finest of `15m`, `30m`, `1h`, `2h`, `3h`, `6h`, `12h`, `1d` and `1w` that keeps the series within the cap. There is no query param for it and the response does not state it: it is the spacing between the `ts` of consecutive points. The windows are aligned to UTC midnight (weeks start on Monday), so every point spans the same time (only the first and the last window can be partial), the boundaries do not move when new data arrives, and summing the points by UTC day (by ISO week for `1w`) gives exact totals. Each point is stamped with its window start, `power_w` is the mean over the window and `live` is `true` when the window holds any live bucket. The cap is soft: the ladder is discrete, so a series usually has fewer points than the cap (a datacenter with 34 days of data and a cap of 500 gets 3 h windows and 346 points), and past the coarsest window (weeks) the cap no longer applies. + Every endpoint that attaches energy also takes `energy` (default `true`). Pass `energy=false` to drop the `energy` block entirely, for a leaner and slightly faster response when the caller does not need it. PAPI itself passes `energy=false` on its internal quota and resource checks. Notes: @@ -33,7 +35,7 @@ If the feature is disabled or has no data, every one of these is `null` (or an a - Timestamps: ISO-8601 UTC, always with the `Z` suffix (`2026-09-07T07:50:00Z`). - Every numeric field is a JSON number (float), returned at full precision (not rounded): the accumulated total always equals the sum of `series`, and idle CPU workloads can legitimately sit at 1e-5 Wh. Round for display on the client. - Every energy, power, carbon and water value is normalized by the datacenter TUE (grid to compute ratio). The raw pre-TUE figure is not sent, but `tue_factor` is: divide by it to get the raw value. On a single-datacenter deployment `tue_factor` is that datacenter's configured TUE; on the per-user aggregate and on a deployment that moved between sites it is the blended `energy_wh / raw_energy_wh`. -- `power_w` in the block is the **current draw**: the last sample read from Mimir (~30 to 60 s old) at every level. `series[].power_w` in the time series is different: the **mean** over that 15-min bucket (so `energy_wh ~= power_w * bucket_hours`). Either way it is a rate, not a total, and is never summed. +- `power_w` in the block is the **current draw**: the last sample read from Mimir (~30 to 60 s old) at every level. `series[].power_w` in the time series is different: the **mean** over that window (so `energy_wh ~= power_w * window_hours`). Either way it is a rate, not a total, and is never summed. ## The `EnergyStats` object @@ -71,7 +73,7 @@ One object, the same shape at every level, always with the live top-up applied: | `coverage_ratio` | number or null | Deployment and per-user only: fraction of the run time that has power data (`1.0` = fully covered). `null` on the datacenter block. | | `datacenters` | string[] or null | Deployment and per-user only: datacenters the deployment(s) ran in. `null` on the datacenter block. | | `deployments` | number or null | Per-user aggregate only: how many deployments went into the sum. | -| `series` | array or null | Per-user aggregate only, and only with `full_info=true`: the merged 15-min time series. `null` otherwise. | +| `series` | array or null | Per-user aggregate only, and only with `full_info=true`: the merged time series, in equal windows. `null` otherwise. | The values already include the live top-up: `energy_wh` is the persisted accumulator plus the slice from the last persisted point up to `live_as_of`, computed live against Mimir on each request (cached ~25 s). That is why the counter moves between refreshes without a background job. @@ -148,18 +150,18 @@ The list endpoint resolves all of the user's deployments with one or two Mimir q | Field | Type | Meaning | |---|---|---| | `deployment_uuid` | string | Echo of the deployment. | -| `start` / `end` | string | Time span covered by `series`. `start` equals `accumulated.since`, `end` the live edge (the last point). Convenience only, both are derivable from the series. | -| `series` | array | Ordered by `ts` ascending. Whole deployment life, downsampled to `energy.deployment_series_points` (default 1000). | +| `start` / `end` | string | Time span covered by `series`. `start` equals `accumulated.since`, `end` the live edge (`accumulated.live_as_of`). Both come from `accumulated`, not from the first and last point (those are stamped with their window start). | +| `series` | array | Ordered by `ts` ascending. Whole deployment life, in windows picked by `energy.deployment_series_points` (see above). | | `series[].ts` | string | Bucket start. | -| `series[].power_w` | number or null | Mean power over the part of the bucket that had data, normalized by TUE. `energy_wh` is about `power_w` times the bucket duration. | -| `series[].energy_wh` | number or null | Energy in that bucket (per-bucket increment). A running total is `cumsum(series[].energy_wh)` and matches `accumulated.energy_wh`. | -| `series[].carbon_g` | number or null | Carbon in that bucket. Same, cumsum matches `accumulated.carbon_g`. | -| `series[].water_l` | number or null | Water in that bucket. Same, cumsum matches `accumulated.water_l`. | +| `series[].power_w` | number or null | Mean power over the part of the window that had data, normalized by TUE. `energy_wh` is about `power_w` times the window duration. | +| `series[].energy_wh` | number or null | Energy in that window (per-window increment). A running total is `cumsum(series[].energy_wh)` and matches `accumulated.energy_wh`. | +| `series[].carbon_g` | number or null | Carbon in that window. Same, cumsum matches `accumulated.carbon_g`. | +| `series[].water_l` | number or null | Water in that window. Same, cumsum matches `accumulated.water_l`. | | `series[].live` | boolean | `true` for the tail points served live from Mimir (not yet persisted). They can change on the next refresh. | | `source` | string | `"store"` (only the persisted archive) or `"mixed"` (archive plus a live tail). Equivalent to `series[-1].live`. | | `accumulated` | EnergyStats | The current totals, same object as the list view. Use it for the headline numbers next to the chart (and its `degraded` flag). | -- The persisted archive is at 15-minute resolution (`energy.series_bucket_seconds`); the response is downsampled to `energy.deployment_series_points` (default 1000) across the whole life. +- The persisted archive is at 15-minute resolution (`energy.series_bucket_seconds`); the response aggregates it into fixed windows across the whole life, the finest that fits `energy.deployment_series_points` (see above). - The deployment is running, so `end` reaches "now" and the last points carry `live: true`. Calling the endpoint again a minute later returns a slightly longer series with updated `live` points. That is what makes the chart grow. - Poll the detail view every 30 to 60 s. @@ -198,7 +200,7 @@ The existing response gains an `energy` key: all of the user's deployments in th - Same fields as the per-deployment `EnergyStats` (see above), aggregated: `since` is the earliest across the user's deployments, `as_of` / `live_as_of` the latest, `complete` is the AND, `degraded` the OR, `power_w` the sum over the running ones, `coverage_ratio` the mean, `datacenters` the union, `tue_factor` the blended `energy_wh / raw_energy_wh`. - `deployments`: how many accounted deployments went into the sum. -- `series`: `null` unless `full_info=true`, then a 15-min time series of the user's deployments summed by timestamp and downsampled once to `energy.user_series_points` (default 1000; `EnergyTimeSeriesPoint` shape, uniform step across the whole series), for a "my usage over time" chart. +- `series`: `null` unless `full_info=true`, then a time series of the user's deployments summed by timestamp and aggregated into windows that fit `energy.user_series_points` (`EnergyTimeSeriesPoint` shape, same step across the whole series), for a "my usage over time" chart. Periods when none of the user's deployments was running are filled with zero points, up to the last complete bucket, except past the last known data of a deployment that is running (there the store knows nothing, so the series simply ends and the live tail continues it). - `null` when the user has no accounted deployments. - Independent of the pre-existing CSV usage stats: if those are missing the endpoint still returns the `energy` block. @@ -214,7 +216,7 @@ Each `datacenters[]` object gains an `energy` block: the same `EnergyStats` - **`power_w`**: the current draw from the last Mimir scrape, refreshed every ~30 s, normalized by TUE. It is `(sum of scaphandre host power + sum of GPU power) * TUE`. - **accumulated** energy and footprint since accounting started (`energy_wh`, `carbon_g`, `water_l`, all TUE-normalized, divide by `tue_factor` for raw), kept indefinitely and topped up live on every read. -- **`series`**: `null` by default; with `full_info=true`, a 15-min series downsampled to `energy.cluster_series_points` (default 500). +- **`series`**: `null` by default; with `full_info=true`, a series in windows that fit `energy.cluster_series_points`. A hole in the datacenter data (a Mimir outage) stays a hole. `?vo=` filters the node lists but not the `energy` blocks: an instrumented datacenter keeps its `energy` even when the VO has no nodes there (its `nodes` map is then empty). diff --git a/etc/main.yaml b/etc/main.yaml index 9fbcd2d5..12eff4bb 100644 --- a/etc/main.yaml +++ b/etc/main.yaml @@ -97,9 +97,9 @@ energy: # per-deployment energy consumption and environmental footprint stats mimir_retention_days: 90 # tune to Mimir's actual retention footprint_revision_hours: 24 # re-compute footprint for buckets newer than this topup_cache_seconds: 25 # live top-up cache (< dashboard poll so the counter moves) - deployment_series_points: 1000 # downsample target for the per-deployment series in /deployments/* - user_series_points: 1000 # downsample target for the merged per-user series in /stats/user (full_info=true) - cluster_series_points: 500 # downsample target for the per-datacenter series in /stats/cluster (full_info=true) + deployment_series_points: 1000 # point cap of the per-deployment series in /deployments/* (soft: see series.pick_resolution) + user_series_points: 1000 # point cap of the merged per-user series in /stats/user (full_info=true) + cluster_series_points: 500 # point cap of the per-datacenter series in /stats/cluster (full_info=true) metrics_datacenters: # datacenters where power metrics exist diff --git a/tests/unit/test_energy.py b/tests/unit/test_energy.py index 25e9d345..00326c54 100644 --- a/tests/unit/test_energy.py +++ b/tests/unit/test_energy.py @@ -213,16 +213,14 @@ def test_downsample(): many = [ compute.Bucket(compute.iso(i * 300), 1.0, 10.0, 0.1, 0.01) for i in range(100) ] - ds = compute.downsample(many, 10) - assert 2 <= len(ds) <= 10 - # fixed time windows: all the same span except the first and last (partial) - _w = { - round(compute.parse_iso(b.ts) - compute.parse_iso(a.ts)) - for a, b in itertools.pairwise(ds) - } - assert len(_w) == 1, _w + ds = compute.downsample(many, 900) + # fixed, epoch-aligned windows: every point spans the same time except the + # last (partial) one, and nothing is lost + assert [b.ts for b in ds[:2]] == [compute.iso(0), compute.iso(900)] + assert len({round(compute.parse_iso(b.ts)) % 900 for b in ds}) == 1 + assert [b.energy_wh for b in ds[:-1]] == [3.0] * (len(ds) - 1) assert abs(sum(b.energy_wh for b in ds) - sum(b.energy_wh for b in many)) < 1e-6 - assert compute.downsample(many, 500) is many # nothing to do + assert ds[0].power_w_avg == 10.0 # power is averaged, not summed def test_merge_buckets(): @@ -303,6 +301,9 @@ def test_sweep_pipeline(monkeypatch): schemas.EnergyStats(**stats) series = energy.get_series(_NS, _UUID) schemas.EnergyTimeSeries(**series) + assert series["start"] == stats["since"] + _acc = series["accumulated"] + assert series["end"] == (_acc["live_as_of"] or _acc["as_of"]) assert series["series"] and all( p["energy_wh"] is not None for p in series["series"] ) @@ -343,9 +344,15 @@ def test_sweep_pipeline(monkeypatch): round((_uts[i + 1] - _uts[i]).total_seconds()) for i in range(len(_uts) - 1) } assert len(_steps) == 1, _steps - # hard cap on the persisted point count, same as the other levels - _capped = energy.get_user_energy(_NS, "user@egi.eu", series=True, target_points=5) - assert len([p for p in _capped["series"] if not p["live"]]) <= 5 + # the point cap picks the window: a tight cap forces a coarser one, and the + # energy adds up either way + monkeypatch.setitem(config.get(), "user_series_points", 5) + _capped = energy.get_user_energy(_NS, "user@egi.eu", series=True) + assert len(_capped["series"]) <= 5 + 1 # + the live window, if any + assert ( + abs(sum(p["energy_wh"] for p in _capped["series"]) - _capped["energy_wh"]) + < 1e-9 + ) assert energy.get_user_energy(_NS, "nobody@egi.eu") is None # live tail: a running deployment's graph keeps growing past the stored series @@ -968,7 +975,7 @@ def b(ts, wh): persisted = [[b("2026-01-01T00:00:00Z", 1.0), b("2026-01-01T00:15:00Z", 2.0)]] live = [[b("2026-01-01T00:15:00Z", 9.0), b("2026-01-01T00:30:00Z", 3.0)]] - points = series.build_series(persisted, live, target_points=100) + points = series.build_series(persisted, live, 1000) assert [(p["ts"], p["energy_wh"], p["live"]) for p in points] == [ ("2026-01-01T00:00:00Z", 1.0, False), ("2026-01-01T00:15:00Z", 2.0, False), @@ -976,7 +983,7 @@ def b(ts, wh): ] # several entities (a user's deployments) are summed per timestamp - two = series.build_series([persisted[0], persisted[0]], [], target_points=100) + two = series.build_series([persisted[0], persisted[0]], [], 1000) assert [p["energy_wh"] for p in two] == [2.0, 4.0] @@ -1183,7 +1190,7 @@ def b(ts, wh): return {"ts": ts, "energy_wh": wh, "power_w_avg": 5.0} persisted = [[b("2026-01-01T00:00:00Z", 1.0), b("2026-01-01T00:45:00Z", 2.0)]] - points = series.build_series(persisted, [], target_points=100, fill_s=900) + points = series.build_series(persisted, [], 1000, fill_s=900) assert [(p["ts"], p["energy_wh"], p["power_w"]) for p in points] == [ ("2026-01-01T00:00:00Z", 1.0, 5.0), ("2026-01-01T00:15:00Z", 0.0, 0.0), @@ -1192,13 +1199,62 @@ def b(ts, wh): ] +def test_build_series_fill_keeps_off_grid_buckets(): + def b(ts, wh): + return {"ts": ts, "energy_wh": wh, "power_w_avg": 5.0} + + persisted = [[b("2026-01-01T00:05:00Z", 1.0), b("2026-01-01T00:30:00Z", 2.0)]] + points = series.build_series(persisted, [], 1000, fill_s=900) + assert sum(p["energy_wh"] for p in points) == 3.0 + + +def test_cluster_stats_series_only_with_full_info(monkeypatch): + from ai4papi.routers.v1.stats import deployments as stats_dep + + dc = schemas.DatacenterStats( + lat=0, lon=0, PUE=1, nodes={}, energy=schemas.EnergyStats() + ) + base = schemas.ClusterStats( + datacenters={"dc": dc}, cluster=schemas.ResourceStats(gpu_models={}) + ) + point = { + "ts": "2026-01-01T00:00:00Z", + "power_w": 1.0, + "energy_wh": 2.0, + "carbon_g": 0.0, + "water_l": 0.0, + "live": False, + } + calls = [] + + def fake_energy(series=True): + calls.append(series) + return {"dc": {"series": [point]}} + + monkeypatch.setattr(stats_dep, "cluster_stats", base) + monkeypatch.setattr( + stats_dep, "cluster_stats_updated_at", __import__("time").time() + ) + monkeypatch.setattr(stats_dep.energy_accounting, "enabled", lambda: True) + monkeypatch.setattr(stats_dep.energy_accounting, "get_cluster_energy", fake_energy) + + assert ( + stats_dep.get_cluster_stats(full_info=False).datacenters["dc"].energy.series + is None + ) + assert calls == [] + out = stats_dep.get_cluster_stats(full_info=True) + assert out.datacenters["dc"].energy.series[0].energy_wh == 2.0 + assert calls == [True] + + def test_build_series_fill_until_now(): def b(ts, wh): return {"ts": ts, "energy_wh": wh, "power_w_avg": 5.0} persisted = [[b("2026-01-01T00:00:00Z", 1.0)]] until = compute.parse_iso("2026-01-01T00:50:00Z") # last complete bucket: 00:30 - points = series.build_series(persisted, [], 100, fill_s=900, until=until) + points = series.build_series(persisted, [], 1000, fill_s=900, until=until) assert [p["ts"] for p in points] == [ "2026-01-01T00:00:00Z", "2026-01-01T00:15:00Z", @@ -1207,7 +1263,7 @@ def b(ts, wh): # a live tail takes over where it starts: no zero is placed over it live = [[b("2026-01-01T00:30:00Z", 7.0)]] - points = series.build_series(persisted, live, 100, fill_s=900, until=until) + points = series.build_series(persisted, live, 1000, fill_s=900, until=until) assert [(p["ts"], p["energy_wh"], p["live"]) for p in points] == [ ("2026-01-01T00:00:00Z", 1.0, False), ("2026-01-01T00:15:00Z", 0.0, False), @@ -1249,14 +1305,13 @@ def test_build_series_uniform_resolution_with_live_tail(): def b(i, wh=1.0): return {"ts": compute.iso(i * 900), "energy_wh": wh, "power_w_avg": 5.0} - # 100 persisted buckets + 3 live ones, squeezed into 10 points: every point - # spans the same time, the live tail included, and nothing is lost + # 100 persisted buckets + 3 live ones under a 30-point cap: the finest window + # that fits is 1 h; every point spans the same time, the live tail included, + # and nothing is lost points = series.build_series( - [[b(i) for i in range(100)]], - [[b(i) for i in range(100, 103)]], - target_points=10, + [[b(i) for i in range(100)]], [[b(i) for i in range(100, 103)]], 30 ) - assert len(points) <= 10 + assert len(points) == 26 # 1 h windows: the finest that fits 30 points steps = { round(compute.parse_iso(q["ts"]) - compute.parse_iso(p["ts"])) for p, q in itertools.pairwise(points) @@ -1291,3 +1346,57 @@ def test_user_series_no_trailing_zeros_when_live_tail_is_missing(monkeypatch): ) pts = energy.get_user_energy(_NS, "u@egi.eu", series=True)["series"] assert [p["ts"] for p in pts] == [compute.iso(base - 5 * step)] + + +@pytest.mark.parametrize("name", list(series.RESOLUTIONS)) +def test_every_resolution_keeps_calendar_totals_exact(name): + """Every window of the ladder lines up with the UTC day (and the week one with + ISO weeks): grouping the points by day, or by week for `1w`, gives the same + totals as the raw buckets. A window that drifts against midnight made one day + in three look lower and the other two higher.""" + w = series.RESOLUTIONS[name] + monday = 1_700_006_400 + 5 * 86400 # a UTC midnight that is a Monday + assert datetime.datetime.fromtimestamp(monday, datetime.timezone.utc).weekday() == 0 + raw = [ + compute.Bucket(compute.iso(monday + i * 900), 1.0 + (i % 7), 10.0, 0.1, 0.01) + for i in range(120 * 96) + ] + unit = 7 * 86400 if name == "1w" else 86400 + + def per_unit(bs): + out = {} + for b in bs: + k = (compute.parse_iso(b.ts) - monday) // unit + out[k] = out.get(k, 0.0) + b.energy_wh + return out + + ds = compute.downsample(raw, w) + assert per_unit(ds) == pytest.approx(per_unit(raw)) + assert ( + len( + { + round(compute.parse_iso(q.ts) - compute.parse_iso(p.ts)) + for p, q in itertools.pairwise(ds) + } + ) + == 1 + ) + + +def test_pick_resolution_respects_the_cap(): + day = 86400 + # a short series keeps the native 15 min, a long one climbs the ladder + assert series.pick_resolution(0, 3 * day, 1000) == "15m" + assert series.pick_resolution(0, 30 * day, 1000) == "1h" + assert series.pick_resolution(0, 365 * day, 1000) == "12h" + assert series.pick_resolution(0, 365 * day, 500) == "1d" + assert series.pick_resolution(0, 3650 * day, 500) == "1w" + # the number of points never goes over the cap (until the coarsest window) + for span_days in (1, 7, 31, 90, 365, 730): + for cap in (100, 500, 1000): + first = 1_700_000_000 + name = series.pick_resolution(first, first + span_days * day, cap) + w = series.RESOLUTIONS[name] + o = compute.WINDOW_ANCHOR_S + n = int((first + span_days * day - o) // w) - int((first - o) // w) + 1 + assert n <= cap or name == "1w", (span_days, cap, name, n)