diff --git a/.gitignore b/.gitignore index 5eb10711..51e45749 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,6 @@ GTAGS # Things specific to this project # ################################### + +# Local energy-accounting store ($ACCOUNTING_PTH), runtime data +.local-accounting/ diff --git a/README.md b/README.md index fca63093..a084f706 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,8 @@ More details can be found in the [API docs](https://api.cloud.ai4eosc.eu/docs). It will serve the contents of the `ai4-accounting/summaries` folder. + Energy consumption and footprint (carbon and water) is attached additively to the existing responses: an `energy` key per deployment, per user in `stats/user` and per datacenter in `stats/cluster`. It needs the `MIMIR_USER` / `MIMIR_PASSWORD` variables and the `energy` section of `etc/main.yaml`. Dashboard integration reference: [docs/energy-api.md](./docs/energy-api.md); architecture and internals: [docs/energy-accounting.md](./docs/energy-accounting.md). +
The API methods can also be accessed by interacting directly with diff --git a/ai4papi/clients/__init__.py b/ai4papi/clients/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai4papi/clients/mimir.py b/ai4papi/clients/mimir.py new file mode 100644 index 00000000..abfc8467 --- /dev/null +++ b/ai4papi/clients/mimir.py @@ -0,0 +1,133 @@ +""" +Minimal client for the Grafana Mimir (Prometheus-compatible) HTTP API. + +Used by the energy-accounting feature to read per-allocation power time series. +API reference: https://prometheus.io/docs/prometheus/latest/querying/api/ +""" + +import datetime +import os +import time + +import requests + +MIMIR_URL = "https://mimir.k8s.cloud.ai4eosc.eu/prometheus" +# Optional: the energy-accounting feature is disabled if these are not set. +MIMIR_USER = os.environ.get("MIMIR_USER") +MIMIR_PASSWORD = os.environ.get("MIMIR_PASSWORD") +if not (MIMIR_USER and MIMIR_PASSWORD): + print("You should define MIMIR_USER / MIMIR_PASSWORD to enable energy stats") + +# Mimir keeps each series under a point cap per query; stay well below it. +MAX_POINTS_PER_QUERY = 11_000 +# Measured against the real Mimir: a 24h range query takes under 1 s and a full +# 11,000-point chunk about 1.5 s, so these leave a wide margin while a hung Mimir +# does not hold a request or the sweep thread for a minute. + +session = requests.Session() + +# After a timeout or connection error Mimir is not asked again for this long (the +# calls fail fast with `MimirError`), so a hung Mimir costs one timeout per minute +# instead of one per query. +BACKOFF_SECONDS = 60 +_down_until = 0.0 + + +class MimirError(RuntimeError): + """Raised when Mimir returns a non-successful response.""" + + +def _auth(): + return (MIMIR_USER, MIMIR_PASSWORD) if MIMIR_USER else None + + +def _get(path: str, params: dict, timeout: int) -> list: + """GET a Prometheus API route and return `data.result`.""" + global _down_until + if time.monotonic() < _down_until: + raise MimirError("Mimir unreachable, retrying shortly") + try: + r = session.get( + f"{MIMIR_URL}{path}", params=params, auth=_auth(), timeout=timeout + ) + except (requests.ConnectionError, requests.Timeout): + _down_until = time.monotonic() + BACKOFF_SECONDS + raise + r.raise_for_status() + body = r.json() + if body.get("status") != "success": + raise MimirError(f"Mimir query failed: {body}") + return body.get("data", {}).get("result", []) + + +def query(promql: str, at: datetime.datetime | None = None, timeout: int = 10): + """ + Instantaneous query (`/api/v1/query`). Returns the raw `data.result` list + (vector: one entry per series, each with `metric` and `value` = [ts, str]). + """ + params = {"query": promql} + if at is not None: + params["time"] = at.timestamp() + return _get("/api/v1/query", params, timeout) + + +def query_range( + promql: str, + start: datetime.datetime, + end: datetime.datetime, + step_s: int = 30, + timeout: int = 15, +): + """ + Range query (`/api/v1/query_range`). Returns the raw `data.result` list + (matrix: one entry per series, each with `metric` and `values` = + [[ts, str], ...]). + """ + params = { + "query": promql, + "start": start.timestamp(), + "end": end.timestamp(), + "step": f"{step_s}s", + } + return _get("/api/v1/query_range", params, timeout) + + +def _split_chunks(start, end, step_s, max_points): + """ + Yield non-overlapping (chunk_start, chunk_end) tuples so each range query + stays under `max_points` samples per series (relevant when backfilling a + deployment whose pointer is far behind). + """ + while start < end: + remaining_steps = int((end - start).total_seconds() / step_s) + chunk_steps = min(remaining_steps, max_points) + if chunk_steps <= 0: + break + chunk_end = start + datetime.timedelta(seconds=chunk_steps * step_s) + yield start, min(chunk_end, end) + start = chunk_end + datetime.timedelta(seconds=step_s) + + +def query_range_chunked( + promql: str, + start: datetime.datetime, + end: datetime.datetime, + step_s: int = 30, + max_points: int = MAX_POINTS_PER_QUERY, +): + """ + Like `query_range` but splits long windows into consecutive chunks and + merges the results, concatenating each series' `values` sorted by timestamp. + """ + merged: dict = {} + for chunk_start, chunk_end in _split_chunks(start, end, step_s, max_points): + for series in query_range(promql, chunk_start, chunk_end, step_s): + key = frozenset(series.get("metric", {}).items()) + merged.setdefault(key, {"metric": series.get("metric", {}), "values": []}) + merged[key]["values"].extend(series.get("values", [])) + + result = [] + for entry in merged.values(): + entry["values"].sort(key=lambda v: float(v[0])) + result.append(entry) + return result diff --git a/ai4papi/conf.py b/ai4papi/conf.py index 6dea1a53..51e4b96f 100644 --- a/ai4papi/conf.py +++ b/ai4papi/conf.py @@ -172,7 +172,8 @@ class AssetConfig(TypedDict): elif k == "country": datacenters[name][k] = v else: - datacenters[name][k] = float(v) + # Numeric columns; allow blank cells (eg. TUE only known for some DCs) + datacenters[name][k] = float(v) if v not in (None, "") else None # Retrieve git info from PAPI, to show current version in the docs papi_commit = subprocess.run( diff --git a/ai4papi/energy/__init__.py b/ai4papi/energy/__init__.py new file mode 100644 index 00000000..f81f3da2 --- /dev/null +++ b/ai4papi/energy/__init__.py @@ -0,0 +1,449 @@ +""" +Energy-accounting read API used by the routers. + +* `get_accumulated` / `get_accumulated_bulk`: per-deployment accumulated energy, + carbon and water (persisted accumulator + live Mimir top-up). +* `get_series`: per-deployment time series for the detail-view graphs (persisted + 15-min archive + a live tail so the graph keeps growing). +* `get_user_energy`: per-user aggregate (all of the user's deployments summed, + each with its live top-up), optionally with a merged time series. +* `get_datacenter_power` / `get_cluster_energy`: per-datacenter instantaneous + power and accumulated energy for `/stats/cluster` (no platform-wide total). + +Everything degrades to `None` / empty when the feature is disabled or the data +is not available; it never raises to the caller. +""" + +import datetime +import logging +import threading + +from cachetools import TTLCache, cached + +from ai4papi.clients import mimir +from ai4papi.energy import compute, config, store +from ai4papi.energy.series import build_series, persisted +from ai4papi.energy.config import CLUSTER_NS, KEYS, enabled +from ai4papi.energy.live import live_series, topup +from ai4papi.energy.ondemand import ensure_swept, ensure_swept_bulk, live_only_doc +from ai4papi.energy.sweep import process_single, run_sweep + +__all__ = [ + "add_to_job", + "add_to_jobs", + "enabled", + "get_accumulated", + "get_accumulated_bulk", + "get_cluster_energy", + "get_datacenter_power", + "get_series", + "get_user_energy", + "process_single", + "run_sweep", +] + +LOG = logging.getLogger(__name__) + + +@cached(cache=TTLCache(maxsize=8192, ttl=60), lock=threading.Lock()) +def _read_doc(namespace: str, uuid: str): + return store.read_accum(namespace, uuid) + + +def _doc_tue(doc: dict) -> float | None: + """`tue_factor` from the doc; for a doc written before that field existed, + the single datacenter's configured TUE (blended docs then read `None`).""" + if doc.get("tue_factor"): + return doc["tue_factor"] + dcs = doc.get("datacenters") or [] + return config.tue(dcs[0]) if len(dcs) == 1 else None + + +def _to_stats(doc: dict) -> dict: + acc = doc.get("accumulated") or {} + # fallback power for a running deployment with no live top-up: the last + # persisted bucket's mean. `_apply_topup` replaces it with the actual last + # Mimir sample (current draw) whenever the top-up succeeds. + tail = doc.get("tail") or [] + running = doc.get("status") == "running" + last = tail[-1] if tail else {} + return { + "energy_wh": acc.get("energy_wh", 0.0), + "tue_factor": _doc_tue(doc), + "carbon_g": acc.get("carbon_g", 0.0), + "water_l": acc.get("water_l", 0.0), + "power_w": last.get("power_w_avg") if running else None, + "since": doc.get("window_start") or doc.get("settled_ts"), + "as_of": doc.get("pointer_ts"), + "live_as_of": None, + "complete": doc.get("complete", True), + "coverage_ratio": doc.get("coverage_ratio", 1.0), + "degraded": False, + "datacenters": doc.get("datacenters", []), + } + + +def _apply_topup(stats: dict, doc: dict) -> dict: + t = topup(doc) + if not t: + return stats + if t.get("degraded"): + stats["degraded"] = True + return stats + # the top-up slice uses the same TUE as the accumulator, so `tue_factor` + # does not move; just add the energy / footprint deltas and refresh power. + for k in KEYS: + stats[k] += t.get(k, 0.0) + if t.get("power_w") is not None: + stats["power_w"] = t["power_w"] + stats["live_as_of"] = t.get("live_as_of") + return stats + + +def _live_only_stats(namespace: str, uuid: str) -> tuple[dict, dict] | None: + """ + `(doc, stats)` for a deployment with no accumulator doc yet, computed purely + from `live_only_doc` + its top-up. `None` whenever there is nothing to show: + no running allocation, Mimir unreachable (`degraded`), or Mimir simply has + not returned a single sample for this allocation yet -- an all-zero top-up + with no instantaneous power is indistinguishable from "no measurement", so + it is treated as not populated rather than shown as a fabricated `0` reading. + """ + doc = live_only_doc(namespace, uuid) + if doc is None: + return None + stats = _apply_topup(_to_stats(doc), doc) + has_data = stats.get("power_w") is not None or any(stats.get(k, 0.0) for k in KEYS) + if not has_data: + return None + return doc, stats + + +def _resolve(namespace: str, uuid: str, live: bool) -> tuple[dict, dict] | None: + """ + `(doc, stats)` of one deployment, or `None` when there is nothing to report. + The stored doc is swept on demand when missing; without a usable doc the + number comes from Mimir alone (`live_only_doc`). + """ + doc = _read_doc(namespace, uuid) + if doc is None: + # never swept: run one sweep pass for just this deployment instead of + # waiting up to `sweep_seconds` for the background thread to reach it. + doc = ensure_swept(namespace, uuid) + if doc is not None and doc.get("metrics_available") is False: + # confirmed: ran a significant time with no usable data. Do not keep + # retrying live -- wait for the real sweep to reassess it. + return None + if doc is None or "accumulated" not in doc: + # nothing settled yet: either too young for a bucket, or an + # inconclusive on-demand stub written before Nomad had even placed the + # allocation (no `metrics_available` verdict either way). Fall back to + # a pure-Mimir number, but only once Mimir actually has a sample for + # it - never a fabricated zero. + return _live_only_stats(namespace, uuid) if live else None + stats = _to_stats(doc) + return doc, (_apply_topup(stats, doc) if live else stats) + + +def _owner_rows(namespace: str, owner: str, live: bool) -> list[tuple[str, dict, dict]]: + """`(uuid, doc, stats)` of every accounted deployment of `owner` in `namespace`, + sweeping the missing ones on demand first.""" + live_jobs = ensure_swept_bulk(namespace, owner) + rows: list[tuple[str, dict, dict]] = [] + covered: set[str] = set() + for uuid, doc in store.iter_owner(namespace, owner): + if doc.get("metrics_available") is False: + covered.add(uuid) + continue + if "accumulated" not in doc: + continue # inconclusive stub: leave uncovered, try live-only below + covered.add(uuid) + stats = _to_stats(doc) + rows.append((uuid, doc, _apply_topup(stats, doc) if live else stats)) + if live: + for stub in live_jobs: + uuid = stub["ID"] + if uuid in covered: + continue + res = _live_only_stats(namespace, uuid) + if res is not None: + rows.append((uuid, *res)) + return rows + + +def get_accumulated(namespace: str, uuid: str, live: bool = True) -> dict | None: + if not enabled(): + return None + res = _resolve(namespace, uuid, live) + return res[1] if res else None + + +def get_accumulated_bulk( + namespace: str, owner: str, live: bool = True +) -> dict[str, dict]: + if not enabled(): + return {} + return {uuid: stats for uuid, _doc, stats in _owner_rows(namespace, owner, live)} + + +def add_to_job(job: dict, namespace: str, full_info: bool) -> None: + """Set `job["energy"]`: the extended time series if `full_info`, else just the + accumulated stats.""" + get = get_series if full_info else get_accumulated + job["energy"] = get(namespace, job["job_ID"]) + + +def add_to_jobs(jobs: list[dict], namespace: str, owner: str, full_info: bool) -> None: + """`add_to_job` for all the `owner`'s jobs of a namespace. Without `full_info` + it uses one batch of cheap file reads and a single cached Mimir top-up; + with it, the extended time series per deployment (heavier).""" + if not jobs: + return + if full_info: + for job in jobs: + add_to_job(job, namespace, full_info) + else: + stats = get_accumulated_bulk(namespace, owner) + for job in jobs: + job["energy"] = stats.get(job["job_ID"]) + + +def _aggregate(stats_list: list[dict]) -> dict: + """Sum the `EnergyStats` of several deployments into one block.""" + agg = dict.fromkeys(KEYS, 0.0) + dcs: set[str] = set() + power = raw_energy = 0.0 # raw_energy: Σ energy_wh / tue, to blend `tue_factor` + since, as_of, live_as_of, coverage = [], [], [], [] + for s in stats_list: + for k in KEYS: + agg[k] += s.get(k, 0.0) or 0.0 + if s.get("tue_factor"): + raw_energy += s["energy_wh"] / s["tue_factor"] + dcs.update(s.get("datacenters") or []) + power += s.get("power_w") or 0.0 + since += [s["since"]] if s.get("since") else [] + as_of += [s["as_of"]] if s.get("as_of") else [] + live_as_of += [s["live_as_of"]] if s.get("live_as_of") else [] + coverage.append(s.get("coverage_ratio", 1.0)) + + return { + # not rounded, same as the per-deployment `EnergyStats`: the values can be + # legitimately tiny (idle CPU workloads) and must reconcile with the sum + # of `series`. + **agg, + "tue_factor": (agg["energy_wh"] / raw_energy) if raw_energy else None, + "power_w": power or None, + # time range these numbers cover: [since, as_of], live top-up to live_as_of + "since": min(since, default=None), + "as_of": max(as_of, default=None), + "live_as_of": max(live_as_of, default=None), + "complete": all(s.get("complete", True) for s in stats_list), + "coverage_ratio": round(sum(coverage) / len(coverage), 3), + "degraded": any(s.get("degraded") for s in stats_list), + "datacenters": sorted(dcs), + "deployments": len(stats_list), + } + + +def get_user_energy( + namespace: str, owner: str, series: bool = False, target_points: int | None = None +) -> dict | None: + """ + All of a user's deployments in a VO rolled into one `EnergyStats`-shaped + block: each deployment's persisted accumulator plus its live Mimir top-up, + summed. With `series=True`, also a 15-min time series merged across the + deployments (`full_info=true` on `GET /stats/user`). `None` when the user has + no accounted deployment. + """ + if not enabled(): + return None + rows = _owner_rows(namespace, owner, live=True) + if not rows: + return None + + out = _aggregate([stats for _uuid, _doc, stats in rows]) + out["series"] = None + if series: + cfg = config.get() + step_s = int(cfg["series_bucket_seconds"]) + if target_points is None: + target_points = int(cfg.get("user_series_points", 1000)) + out["series"] = ( + build_series( + [ + persisted(store.read_series(namespace, uuid, None, None), doc) + for uuid, doc, _stats in rows + ], + [live_series(doc, bucket_s=step_s) for _uuid, doc, _stats in rows], + target_points, + ) + or None + ) + return out + + +# --------------------------------------------------------------------------- +# cluster-level instantaneous power / footprint (for /stats/cluster) +# --------------------------------------------------------------------------- + + +def _sum_by_datacenter(promql: str, scale: float) -> dict[str, float]: + """Instant Mimir query grouped `by (datacenter)` -> {dc: value * scale}.""" + out: dict[str, float] = {} + for series in mimir.query(promql): + dc = series.get("metric", {}).get("datacenter") + if not dc: + continue + try: + out[dc] = float(series["value"][1]) * scale + except (KeyError, IndexError, ValueError, TypeError): + continue + return out + + +@cached(cache=TTLCache(maxsize=1, ttl=25), lock=threading.Lock()) +def get_datacenter_power() -> dict[str, dict]: + """ + Instantaneous power draw per datacenter, normalized by TUE: + `power_w = (sum host power + sum GPU power) * TUE[dc]`. Best-effort: an + unreachable Mimir yields `{}`, and a datacenter with no power series at all + is simply absent, never reported as zero. A datacenter listed in + `energy.metrics_datacenters` counts with whatever it has (host-only, + GPU-only or both); without that allowlist a host-power series is required so + a stray GPU metric does not fabricate a total. `{}` when disabled. + """ + if not enabled(): + return {} + + metrics_dcs = set(config.get().get("metrics_datacenters") or []) + try: + host_w = _sum_by_datacenter( + "sum by (datacenter) (scaph_host_power_microwatts)", 1e-6 + ) + gpu_w = _sum_by_datacenter("sum by (datacenter) (DCGM_FI_DEV_POWER_USAGE)", 1.0) + except (mimir.MimirError, OSError): + LOG.warning("mimir cluster-power query failed") + return {} + + as_of = compute.iso(datetime.datetime.now(datetime.timezone.utc).timestamp()) + out: dict[str, dict] = {} + for dc in set(host_w) | set(gpu_w): + if metrics_dcs: + if dc not in metrics_dcs: + continue + # declared site: host-only, GPU-only or both are all valid totals + elif dc not in host_w: + # no allowlist: require host power so a stray GPU metric for an + # unmonitored datacenter does not fabricate a total + continue + tue = config.tue(dc) + out[dc] = { + "power_w": (host_w.get(dc, 0.0) + gpu_w.get(dc, 0.0)) * tue, + "tue_factor": tue, + "as_of": as_of, + } + return out + + +@cached(cache=TTLCache(maxsize=16, ttl=300), lock=threading.Lock()) +def _cluster_archive_series(dc: str) -> list[dict]: + """ + Consolidated 15-min buckets of a datacenter (the `.series.jsonl`). Cached for + ~5 min: it only changes when a bucket ages out of the 24h tail, which happens + at sweep cadence, so the 30s `get_cluster_stats` thread need not re-read it. + """ + return store.read_series(CLUSTER_NS, dc, None, None) + + +def get_cluster_energy( + series: bool = True, target_points: int | None = None +) -> dict[str, dict]: + """ + Per-datacenter energy for `/stats/cluster`: the instantaneous power from + `get_datacenter_power()` merged with the persisted accumulator plus a live + top-up, plus a downsampled 15-min `series`. `{}` when the feature is disabled. + A datacenter shows up if it has instantaneous power or an accumulator doc. + """ + if not enabled(): + return {} + cfg = config.get() + if target_points is None: + target_points = int(cfg.get("cluster_series_points", 500)) + bucket_s = int(cfg["series_bucket_seconds"]) + + inst = get_datacenter_power() + metrics_dcs = cfg.get("metrics_datacenters") or [] + out: dict[str, dict] = {} + + for dc in set(inst) | set(metrics_dcs): + entry = dict(inst.get(dc) or {}) + doc = store.read_accum(CLUSTER_NS, dc) + if doc and doc.get("accumulated") and doc.get("metrics_available") is not False: + acc = {k: doc["accumulated"].get(k, 0.0) for k in KEYS} + live_as_of, degraded = None, False + t = topup(doc) + if t and t.get("degraded"): + degraded = True + elif t: + for k in KEYS: + acc[k] += t.get(k, 0.0) + live_as_of = t.get("live_as_of") + # not rounded: consistent with the per-deployment / per-user blocks, + # and the total must reconcile with the sum of `series`. + entry.update( + energy_wh=acc["energy_wh"], + carbon_g=acc["carbon_g"], + water_l=acc["water_l"], + tue_factor=doc.get("tue_factor") or config.tue(dc), + complete=doc.get("complete", True), + since=doc.get("window_start") or doc.get("settled_ts"), + as_of=doc.get("pointer_ts"), + live_as_of=live_as_of, + degraded=degraded, + ) + if series: + entry["series"] = build_series( + [persisted(_cluster_archive_series(dc), doc)], + [live_series(doc, bucket_s=bucket_s)], + target_points, + ) + if entry: + out[dc] = entry + return out + + +# --------------------------------------------------------------------------- +# time series helpers +# --------------------------------------------------------------------------- + + +def get_series( + namespace: str, uuid: str, target_points: int | None = None +) -> dict | None: + if not enabled(): + return None + res = _resolve(namespace, uuid, live=True) + if res is None: + return None + doc, stats = res + + cfg = config.get() + if target_points is None: + target_points = int(cfg.get("deployment_series_points", 1000)) + step_s = int(cfg["series_bucket_seconds"]) + series = build_series( + [persisted(store.read_series(namespace, uuid, None, None), doc)], + [live_series(doc, bucket_s=step_s)], + target_points, + ) + + return { + "deployment_uuid": uuid, + "start": series[0]["ts"] if series else (stats.get("since") or ""), + "end": series[-1]["ts"] + if series + else (stats.get("live_as_of") or stats.get("as_of") or ""), + "source": "mixed" if any(p["live"] for p in series) else "store", + "series": series, + "accumulated": stats, + } diff --git a/ai4papi/energy/compute.py b/ai4papi/energy/compute.py new file mode 100644 index 00000000..542b3423 --- /dev/null +++ b/ai4papi/energy/compute.py @@ -0,0 +1,411 @@ +""" +Pure(ish) helpers for the energy-accounting feature: resolve a deployment's +allocations, build the PromQL, integrate power into energy buckets, apply the +footprint, and downsample. + +The only side effect here is reading from Nomad in `resolve_alloc_ids` (and its +cached node->datacenter lookup); everything else is pure and unit-testable. +""" + +import bisect +import datetime +import threading +import statistics +from collections import namedtuple + +from cachetools import TTLCache, cached + +import ai4papi.conf as papiconf +from ai4papi.nomad_utils import Nomad +from ai4papi.wattnet import GreenDirector + +# Bucket start `ts` is an ISO-8601 UTC string ("...Z"). Energy in Wh (normalized +# by the datacenter TUE), power in W, footprint per bucket. +Bucket = namedtuple("Bucket", ["ts", "energy_wh", "power_w_avg", "carbon_g", "water_l"]) + +DEFAULT_CARBON = GreenDirector.DEFAULTS["carbon"] # gCO2eq/kWh +DEFAULT_WATER = GreenDirector.DEFAULTS["water"] # L/kWh + +_TERMINAL_STATUSES = {"complete", "failed", "lost"} + + +def iso(ts: float) -> str: + return ( + datetime.datetime.fromtimestamp(ts, datetime.timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + + +def parse_iso(s: str) -> float: + return datetime.datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() + + +# --------------------------------------------------------------------------- +# Nomad: deployment -> allocations +# --------------------------------------------------------------------------- + +AllocWindow = namedtuple("AllocWindow", ["alloc_id", "start_ts", "end_ts"]) + + +@cached(cache=TTLCache(maxsize=4096, ttl=60 * 60), lock=threading.Lock()) +def _node_datacenter(node_id: str) -> str: + return Nomad.node.get_node(node_id)["Datacenter"] + + +def resolve_alloc_ids(job_id: str, namespace: str): + """ + Return `(allocs_by_dc, alloc_windows, running_allocs, partial_dc)` for every + allocation the job ever had, keeping only datacenters that have power metrics + (`energy.metrics_datacenters`). + + * allocs_by_dc: {datacenter: [alloc_id, ...]} (whole lifetime) + * alloc_windows: [AllocWindow(alloc_id, start_ts, end_ts|None)] (all allocs, + to estimate how long the deployment actually ran) + * running_allocs: [{"alloc_id": ..., "datacenter": ...}] (currently running, + for the live top-up) + * partial_dc: True if some allocation ran in a datacenter without metrics + """ + metrics_dcs = set(papiconf.MAIN_CONF["energy"]["metrics_datacenters"]) + + allocs = Nomad.job.get_allocations(id_=job_id, namespace=namespace) + + allocs_by_dc: dict[str, list[str]] = {} + alloc_windows: list[AllocWindow] = [] + running_allocs: list[dict] = [] + partial_dc = False + + for a in allocs: + try: + dc = _node_datacenter(a["NodeID"]) + except Exception: + continue + + start_ts = a["CreateTime"] / 1e9 if a.get("CreateTime") else None + end_ts = None + if a.get("ClientStatus") in _TERMINAL_STATUSES and a.get("ModifyTime"): + end_ts = a["ModifyTime"] / 1e9 + if start_ts is not None: + alloc_windows.append(AllocWindow(a["ID"], start_ts, end_ts)) + + if dc in metrics_dcs: + allocs_by_dc.setdefault(dc, []).append(a["ID"]) + if a.get("ClientStatus") == "running": + running_allocs.append({"alloc_id": a["ID"], "datacenter": dc}) + else: + partial_dc = True + + return allocs_by_dc, alloc_windows, running_allocs, partial_dc + + +def running_seconds(alloc_windows, start_ts: float, end_ts: float) -> float: + """ + Union of the alloc windows clipped to [start_ts, end_ts]. Approximates how + long the deployment was actually running in that interval (to compare + against the time we have power data for). + """ + spans = [] + for w in alloc_windows: + s = max(w.start_ts, start_ts) + e = min(w.end_ts if w.end_ts else end_ts, end_ts) + if e > s: + spans.append((s, e)) + if not spans: + return 0.0 + spans.sort() + total = 0.0 + cur_s, cur_e = spans[0] + for s, e in spans[1:]: + if s <= cur_e: + cur_e = max(cur_e, e) + else: + total += cur_e - cur_s + cur_s, cur_e = s, e + total += cur_e - cur_s + return total + + +# --------------------------------------------------------------------------- +# PromQL +# --------------------------------------------------------------------------- + + +def power_promql(alloc_ids: list[str]) -> tuple[str, str]: + """ + (cpu_query, gpu_query) for a set of allocation ids (all in the same + datacenter). + + Both are grouped, not fully summed, so the data-quality filter runs on the + original per-source series (per allocation for CPU, per GPU for GPU power): + a single bad reading is dropped without discarding the other sources at the + same instant, and the ceiling / spike checks compare against a physically + bounded signal. The integrator then sums the per-series energy. + """ + regex = "|".join(alloc_ids) + cpu_q = ( + "sum by (container_label_com_hashicorp_nomad_alloc_id) (" + "scaph_process_power_consumption_microwatts" + f'{{container_label_com_hashicorp_nomad_alloc_id=~"{regex}"}})' + ) + gpu_q = ( + "DCGM_FI_DEV_POWER_USAGE" + " * on(UUID, instance) group_left(alloc_id)" + f' nomad_gpu_allocation_info{{alloc_id=~"{regex}"}}' + ) + return cpu_q, gpu_q + + +def cluster_promql(dc: str) -> tuple[str, str]: + """ + (host_query, gpu_query) for a whole datacenter: one series per host and one + per GPU (grouped, not summed) so `integrate` runs the data-quality filter on + each raw Mimir series before summing the energy. + """ + host_q = f'sum by (node) (scaph_host_power_microwatts{{datacenter="{dc}"}})' + gpu_q = f'sum by (UUID) (DCGM_FI_DEV_POWER_USAGE{{datacenter="{dc}"}})' + return host_q, gpu_q + + +# --------------------------------------------------------------------------- +# Integration: power samples -> energy buckets +# --------------------------------------------------------------------------- + + +def _series_samples( + result: list[dict], scale: float +) -> list[list[tuple[float, float]]]: + """A Mimir matrix result -> one sorted `[(ts, watts)]` list per series.""" + out: list[list[tuple[float, float]]] = [] + for series in result: + s: list[tuple[float, float]] = [] + for ts, val in series.get("values", []): + try: + s.append((float(ts), float(val) * scale)) + except (TypeError, ValueError): + continue + if s: + s.sort() + out.append(s) + return out + + +def sanitize( + samples: list[tuple[float, float]], dq: dict | None +) -> list[tuple[float, float]]: + """ + Data-quality filter on one original power series (per allocation for CPU, + per GPU for GPU power), before integration. Anomalous samples are *dropped* + (the integrator then treats it as a gap), never clamped, so a bad reading + never inflates the accumulated energy. Every check is skippable via + `energy.data_quality`: + + * `min_power_w` : drop samples below this (default 0, kills negatives) + * `max_power_w` : drop samples above this hard ceiling (0 = off). Physically + bounded because it applies per source (one alloc / one GPU). + * `spike_factor` : drop a sample above `spike_factor` * rolling median of its + `spike_window` neighbours (0 = off) + """ + if not dq or not dq.get("enabled", True): + return samples + + lo = dq.get("min_power_w", 0.0) + hi = dq.get("max_power_w", 0.0) + out = [(t, p) for t, p in samples if p >= lo and (not hi or p <= hi)] + + factor = dq.get("spike_factor", 0.0) + window = int(dq.get("spike_window", 11)) or 11 + if factor and len(out) > window: + half = window // 2 + vals = [p for _, p in out] + kept = [] + for i, (t, p) in enumerate(out): + lo_i, hi_i = max(0, i - half), min(len(vals), i + half + 1) + neighbours = vals[lo_i:i] + vals[i + 1 : hi_i] + med = statistics.median(neighbours) if neighbours else 0.0 + if med > 1e-6 and p > factor * med: + continue # spike -> drop + kept.append((t, p)) + out = kept + + return out + + +def _integrate_series(samples, step_s, bucket_s, gap_factor): + """One power series -> {bucket_ts: [energy_wh, covered_s]} (trapezoidal).""" + max_gap = gap_factor * step_s + out: dict[int, list[float]] = {} + for (t0, p0), (t1, p1) in zip(samples, samples[1:]): + dt = t1 - t0 + if dt <= 0 or dt > max_gap: + continue + wh = (p0 + p1) / 2 * dt / 3600 + b = int(t0 // bucket_s) * bucket_s + cur = out.setdefault(b, [0.0, 0.0]) + cur[0] += wh + cur[1] += dt + return out + + +def integrate( + cpu_result: list[dict], + gpu_result: list[dict], + step_s: int = 30, + bucket_s: int = 900, + tue_factor: float = 1.0, + gap_factor: float = 3.0, + data_quality: dict | None = None, +) -> list[Bucket]: + """ + Per-source data-quality filter, then trapezoidal integral of every power + series (CPU per allocation, GPU per GPU) into fixed `bucket_s` energy + buckets, summed. Gaps longer than `gap_factor * step_s` are not integrated + across (deployment stopped / scrape hole / dropped anomaly). `carbon_g` / + `water_l` are left at 0 (see `apply_footprint`). + """ + acc: dict[int, list[float]] = {} # bucket_ts -> [raw_energy_wh, covered_s] + + series_lists = _series_samples(cpu_result, 1e-6) + _series_samples(gpu_result, 1.0) + for samples in series_lists: + clean = sanitize(samples, data_quality) + if len(clean) < 2: + continue + for b, (e, c) in _integrate_series(clean, step_s, bucket_s, gap_factor).items(): + cur = acc.setdefault(b, [0.0, 0.0]) + cur[0] += e + cur[1] = max(cur[1], c) # max, not sum: covered time of the busiest source + + buckets = [] + for b in sorted(acc): + raw_wh, covered = acc[b] + energy_wh = raw_wh * tue_factor + hours = covered / 3600 if covered else 0.0 + buckets.append( + Bucket( + ts=iso(b), + energy_wh=energy_wh, + # mean power over the covered part of the bucket, normalized like + # energy_wh so `energy_wh ~= power_w_avg * bucket_hours` + power_w_avg=(energy_wh / hours) if hours else 0.0, + carbon_g=0.0, + water_l=0.0, + ) + ) + return buckets + + +def instant_power( + cpu_result: list[dict], + gpu_result: list[dict], + tue_factor: float, + data_quality: dict | None = None, +) -> float | None: + """ + Current draw from a range query: the sum of the most recent (sanitized) + sample of every power series, normalized by TUE. Mimir data is immutable + once ingested, so the actual last sample is always used, however old it + is -- `None` only when a series has no sample at all (nothing ingested + yet, or dropped entirely by `sanitize`). + """ + raw_w, fresh = 0.0, False + for scale, result in ((1e-6, cpu_result), (1.0, gpu_result)): + for samples in _series_samples(result, scale): + clean = sanitize(samples, data_quality) + if clean: + raw_w += clean[-1][1] + fresh = True + return raw_w * tue_factor if fresh else None + + +def merge_buckets(bucket_lists: list[list[Bucket]]) -> list[Bucket]: + """ + Sum buckets from several sources by timestamp: the datacenters a deployment + ran in, or the deployments of one user. Inputs share the 15-min grid. + """ + acc: dict[str, Bucket] = {} + for buckets in bucket_lists: + for bk in buckets: + cur = acc.get(bk.ts) + if cur is None: + acc[bk.ts] = bk + else: + acc[bk.ts] = cur._replace( + energy_wh=cur.energy_wh + bk.energy_wh, + power_w_avg=cur.power_w_avg + bk.power_w_avg, + carbon_g=cur.carbon_g + bk.carbon_g, + water_l=cur.water_l + bk.water_l, + ) + return [acc[k] for k in sorted(acc)] + + +# --------------------------------------------------------------------------- +# Footprint +# --------------------------------------------------------------------------- + + +def _prep_series(series): + """[[iso_ts, value], ...] -> (sorted ts floats, values) for bisect lookup.""" + if not series: + return [], [] + parsed = sorted((parse_iso(t), float(v)) for t, v in series) + return [p[0] for p in parsed], [p[1] for p in parsed] + + +def intensity_at(ts_arr, val_arr, ts: float, default: float) -> float: + """Value of the last sample with sample_ts <= ts; `default` if none.""" + if not ts_arr: + return default + i = bisect.bisect_right(ts_arr, ts) + return val_arr[i - 1] if i > 0 else default + + +def apply_footprint( + buckets: list[Bucket], + carbon_series=None, + water_series=None, +) -> list[Bucket]: + """ + Fill `carbon_g` / `water_l` per bucket = (energy_wh / 1000) * intensity(ts), + with the Wattnet 15-min series (already normalized energy). Falls back to + GreenDirector.DEFAULTS when a series is missing or predates the bucket. + """ + c_ts, c_val = _prep_series(carbon_series) + w_ts, w_val = _prep_series(water_series) + + out = [] + for bk in buckets: + ts = parse_iso(bk.ts) + kwh = bk.energy_wh / 1000 + out.append( + bk._replace( + carbon_g=kwh * intensity_at(c_ts, c_val, ts, DEFAULT_CARBON), + water_l=kwh * intensity_at(w_ts, w_val, ts, DEFAULT_WATER), + ) + ) + return out + + +# --------------------------------------------------------------------------- +# Downsampling (for the time-series responses) +# --------------------------------------------------------------------------- + + +def downsample(buckets: list[Bucket], target_points: int) -> list[Bucket]: + """Aggregate consecutive buckets so the result has at most `target_points`.""" + n = len(buckets) + if n <= target_points or target_points < 1: + return buckets + + group = -(-n // target_points) # ceil + out = [] + for i in range(0, n, group): + chunk = buckets[i : i + group] + out.append( + Bucket( + ts=chunk[0].ts, + energy_wh=sum(b.energy_wh for b in chunk), + power_w_avg=sum(b.power_w_avg for b in chunk) / len(chunk), + carbon_g=sum(b.carbon_g for b in chunk), + water_l=sum(b.water_l for b in chunk), + ) + ) + return out diff --git a/ai4papi/energy/config.py b/ai4papi/energy/config.py new file mode 100644 index 00000000..70bcbe3e --- /dev/null +++ b/ai4papi/energy/config.py @@ -0,0 +1,81 @@ +""" +Shared configuration and tiny helpers of the energy-accounting feature. +""" + +import datetime + +from cachetools import TTLCache + +import ai4papi.conf as papiconf +from ai4papi.clients import mimir +from ai4papi.energy import store + +# Nomad job-name prefixes that are deployments (as opposed to e.g. traefik) +JOB_PREFIXES = ("module", "tool", "batch", "try") +KEYS = ("energy_wh", "carbon_g", "water_l") # TUE-normalized totals kept per accum +EMPTY = dict.fromkeys(KEYS, 0.0) + +# Pseudo-namespace for the datacenter-level documents. Does not collide with any +# VO namespace nor with the deployment sweep's namespace loop. +CLUSTER_NS = "_cluster" + + +def get() -> dict: + """The `energy` section of the main config.""" + return papiconf.MAIN_CONF["energy"] + + +def enabled() -> bool: + return ( + bool(get().get("enabled")) + and store.base_dir() is not None + and bool(mimir.MIMIR_USER) + ) + + +def topup_cache(maxsize: int) -> TTLCache: + """Cache for the live path: shorter than the dashboard poll, so the counter moves.""" + return TTLCache(maxsize=maxsize, ttl=get().get("topup_cache_seconds", 25)) + + +def now() -> datetime.datetime: + return datetime.datetime.now(datetime.timezone.utc) + + +def dt(ts: float) -> datetime.datetime: + return datetime.datetime.fromtimestamp(ts, datetime.timezone.utc) + + +def floor(ts: float, step: int) -> float: + return (int(ts) // step) * step + + +def tue(dc: str) -> float: + return (papiconf.datacenters.get(dc) or {}).get("TUE") or get()[ + "default_tue_factor" + ] + + +def cluster_dq() -> dict: + """ + Data-quality config for the datacenter sweep: reuse `energy.data_quality` + but drop the absolute `max_power_w` ceiling (it is calibrated per allocation + / per GPU, far below a whole host). Negative-drop and the scale-free spike + check still apply, per host / per GPU series. + """ + return {**(get().get("data_quality") or {}), "max_power_w": 0} + + +def sum_totals(buckets) -> dict: + return {k: sum(b[k] for b in buckets) for k in KEYS} + + +def add_totals(a: dict, b: dict) -> dict: + return {k: a.get(k, 0.0) + b.get(k, 0.0) for k in KEYS} + + +def log_safe(value: str) -> str: + """Strip CR/LF from an identifier that may come straight from a request + path or auth token before it is interpolated into a log message, so it + cannot be used to forge log lines (CWE-117).""" + return str(value).replace("\r", "").replace("\n", "") diff --git a/ai4papi/energy/live.py b/ai4papi/energy/live.py new file mode 100644 index 00000000..082afd5e --- /dev/null +++ b/ai4papi/energy/live.py @@ -0,0 +1,169 @@ +""" +Live top-up of the energy-accounting feature (read path). + +The sweep only reaches `pointer_ts` (up to one sweep interval behind). On each +API read, `topup(doc)` adds the `pointer_ts -> now` slice live from Mimir so the +dashboard counter moves every ~30s, and `live_series(doc)` returns that same +slice as time-series points. Both work on a deployment doc and on a datacenter +doc (`namespace == CLUSTER_NS`). +""" + +import logging +import threading + +from cachetools import cached + +from ai4papi.energy import compute, config, query +from ai4papi.wattnet import green_director + +LOG = logging.getLogger(__name__) + +_live_cache = config.topup_cache(8192) + + +def _live_window(pointer_iso: str) -> tuple[float, float]: + """ + `[pointer_ts, now]`, clamped on the left so a long-dormant pointer does not + trigger a huge query. No ingestion-lag cushion: Mimir data is immutable + once ingested (never revised), so querying right up to "now" cannot return + a wrong value -- worst case, a sample scraped a moment ago is not visible + *yet* and simply is not the one returned as "last", which self-corrects on + the very next call. Used by `topup` for both a real, persisted accumulator + and `live_only_doc`'s transient one (nothing persisted): both get the + freshest number Mimir can currently give. + """ + cfg = config.get() + end_ts = config.now().timestamp() + start_ts = max( + compute.parse_iso(pointer_iso), + end_ts - (cfg["sweep_seconds"] + 4 * cfg["series_bucket_seconds"]), + ) + return start_ts, end_ts + + +def _running_key(doc: dict | None) -> tuple | None: + """Hashable summary of what a live query depends on; `None` if there is + nothing to query (no data yet, or a deployment with no running allocation).""" + if not doc: + return None + pointer = doc.get("pointer_ts") or doc.get("settled_ts") + if not doc.get("metrics_available", False) or not pointer: + return None + if doc.get("namespace") == config.CLUSTER_NS: + return (config.CLUSTER_NS, doc["deployment_uuid"], pointer, ()) + running = doc.get("active_allocs") or [] + if not running: + return None + return ( + doc["namespace"], + doc["deployment_uuid"], + pointer, + tuple(sorted((a["alloc_id"], a["datacenter"]) for a in running)), + ) + + +def _sources(key: tuple) -> list[tuple]: + """`(dc, cpu_query, gpu_query, data_quality)` per datacenter.""" + ns, uuid, _pointer, alloc_pairs = key + if ns == config.CLUSTER_NS: + return [(uuid, *compute.cluster_promql(uuid), config.cluster_dq())] + + by_dc: dict[str, list[str]] = {} + for alloc_id, dc in alloc_pairs: + by_dc.setdefault(dc, []).append(alloc_id) + dq = config.get().get("data_quality") + return [(dc, *compute.power_promql(ids), dq) for dc, ids in by_dc.items()] + + +def _live_buckets(key: tuple, start_ts: float, end_ts: float, bucket_s: int): + """ + Integrate the live window from Mimir into `bucket_s` buckets (per DC, merged), + plus the current draw from the last sample of that same query. Returns + `(buckets, power_w)`; `power_w` is `None` only when a series has no sample + at all in the window (nothing to read yet), never because the last sample + is "too old" -- Mimir data is immutable once ingested, so the actual last + point is always the right one to show, however old it is. + """ + step_s = int(config.get()["query_step_seconds"]) + if end_ts - start_ts < step_s: + return [], None + + per_dc = [] + inst_w = 0.0 + any_inst = False + for dc, cpu_q, gpu_q, dq in _sources(key): + cpu, gpu = query.fetch_power(cpu_q, gpu_q, start_ts, end_ts, step_s) + tue = config.tue(dc) + buckets = compute.integrate(cpu, gpu, step_s, bucket_s, tue, data_quality=dq) + metrics = green_director.metrics.get(dc, {}) + per_dc.append( + compute.apply_footprint( + buckets, metrics.get("carbon"), metrics.get("water") + ) + ) + iw = compute.instant_power(cpu, gpu, tue, dq) + if iw is not None: + inst_w += iw + any_inst = True + + return compute.merge_buckets(per_dc), (inst_w if any_inst else None) + + +def topup(doc: dict | None) -> dict | None: + """ + Live `pointer_ts -> now` slice as a single delta: dict with the summed energy + deltas, `power_w` (the last sample read, i.e. the current draw) and + `live_as_of`; `{"degraded": True}` on failure; None if not applicable. + Cached for `topup_cache_seconds`. + """ + key = _running_key(doc) + if key is None: + return None + try: + return _compute_topup(key) + except Exception: + LOG.warning( + "energy top-up failed for %s", + config.log_safe(doc.get("deployment_uuid")), + exc_info=True, + ) + return {"degraded": True} + + +@cached(_live_cache, lock=threading.Lock()) +def _compute_topup(key: tuple) -> dict: + start_ts, end_ts = _live_window(key[2]) + result = { + **config.EMPTY, + "power_w": None, + "live_as_of": compute.iso(end_ts), + "degraded": False, + } + span = int(end_ts - start_ts) + int(config.get()["query_step_seconds"]) + buckets, inst_w = _live_buckets(key, start_ts, end_ts, bucket_s=max(span, 1)) + for b in buckets: + for k in config.KEYS: + result[k] += getattr(b, k) + # `power_w` is the last sample read (current draw), not a window mean + result["power_w"] = inst_w + return result + + +def live_series(doc: dict | None, bucket_s: int) -> list[dict]: + """ + Live tail as time-series points (so the detail-view graph keeps growing). + `[]` if not applicable or on any failure. + """ + key = _running_key(doc) + if key is None: + return [] + start_ts, end_ts = _live_window(key[2]) + try: + buckets, _ = _live_buckets(key, start_ts, end_ts, bucket_s=max(bucket_s, 1)) + except Exception: + LOG.warning( + "energy live_series failed for %s", + config.log_safe(doc.get("deployment_uuid")), + ) + return [] + return [{**b._asdict(), "live": True} for b in buckets] diff --git a/ai4papi/energy/ondemand.py b/ai4papi/energy/ondemand.py new file mode 100644 index 00000000..0fd1e838 --- /dev/null +++ b/ai4papi/energy/ondemand.py @@ -0,0 +1,185 @@ +""" +On-demand sweep for the read path of the energy-accounting feature. + +When a deployment has no accumulator doc yet (just deployed, or the background +sweep has not reached it), the read API sweeps it on the spot instead of +waiting up to `sweep_seconds`: `ensure_swept` for a single deployment, +`ensure_swept_bulk` for the list / per-user reads. `live_only_doc` covers the +last gap, a deployment too young for even one completed bucket. +""" + +import logging +import threading + +from cachetools import cached + +from ai4papi.clients import mimir +from ai4papi.energy import compute, config, store, sweep +from ai4papi.nomad_utils import Nomad + +LOG = logging.getLogger(__name__) + + +def _sweep_one(ns: str, uuid: str, now) -> bool: + """Fetch one Nomad job and run `process_single` on it. `False` when the job + cannot be found (not a deployment, or already gone) or the sweep failed.""" + try: + j = Nomad.job.get_job(id_=uuid, namespace=ns) + except Exception: + return False + try: + sweep.process_single(sweep.info_from_job(j, ns), now) + except mimir.MimirError as e: + LOG.warning("on-demand energy sweep skipped, Mimir unavailable: %s", e) + return False + except Exception: + LOG.warning( + "on-demand energy sweep failed for %s/%s", + config.log_safe(ns), + config.log_safe(uuid), + exc_info=True, + ) + return False + return True + + +def ensure_swept(ns: str, uuid: str) -> dict | None: + """ + On-demand equivalent of one sweep pass for a single deployment, called from + the read path when no accumulator doc exists yet: a deployment that already + has real samples in Mimir does not have to wait up to `sweep_seconds` for + the background thread to reach it before the API can report anything. + + A no-op (returns the existing doc untouched, no Nomad/Mimir call) once the + background sweep has processed this deployment at least once, so it never + duplicates the periodic sweep's work. `None` if the job cannot be found (not + a deployment, or already gone). + """ + if not config.enabled(): + return None + doc = store.read_accum(ns, uuid) + if doc is not None: + return doc + if not _sweep_one(ns, uuid, config.now()): + return None + return store.read_accum(ns, uuid) + + +_bulk_sweep_cache = config.topup_cache(2048) + + +@cached(_bulk_sweep_cache, lock=threading.Lock()) +def ensure_swept_bulk(ns: str, owner: str) -> list[dict]: + """ + On-demand equivalent of one sweep pass, restricted to one owner's active + deployments in `ns`: the bulk counterpart of `ensure_swept`, used by the + list (`get_accumulated_bulk`) and per-user (`get_user_energy`) read paths so + a just-deployed job shows up there too, not only on its own detail endpoint. + + Lists the owner's active jobs (cheap, no per-job fetch) and only runs + `process_single` for the ones with no accumulator doc yet; a no-op once the + background sweep (or a prior on-demand call) has reached all of them. + Cached for `topup_cache_seconds`: without it, every list/`/stats/user` read + would list Nomad on every single request, forever, not just during the + bootstrap window -- a few seconds of staleness for a listing that itself + only matters for brand-new deployments is a good trade. + + Returns the filtered job stubs (`[{"ID": ..., "Name": ...}, ...]`) so the + caller can also try `live_only_doc` for whichever of them still have no doc + afterwards (a deployment too young for even one completed bucket), without + listing Nomad a second time. `[]` when disabled or the listing itself fails. + """ + if not config.enabled(): + return [] + try: + jobs = Nomad.jobs.get_jobs( + namespace=ns, + filter_=f'Meta.owner == "{owner}" and Status != "dead"', + ) + except Exception: + LOG.warning( + "could not list jobs for on-demand sweep (%s/%s)", + config.log_safe(ns), + config.log_safe(owner), + ) + return [] + now = config.now() + stubs = [s for s in jobs if s["Name"].startswith(config.JOB_PREFIXES)] + for stub in stubs: + if store.read_accum(ns, stub["ID"]) is not None: + continue + _sweep_one(ns, stub["ID"], now) + return stubs + + +_live_only_doc_cache = config.topup_cache(8192) + + +@cached(_live_only_doc_cache, lock=threading.Lock()) +def live_only_doc(ns: str, uuid: str) -> dict | None: + """ + Transient stand-in for a deployment `process_single` has not written any doc + for at all yet -- typically younger than one `series_bucket_seconds`, so it + has no completed bucket to fold into an accumulator (see `ensure_swept`, + which still leaves such a deployment with no doc by design: there is + nothing settled to persist). Never written to the store. + + Cached for `topup_cache_seconds`: without it, every read of a deployment + still in this bootstrap window would fetch its Nomad job and resolve its + allocations from scratch on every single request for as long as it stays + doc-less (which can be several minutes). A few seconds of staleness on + which allocation is "current" is a good trade against that. + + Shaped so the normal `topup` / `live_series` machinery -- and therefore + `energy._to_stats` / `_apply_topup` / `_build_series` -- treats it like + a real doc with an all-zero accumulator and `pointer_ts = submit_time`: the + whole reported number then comes from the live top-up's Mimir query, capped + at the same lookback `topup` itself uses. `None` if the job cannot be + resolved to a running, monitored allocation (not found, queued, or no + allocation yet). + """ + if not config.enabled(): + return None + try: + j = Nomad.job.get_job(id_=uuid, namespace=ns) + except Exception: + return None + info = sweep.info_from_job(j, ns) + try: + allocs_by_dc, _windows, running_allocs, _partial = compute.resolve_alloc_ids( + uuid, ns + ) + except Exception: + return None + if not running_allocs: + return None + submit = info.get("submit_time") + submit_ts = submit or config.now().timestamp() + # `window_start` below is pinned to the exact submit time, same as + # `process_single`'s first-encounter `window_start` -- not started late or + # clamped, so `complete` is `True` unless we had no real `submit_time` to + # pin it to at all (job fetched with no `SubmitTime`, an edge case) or it + # somehow predates the lookback window (impossible in practice: this path + # only ever runs for a deployment too young to have a doc yet). + complete = ( + bool(submit) + and submit + >= config.now().timestamp() - config.get()["initial_lookback_hours"] * 3600 + ) + return { + "deployment_uuid": uuid, + "namespace": ns, + "job_name": info.get("job_name"), + "owner": info.get("owner"), + "status": "running", + "metrics_available": True, + "complete": complete, + "window_start": compute.iso(submit_ts), + "settled_ts": compute.iso(submit_ts), + "pointer_ts": compute.iso(submit_ts), + "active_allocs": running_allocs, + "datacenters": sorted(allocs_by_dc), + "tail": [], + "settled": dict(config.EMPTY), + "accumulated": dict(config.EMPTY), + } diff --git a/ai4papi/energy/query.py b/ai4papi/energy/query.py new file mode 100644 index 00000000..1aa5da71 --- /dev/null +++ b/ai4papi/energy/query.py @@ -0,0 +1,22 @@ +""" +Mimir power queries shared by the sweep (write path) and the live top-up. +""" + +from ai4papi.clients import mimir +from ai4papi.energy import config + + +def fetch_power( + cpu_q: str, gpu_q: str, start_ts: float, end_ts: float, step_s: int +) -> tuple[list, list]: + """ + `(cpu_result, gpu_result)` range queries over [start, end]. A missing metric + comes back as an empty result, not an error (a GPU-only site has no `scaph_*` + series), so a query that fails is a real failure and propagates: the sweep + skips that deployment and retries on the next pass, the live path reports + `degraded`. Long windows are split in chunks. + """ + start, end = config.dt(start_ts), config.dt(end_ts) + chunked = end_ts - start_ts > mimir.MAX_POINTS_PER_QUERY * step_s + qfn = mimir.query_range_chunked if chunked else mimir.query_range + return qfn(cpu_q, start, end, step_s), qfn(gpu_q, start, end, step_s) diff --git a/ai4papi/energy/series.py b/ai4papi/energy/series.py new file mode 100644 index 00000000..98a30d38 --- /dev/null +++ b/ai4papi/energy/series.py @@ -0,0 +1,61 @@ +""" +Time-series building of the energy-accounting feature: turns persisted buckets +and live Mimir tails into the points served by the API. +""" + +from ai4papi.energy import compute + + +def _bucket(b: dict) -> compute.Bucket: + return compute.Bucket( + ts=b["ts"], + energy_wh=b.get("energy_wh", 0.0), + power_w_avg=b.get("power_w_avg", 0.0), + carbon_g=b.get("carbon_g", 0.0), + water_l=b.get("water_l", 0.0), + ) + + +def _point(b: compute.Bucket, live: bool) -> dict: + return { + "ts": b.ts, + "power_w": b.power_w_avg, + "energy_wh": b.energy_wh, + "carbon_g": b.carbon_g, + "water_l": b.water_l, + "live": live, + } + + +def persisted(archive: list[dict], doc: dict) -> list[dict]: + """Persisted 15-min buckets of one entity: the consolidated archive plus the + still-mutable 24h tail.""" + return list(archive) + list(doc.get("tail") or []) + + +def build_series( + persisted_lists: list[list[dict]], live: list[list[dict]], target_points: int +) -> list[dict]: + """ + Time series of one or several entities (a deployment, a user's deployments, + a datacenter). The persisted buckets are merged by timestamp across entities + and `compute.downsample`d once, so the resolution is uniform and capped at + `target_points`; the live Mimir tails are merged the same way and appended + un-downsampled. The points carry the per-bucket increments only; a running + total is `cumsum(series[].energy_wh)` and matches `accumulated.energy_wh`. + """ + + def merged(lists: list[list[dict]]) -> list[compute.Bucket]: + out = compute.merge_buckets([[_bucket(b) for b in bs] for bs in lists]) + return sorted(out, key=lambda b: b.ts) + + series = [ + _point(b, live=False) + for b in compute.downsample(merged(persisted_lists), target_points) + ] + series += [ + _point(b, live=True) + for b in merged(live) + if not series or b.ts > series[-1]["ts"] + ] + return series diff --git a/ai4papi/energy/store.py b/ai4papi/energy/store.py new file mode 100644 index 00000000..1009bfe9 --- /dev/null +++ b/ai4papi/energy/store.py @@ -0,0 +1,161 @@ +""" +File-based store for the energy-accounting feature. + +One JSON document per deployment under `$ACCOUNTING_PTH/energy//`, +plus an append-only `.series.jsonl` with the consolidated 15-min buckets +(`energy.series_bucket_seconds`). The per-datacenter accumulators live the same +way under the `_cluster` pseudo-namespace. Same convention as +`ai4papi.utils.retrieve_from_snapshots`. The interface is deliberately small so +it can be swapped for a database later. +""" + +import json +import os +import re +import threading +import time +from pathlib import Path + + +def base_dir() -> Path | None: + """`$ACCOUNTING_PTH/energy`, or None if accounting storage is not configured.""" + root = os.environ.get("ACCOUNTING_PTH") + if not root: + return None + return Path(root) / "energy" + + +# A namespace or deployment id is used as a file name: only a plain name is allowed +# (no separators, no leading dot), so an id can never point outside the store. +_SAFE_NAME = re.compile(r"[A-Za-z0-9_][A-Za-z0-9._-]*") + + +def _name(value: str) -> str | None: + return os.path.basename(value) if _SAFE_NAME.fullmatch(value) else None + + +def _ns_dir(ns: str) -> Path | None: + base, name = base_dir(), _name(ns) + return (base / name) if base and name else None + + +def _accum_path(ns: str, uuid: str) -> Path | None: + d, name = _ns_dir(ns), _name(uuid) + return (d / f"{name}.accum.json") if d and name else None + + +def _series_path(ns: str, uuid: str) -> Path | None: + d, name = _ns_dir(ns), _name(uuid) + return (d / f"{name}.series.jsonl") if d and name else None + + +def _atomic_write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + with open(tmp, "w", encoding="utf-8") as f: + f.write(text) + os.replace(tmp, path) + + +# --------------------------------------------------------------------------- +# accumulator document +# --------------------------------------------------------------------------- + + +def read_accum(ns: str, uuid: str) -> dict | None: + path = _accum_path(ns, uuid) + if not path or not path.is_file(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + + +def write_accum(ns: str, uuid: str, doc: dict) -> None: + path = _accum_path(ns, uuid) + if not path: + return + _atomic_write(path, json.dumps(doc, separators=(",", ":"))) + with _owner_lock: + entry = _owner_idx.get(ns) + if entry: + entry[1].setdefault(doc.get("owner"), set()).add(uuid) + + +def iter_namespace(ns: str): + """Yield (uuid, doc) for every accumulator document in the namespace.""" + d = _ns_dir(ns) + if not d or not d.is_dir(): + return + for path in d.glob("*.accum.json"): + uuid = path.name[: -len(".accum.json")] + try: + yield uuid, json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + + +# owner -> uuids of a namespace, so listing one user's deployments does not parse +# every document of the namespace. Kept in sync by `write_accum` and rebuilt from +# disk every `_OWNER_IDX_TTL` seconds (heals anything written by another process). +# shortcut: assumes one PAPI process writes the store, upgrade to a database index +# if PAPI runs as several replicas on the same storage. +_OWNER_IDX_TTL = 300 +_owner_idx: dict[str, tuple[float, dict[str | None, set[str]]]] = {} +_owner_lock = threading.Lock() + + +def _owner_uuids(ns: str, owner: str) -> list[str]: + with _owner_lock: + entry = _owner_idx.get(ns) + if entry is None or time.monotonic() - entry[0] > _OWNER_IDX_TTL: + idx: dict[str | None, set[str]] = {} + for uuid, doc in iter_namespace(ns): + idx.setdefault(doc.get("owner"), set()).add(uuid) + entry = _owner_idx[ns] = (time.monotonic(), idx) + return sorted(entry[1].get(owner, ())) + + +def iter_owner(ns: str, owner: str): + """Yield (uuid, doc) for the accumulator documents owned by `owner`.""" + for uuid in _owner_uuids(ns, owner): + doc = read_accum(ns, uuid) + if doc is not None and doc.get("owner") == owner: + yield uuid, doc + + +# --------------------------------------------------------------------------- +# consolidated time series (append-only .series.jsonl) +# --------------------------------------------------------------------------- + + +def append_series(ns: str, uuid: str, buckets: list[dict]) -> None: + path = _series_path(ns, uuid) + if not path or not buckets: + return + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "a", encoding="utf-8") as f: + f.writelines(json.dumps(b, separators=(",", ":")) + "\n" for b in buckets) + + +def read_series(ns: str, uuid: str, start_ts: str | None, end_ts: str | None): + path = _series_path(ns, uuid) + if not path or not path.is_file(): + return [] + out: dict[str, dict] = {} # by ts: a bucket appended twice counts once + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + b = json.loads(line) + except json.JSONDecodeError: + continue + if start_ts and b["ts"] < start_ts: + continue + if end_ts and b["ts"] > end_ts: + continue + out[b["ts"]] = b + return [out[ts] for ts in sorted(out)] diff --git a/ai4papi/energy/sweep.py b/ai4papi/energy/sweep.py new file mode 100644 index 00000000..41323bc2 --- /dev/null +++ b/ai4papi/energy/sweep.py @@ -0,0 +1,447 @@ +""" +Background sweep of the energy-accounting feature (write path). + +`run_sweep()` runs on a `@repeat_every` thread in `ai4papi.main`. Per deployment +(and per datacenter for `/stats/cluster`) it integrates the un-consolidated +window of Mimir power into energy buckets (`energy.series_bucket_seconds`, +15 min), applies the per-datacenter TUE and the Wattnet footprint, keeps a +running accumulator plus a ~24h "tail" that is re-computed every sweep (so the +footprint self-heals as Wattnet settles), and persists a small JSON document. + +The read-side counterparts live in `ai4papi.energy.live` (live top-up) and +`ai4papi.energy.ondemand` (sweep a deployment from the read path). +""" + +import logging +from typing import NamedTuple + +import ai4papi.conf as papiconf +from ai4papi.clients import mimir +from ai4papi.energy import compute, config, query, store +from ai4papi.nomad_utils import Nomad +from ai4papi.wattnet import green_director + +LOG = logging.getLogger(__name__) + + +class Clock(NamedTuple): + """Time references of one sweep pass (all epoch seconds unless noted).""" + + now_ts: float + bucket_s: int + step_s: int + extract_to: float # newest instant to integrate (now minus the ingestion lag) + revision_cut: float # buckets older than this are final (footprint settled) + retention_cut: float # oldest instant Mimir still holds + lookback_ts: float # how far back a first-seen deployment is integrated + + +def _clock(now) -> Clock: + cfg = config.get() + now_ts = now.timestamp() + bucket_s = int(cfg["series_bucket_seconds"]) + return Clock( + now_ts=now_ts, + bucket_s=bucket_s, + step_s=int(cfg["query_step_seconds"]), + extract_to=config.floor(now_ts - cfg["query_lag_seconds"], bucket_s), + revision_cut=now_ts - cfg["footprint_revision_hours"] * 3600, + retention_cut=now_ts - cfg["mimir_retention_days"] * 86400, + lookback_ts=now_ts - cfg["initial_lookback_hours"] * 3600, + ) + + +def _resume_point(doc: dict, t: Clock, submit: float | None = None): + """ + `(settled_ts, complete)`: where integration resumes. On first encounter never + look back further than `initial_lookback_hours` (pre-existing deployments are + not fully backfilled); never further than Mimir retention either. + """ + if doc.get("settled_ts"): + settled_ts = compute.parse_iso(doc["settled_ts"]) + complete = doc.get("complete", True) + else: + settled_ts = max(submit or t.lookback_ts, t.lookback_ts) + complete = bool(submit) and submit >= t.lookback_ts + if settled_ts < t.retention_cut: + settled_ts, complete = t.retention_cut, False + return config.floor(settled_ts, t.bucket_s), complete + + +def _earliest_bucket_ts(ns: str, uuid: str, doc: dict) -> float | None: + """Timestamp of the oldest 15-min bucket on record: the consolidated + `.series.jsonl` if there is one, else the oldest still in `doc["tail"]`.""" + archive = store.read_series(ns, uuid, None, None) + if archive: + return compute.parse_iso(archive[0]["ts"]) + tail = doc.get("tail") or [] + if tail: + return min(compute.parse_iso(b["ts"]) for b in tail) + return None + + +def _window_start(ns: str, uuid: str, doc: dict, settled_ts: float, t: Clock) -> float: + """ + True start of the accumulation window, exposed to the API as `since`. + + `settled_ts` cannot serve this role: it marches forward every sweep as + buckets age into `doc["settled"]`, while the accumulated totals (and the + persisted series) keep every bucket since the beginning. + + Pinned once and only moved forward if it falls behind Mimir retention: + - fresh doc: the first-sweep `settled_ts` is the window start; + - doc written before this field existed (`settled` / `tail` already there, + `settled_ts` already advanced): recover it from the oldest bucket on record. + """ + retention_floor = config.floor(t.retention_cut, t.bucket_s) + if doc.get("window_start"): + return max(compute.parse_iso(doc["window_start"]), retention_floor) + if "settled" in doc or doc.get("tail"): + earliest = _earliest_bucket_ts(ns, uuid, doc) + if earliest is not None: + return max(min(earliest, settled_ts), retention_floor) + return max(settled_ts, retention_floor) + + +def _footprint_series(dc: str, start_ts: float, end_ts: float): + """ + (carbon, water) intensity series for a datacenter over [start, end]. + In steady state uses the 7-day window that `green_director` already keeps; + only fetches a wider range from Wattnet when backfilling further back. + `apply_footprint` falls back to DEFAULTS when a series is empty/None. + """ + metrics = green_director.metrics.get(dc, {}) + if start_ts >= config.now().timestamp() - 7 * 86400: + return metrics.get("carbon"), metrics.get("water") + try: + fetched = green_director.footprint_series( + dc, config.dt(start_ts), config.dt(end_ts) + ) + except Exception: + LOG.warning("wattnet footprint_series failed for %s", dc) + fetched = {} + return ( + fetched.get("carbon") or metrics.get("carbon"), + fetched.get("water") or metrics.get("water"), + ) + + +def _with_footprint(dc: str, buckets: list, settled_ts: float, t: Clock) -> list: + if not buckets: + return buckets + carbon, water = _footprint_series(dc, settled_ts, t.now_ts) + return compute.apply_footprint(buckets, carbon, water) + + +def _complete_buckets(buckets, t: Clock) -> list: + """Only buckets that are fully inside the integrated window.""" + return [b for b in buckets if compute.parse_iso(b.ts) + t.bucket_s <= t.extract_to] + + +# --------------------------------------------------------------------------- +# sweep driver +# --------------------------------------------------------------------------- + + +def run_sweep() -> None: + if not config.enabled(): + return + now = config.now() + for ns in set(papiconf.MAIN_CONF["nomad"]["namespaces"].values()): + try: + _sweep_namespace(ns, now) + except Exception: + LOG.exception("energy sweep failed for namespace %s", ns) + try: + _sweep_cluster(now) + except Exception: + LOG.exception("cluster energy sweep failed") + + +def info_from_job(j: dict, ns: str) -> dict: + """`info` dict `process_single` expects, from a raw Nomad job.""" + meta = j.get("Meta") or {} + return { + "uuid": j["ID"], + "job_name": j["Name"], + "namespace": ns, + "owner": meta.get("owner", ""), + "submit_time": (j["SubmitTime"] / 1e9) if j.get("SubmitTime") else None, + "status": "running", + } + + +def _sweep_namespace(ns: str, now) -> None: + try: + jobs = Nomad.jobs.get_jobs(namespace=ns, filter_='Status != "dead"') + except Exception: + LOG.exception("could not list jobs for namespace %s", ns) + return + + seen = set() + for stub in jobs: + if not stub["Name"].startswith(config.JOB_PREFIXES): + continue + seen.add(stub["ID"]) + # the job list stub does not carry Meta, fetch the full job for `owner` + try: + j = Nomad.job.get_job(id_=stub["ID"], namespace=ns) + except Exception: + LOG.exception("could not fetch job %s", stub["ID"]) + continue + try: + process_single(info_from_job(j, ns), now) + except mimir.MimirError as e: + LOG.warning("Mimir unavailable, stopping the sweep of %s: %s", ns, e) + return + except Exception: + LOG.exception("energy sweep failed for deployment %s", stub["ID"]) + + # close deployments whose Nomad job has been garbage-collected + for uuid, doc in store.iter_namespace(ns): + if uuid in seen or doc.get("closed_at"): + continue + info = { + "uuid": uuid, + "job_name": doc.get("job_name"), + "namespace": ns, + "owner": doc.get("owner", ""), + "submit_time": doc.get("submit_time_epoch"), + "status": "closed", + } + try: + process_single(info, now, closing=True) + except mimir.MimirError as e: + LOG.warning("Mimir unavailable, stopping the sweep of %s: %s", ns, e) + return + except Exception: + LOG.exception("energy close failed for deployment %s", uuid) + + +# --------------------------------------------------------------------------- +# per-deployment sweep +# --------------------------------------------------------------------------- + + +def _deployment_buckets( + doc: dict, allocs_by_dc: dict, alloc_windows: list, settled_ts: float, t: Clock +) -> list: + """ + Energy buckets of [settled_ts, extract_to] summed over the deployment's + datacenters, clipped to its actual life. Also records on `doc` the + energy-weighted mean TUE of what was integrated. + """ + if settled_ts >= t.extract_to: + return [] + + dq = config.get().get("data_quality") + per_dc = [] + tue_norm = tue_raw = 0.0 + for dc, alloc_ids in allocs_by_dc.items(): + cpu, gpu = query.fetch_power( + *compute.power_promql(alloc_ids), settled_ts, t.extract_to, t.step_s + ) + if not cpu and not gpu: + continue + tue = config.tue(dc) + buckets = compute.integrate( + cpu, gpu, t.step_s, t.bucket_s, tue, data_quality=dq + ) + e = sum(b.energy_wh for b in buckets) + tue_norm += e + tue_raw += e / tue + per_dc.append(_with_footprint(dc, buckets, settled_ts, t)) + + if tue_raw: + doc["tue_factor"] = tue_norm / tue_raw + elif "tue_factor" not in doc and allocs_by_dc: + doc["tue_factor"] = config.tue(next(iter(allocs_by_dc))) + + new_buckets = _complete_buckets(compute.merge_buckets(per_dc), t) + # clip to the deployment's actual life: drop buckets with no overlap with + # any allocation window. Guards against metric staleness (the per-alloc + # power series and `nomad_gpu_allocation_info` can linger a few minutes + # after an allocation ends, which would otherwise attribute idle GPU / + # host power to a finished deployment). + if alloc_windows: + new_buckets = [ + b + for b in new_buckets + if compute.running_seconds( + alloc_windows, + compute.parse_iso(b.ts), + compute.parse_iso(b.ts) + t.bucket_s, + ) + > 0 + ] + return new_buckets + + +def process_single(info: dict, now, closing: bool = False) -> None: + t = _clock(now) + ns, uuid = info["namespace"], info["uuid"] + + doc = store.read_accum(ns, uuid) or {} + doc.pop("partial", None) # dropped in favour of `coverage_ratio` (always a float) + + settled_ts, complete = _resume_point(doc, t, info.get("submit_time")) + window_start = _window_start(ns, uuid, doc, settled_ts, t) + + try: + allocs_by_dc, alloc_windows, running_allocs, partial_dc = ( + compute.resolve_alloc_ids(uuid, ns) + ) + except Exception: + allocs_by_dc, alloc_windows, running_allocs, partial_dc = {}, [], [], False + + # metadata (always refreshed) + doc.update( + { + "deployment_uuid": uuid, + "job_name": info.get("job_name") or doc.get("job_name"), + "namespace": ns, + "owner": info.get("owner") or doc.get("owner", ""), + "status": "closed" if closing else info.get("status", "running"), + "complete": complete, + "updated_at": compute.iso(t.now_ts), + "active_allocs": running_allocs, + } + ) + if info.get("submit_time"): + doc["submit_time"] = compute.iso(info["submit_time"]) + doc["submit_time_epoch"] = info["submit_time"] + if allocs_by_dc: + doc["datacenters"] = list(allocs_by_dc) + if closing and not doc.get("closed_at"): + doc["closed_at"] = compute.iso(t.now_ts) + + run_s = compute.running_seconds(alloc_windows, settled_ts, t.extract_to) + + # no monitored allocations: queued, or ran only in unmonitored sites + if not allocs_by_dc: + if "settled" not in doc and partial_dc and run_s > 2 * t.bucket_s: + doc.update({"metrics_available": False, "coverage_ratio": 0.0}) + store.write_accum(ns, uuid, doc) + return + + new_buckets = _deployment_buckets(doc, allocs_by_dc, alloc_windows, settled_ts, t) + + # best-effort coverage: no data while the deployment was clearly running + # means the node has no monitoring stack yet -> null, retry next sweep. + if not new_buckets and "settled" not in doc: + if run_s > 2 * t.bucket_s: + doc.update({"metrics_available": False, "coverage_ratio": 0.0}) + store.write_accum(ns, uuid, doc) + return + + doc["metrics_available"] = True + doc["window_start"] = compute.iso(window_start) + + aged_n, tail_n = _fold_and_accumulate(ns, uuid, doc, new_buckets, settled_ts, t) + + # coverage: fraction of the run time (including time on unmonitored sites, so + # `partial_dc` is captured implicitly) that ended up with power data. + covered_s = (tail_n + aged_n) * t.bucket_s + doc["coverage_ratio"] = round(min(1.0, covered_s / run_s), 3) if run_s > 0 else 1.0 + + store.write_accum(ns, uuid, doc) + + +def _fold_and_accumulate( + ns: str, + uuid: str, + doc: dict, + new_buckets: list, + settled_ts: float, + t: Clock, +) -> tuple[int, int]: + """ + Merge `new_buckets` (Bucket namedtuples) over `doc["tail"]` keyed by `ts`, + age the buckets older than `revision_cut` into `doc["settled"]` (appending + them to the persisted series), and recompute `settled_ts` / `accumulated` / + `pointer_ts` on `doc`. Returns `(aged_count, tail_count)`. + + Shared by the per-deployment (`process_single`) and per-datacenter + (`process_datacenter`) sweeps. + """ + settled = dict(doc.get("settled") or config.EMPTY) + + by_ts = {b["ts"]: b for b in (doc.get("tail") or [])} + for b in new_buckets: + by_ts[b.ts] = b._asdict() + + tail, aged_out = [], [] + for ts in sorted(by_ts): + (aged_out if compute.parse_iso(ts) < t.revision_cut else tail).append(by_ts[ts]) + + if aged_out: + settled = config.add_totals(settled, config.sum_totals(aged_out)) + store.append_series(ns, uuid, aged_out) + settled_ts = max(compute.parse_iso(b["ts"]) for b in aged_out) + t.bucket_s + + doc["settled"] = settled + doc["tail"] = tail + doc["settled_ts"] = compute.iso(settled_ts) + doc["accumulated"] = config.add_totals(settled, config.sum_totals(tail)) + doc["pointer_ts"] = compute.iso( + (compute.parse_iso(tail[-1]["ts"]) + t.bucket_s) if tail else settled_ts + ) + return len(aged_out), len(tail) + + +# --------------------------------------------------------------------------- +# per-datacenter sweep (cluster-level energy accumulator) +# --------------------------------------------------------------------------- + + +def _sweep_cluster(now) -> None: + for dc in config.get()["metrics_datacenters"]: + try: + process_datacenter(dc, now) + except Exception: + LOG.exception("cluster energy sweep failed for datacenter %s", dc) + + +def _cluster_buckets(dc: str, settled_ts: float, t: Clock) -> list: + if settled_ts >= t.extract_to: + return [] + cpu, gpu = query.fetch_power( + *compute.cluster_promql(dc), settled_ts, t.extract_to, t.step_s + ) + buckets = compute.integrate( + cpu, gpu, t.step_s, t.bucket_s, config.tue(dc), data_quality=config.cluster_dq() + ) + return _complete_buckets(_with_footprint(dc, buckets, settled_ts, t), t) + + +def process_datacenter(dc: str, now) -> None: + t = _clock(now) + ns = config.CLUSTER_NS + + doc = store.read_accum(ns, dc) or {} + settled_ts, complete = _resume_point(doc, t) + window_start = _window_start(ns, dc, doc, settled_ts, t) + + doc.update( + { + "datacenter": dc, + "deployment_uuid": dc, # reuse the same key name as deployment docs + "namespace": ns, + "complete": complete, + "updated_at": compute.iso(t.now_ts), + "tue_factor": config.tue(dc), + } + ) + + new_buckets = _cluster_buckets(dc, settled_ts, t) + + if not new_buckets and "settled" not in doc: + # datacenter has no host-power series yet: retry next sweep, no zeros + doc["metrics_available"] = False + store.write_accum(ns, dc, doc) + return + + doc["metrics_available"] = True + doc["window_start"] = compute.iso(window_start) + _fold_and_accumulate(ns, dc, doc, new_buckets, settled_ts, t) + store.write_accum(ns, dc, doc) diff --git a/ai4papi/main.py b/ai4papi/main.py index 50ccd7e9..06e3495e 100644 --- a/ai4papi/main.py +++ b/ai4papi/main.py @@ -11,6 +11,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi_utils.tasks import repeat_every +from ai4papi import energy as energy_accounting from ai4papi.conf import MAIN_CONF, paths, papi_branch, papi_commit from ai4papi.routers import v1 from ai4papi.routers.v1.stats.deployments import get_cluster_stats_bg @@ -48,6 +49,7 @@ async def lifespan(app: fastapi.FastAPI): # on startup await get_cluster_stats_thread() + await energy_accounting_thread() yield # on shutdown # (nothing to do) @@ -152,5 +154,19 @@ def get_cluster_stats_thread(): logging.exception("Error in background task get_cluster_stats_bg") +# Accumulate per-deployment energy / footprint stats in background task. +# No-op unless the `energy` feature is enabled and Mimir credentials are set. +@repeat_every(seconds=MAIN_CONF.get("energy", {}).get("sweep_seconds", 900)) +def energy_accounting_thread(): + """ + Incrementally integrate Mimir power into per-deployment energy/carbon/water. + Not async, same reasoning as get_cluster_stats_thread. + """ + try: + energy_accounting.run_sweep() + except Exception: + logging.exception("Error in background task energy_accounting") + + if __name__ == "__main__": run() diff --git a/ai4papi/routers/v1/batch.py b/ai4papi/routers/v1/batch.py index 82f3d293..d9e18e40 100644 --- a/ai4papi/routers/v1/batch.py +++ b/ai4papi/routers/v1/batch.py @@ -11,6 +11,7 @@ from fastapi.security import HTTPBearer from ai4papi import auth, module_patches, quotas, schemas, utils +from ai4papi import energy as energy_accounting import ai4papi.conf as papiconf import ai4papi.nomad_utils as nomad_utils from ai4papi.routers import v1 @@ -29,6 +30,7 @@ def get_deployments( vos: schemas.VoList = None, full_info: bool = False, + energy: bool = True, authorization=Depends(security), ): """ @@ -40,6 +42,9 @@ def get_deployments( * **full_info**: retrieve the full information of each deployment. Disabled by default, as it will increase latency too much if there are many deployments. + * **energy**: attach energy/footprint stats to each deployment (`energy` + key). Enabled by default. `full_info=false` -> accumulated `EnergyStats`; + `full_info=true` -> extended `EnergyTimeSeries`. """ # Retrieve authenticated user info auth_info = auth.get_user_info(token=authorization.credentials) @@ -58,6 +63,8 @@ def get_deployments( user_jobs = [] for vo in user_vos: + namespace = papiconf.MAIN_CONF["nomad"]["namespaces"][vo] + # Retrieve all jobs in namespace (including dead jobs) job_filter = ( 'Name matches "^batch" and ' @@ -65,17 +72,19 @@ def get_deployments( + f'Meta.owner == "{auth_info["id"]}"' ) jobs = nomad_utils.Nomad.jobs.get_jobs( - namespace=papiconf.MAIN_CONF["nomad"]["namespaces"][vo], + namespace=namespace, filter_=job_filter, ) # Retrieve info for jobs in namespace + vo_jobs = [] for j in jobs: try: job_info = get_deployment( vo=vo, deployment_uuid=j["ID"], full_info=full_info, + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), @@ -83,7 +92,16 @@ def get_deployments( except HTTPException: # not a module continue - user_jobs.append(job_info) + vo_jobs.append(job_info) + + # `full_info=False` (default): one cheap batch. `full_info=True`: the + # extended time series per deployment (heavier). + if energy: + energy_accounting.add_to_jobs( + vo_jobs, namespace, auth_info["id"], full_info + ) + + user_jobs.extend(vo_jobs) # Sort deployments by submission time in descending order sorted_jobs = sorted(user_jobs, key=lambda x: x["submit_time"], reverse=True) @@ -96,6 +114,7 @@ def get_deployment( vo: str, deployment_uuid: str, full_info: bool = True, + energy: bool = True, authorization=Depends(security), ): """ @@ -107,6 +126,9 @@ def get_deployment( * **deployment_uuid**: uuid of deployment to gather info about * **full_info**: retrieve the full information of that deployment (may increase latency) + * **energy**: attach energy/footprint stats (`energy` key). Enabled by + default. `full_info=true` (default here) -> extended `EnergyTimeSeries`; + `full_info=false` -> accumulated `EnergyStats`. Returns a dict with info """ @@ -132,6 +154,9 @@ def get_deployment( detail="This deployment is not a batch job.", ) + if energy: + energy_accounting.add_to_job(job, namespace, full_info) + return job @@ -202,6 +227,7 @@ async def create_deployment( # jobs) batch_deps = get_deployments( vos=[vo], + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), diff --git a/ai4papi/routers/v1/deployments/modules.py b/ai4papi/routers/v1/deployments/modules.py index 65ab1d70..271ef2ca 100644 --- a/ai4papi/routers/v1/deployments/modules.py +++ b/ai4papi/routers/v1/deployments/modules.py @@ -9,6 +9,7 @@ from fastapi.security import HTTPBearer from ai4papi import auth, module_patches, quotas, schemas, utils +from ai4papi import energy as energy_accounting from ai4papi.wattnet import green_director import ai4papi.conf as papiconf import ai4papi.nomad_utils as nomad_utils @@ -33,6 +34,7 @@ def get_deployments( vos: schemas.VoList = None, full_info: bool = False, + energy: bool = True, authorization=Depends(security), ): """ @@ -44,6 +46,9 @@ def get_deployments( * **full_info**: retrieve the full information of each deployment. Disabled by default, as it will increase latency too much if there are many deployments. + * **energy**: attach energy/footprint stats to each deployment (`energy` + key). Enabled by default. With `full_info=false` it is the accumulated + `EnergyStats`; with `full_info=true` it is the extended `EnergyTimeSeries`. """ # Retrieve authenticated user info auth_info = auth.get_user_info(token=authorization.credentials) @@ -62,20 +67,24 @@ def get_deployments( user_jobs = [] for vo in user_vos: + namespace = papiconf.MAIN_CONF["nomad"]["namespaces"][vo] + # Retrieve all jobs in namespace jobs = nomad_utils.get_deployments( - namespace=papiconf.MAIN_CONF["nomad"]["namespaces"][vo], + namespace=namespace, owner=auth_info["id"], prefix="module", ) # Retrieve info for jobs in namespace + vo_jobs = [] for j in jobs: try: job_info = get_deployment( vo=vo, deployment_uuid=j["ID"], full_info=full_info, + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), @@ -83,7 +92,17 @@ def get_deployments( except HTTPException: # not a module continue - user_jobs.append(job_info) + vo_jobs.append(job_info) + + # Attach energy/footprint stats. `full_info=False` (the default) uses one + # batch of cheap file reads + a single cached Mimir top-up; `full_info=True` + # attaches the extended time series per deployment (heavier). + if energy: + energy_accounting.add_to_jobs( + vo_jobs, namespace, auth_info["id"], full_info + ) + + user_jobs.extend(vo_jobs) # Sort deployments by submission time in descending order sorted_jobs = sorted(user_jobs, key=lambda x: x["submit_time"], reverse=True) @@ -96,6 +115,7 @@ def get_deployment( vo: str, deployment_uuid: str, full_info: bool = True, + energy: bool = True, authorization=Depends(security), ): """ @@ -107,6 +127,9 @@ def get_deployment( * **deployment_uuid**: uuid of deployment to gather info about * **full_info**: retrieve the full information of that deployment (may increase latency) + * **energy**: attach energy/footprint stats (`energy` key). Enabled by + default. With `full_info=true` (the default here) it is the extended + `EnergyTimeSeries`; with `full_info=false` the accumulated `EnergyStats`. Returns a dict with info """ @@ -135,6 +158,9 @@ def get_deployment( detail="This deployment is not a module.", ) + if energy: + energy_accounting.add_to_job(job, namespace, full_info) + return job @@ -195,12 +221,14 @@ async def create_deployment( # tools) modules_deps = get_deployments( vos=[vo], + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), ) tools_deps = ai4_deployments.tools.get_deployments( vos=[vo], + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), @@ -289,7 +317,7 @@ async def create_deployment( # Add affinity for greener nodes nomad_conf = green_director.add_green_affinities( nomad_conf=nomad_conf, - stats=ai4_stats.get_cluster_stats(vo), + stats=ai4_stats.get_cluster_stats(vo, energy=False), workload_type="cpu" if user_conf["hardware"]["gpu_num"] == 0 else "gpu", ) diff --git a/ai4papi/routers/v1/deployments/tools.py b/ai4papi/routers/v1/deployments/tools.py index 0bbcd762..2b49a737 100644 --- a/ai4papi/routers/v1/deployments/tools.py +++ b/ai4papi/routers/v1/deployments/tools.py @@ -12,6 +12,7 @@ from fastapi.security import HTTPBearer from ai4papi import auth, quotas, schemas, utils +from ai4papi import energy as energy_accounting from ai4papi.wattnet import green_director import ai4papi.conf as papiconf import ai4papi.nomad_utils as nomad_utils @@ -34,6 +35,7 @@ def get_deployments( vos: schemas.VoList = None, full_info: bool = False, + energy: bool = True, authorization=Depends(security), ): """ @@ -45,6 +47,9 @@ def get_deployments( * **full_info**: retrieve the full information of each deployment. Disabled by default, as it will increase latency too much if there are many deployments. + * **energy**: attach energy/footprint stats to each deployment (`energy` + key). Enabled by default. `full_info=false` -> accumulated `EnergyStats`; + `full_info=true` -> extended `EnergyTimeSeries`. """ # Retrieve authenticated user info auth_info = auth.get_user_info(token=authorization.credentials) @@ -63,20 +68,24 @@ def get_deployments( user_jobs = [] for vo in user_vos: + namespace = papiconf.MAIN_CONF["nomad"]["namespaces"][vo] + # Retrieve all jobs in namespace jobs = nomad_utils.get_deployments( - namespace=papiconf.MAIN_CONF["nomad"]["namespaces"][vo], + namespace=namespace, owner=auth_info["id"], prefix="tool", ) # Retrieve info for jobs in namespace + vo_jobs = [] for j in jobs: try: job_info = get_deployment( vo=vo, deployment_uuid=j["ID"], full_info=full_info, + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), @@ -84,7 +93,16 @@ def get_deployments( except HTTPException: # not a tool continue - user_jobs.append(job_info) + vo_jobs.append(job_info) + + # `full_info=False` (default): one cheap batch. `full_info=True`: the + # extended time series per deployment (heavier). + if energy: + energy_accounting.add_to_jobs( + vo_jobs, namespace, auth_info["id"], full_info + ) + + user_jobs.extend(vo_jobs) # Sort deployments by submission time in descending order sorted_jobs = sorted(user_jobs, key=lambda x: x["submit_time"], reverse=True) @@ -97,6 +115,7 @@ def get_deployment( vo: str, deployment_uuid: str, full_info: bool = True, + energy: bool = True, authorization=Depends(security), ): """ @@ -108,6 +127,9 @@ def get_deployment( * **deployment_uuid**: uuid of deployment to gather info about * **full_info**: retrieve the full information of that deployment (may increase latency) + * **energy**: attach energy/footprint stats (`energy` key). Enabled by + default. `full_info=true` (default here) -> extended `EnergyTimeSeries`; + `full_info=false` -> accumulated `EnergyStats`. Returns a dict with info """ @@ -165,6 +187,9 @@ def get_deployment( k for k in job["active_endpoints"] if k not in ignore ] + if energy: + energy_accounting.add_to_job(job, namespace, full_info) + return job @@ -246,9 +271,9 @@ def create_deployment( # Check if requested hardware is within the user total quota (summing modules and # tools) - tools_deps = get_deployments(vos=[vo], authorization=auth_arg) + tools_deps = get_deployments(vos=[vo], energy=False, authorization=auth_arg) modules_deps = ai4_deployments.modules.get_deployments( - vos=[vo], authorization=auth_arg + vos=[vo], energy=False, authorization=auth_arg ) quotas.check_userwise( conf=user_conf, @@ -755,7 +780,7 @@ def create_deployment( # Add affinity for greener nodes nomad_conf = green_director.add_green_affinities( nomad_conf=nomad_conf, - stats=ai4_stats.get_cluster_stats(vo), + stats=ai4_stats.get_cluster_stats(vo, energy=False), workload_type="cpu" if user_conf.get("hardware", {}).get("gpu_num", 0) == 0 else "gpu", diff --git a/ai4papi/routers/v1/stats/deployments.py b/ai4papi/routers/v1/stats/deployments.py index d14a31d8..5100b5b3 100644 --- a/ai4papi/routers/v1/stats/deployments.py +++ b/ai4papi/routers/v1/stats/deployments.py @@ -4,6 +4,7 @@ import copy import csv +import logging import time from datetime import datetime, timedelta import os @@ -14,6 +15,7 @@ from fastapi.security import HTTPBearer from ai4papi import auth, schemas +from ai4papi import energy as energy_accounting from ai4papi.wattnet import green_director import ai4papi.conf as papiconf from ai4papi.nomad_utils import Nomad @@ -90,6 +92,8 @@ def load_stats( @router.get("/user") def get_user_stats( vo: str, + full_info: bool = False, + energy: bool = True, authorization=Depends(security), ): """ @@ -100,6 +104,11 @@ def get_user_stats( Parameters: * **vo**: Virtual Organization where you want the stats from. + * **energy**: attach the `energy` accounting block (the user's accumulated + consumption and footprint, each deployment's live Mimir top-up included). + Enabled by default. + * **full_info**: when `true`, the `energy` block also carries a merged 15-min + time series of the user's deployments. Off by default (accumulated only). """ # Retrieve authenticated user info @@ -109,18 +118,31 @@ def get_user_stats( # Retrieve the associated namespace to that VO namespace = papiconf.MAIN_CONF["nomad"]["namespaces"][vo] - # Load proper namespace stats - full_stats = load_stats(namespace=namespace) - - # Keep only stats from the current user - user_stats = copy.deepcopy(full_stats) + # Load proper namespace stats (CSV usage stats from the `ai4-accounting` repo). + # These are independent of the energy accounting below: if they are missing, + # still serve the energy block instead of failing the whole endpoint. try: - idx = full_stats["users-agg"]["owner"].index(auth_info["id"]) - user_stats["users-agg"] = { - k: v[idx] for k, v in full_stats["users-agg"].items() - } - except ValueError: # user has still no recorded stats - user_stats["users-agg"] = None + full_stats = load_stats(namespace=namespace) + user_stats = copy.deepcopy(full_stats) + try: + idx = full_stats["users-agg"]["owner"].index(auth_info["id"]) + user_stats["users-agg"] = { + k: v[idx] for k, v in full_stats["users-agg"].items() + } + except ValueError: # user has still no recorded stats + user_stats["users-agg"] = None + except HTTPException: + if not (energy and energy_accounting.enabled()): + raise + user_stats = {} + + # Accumulated energy consumption + footprint of the user in that VO (each + # deployment's accumulator + its live top-up, summed). `full_info` adds the + # merged time series. + if energy and energy_accounting.enabled(): + user_stats["energy"] = energy_accounting.get_user_energy( + namespace, auth_info["id"], series=full_info + ) return user_stats @@ -160,6 +182,8 @@ def get_proper_allocation(allocs): @cached(cache=TTLCache(maxsize=1024, ttl=30)) def get_cluster_stats( vo: str | None = None, + full_info: bool = False, + energy: bool = True, ) -> schemas.ClusterStats: """ Returns the following stats of the nodes and the cluster (per resource type): @@ -170,6 +194,14 @@ def get_cluster_stats( ---------- vo: string Keep only the nodes supporting a specific VO. If not provided, returns all nodes. + energy: bool + Attach the per-datacenter `energy` block (site infrastructure power draw plus + the accumulated energy/footprint). Enabled by default. + full_info: bool + When `true`, each `datacenters[].energy` block also carries its 15-min + time series. Off by default (accumulated numbers only); the live top-up is + always applied either way. Energy is reported per datacenter, there is no + platform-wide total. """ global cluster_stats, cluster_stats_updated_at @@ -180,7 +212,7 @@ def get_cluster_stats( # So if None, we need to initialize it cluster_stats = get_cluster_stats_bg() - # If the background task fails for some reason (failed Nomad calls, failed WattNet + # If the background task fails for some reason (failed Nomad calls, failed Wattnet # calls, etc), the stats won't be updated and this endpoint will keep serving the # same (old) stats, which can be misleading because it gives the impression that # everything works normally. So we give a 1 hour grace time and then raise an Error. @@ -192,6 +224,12 @@ def get_cluster_stats( stats = copy.deepcopy(cluster_stats) + # Drop the energy blocks when not requested (the background thread computes + # them regardless; this just keeps them out of the response). + if not energy: + for dc_stats in stats.datacenters.values(): + dc_stats.energy = None + namespace = papiconf.MAIN_CONF["nomad"]["namespaces"][vo] if vo else "all" for k, v in list(stats.datacenters.items()): # we make an object copy with list() @@ -201,8 +239,9 @@ def get_cluster_stats( if namespace == "all" or namespace in n_stats.namespaces: nodes[n_id] = n_stats - # Ignore datacenters with no nodes - if not nodes: + # Ignore datacenters with no nodes, unless they carry an energy block + # (site infrastructure energy is reported regardless of the VO filter). + if not nodes and stats.datacenters[k].energy is None: del stats.datacenters[k] else: stats.datacenters[k].nodes = nodes @@ -232,6 +271,14 @@ def get_cluster_stats( stats.cluster.gpu_models[model_name].gpu_total += g_stats.gpu_total stats.cluster.gpu_models[model_name].gpu_used += g_stats.gpu_used + # Per-datacenter energy time series only with `full_info=true` (it is large: + # ~500 points per datacenter). The accumulated numbers and the live top-up + # are always present. + if not full_info: + for dc_stats in stats.datacenters.values(): + if dc_stats.energy is not None: + dc_stats.energy.series = None + # Add update time stats.updated_at = ( datetime.fromtimestamp(cluster_stats_updated_at).isoformat() + "Z" # ty: ignore[invalid-argument-type] @@ -419,6 +466,24 @@ def get_cluster_stats_bg() -> schemas.ClusterStats: for g_stats in n_stats.gpu_models.values(): g_stats.gpu_total = n_stats.gpu_used + # Per-datacenter energy (energy feature): instantaneous power draw + the + # accumulated energy/footprint since accounting started + a 15-min series. + # Reported per datacenter only, there is no platform-wide total (clients sum + # the datacenters they care about). Best-effort: a failure here must not + # break the cluster stats. + if energy_accounting.enabled(): + try: + for dc_name, dc_energy in energy_accounting.get_cluster_energy( + series=True + ).items(): + if dc_name not in stats.datacenters: + stats.datacenters[dc_name] = schemas.DatacenterStats( + lat=0.0, lon=0.0, PUE=0.0, nodes={} + ) + stats.datacenters[dc_name].energy = schemas.EnergyStats(**dc_energy) + except Exception: + logging.exception("Error computing cluster energy stats") + # Set the new shared variable global cluster_stats, cluster_stats_updated_at cluster_stats = stats diff --git a/ai4papi/routers/v1/try_me/nomad.py b/ai4papi/routers/v1/try_me/nomad.py index 448d887d..943bdc6a 100644 --- a/ai4papi/routers/v1/try_me/nomad.py +++ b/ai4papi/routers/v1/try_me/nomad.py @@ -11,6 +11,7 @@ from fastapi.security import HTTPBearer from ai4papi import auth +from ai4papi import energy as energy_accounting import ai4papi.conf as papiconf from ai4papi.routers.v1.catalog.modules import Modules from ai4papi.routers.v1.stats.deployments import get_cluster_stats @@ -39,6 +40,7 @@ @router.get("") def get_deployments( full_info: bool = False, + energy: bool = True, authorization=Depends(security), ): """ @@ -50,6 +52,9 @@ def get_deployments( * **full_info**: retrieve the full information of each deployment. Disabled by default, as it will increase latency too much if there are many deployments. + * **energy**: attach energy/footprint stats (`energy` key). Enabled by + default. `full_info=false` -> accumulated `EnergyStats`; `full_info=true` + -> extended `EnergyTimeSeries`. """ # Retrieve authenticated user info auth_info = auth.get_user_info(token=authorization.credentials) @@ -67,6 +72,7 @@ def get_deployments( job_info = get_deployment( deployment_uuid=j["ID"], full_info=full_info, + energy=False, authorization=types.SimpleNamespace( credentials=authorization.credentials # token ), @@ -76,6 +82,9 @@ def get_deployments( user_jobs.append(job_info) + if energy: + energy_accounting.add_to_jobs(user_jobs, NAMESPACE, auth_info["id"], full_info) + # Sort deployments by submission time in descending order sorted_jobs = sorted(user_jobs, key=lambda x: x["submit_time"], reverse=True) @@ -86,6 +95,7 @@ def get_deployments( def get_deployment( deployment_uuid: str, full_info: bool = True, + energy: bool = True, authorization=Depends(security), ): """ @@ -95,6 +105,9 @@ def get_deployment( Parameters: * **deployment_uuid**: uuid of deployment to gather info about + * **energy**: attach energy/footprint stats (`energy` key). Enabled by + default. `full_info=true` (default here) -> extended `EnergyTimeSeries`; + `full_info=false` -> accumulated `EnergyStats`. Returns a dict with info """ @@ -112,6 +125,9 @@ def get_deployment( # Rewrite main endpoint, otherwise it automatically selects DEEPaaS API job["main_endpoint"] = "ui" + if energy: + energy_accounting.add_to_job(job, NAMESPACE, full_info) + return job @@ -188,7 +204,7 @@ def create_deployment( # Check that the target node (ie. tag='tryme') resources are available because # these jobs cannot be left queueing # We check for every resource metric (cpu, disk, ram) - cluster_stats = get_cluster_stats(vo=VO) + cluster_stats = get_cluster_stats(vo=VO, energy=False) resources = ["cpu", "ram", "disk"] keys = [f"{i}_used" for i in resources] + [f"{i}_total" for i in resources] status = dict.fromkeys(keys, 0) diff --git a/ai4papi/schemas.py b/ai4papi/schemas.py index da6b5ce2..b797a91b 100644 --- a/ai4papi/schemas.py +++ b/ai4papi/schemas.py @@ -38,6 +38,60 @@ class NodeInfo(ResourceStats): cpu_model: str +class EnergyTimeSeriesPoint(BaseModel): + # Per-bucket increments only. A running total is `cumsum(series[].energy_wh)` + # and matches `accumulated.energy_wh` (same for carbon and water). + ts: str + power_w: Optional[float] = None # mean over the bucket, normalized (x TUE) + energy_wh: Optional[float] = None # normalized, incremental for the bucket + carbon_g: Optional[float] = None + water_l: Optional[float] = None + live: bool = False # point from the live tail (not yet persisted) + + +class EnergyStats(BaseModel): + """ + Energy consumption and footprint (carbon and water, scope 2) of one entity, + always with the live Mimir top-up applied. The same shape at every level: + `energy` per deployment (list endpoints), `accumulated` in the per-deployment + time series, `energy` per datacenter in `GET /stats/cluster`, and `energy` + per user in `GET /stats/user`. + + Every value is normalized by the datacenter TUE; `tue_factor` is reported so a + client that wants the raw meter figure divides by it (the raw is not sent). + `coverage_ratio` / `datacenters` are `null` on the datacenter block; + `deployments` is set only on the per-user aggregate. + """ + + energy_wh: Optional[float] = None # accumulated, normalized (x TUE) + tue_factor: Optional[float] = None # normalized / raw (per-datacenter or blended) + carbon_g: Optional[float] = None # gCO2eq + water_l: Optional[float] = None # L + power_w: Optional[float] = ( + None # current draw (last Mimir sample, ~30-60s old); None if stopped + ) + since: Optional[str] = None # start of the accumulation window + as_of: Optional[str] = None # store pointer timestamp + live_as_of: Optional[str] = None # instant of the live top-up + complete: bool = True # False if it started late / clamped by Mimir retention + degraded: bool = False # the live Mimir top-up failed + coverage_ratio: Optional[float] = None # per deployment / user: fraction with data + datacenters: Optional[list[str]] = None # per deployment / user + deployments: Optional[int] = None # per-user aggregate only + series: Optional[list[EnergyTimeSeriesPoint]] = ( + None # 15-min buckets; only with full_info=true + ) + + +class EnergyTimeSeries(BaseModel): + deployment_uuid: str + start: str # first point (equivalently `accumulated.since`) + end: str # last point (the live edge while running) + source: str # "store" (archive only) | "mixed" (archive + live tail) + series: list[EnergyTimeSeriesPoint] + accumulated: EnergyStats + + class DatacenterStats(BaseModel): # CSV metadata lat: float @@ -48,7 +102,8 @@ class DatacenterStats(BaseModel): nodes: Dict[str, NodeInfo] # Optional fields (added dynamically) - footprints: Optional[Dict[str, Any]] = None # fromwattnet.GreenDirector + footprints: Optional[Dict[str, Any]] = None # from wattnet.GreenDirector + energy: Optional[EnergyStats] = None # current power + accumulated energy/footprint class ClusterStats(BaseModel): @@ -60,3 +115,6 @@ class ClusterStats(BaseModel): # Added in get_cluster_stats() updated_at: Optional[str] = None + + # Energy is reported per datacenter (`datacenters[].energy`); there is no + # platform-wide total, clients sum the datacenters they care about. diff --git a/ai4papi/wattnet.py b/ai4papi/wattnet.py index 9b307b61..3512a2da 100644 --- a/ai4papi/wattnet.py +++ b/ai4papi/wattnet.py @@ -1,25 +1,32 @@ """ -Utilities for the integration with WattNet. +Utilities for the integration with Wattnet. API reference: https://api.wattnet.eu/v1/docs """ -from collections.abc import Iterable import datetime import json -import requests +import logging import os import statistics import warnings +from collections.abc import Iterable + +import requests +from cachetools import TTLCache, cached -from cachetools import cached, TTLCache -from ai4papi import utils import ai4papi.conf as papiconf +from ai4papi import utils +LOG = logging.getLogger(__name__) session = requests.Session() WATTNET_URL = "https://api.wattnet.eu" -WATTNET_EMAIL = "bot@ai4eosc.eu" +TIMEOUT = 30 # seconds; without it a stalled Wattnet would block the background threads +FAILURE_BACKOFF = 5 * 60 # seconds a failing datacenter is not asked again +WATTNET_EMAIL = os.environ.get("WATTNET_EMAIL") +if not WATTNET_EMAIL: + print("You should define a WATTNET_EMAIL") WATTNET_PASS = os.environ.get("WATTNET_PASSWORD") if not WATTNET_PASS: print("You should define a WATTNET_PASSWORD") @@ -32,11 +39,11 @@ def algorithm(func): class GreenDirector: - # Define sensible default footprint values for datacenter outside WattNet scope (Europe) + # Define sensible default footprint values for datacenter outside Wattnet scope (Europe) DEFAULTS = { "carbon": 301, # default energy quality in gCO2/kWh "water": 12, # default water usage in L/kWh - "green-score": 50, # default green score (combining carbon and water). + "environmental-score": 50, # default environmental score (combining carbon and water). } def __init__(self, algorithm: str = "linear_rank"): @@ -64,24 +71,28 @@ def __init__(self, algorithm: str = "linear_rank"): # Init vars self.algorithm_name = algorithm self.metrics = {} + self._backoff = TTLCache(maxsize=1024, ttl=FAILURE_BACKOFF) + self._failing = set() # datacenters whose last refresh failed (log on change) @cached(cache=TTLCache(maxsize=1024, ttl=20 * 60 * 60)) def _retrieve_token(self): """ - WattNet tokens last only one day, so we cache the response for 20 hours. + Wattnet tokens last only one day, so we cache the response for 20 hours. """ url = f"{WATTNET_URL}/token-request/get_token" headers = {"Content-Type": "application/json"} data = {"email": WATTNET_EMAIL, "password": WATTNET_PASS} - response = requests.post(url, headers=headers, data=json.dumps(data)) + response = requests.post( + url, headers=headers, data=json.dumps(data), timeout=TIMEOUT + ) response.raise_for_status() return response.json()["access_token"] @cached(cache=TTLCache(maxsize=1024, ttl=15 * 60)) def _fetch_footprint_data(self, lat, lon): """ - Fetch footprint data and green score data from WattNet for a specific - lat-lon location. WattNet has a temporal resolution of 15 minutes, so we + Fetch footprint data and environmental score data from Wattnet for a specific + lat-lon location. Wattnet has a temporal resolution of 15 minutes, so we cache for that amount of time. """ end = datetime.datetime.now(datetime.timezone.utc) @@ -96,24 +107,78 @@ def _fetch_footprint_data(self, lat, lon): token = self._retrieve_token() headers = {"Authorization": f"Bearer {token}"} - r1 = session.get(f"{WATTNET_URL}/v1/footprints", headers=headers, params=params) + r1 = session.get( + f"{WATTNET_URL}/v1/footprints", + headers=headers, + params=params, + timeout=TIMEOUT, + ) r2 = session.get( - f"{WATTNET_URL}/v1/green-score", headers=headers, params=params + f"{WATTNET_URL}/v1/environmental-score", + headers=headers, + params=params, + timeout=TIMEOUT, ) + # Raise instead of returning [] so a failure is not cached for 15 min + # (cachetools only caches return values); the caller falls back to DEFAULTS if not (r1.ok and r2.ok): - warnings.warn( - f"[wattnet] Failed to retrieve WattNet data for coordinates ({lat}, {lon})" + raise RuntimeError( + f"Wattnet request failed for coordinates ({lat}, {lon}): " + f"{r1.status_code}, {r2.status_code}" ) - return [] footprints = r1.json() score = r2.json() if (not footprints) or (not score): - raise Exception("Error retrieving information from Wattnet.") - score[0].update({"footprint_type": "green-score"}) + raise RuntimeError(f"Empty Wattnet data for coordinates ({lat}, {lon})") + score[0].update({"footprint_type": "environmental-score"}) return footprints + score + def footprint_series(self, datacenter: str, start, end): + """ + Fetch the carbon/water intensity series for a datacenter over an + arbitrary [start, end] window (Wattnet serves historical data well + beyond the 7-day window kept in `self.metrics`). Used by the + energy-accounting sweep when backfilling a deployment. + + Returns `{"carbon": [[iso_ts, gCO2/kWh], ...], "water": [[iso_ts, L/kWh], ...]}` + or `{}` if Wattnet has no data for that location (caller falls back to + DEFAULTS). + """ + dc = papiconf.datacenters.get(datacenter, {}) + lat, lon = dc.get("lat"), dc.get("lon") + if lat is None or lon is None: + return {} + + params = { + "lat": lat, + "lon": lon, + "start": start.isoformat().replace("+00:00", "Z"), + "end": end.isoformat().replace("+00:00", "Z"), + "aggregate": "false", + } + headers = {"Authorization": f"Bearer {self._retrieve_token()}"} + r = session.get( + f"{WATTNET_URL}/v1/footprints", + headers=headers, + params=params, + timeout=TIMEOUT, + ) + if not r.ok: + warnings.warn( + f"[wattnet] Failed to retrieve footprint series for {datacenter}" + ) + return {} + + out = {} + for footprint in r.json(): + series = [] + for sublist in footprint.get("series", []): + series += sublist["values"] + out[footprint["footprint_type"]] = sorted(series, key=lambda x: x[0]) + return out + def retrieve_footprints(self): """ Retrieve the footprints for all datacenters. @@ -128,7 +193,30 @@ def retrieve_footprints(self): start = end - datetime.timedelta(days=7) for k, v in datacenters.items(): - data = self._fetch_footprint_data(v["lat"], v["lon"]) + if k in self._backoff: + continue # failed a moment ago: do not wait on a down Wattnet again + try: + data = self._fetch_footprint_data(v["lat"], v["lon"]) + except Exception as e: + self._backoff[k] = True + if k not in self._failing: + self._failing.add(k) + kept = self.metrics.get(k, {}).get("carbon") + LOG.warning( + "Wattnet refresh failed for %s (%s); %s", + k, + e, + f"keeping data up to {kept[-1][0]}" + if kept + else "using default values", + ) + if k in self.metrics: + continue # keep the last good data rather than overwrite it + data = [] # never had data: fall back to DEFAULTS + else: + if k in self._failing: + self._failing.discard(k) + LOG.info("Wattnet refresh recovered for %s", k) # For each footprint type, we concatenate all timeseries, irrespectively of # whether they are [valid=True] (meaning their values are final) or @@ -143,7 +231,7 @@ def retrieve_footprints(self): series = sorted(series, key=lambda data: data[0]) self.metrics[k][fp_type] = series - # For datacenters outside Europe (e.g. Tubitak), WattNet offers no data + # For datacenters outside Europe, Wattnet offers no data # Therefore we return timeseries with default values if not data: round_end = end.replace( @@ -184,13 +272,13 @@ def normalize_list( return [target_min + ((x - val_min) * target_span) / range_span for x in values] @algorithm - def _linear_rank(self, stats: dict, metric: str = "green-score"): + def _linear_rank(self, stats: dict, metric: str = "environmental-score"): """ - Computes a DIRAC-like score for each node: + Computes a DIRAC-like green_score for each node: - score = processor_efficiency / (datacenter_PUE * datacenter_footprint) + green_score = processor_efficiency / (datacenter_PUE * datacenter_footprint) """ - metrics = ["carbon", "water", "green-score"] + metrics = ["carbon", "water", "environmental-score"] if metric not in metrics: raise ValueError(f"Invalid metric: {metric}. Must be one of: {metrics}") @@ -229,21 +317,21 @@ def _linear_rank(self, stats: dict, metric: str = "green-score"): scaled_values = self.normalize_list(gpu_eff.values(), target_range=(1, 2)) gpu_eff = dict(zip(gpu_eff.keys(), scaled_values)) - # Compute the individual node scores (the higher the better) - scores = {} + # Compute the individual node green_scores (the higher the better) + green_scores = {} for dc_name, dc in stats.datacenters.items(): # Compute the overall datacenter footprint mean = statistics.mean([i[1] for i in self.metrics[dc_name][metric]]) - # In the case of green score, high values lower the footprint - mean = 1 / mean if metric in ["green-score"] else mean + # In the case of environmental score, high values lower the footprint + mean = 1 / mean if metric in ["environmental-score"] else mean - # Now compute per node Dirac-style green-score, weighting the datacenter + # Now compute per node Dirac-style environmental-score, weighting the datacenter # footprint with the node specific efficiency for nid, node in dc.nodes.items(): if nid not in cpu_eff.keys(): continue - scores[nid] = {} - scores[nid]["cpu"] = cpu_eff[nid] / (mean * dc.PUE) + green_scores[nid] = {} + green_scores[nid]["cpu"] = cpu_eff[nid] / (mean * dc.PUE) if nid not in gpu_eff.keys(): continue @@ -251,9 +339,9 @@ def _linear_rank(self, stats: dict, metric: str = "green-score"): # We *assume* that a typical GPU workload consumes 80% of its power on # the GPU and 20% on a CPU combined_eff = 0.2 * cpu_eff[nid] + 0.8 * gpu_eff[nid] - scores[nid]["gpu"] = combined_eff / (mean * dc.PUE) + green_scores[nid]["gpu"] = combined_eff / (mean * dc.PUE) - return scores + return green_scores def rank(self, stats: dict): """ diff --git a/docs/energy-accounting.md b/docs/energy-accounting.md new file mode 100644 index 00000000..3b36e2fa --- /dev/null +++ b/docs/energy-accounting.md @@ -0,0 +1,356 @@ +# Per-deployment energy consumption and footprint + +This feature exposes, for every PAPI deployment, its accumulated energy consumption and footprint (carbon and water, scope 2), plus time series for the dashboard graphs. + +- Accumulated (indefinite retention): energy (Wh), carbon (gCO2eq), water (L). +- Time series of the same three variables, for the deployment detail view. + +It is fully additive: nothing in the existing API changes behaviour and no new endpoints are added. Deployments get a new `energy` key, `/v1/deployments/stats/user` gains an `energy` block (the user's deployments summed), `/v1/deployments/stats/cluster` gains an `energy` block per datacenter (no platform-wide total, clients sum), and there is one extra background thread. If the feature is disabled or its inputs are missing, `energy` is `null` (or absent) and the rest of the API is identical. + +## Inputs + +| Input | Native resolution | Availability | Notes | +|---|---|---|---| +| CPU power: `scaph_process_power_consumption_microwatts` (uW) | 30 s | Mimir retention (tunable, will drop to ~3 months) | label `container_label_com_hashicorp_nomad_alloc_id`; aggregates every container of the alloc (task `main` + sidecars). The Nomad nodes are VMs, so scaphandre's per-process figures are small: real CPU energy only shows up under real load. | +| GPU power: `DCGM_FI_DEV_POWER_USAGE` (W) | 30 s | Mimir retention | board-level power, labels `UUID`, `instance` | +| GPU to alloc mapping: `nomad_gpu_allocation_info` | 30 s | only while a GPU deployment runs | info gauge, label `alloc_id`; kept as the mapping mechanism on purpose (other accounting software relies on it to avoid depending on the Nomad API) | +| Carbon and water intensity (Wattnet) | 15 min | full history on demand; `green_director` keeps a rolling 7-day window | already integrated in `ai4papi/wattnet.py`. Recent intervals are provisional and get revised every ~15 min until settled. `DEFAULTS` (301 gCO2/kWh, 12 L/kWh) outside coverage. | +| Allocations of a deployment | event | while the Nomad job is not garbage-collected (hours) | `Nomad.job.get_allocations`; several allocations over a job's life due to reallocations | + +Power metrics currently exist only in the `ifca-*` datacenters, and not on every node. Availability is effectively per allocation: it is decided by whether Mimir has data for that `alloc_id`, not by the datacenter alone. The CPU (scaphandre) and GPU (DCGM) sources are independent: whichever one has data contributes, so a site that only monitors its GPUs still accounts the GPU energy (a missing metric comes back as an empty Mimir result, not an error). The feature is best-effort: if there is data it is reported; if there is none, `energy` is `null` (never zeros); if there is data for only part of the deployment's life, that part is accounted and `coverage_ratio` drops below `1.0`. + +Mimir endpoint: `https://mimir.k8s.cloud.ai4eosc.eu/prometheus/api/v1/query_range`, HTTP Basic auth, no `X-Scope-OrgID` header. + +## Normalization + +`energy_wh = integrated_Wh * TUE[datacenter]`, where `integrated_Wh` is the raw trapezoidal integral of `CPU + GPU` power. + +The TUE (Total Usage Effectiveness, grid to compute ratio) is per datacenter and already covers CPU plus GPU, so it is applied once, before the footprint (which is scope 2, so it must reflect the whole facility draw). It is a single empirical factor per datacenter (default 2.73), consistent with the platform's existing energy accounting. Every energy, power, carbon and water value in an API response is already TUE-normalized; the applied factor is reported as `tue_factor` (per datacenter, or blended `energy_wh / raw` on multi-datacenter and per-user aggregates) so a client that wants the raw meter figure divides by it. The accumulator stores `tue_factor` per document. + +The TUE is read from the `TUE` column of `var/datacenters.csv`, only populated for the datacenters that have a value (currently `ifca-ai4eosc` and `ifca-imagine`). Blank cells load as `None` and `energy.default_tue_factor` is used as fallback. + +## Architecture + +Four layers: a durable background sweep that owns the persisted record, a live read-time top-up on top of it, an on-demand version of the sweep for a deployment the background thread has not reached yet, and a pure-Mimir fallback for one still too young to have anything settled at all. + +``` + background thread in ai4papi.main: @repeat_every(energy.sweep_seconds) + energy.run_sweep() + per deployment: Nomad (jobs + allocations + node -> datacenter) + Mimir query_range (per-alloc power, un-consolidated window) + per datacenter: Mimir query_range (per-host + per-GPU power) + green_director.metrics (Wattnet 15-min intensity) + integrate 30 s samples -> 15-min buckets, x TUE, x intensity + | + v atomic write, one JSON per deployment / per datacenter + $ACCOUNTING_PTH/energy//.accum.json meta + accumulator + 24h tail + $ACCOUNTING_PTH/energy//.series.jsonl consolidated 15-min buckets, append-only + $ACCOUNTING_PTH/energy/_cluster/.accum.json per-datacenter accumulator + 24h tail + $ACCOUNTING_PTH/energy/_cluster/.series.jsonl + | + v cached read + PAPI API (stateless) + energy block = stored accumulator + live top-up (Mimir, short window, cache 25 s) + GET /v1/deployments/{modules,tools,batch,try_me/nomad} -> + energy (EnergyStats, list view) + GET /v1/deployments/{modules,tools,batch,try_me/nomad}/{uuid} -> + energy (EnergyTimeSeries, detail view) + GET /v1/deployments/stats/user -> + energy (per-user total) + GET /v1/deployments/stats/cluster -> + energy (per-datacenter: power + accumulated + series) +``` + +The shape of the deployment `energy` block follows the existing `full_info` query param: `full_info=false` (the list default) attaches the accumulated `EnergyStats`; `full_info=true` (the individual-endpoint default) attaches the extended `EnergyTimeSeries`. + +The diagram above is the steady state, once a deployment has a persisted doc. Before that, the read path itself does the work the diagram shows the background thread doing (Layer 3), and short of that, falls back to querying Mimir directly with nothing persisted at all (Layer 4) -- see both below. + +### Layer 1: durable sweep + +`energy.run_sweep()` runs on a `@repeat_every(energy.sweep_seconds)` thread started in `ai4papi.main`'s `lifespan`, next to `get_cluster_stats_thread`. Same posture: not async, one `try/except` per deployment, no-op if the feature is disabled. + +Per namespace it lists the active Nomad jobs (`Status != "dead"`, names matching `^(module|tool|batch|try)`) and also every existing `.accum.json` whose job has vanished, to close it out. Per deployment it calls `process_single(info, now)`: + +1. Load `.accum.json`. The integration pointer `settled_ts` comes from the doc; on first encounter it is clamped to `now - initial_lookback_hours` (pre-existing deployments are not fully backfilled) and to `now - mimir_retention_days` (older than that, `complete` is `false`). `window_start` (exposed as `since`) is pinned to that first-encounter value and never moves afterwards unless it falls behind `now - mimir_retention_days`; unlike `settled_ts` it is not a moving pointer, so it always marks the true start of the span the totals cover. A doc written before this field existed has it backfilled once, from the oldest bucket on record (`.series.jsonl`, else the oldest still in `tail`). +2. Resolve allocations via `compute.resolve_alloc_ids` into `{datacenter: [alloc_id]}` (whole lifetime), the currently-running allocs (for the top-up), the alloc time windows (to estimate `run_seconds`, which spans the whole life including time on datacenters without metrics), and `partial_dc` (an alloc ran in a datacenter without metrics). +3. Per datacenter, query Mimir for `[settled_ts, now - query_lag_seconds]` (chunked only if the span is huge), `sanitize` the samples, `integrate` trapezoidally into 15-min buckets applying that datacenter's TUE, then `apply_footprint` with the Wattnet intensity of that datacenter. Merge the per-datacenter buckets by timestamp. +4. Best-effort coverage: if the deployment was clearly running (`run_seconds` significant) but Mimir returned nothing, mark `metrics_available: false` and retry next sweep. Otherwise `metrics_available: true` and `coverage_ratio = covered_time / run_seconds` (`1.0` = full; lower means part of the life ran unmonitored). +5. Merge the new buckets over the existing 24h tail. Buckets older than `now - footprint_revision_hours` are consolidated: their totals are added to the running `settled` accumulator, `settled_ts` advances past them, and they are appended to `.series.jsonl`. The rest stay in `tail` and are recomputed in full every sweep, so the footprint self-heals as Wattnet settles. `settled_ts` therefore trails `now` by `footprint_revision_hours` in steady state and must not be read as the window start; that is `window_start` / `since`. +6. `accumulated = settled + sum(tail)`. `pointer_ts` is the end of the last tail bucket. Write the doc atomically. + +The sweep is idempotent: re-running with the same `now` produces the same document (buckets keyed by timestamp, `settled` only advances over buckets that leave the 24h window). + +The consolidation step (fold new buckets over the tail, age the old ones into `settled` + `.series.jsonl`, recompute `accumulated` / `pointer_ts`) is `sweep._fold_and_accumulate` (`ai4papi/energy/sweep.py`), shared with the datacenter sweep below. + +### Layer 1b: per-datacenter sweep + +After the deployment loop, `run_sweep()` calls `_sweep_cluster(now)`, which runs `process_datacenter(dc, now)` for every `energy.metrics_datacenters`. Documents live under the pseudo-namespace `_cluster` (`$ACCOUNTING_PTH/energy/_cluster/.accum.json` + `.series.jsonl`), reusing `energy.store` unchanged. + +`process_datacenter` is `process_single` without the Nomad parts: no allocation resolution, no `coverage_ratio`, no owner/close handling. + +1. Load the doc; `settled_ts` from it, else clamped to `initial_lookback_hours` / `mimir_retention_days` (retention clamp sets `complete: false`). `window_start` / `since` is pinned on the first sweep, same as the per-deployment path. +2. Query Mimir over `[settled_ts, now - query_lag_seconds]` with `compute.cluster_promql(dc)`: `sum by (node) (scaph_host_power_microwatts{datacenter=""})` (one series per host) and `sum by (UUID) (DCGM_FI_DEV_POWER_USAGE{datacenter=""})` (one series per GPU). Grouped, not summed, so `compute.integrate` runs the data-quality filter on each raw host / GPU series before summing, exactly as the per-deployment path does per allocation. +3. Data quality is `config.cluster_dq()`: `energy.data_quality` with `max_power_w` forced to `0` (the per-alloc / per-GPU ceiling of ~1000 W is meaningless for a whole host). Negative-drop and the scale-free rolling-median spike check still apply. +4. `integrate` with `TUE[dc]`, `apply_footprint` with the datacenter's Wattnet intensity, then `_fold_and_accumulate`. If the datacenter has no host-power series at all: `metrics_available: false`, retry next sweep, no zeros. + +Note on double counting: scaphandre host power is RAPL (CPU package + DRAM), it does not include the GPU board, so `host + GPU` is correct. + +### Layer 2: read-time top-up + +The stored accumulator only reaches `pointer_ts` (up to one sweep interval behind). On each API read, `energy.live.topup(doc)` queries Mimir live for `[pointer_ts, now]` (a single `query_range`, no chunking, no ingestion-lag cushion -- Mimir data is immutable once ingested, so querying right up to "now" cannot return a wrong value, only occasionally miss a sample scraped a moment ago that is not visible *yet*, which self-corrects on the very next call), applies the TUE and the current Wattnet intensity, and returns the energy deltas plus `live_as_of` and `power_w`. `power_w` is the **current draw**: the TUE-normalized sum of the actual last sample of every series in that same query, however old that sample is, not a window mean. The result is cached for `topup_cache_seconds` (25 s, below the dashboard's 30 s poll, so the number visibly moves). Any failure returns `{"degraded": true}` and the API serves the stored value. + +`topup` / `live_series` also accept a datacenter doc (`namespace == _cluster`), same window logic and `compute.cluster_promql` instead of the per-alloc query). They are computed inside `get_cluster_stats_bg()` (which runs every 30 s anyway), so the datacenter counter moves at the same cadence as the per-deployment one. + +`accumulated (stored, frozen for up to a sweep) + top-up (growing)` stays continuous: when the next sweep advances `pointer_ts` and folds that slice into `accumulated`, the top-up window shrinks by the same amount. + +### Layer 3: on-demand sweep + +A deployment the background thread has not reached yet has no `.accum.json`, so the top-up (which reads `pointer_ts` off the stored doc) has nothing to attach to. Rather than making a fresh deployment wait up to `sweep_seconds` for its first numbers, every read path runs one `process_single` pass synchronously (the same function and same write path the background thread uses) for whatever it is about to serve and has no doc yet: + +- Single-deployment reads (`get_accumulated`, `get_series`) call `energy.ondemand.ensure_swept(namespace, uuid)`: fetch that one Nomad job, sweep it, return the resulting doc. +- Bulk/per-user reads (`get_accumulated_bulk`, `get_user_energy`) call `energy.ondemand.ensure_swept_bulk(namespace, owner)` first: list the owner's active jobs (one cheap Nomad call, no per-job fetch) and sweep only the ones still missing a doc, before reading the owner's documents with `store.iter_owner`, which uses an in-memory owner to uuids index instead of scanning the namespace (see [Persistence](#persistence)). + +Once a doc exists, both are a single cheap `store.read_accum` check (or, for the bulk path, one job-list call) and defer entirely to the background sweep; neither re-queries Mimir beyond that first pass. A job Nomad no longer knows about (or a listing failure) degrades to `energy: null` / that deployment simply absent from the aggregate. `/stats/cluster` (`get_cluster_energy`, per-datacenter) is not covered by this: its instantaneous power is already live regardless (`get_datacenter_power`), and its accumulator is seeded by the small, fixed `metrics_datacenters` list, not by individual deployments. + +`process_single` also persists a bare stub (no `accumulated` key, no `metrics_available` verdict either way) the first time it runs for a job with no allocation resolved at all yet (eg. queued, or Nomad has not placed the allocation on a node). That stub is not a negative result, just "too early to tell": the read path treats "doc is `None`" and "doc exists but has no `accumulated` and `metrics_available` is not explicitly `False`" the same way, both falling through to Layer 4. Only an explicit `metrics_available: False` (a confirmed gap, set after the deployment has run long enough that the absence of data is meaningful) stops the fallback and returns `null` outright, deferring to the next real sweep. + +### Layer 4: pure-Mimir fallback + +`process_single` leaves a deployment with **no** doc when it is younger than one `series_bucket_seconds`: there is nothing settled to fold into an accumulator yet, so Layer 3 alone still reports `energy: null` for the first few minutes of a deployment's life. `ondemand.live_only_doc(ns, uuid)` closes that gap: a transient, **never persisted** stand-in doc (`accumulated` all zero, `pointer_ts = submit_time`, the currently-running allocations) shaped so the normal `topup` / `live_series` machinery treats it exactly like a real doc -- the whole reported number then comes straight from Mimir, through the same lag-free `_live_window` as Layer 2. `energy._live_only_stats()` wraps it and only returns something when the top-up actually found data (`power_w` set, or a non-zero energy/carbon/water delta) -- an all-zero top-up is indistinguishable from "no measurement yet", so it is treated as not populated (`null`) rather than shown as a fabricated `0` reading. Used by all four read functions whenever they still have no usable doc after Layer 3. + +## Persistence + +One small JSON per deployment under `$ACCOUNTING_PTH/energy//`, atomic writes (temp file + `os.replace`). `process_single` / `process_datacenter` are the only writers, run either by the periodic sweep thread or, for a single deployment's very first doc, synchronously from the read path (see [Layer 3](#layer-3-on-demand-sweep)). Same convention as `ai4papi.utils.retrieve_from_snapshots`. No database. The `energy.store` interface is deliberately small so it can move to PostgreSQL later without touching the routers. + +`.accum.json` (rewritten every sweep, ~25 KB with a full 24h tail): + +```json +{ + "deployment_uuid": "...", "job_name": "module-...", "namespace": "vo-...", + "owner": "...@egi.eu", "submit_time": "2026-01-05T10:00:00Z", + "submit_time_epoch": 1736071200.0, + "status": "running", "closed_at": null, + "metrics_available": true, "complete": true, "coverage_ratio": 1.0, + "datacenters": ["ifca-imagine"], + "active_allocs": [{"alloc_id": "...", "datacenter": "ifca-imagine"}], + "window_start": "2026-09-02T12:00:00Z", + "settled_ts": "2026-09-02T12:00:00Z", "pointer_ts": "2026-09-03T12:45:00Z", + "updated_at": "2026-09-03T12:50:00Z", "tue_factor": 2.73, + "settled": {"energy_wh": 41200.0, "carbon_g": 12100.0, "water_l": 480.0}, + "tail": [{"ts": "2026-09-02T12:00:00Z", "energy_wh": 4.1, "power_w_avg": 24.3, "carbon_g": 1.2, "water_l": 0.05}], + "accumulated": {"energy_wh": 42873.4, "carbon_g": 12550.2, "water_l": 498.1} +} +``` + +`.series.jsonl`: one line per consolidated bucket (`energy.series_bucket_seconds`, 15 min), append-only (a bucket is written once, when it leaves the 24h tail, never rewritten). About 35k lines (~3 MB) per year for an always-on deployment, a few KB appended per sweep. If the process dies between appending the buckets and writing the doc, the next sweep appends them again; `read_series` keeps one line per `ts`, so a repeated bucket counts once. + +There is no summary file. Per-user and per-datacenter aggregates are computed on read from the `.accum.json` documents (`get_user_energy`, `get_cluster_energy`), because they need the live top-up anyway. Listing one user's deployments does not parse the whole namespace: `store.iter_owner` uses an in-memory `owner -> uuids` index, built with one scan, kept in sync by every `write_accum` and rebuilt from disk every 5 minutes. It assumes a single PAPI process writes the store (marked with a `shortcut:` comment); with several replicas on the same storage a replica would see another one's new documents up to 5 minutes late, and the index should move to a database. + +## Resolution strategy + +| Layer | Resolution | Why | +|---|---|---| +| Integration for the accumulator | 30 s (native) | the accumulator is as precise as the source, independent of bucket size | +| Persisted archive series | 15 min (`energy.series_bucket_seconds`) | matches Wattnet's 15-min cadence, ~3 MB/year for a long deployment, append-only file | +| Recent-window graph | 30 s from Mimir, downsampled to ~1000 points | full resolution and always current while the data is in Mimir | +| Old-window graph (past Mimir retention) | 15 min from the store | the only source left, enough to see the shape | +| Live counter tail (`pointer_ts -> now`) | 30 s | the number moves at scrape cadence | + +The store is the permanent low-resolution archive; Mimir is the high-resolution source for anything recent. + +What the `series` in an API response actually is: the persisted 15-min archive (plus the still-mutable tail) `compute.downsample`d to a `target_points` cap, then the live 15-min tail appended un-downsampled. Caps: deployment detail `energy.deployment_series_points` (1000), datacenter `energy.cluster_series_points` (500), per-user `energy.user_series_points` (1000, merged across the user's deployments first, then one downsample). Each function falls back to its hardcoded default when the key is absent from the config. Below the cap you get the raw 15-min buckets; above it, `downsample` groups `ceil(n / cap)` consecutive buckets, so the effective step is `ceil(n / cap) * 15 min`, uniform across the series. `get_series(namespace, uuid)` takes no time-range or resolution arguments (the routers always call it with the two positional args): the response is always the whole life. + +## Computation details + +Module `ai4papi/energy/compute.py`, all pure and unit-testable except the cached Nomad lookup: + +- `resolve_alloc_ids(job_id, namespace)`: `Nomad.job.get_allocations` plus `_node_datacenter` (TTLCache), filtered to `energy.metrics_datacenters`. +- `power_promql(alloc_ids)`: the CPU and GPU queries for one datacenter's alloc subset. Grouped, not fully summed, so the data-quality filter runs on the original per-source series. + - CPU: `sum by (container_label_com_hashicorp_nomad_alloc_id) (scaph_process_power_consumption_microwatts{container_label_com_hashicorp_nomad_alloc_id=~"id1|id2"})` (one series per allocation) + - GPU: `DCGM_FI_DEV_POWER_USAGE * on(UUID, instance) group_left(alloc_id) nomad_gpu_allocation_info{alloc_id=~"id1|id2"}` (one series per GPU) +- `cluster_promql(dc)`: the host and GPU queries of a whole datacenter (one series per host and per GPU), used by the datacenter sweep and its live top-up. +- `sanitize(samples, dq)`: data-quality filter on one power series (see below). +- `integrate(cpu_result, gpu_result, step_s, bucket_s, tue_factor, gap_factor, data_quality)`: CPU uW/1e6 plus GPU W, `sanitize` per source series, trapezoidal integral of each into `bucket_s` energy buckets, summed, then `energy_wh = raw_integral * tue_factor`. Gaps longer than `gap_factor * step_s` are not integrated across (deployment stopped, scrape hole, dropped anomaly). +- `apply_footprint(buckets, carbon_series, water_series)`: `carbon_g = (energy_wh / 1000) * intensity_at(ts)`, same for water. `intensity_at` is a bisect for the last Wattnet sample at or before the bucket, `DEFAULTS` otherwise. +- `merge_buckets`, `downsample`, `running_seconds`. + +`Bucket` is a namedtuple `(ts, energy_wh, power_w_avg, carbon_g, water_l)` with `ts` an ISO-8601 UTC string and every value already TUE-normalized. + +### Footprint consolidation + +The sweep uses `green_director.metrics[dc]` (Wattnet, already refreshed every 30 s for the cluster stats) with `DEFAULTS` as fallback. For a backfill older than the 7-day window it calls `green_director.footprint_series(dc, start, end)`, which reuses the Wattnet token and hits `/v1/footprints` for the requested range. + +Every sweep recomputes carbon and water for all buckets newer than `now - footprint_revision_hours` (24 h) with the current intensity, so a bucket's footprint firms up as Wattnet settles it. Older buckets are frozen. The accumulated footprint is always `sum` over buckets, so it self-corrects. + +Wattnet failures are contained in `GreenDirector.retrieve_footprints` (`ai4papi/wattnet.py`). Every request has a 30 s timeout, so a stalled Wattnet cannot block the background threads. A failed or empty answer raises inside `_fetch_footprint_data`, so the 15-minute cache never stores it. The datacenter keeps its last good data (or gets `DEFAULTS` if it never had any), and is not asked again for 5 minutes (`FAILURE_BACKOFF`), so a Wattnet outage costs one round of timeouts per 5 minutes. The log has one warning when a datacenter starts failing (with the date up to which data is kept) and one info line when it recovers, not one per refresh. + +The live top-up multiplies its `delta_wh` by the current provisional intensity; the next sweep reconciles it (typical correction under 1-2%). + +### Data quality + +`compute.sanitize` runs on each original power series (per allocation for CPU, per GPU for GPU power), before integration and before the per-series energies are summed. Anomalous samples are dropped, never clamped, so a bad reading never inflates the accumulated energy (the integrator sees a gap in that one source) and the other sources at the same instant are untouched. Config `energy.data_quality`, every check individually skippable: + +- `min_power_w`: drop samples below this (kills negatives). +- `max_power_w`: drop samples above this hard ceiling (`0` disables). Applies per source, so it is a physically bounded signal (one allocation, one GPU), not the deployment total. +- `spike_factor` plus `spike_window`: drop a sample above `spike_factor` times the rolling median of its `spike_window` neighbours (`0` disables). + +## API surface + +The consumer-facing reference (response formats field by field, examples, polling and UX notes) is in [energy-api.md](./energy-api.md). This section is the maintainer view. + +### `energy` embedded in the deployment info + +The four deployment routers (`modules`, `tools`, `batch`, `try_me/nomad`), in both `get_deployment` and `get_deployments`, attach an `energy` object built from the store plus the live top-up, through two helpers of the read API: `energy.add_to_job(job, namespace, full_info)` for a single deployment and `energy.add_to_jobs(jobs, namespace, owner, full_info)` for a list (a no-op on an empty list). The router only keeps the `if energy:` check. Its shape follows the existing `full_info` param: + +- `full_info=false` (the `get_deployments` default): `energy` = `energy.get_accumulated(...)` for the individual endpoint, or one batch `energy.get_accumulated_bulk(namespace, owner)` mapped over the list. `EnergyStats`. One or two Mimir queries total for a whole list. +- `full_info=true` (the `get_deployment` default): `energy` = `energy.get_series(namespace, uuid)` (whole life). `EnergyTimeSeries` with `accumulated` nested. On a list this is one `get_series` per deployment, matching the "may increase latency" note that `full_info` already carries. + +A `energy: bool = True` query param lets a client opt out; the internal quota-check calls pass `False`. `nomad_utils` is not touched. + +`energy` is `null` when there is no rollup or `metrics_available` is `false` (deployment in a site or node without monitoring). When `coverage_ratio < 1.0` the available data is still returned, flagged. + +`schemas.EnergyStats`: + +``` +energy_wh accumulated energy, normalized (x TUE); Wh so the dashboard counter visibly moves +tue_factor the normalization applied (energy_wh / raw); divide by it for the raw figure +carbon_g gCO2eq, scope 2 +water_l L, scope 2 +power_w current draw: last Mimir sample (~30-60s old); 0 if running but idle, null if not running +since start of the accumulation window (`window_start`): pinned on the first sweep, only advances if it falls behind Mimir retention +as_of store pointer timestamp +live_as_of instant of the live top-up +complete False if it started late or was clamped by Mimir retention +coverage_ratio fraction of run time that has power data (float, 1.0 = full); null on the per-datacenter block +degraded the live Mimir top-up failed (the stored value is served) +datacenters list of datacenters the deployment or user has run in; null on the per-datacenter block +deployments set only on the per-user aggregate: number of deployments summed +series 15-min EnergyTimeSeriesPoint list; null unless full_info=true (per-user aggregate only; the per-deployment full_info uses EnergyTimeSeries instead) +``` + +`power_w` is a rate, not an accumulator, so it is never summed. The `power_w` in the block (deployment, user, datacenter) is the **current draw**: the last power sample read from Mimir, however old it is (`compute.instant_power` does not drop it for being stale -- Mimir data is immutable once ingested, so the actual last sample is always the right one to show; it does still drop spiky samples via `sanitize`'s data-quality filter). Deployment and user take it from the last point of the top-up range query; the datacenter takes it from a dedicated instant query (`get_datacenter_power`). `series[].power_w` in the time series is different: a **mean** over its 15-min bucket (so `energy_wh ~= power_w * bucket_hours`). + +### Time series, for the detail view + +The detail view calls the deployment's own individual endpoint (`GET /v1/deployments/{type}/{uuid}`, which defaults to `full_info=true`) and reads `energy` from the response. That `energy` is `schemas.EnergyTimeSeries` from `energy.get_series(namespace, uuid)`: `{deployment_uuid, start, end, source, series, accumulated}`, where `series` is a list of points (`energy` / `carbon` / `water` per-bucket increment, `power_w` per point; a running total is `cumsum` client-side and matches `accumulated`) and `accumulated` is the `EnergyStats`. It serves the persisted 15-min archive downsampled to ~1000 points plus the live tail (`pointer_ts -> now` from Mimir, `live: true`) appended un-downsampled, so the graph keeps growing on every refresh; `source` is then `"mixed"`. Ownership and 404 handling are the individual endpoint's usual behaviour; `energy` is simply `null` when there is no rollup. + +### Per-datacenter energy, in the cluster stats + +`get_cluster_stats_bg()` (the 30 s background thread, which already refreshes `green_director`) calls `energy.get_cluster_energy(series=True)`, which merges three things per datacenter into `schemas.EnergyStats`: + +1. **Instantaneous power** from `energy.get_datacenter_power()`: two instant Mimir queries, `power_w = (sum by (datacenter) scaph_host_power_microwatts + sum by (datacenter) DCGM_FI_DEV_POWER_USAGE) * TUE[dc]`. The host / GPU split is not exposed (on the ifca VMs `scaph_host_power` reads near zero, so it would be misleading; in practice `power_w` is GPU-dominated). A datacenter listed in `energy.metrics_datacenters` is reported with whatever it has (host-only, GPU-only or both); without that allowlist a host-power series is required so a stray GPU metric for an unmonitored site does not fabricate a total. +2. **Accumulated energy** from the `_cluster/.accum.json` document (persisted by the datacenter sweep) plus a live top-up slice (`live.topup` on the datacenter doc): `energy_wh`, `carbon_g`, `water_l`, `since`, `complete`, `live_as_of`. +3. **Time series**: the consolidated `.series.jsonl` (`_cluster_archive_series`, cached ~5 min so the 30 s thread does not re-parse it) plus the 24h tail, downsampled to `energy.cluster_series_points`, with the live tail (`live.live_series` on the datacenter doc) appended. Points carry per-bucket increments only. + +Attached as `stats.datacenters[].energy`. There is **no** response-root total: `schemas.ClusterStats` has no `energy` field, and clients sum the datacenter blocks they want. `get_cluster_stats()` (the VO-filtered endpoint) prunes each datacenter's node list but keeps the `energy` block, so an instrumented datacenter with no nodes for that VO is still returned (empty `nodes`). The background thread always computes the series; `get_cluster_stats(full_info=false)` (the default) strips it from every datacenter block before returning, so the series only ships when the caller asks for it. Any failure in this block is caught and leaves the datacenters without `energy`. A datacenter that is not in `energy.metrics_datacenters` and has no accumulator doc simply has no `energy` block; a monitored GPU-only site does get one (accumulator plus series, and an instantaneous `power_w` from its GPU draw). + +### Per-user aggregate + +`stats.deployments.get_user_stats` gains an `energy` key from `energy.get_user_energy(namespace, owner, series=full_info)`: it reads the owner's `.accum.json` documents (through the owner index), applies each one's live top-up, and sums them into one `EnergyStats`-shaped block (`since` = earliest, `as_of` / `live_as_of` = latest, `complete` = AND, `degraded` = OR, `power_w` = sum of the running ones, `coverage_ratio` = mean, `tue_factor` = blended `energy_wh / raw`, `datacenters` = union, plus a `deployments` count). With `full_info=true` it also returns a merged time series (`series.build_series`): the raw 15-min buckets of every deployment (archive + tail) are summed by timestamp first, then `compute.downsample`d once to `target_points` (1000). Merging before downsampling keeps the resolution uniform across the whole series and gives the same hard cap as the other levels, regardless of how long each deployment individually has been running; the live tails are merged and appended un-downsampled, as elsewhere. The energy block is independent of the pre-existing CSV usage stats: if those are missing, the endpoint still returns the `energy` block instead of failing. + +## Configuration + +`etc/main.yaml`, `energy` section: + +```yaml +energy: + enabled: true + sweep_seconds: 900 # background thread cadence + series_bucket_seconds: 900 # persisted time-series bucket size (matches Wattnet's 15-min cadence) + query_step_seconds: 30 # Mimir scrape interval + query_lag_seconds: 120 # ignore the most recent data (ingestion lag) + initial_lookback_hours: 24 # window covered on a deployment's first sweep + mimir_retention_days: 90 # tune to Mimir's actual retention + footprint_revision_hours: 24 # re-compute footprint for buckets newer than this + topup_cache_seconds: 25 # live top-up cache (< dashboard poll) + deployment_series_points: 1000 # downsample target for the per-deployment series in /deployments/* + user_series_points: 1000 # downsample target for the merged per-user series in /stats/user + cluster_series_points: 500 # downsample target for the per-datacenter series in /stats/cluster + metrics_datacenters: [ifca-ai4eosc, ifca-imagine] + default_tue_factor: 2.73 # fallback when a datacenter has no TUE in datacenters.csv + data_quality: {enabled: true, min_power_w: 0, max_power_w: 1000, spike_factor: 12, spike_window: 11} +``` + +`max_power_w` is a per-source ceiling (one allocation's CPU, or one GPU), not the deployment total, so `1000` W is safe for any single GPU and any CPU allocation on a VM. A dropped sample becomes a gap in that source only. The datacenter sweep reuses this `data_quality` block but forces `max_power_w` to `0` (a whole host draws far more than any single alloc), keeping only the negative-drop and the rolling-median spike check. + +- `ai4papi/clients/mimir.py`: `MIMIR_URL` is a module constant; `BACKOFF_SECONDS` too (60 s). Request timeouts are 10 s (instant query) and 15 s (range query): measured against the real Mimir, a 24h range query takes under 1 s and a full 11,000-point backfill chunk about 1.5 s. `MIMIR_USER` and `MIMIR_PASSWORD` come from the environment (`os.environ.get`, like `wattnet.py`, not `load_env`, so a missing value disables the feature instead of breaking PAPI startup). Add both to `template.env` and to the PAPI deployment. +- `var/datacenters.csv`: `TUE` column, populated only for the datacenters that have a value. `ai4papi/conf.py` tolerates a blank numeric cell (loads it as `None`). +- The Prometheus `datacenter` label values must match the `name` column of `datacenters.csv`. + +## Cadences + +| What | Value | Reason | +|---|---|---| +| Mimir scrape | 30 s | given | +| Sweep thread | `sweep_seconds` (900) | keeps the accumulator current, consolidates the footprint at Wattnet's revision cadence, short enough to catch short jobs before Nomad GC | +| Top-up cache | `topup_cache_seconds` (25) | below the dashboard's 30 s poll, so the counter moves | +| Cluster archive-series cache | 300 s | the `.series.jsonl` only changes at sweep cadence; keeps the 30 s thread from re-parsing it | +| Mimir failure backoff | 60 s (`BACKOFF_SECONDS`, constant in `clients/mimir.py`) | after a timeout or connection error every Mimir call fails fast with `MimirError` | +| Wattnet failure backoff | 300 s (`FAILURE_BACKOFF`, constant in `wattnet.py`) | a datacenter whose refresh failed is not asked again meanwhile | +| Owner index rebuild | 300 s | heals anything written outside this process; `write_accum` keeps it current in between | + +Every TTL cache in `energy/` is built with `cachetools.cached(..., lock=threading.Lock())`: endpoints run in a thread pool next to the sweep and stats threads, and an unlocked `cachetools` cache raises `KeyError` / `RuntimeError` under concurrent access. + +For the user to perceive the increment every 30 s the frontend must show Wh (or kWh with 3 decimals) or animate the counter: at 50 W, 30 s is ~0.4 Wh. + +## Degradation + +Same posture as `get_cluster_stats`. + +- Mimir down: the sweep logs and skips (store frozen); the top-up returns `degraded: true`; the API serves the stored accumulator with its `as_of`. A Mimir that hangs or refuses connections costs one timeout, then `clients/mimir.py` fails fast for 60 s (`BACKOFF_SECONDS`): the sweep stops the pass of the namespace with one warning instead of waiting two timeouts per deployment, and the on-demand sweep in a list request skips instead of blocking the page. +- Wattnet down: energy is still computed; the footprint keeps using the last good intensity (or `DEFAULTS` for a datacenter that never had data) and `degraded` stays `false`; buckets still in the 24h tail are recomputed once Wattnet is back. See Footprint consolidation for the timeout and backoff. +- Unreadable file: `get_accumulated` returns `None`, so `energy` is `null`. +- `energy.enabled: false` or no Mimir credentials: `get_accumulated` / `get_series` / `get_datacenter_power` return `None` / `{}`, so `energy` is `null` everywhere and the rest of the API is unchanged. +- Deployment on a site or node without power monitoring: Mimir returns nothing, `metrics_available` is `false`, `energy` is `null` (best-effort, no misleading zeros); retried every sweep in case monitoring appears or the alloc moves. +- Partial coverage: the available data is returned with `coverage_ratio < 1.0`. +- Cluster stats: a Mimir failure leaves `DatacenterStats.energy` `null` without affecting the rest of the cluster stats. The datacenter sweep logs and skips per datacenter; the persisted accumulator is served with its `as_of`. +- Datacenter with no host-power series: `metrics_available: false`, retried every sweep. +- Job garbage-collected before the first sweep: `complete: false`; mitigated by the short sweep interval. A deleted running deployment is deregistered without purge, so it stays queryable for ~4 h and the next sweep's close pass captures the final window. + +## Files + +New: + +- `ai4papi/clients/mimir.py` +- `ai4papi/energy/__init__.py` (read API: `get_accumulated*`, `get_series`, `get_user_energy`, `get_datacenter_power`, `get_cluster_energy`; `add_to_job` / `add_to_jobs` are what the deployment routers call to set `job["energy"]`) +- `ai4papi/energy/store.py` (file store) +- `ai4papi/energy/series.py` (builds the time-series points of the API responses) +- `ai4papi/energy/compute.py` (pure functions) +- `ai4papi/energy/config.py` (config section, `enabled()`, TUE, shared constants and tiny helpers) +- `ai4papi/energy/query.py` (Mimir power queries shared by the sweep and the live top-up) +- `ai4papi/energy/sweep.py` (write path: deployment + datacenter sweep) +- `ai4papi/energy/live.py` (read path: live top-up and live series, for deployments and datacenters) +- `ai4papi/energy/ondemand.py` (read path: on-demand sweep and `live_only_doc`) +- `tests/test_energy.py` + +Modified (all additive): + +- `ai4papi/main.py`: the `energy_accounting_thread` `@repeat_every` and its call in `lifespan` +- `ai4papi/routers/v1/deployments/{modules,tools}.py`, `ai4papi/routers/v1/batch.py`, `ai4papi/routers/v1/try_me/nomad.py`: the `energy` key and param (`EnergyStats` or `EnergyTimeSeries` per `full_info`) +- `ai4papi/routers/v1/stats/deployments.py`: the per-user `energy` aggregate in `get_user_stats` (`full_info` for the series) and the per-datacenter `energy` block in `get_cluster_stats` (`full_info` for the series); no response-root total +- `ai4papi/schemas.py`: `EnergyStats` (per deployment and per user), `EnergyTimeSeriesPoint`, `EnergyTimeSeries` (its point list is `series`); `energy` on `DatacenterStats` (also `EnergyStats`) +- `ai4papi/wattnet.py`: `GreenDirector.footprint_series`; request timeouts, failures raised instead of cached, last good data kept and failure backoff in `retrieve_footprints` +- `pyproject.toml`: optional `test` extra (`pytest`); `tests/main.py`: runs `test_energy` through `pytest.main` +- `ai4papi/conf.py`: blank numeric CSV cell tolerated +- `var/datacenters.csv`: `TUE` column +- `etc/main.yaml` (`energy` section incl. `deployment_series_points` / `user_series_points` / `cluster_series_points`), `template.env`, `tests/test_routes.py` + +## Testing + +`tests/test_energy.py` (pytest, unlike the plain-script tests; `tests/main.py` runs it through `pytest.main`): + +- Pure-function checks, no infrastructure: `integrate` (trapezoidal, TUE, gaps, single sample), `sanitize` (negatives, ceiling, spike detection, disabled passthrough), `apply_footprint` (Wattnet series and `DEFAULTS` fallback), `intensity_at`, `downsample`, `merge_buckets`, `running_seconds`. +- Sweep pipeline, Mimir and Nomad monkeypatched: `process_single` writes a correct `.accum.json`, idempotency (same `now` gives a byte-identical document), monotonicity across sweeps, buckets aging into `.series.jsonl`, and `get_accumulated` / `get_series` validate against the pydantic models. +- Cluster energy, `mimir.query` / `mimir.query_range` monkeypatched (several raw host series): `get_datacenter_power()` gives `power_w = (host + gpu) * TUE`; `process_datacenter` writes a correct `_cluster/.accum.json` with per-series sanitize then sum, monotonic across sweeps, buckets aging into `.series.jsonl`; `get_cluster_energy(series=True)` merges power + accumulator + a series that validates as `EnergyTimeSeriesPoint` and whose `cumsum` reconciles with the accumulated total. +- Per-user aggregate: `get_user_energy` sums the owner's deployments (`EnergyStats`-shaped), with and without the merged series, and its total reconciles with the merged series. +- On-demand paths: a never-swept deployment is swept by the read, an existing doc triggers no Nomad call, an unknown job degrades to `None`; the bulk variant for lists and per-user reads; the pure-Mimir fallback for a deployment younger than one bucket (never a fabricated zero); an inconclusive stub that does not block it. +- `add_to_job` / `add_to_jobs` (list and detail shape, empty list), `series.build_series` (live tail only after the persisted points, several entities summed), and the store (a repeated bucket counts once, the owner index scans once and sees later writes). +- Live top-up: a failing host query is `degraded` on a datacenter doc; a confirmed `metrics_available: false` is reported as `None`. +- Mimir backoff: one timeout, then calls fail fast without a request until the backoff is over; with three deployments and a hung Mimir the sweep pays one timeout and stops the pass. +- Wattnet failure handling with a fake session: defaults while it is down and nothing cached, no new request during the backoff, real data after it recovers, last good data kept on a later blip, one log line per outage, timeouts present. +- An optional live Mimir check, gated on `PAPI_TESTS_MIMIR` plus real credentials. + +Run with the repo venv (`pip install -e .[test]` installs `pytest`): + +```bash +IS_PROD=False env/bin/python -m pytest tests/test_energy.py +cd tests && IS_PROD=False ../env/bin/python test_routes.py && cd .. +``` + +Note: install `fastapi < 0.116`. From 0.116 `include_router` is lazy and `test_routes.py` does not see the routes until the app starts. diff --git a/docs/energy-api.md b/docs/energy-api.md new file mode 100644 index 00000000..23cd4124 --- /dev/null +++ b/docs/energy-api.md @@ -0,0 +1,265 @@ +# Energy stats API (dashboard integration) + +Reference for the energy consumption and footprint (carbon and water, scope 2) that PAPI exposes. Architecture and internals are in [energy-accounting.md](./energy-accounting.md). + +All of this is additive and no new endpoints were created. The data rides on endpoints that already exist, at three levels. The accumulated numbers (and the live Mimir top-up) are **always** present at every level; the time series is only included when `full_info=true`, because it is large (hundreds to a thousand points per entity). + +| Level | Where | Accumulated (+ top-up) | Time series (`full_info=true`) | +|---|---|---|---| +| Datacenter / site | `GET /v1/deployments/stats/cluster` -> `datacenters[].energy` | always | per site, downsampled to `energy.cluster_series_points` | +| User | `GET /v1/deployments/stats/user?vo=` -> `energy` | always | merged across the user's deployments | +| Deployment | `energy` key in the deployment info (list and individual endpoints, all four types) | always | per deployment. On the individual endpoint `full_info` defaults to `true`, so it comes back as `EnergyTimeSeries`; the list form is `EnergyStats` | + +There is **no platform-wide / VO total** in the response: `/stats/cluster` returns one `energy` block per datacenter and the client sums the datacenters it cares about. + +`full_info` defaults to `false` on the list-shaped endpoints (`/stats/cluster`, `/stats/user`, the deployment list) and to `true` on the single-deployment endpoint. Pass it explicitly to override. + +Every endpoint that attaches energy also takes `energy` (default `true`). Pass `energy=false` to drop the `energy` block entirely, for a leaner and slightly faster response when the caller does not need it. PAPI itself passes `energy=false` on its internal quota and resource checks. + +Notes: + +- A datacenter carries an `energy` block only if it is in `energy.metrics_datacenters` (has a power exporter). `/stats/cluster?vo=` still returns every instrumented datacenter's `energy` even when that VO has no nodes there (site infrastructure energy is a property of the site); the VO filter only prunes the node lists. +- Datacenter energy measures **infrastructure** power (host + GPU of the whole site, times TUE). User and deployment energy is **workload** attribution (power of that deployment's allocations). They are not expected to add up. +- There is no VO-level energy block. The `full-agg` / `timeseries` keys already in `GET /stats/user` are the older CPU/GPU usage stats, unrelated to energy. + +If the feature is disabled or has no data, every one of these is `null` (or an absent key) and the API behaves exactly as before. + +## Units and conventions + +- Energy: `energy_wh` in watt-hours. Divide by 1000 for kWh. +- Carbon: `carbon_g` in grams of CO2 equivalent. +- Water: `water_l` in liters. +- Power: `power_w` in watts. +- Timestamps: ISO-8601 UTC, always with the `Z` suffix (`2026-09-07T07:50:00Z`). +- Every numeric field is a JSON number (float), returned at full precision (not rounded): the accumulated total always equals the sum of `series`, and idle CPU workloads can legitimately sit at 1e-5 Wh. Round for display on the client. +- Every energy, power, carbon and water value is normalized by the datacenter TUE (grid to compute ratio). The raw pre-TUE figure is not sent, but `tue_factor` is: divide by it to get the raw value. On a single-datacenter deployment `tue_factor` is that datacenter's configured TUE; on the per-user aggregate and on a deployment that moved between sites it is the blended `energy_wh / raw_energy_wh`. +- `power_w` in the block is the **current draw**: the last sample read from Mimir (~30 to 60 s old) at every level. `series[].power_w` in the time series is different: the **mean** over that 15-min bucket (so `energy_wh ~= power_w * bucket_hours`). Either way it is a rate, not a total, and is never summed. + +## The `EnergyStats` object + +One object, the same shape at every level, always with the live top-up applied: `energy` per deployment (list endpoints), `accumulated` inside the per-deployment time series, `energy` per datacenter in `GET /stats/cluster`, and `energy` per user in `GET /stats/user`. `coverage_ratio` and `datacenters` are `null` on the datacenter block; `deployments` is set only on the per-user aggregate. + +```json +{ + "energy_wh": 3088.98, + "tue_factor": 2.03, + "carbon_g": 929.78, + "water_l": 37.07, + "power_w": 220.0, + "since": "2026-09-01T10:00:00Z", + "as_of": "2026-09-07T07:50:00Z", + "live_as_of": "2026-09-07T07:54:58Z", + "complete": false, + "coverage_ratio": 1.0, + "degraded": false, + "datacenters": ["ifca-imagine"] +} +``` + +| Field | Type | Meaning | +|---|---|---| +| `energy_wh` | number | Accumulated energy, normalized by the datacenter TUE. This is the headline number. Divide by `tue_factor` for the raw figure at the meter. | +| `tue_factor` | number or null | The normalization applied: `energy_wh / raw_energy_wh`. On a single-datacenter deployment it is that datacenter's configured TUE. `null` only before there is any energy. | +| `carbon_g` | number | Accumulated carbon footprint (gCO2eq). | +| `water_l` | number | Accumulated water footprint (L). | +| `power_w` | number or null | Current draw: the last power sample read from Mimir (~30 to 60 s old), normalized by TUE. `0` for a running deployment drawing nothing measurable; `null` if it is not running. | +| `since` | string or null | Start of the accumulation window: the point `energy_wh` and the footprint are counted from. Pinned when accounting first picks up the deployment (its submit time, or as far back as `initial_lookback_hours` / Mimir retention allow) and stable afterwards, so it is not the same as `as_of`. Pair it with `complete: false` for a "since \" note. | +| `as_of` | string or null | Timestamp of the last persisted point. | +| `live_as_of` | string or null | Instant the numbers actually reflect, once the live top-up is applied. Use this for "updated N ago". Null when the values come straight from the store with no live top-up. | +| `complete` | boolean | `false` if accounting started after the deployment was created, or the window was clamped by Mimir retention. When `false`, treat `energy_wh` and the footprint as a lower bound. | +| `degraded` | boolean | `true` if the live Mimir part failed. The stored value is still served, it may just be slightly stale. A Wattnet outage does not set it: the footprint then uses the last good intensity (or the default values) and the numbers stay available. | +| `coverage_ratio` | number or null | Deployment and per-user only: fraction of the run time that has power data (`1.0` = fully covered). `null` on the datacenter block. | +| `datacenters` | string[] or null | Deployment and per-user only: datacenters the deployment(s) ran in. `null` on the datacenter block. | +| `deployments` | number or null | Per-user aggregate only: how many deployments went into the sum. | +| `series` | array or null | Per-user aggregate only, and only with `full_info=true`: the merged 15-min time series. `null` otherwise. | + +The values already include the live top-up: `energy_wh` is the persisted accumulator plus the slice from the last persisted point up to `live_as_of`, computed live against Mimir on each request (cached ~25 s). That is why the counter moves between refreshes without a background job. + +## 1. `energy` in the deployment info + +Added to the existing deployment object. Its shape follows the endpoint's existing `full_info` query param: + +- `full_info=false` -> `energy` is an `EnergyStats` (or `null`). +- `full_info=true` -> `energy` is an `EnergyTimeSeries` (or `null`): the accumulated stats plus the full time series for the detail-view graphs. + +The defaults line up with how the dashboard uses each endpoint: + +| Endpoint | `full_info` default | `energy` shape | +|---|---|---| +| `GET /v1/deployments/modules?vos=` (and `tools` / `batch` / `try_me/nomad` list variants) | `false` | `EnergyStats` per deployment | +| `GET /v1/deployments/modules/{uuid}?vo=` (and `tools` / `batch` variants) | `true` | `EnergyTimeSeries` | +| `GET /v1/deployments/try_me/nomad/{uuid}` (no `vo`, fixed VO) | `true` | `EnergyTimeSeries` | + +You can override in either direction: `...?vos=&full_info=true` gives the full series per deployment in the list (heavier, one Mimir top-up per deployment), and `.../{uuid}?vo=&full_info=false` gives just the accumulated `EnergyStats`. + +- `energy` is `null` when the deployment has no data (site or node without monitoring, still queued, or the feature is off). Show "not available" or hide the widget, never a zero. +- Query param `energy` (default `true`). Pass `?energy=false` to skip the block entirely, for example when you only need the deployment list fast. + +### List view (`full_info=false`) + +```json +{ + "job_ID": "aaaaaaaa-...", + "name": "module-aaaaaaaa-...", + "status": "running", + "energy": { + "energy_wh": 3088.98, + "carbon_g": 929.78, + "water_l": 37.07, + "power_w": 220.0, + "live_as_of": "2026-09-07T07:54:58Z", + "complete": false, "coverage_ratio": 1.0, "degraded": false, + "datacenters": ["ifca-imagine"], + "...": "see EnergyStats above" + } +} +``` + +The list endpoint resolves all of the user's deployments with one or two Mimir queries total, regardless of how many deployments there are. Poll it every 30 to 60 s for the list view. + +### Detail view (`full_info=true`) + +```json +{ + "job_ID": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "name": "module-aaaaaaaa-...", + "status": "running", + "energy": { + "deployment_uuid": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "start": "2026-09-07T00:55:00Z", + "end": "2026-09-07T07:54:58Z", + "source": "mixed", + "series": [ + { + "ts": "2026-09-07T00:55:00Z", + "power_w": 220.0, + "energy_wh": 37.22, + "carbon_g": 11.20, + "water_l": 0.45, + "live": false + }, + { "ts": "2026-09-07T01:00:00Z", "power_w": 220.0, "energy_wh": 37.22, "...": "..." } + ], + "accumulated": { "...": "EnergyStats, exactly the list-view object" } + } +} +``` + +| Field | Type | Meaning | +|---|---|---| +| `deployment_uuid` | string | Echo of the deployment. | +| `start` / `end` | string | Time span covered by `series`. `start` equals `accumulated.since`, `end` the live edge (the last point). Convenience only, both are derivable from the series. | +| `series` | array | Ordered by `ts` ascending. Whole deployment life, downsampled to `energy.deployment_series_points` (default 1000). | +| `series[].ts` | string | Bucket start. | +| `series[].power_w` | number or null | Mean power over the part of the bucket that had data, normalized by TUE. `energy_wh` is about `power_w` times the bucket duration. | +| `series[].energy_wh` | number or null | Energy in that bucket (per-bucket increment). A running total is `cumsum(series[].energy_wh)` and matches `accumulated.energy_wh`. | +| `series[].carbon_g` | number or null | Carbon in that bucket. Same, cumsum matches `accumulated.carbon_g`. | +| `series[].water_l` | number or null | Water in that bucket. Same, cumsum matches `accumulated.water_l`. | +| `series[].live` | boolean | `true` for the tail points served live from Mimir (not yet persisted). They can change on the next refresh. | +| `source` | string | `"store"` (only the persisted archive) or `"mixed"` (archive plus a live tail). Equivalent to `series[-1].live`. | +| `accumulated` | EnergyStats | The current totals, same object as the list view. Use it for the headline numbers next to the chart (and its `degraded` flag). | + +- The persisted archive is at 15-minute resolution (`energy.series_bucket_seconds`); the response is downsampled to `energy.deployment_series_points` (default 1000) across the whole life. +- The deployment is running, so `end` reaches "now" and the last points carry `live: true`. Calling the endpoint again a minute later returns a slightly longer series with updated `live` points. That is what makes the chart grow. +- Poll the detail view every 30 to 60 s. + +## 2. `energy` in the per-user stats + +``` +GET /v1/deployments/stats/user?vo= +GET /v1/deployments/stats/user?vo=&full_info=true +``` + +The existing response gains an `energy` key: all of the user's deployments in that VO, each with its live top-up, summed into one `EnergyStats` object (plus a `deployments` count): + +```json +{ + "full-agg": { "...": "existing compute-usage stats" }, + "timeseries": { "...": "existing" }, + "users-agg": { "...": "existing" }, + "energy": { + "energy_wh": 3088.98, + "tue_factor": 2.03, + "carbon_g": 929.78, + "water_l": 37.07, + "power_w": 220.0, + "since": "2026-09-01T10:00:00Z", + "as_of": "2026-09-09T07:00:00Z", + "live_as_of": "2026-09-09T07:04:58Z", + "complete": false, + "coverage_ratio": 0.98, + "degraded": false, + "datacenters": ["ifca-imagine"], + "deployments": 3, + "series": null + } +} +``` + +- Same fields as the per-deployment `EnergyStats` (see above), aggregated: `since` is the earliest across the user's deployments, `as_of` / `live_as_of` the latest, `complete` is the AND, `degraded` the OR, `power_w` the sum over the running ones, `coverage_ratio` the mean, `datacenters` the union, `tue_factor` the blended `energy_wh / raw_energy_wh`. +- `deployments`: how many accounted deployments went into the sum. +- `series`: `null` unless `full_info=true`, then a 15-min time series of the user's deployments summed by timestamp and downsampled once to `energy.user_series_points` (default 1000; `EnergyTimeSeriesPoint` shape, uniform step across the whole series), for a "my usage over time" chart. +- `null` when the user has no accounted deployments. +- Independent of the pre-existing CSV usage stats: if those are missing the endpoint still returns the `energy` block. + +## 3. `energy` in the cluster stats + +``` +GET /v1/deployments/stats/cluster +GET /v1/deployments/stats/cluster?vo= +GET /v1/deployments/stats/cluster?full_info=true +``` + +Each `datacenters[]` object gains an `energy` block: the same `EnergyStats` object as everywhere else (`coverage_ratio`, `datacenters`, `deployments` are `null` here). There is **no** response-root `energy`: to get a platform-wide or per-VO figure the client sums the datacenter blocks it wants. + +- **`power_w`**: the current draw from the last Mimir scrape, refreshed every ~30 s, normalized by TUE. It is `(sum of scaphandre host power + sum of GPU power) * TUE`. +- **accumulated** energy and footprint since accounting started (`energy_wh`, `carbon_g`, `water_l`, all TUE-normalized, divide by `tue_factor` for raw), kept indefinitely and topped up live on every read. +- **`series`**: `null` by default; with `full_info=true`, a 15-min series downsampled to `energy.cluster_series_points` (default 500). + +`?vo=` filters the node lists but not the `energy` blocks: an instrumented datacenter keeps its `energy` even when the VO has no nodes there (its `nodes` map is then empty). + +```json +{ + "datacenters": { + "ifca-imagine": { + "lat": 43.47, "lon": -3.8, "PUE": 1.31, + "nodes": { "...": "..." }, + "footprints": { "...": "existing" }, + "energy": { + "power_w": 1421.0, + "energy_wh": 184203.5, + "carbon_g": 55450.2, + "water_l": 2210.9, + "tue_factor": 2.03, + "complete": false, + "since": "2026-06-10T00:00:00Z", + "as_of": "2026-09-08T09:10:00Z", + "live_as_of": "2026-09-08T09:12:35Z", + "degraded": false, + "coverage_ratio": null, "datacenters": null, "deployments": null, + "series": null + } + } + }, + "cluster": { "...": "existing ResourceStats" }, + "updated_at": "2026-09-08T09:12:05Z" +} +``` + +Field meanings: see the `EnergyStats` table above. + +- `energy` is `null` for datacenters outside `energy.metrics_datacenters` and whenever Mimir is unreachable; never shown as zero. +- A `vo` filter does not touch the `energy` blocks; it only prunes each datacenter's `nodes`. An instrumented datacenter with no nodes for that VO is still returned, with its `energy` and an empty `nodes`. +- Refreshed by the same 30 s background task as the rest of the cluster stats. The persisted accumulator advances every sweep (~15 min); the live top-up keeps `energy_wh` moving between sweeps. The endpoint itself is cached ~30 s. + +## UX notes + +- Values come at full float precision; round on the client. Show `energy_wh` in Wh, or kWh with enough decimals to move, or animate the counter. At 50 W the increment per 30 s poll is about 0.4 Wh, invisible if you round to whole kWh. +- The displayed value trails real time by about 25 to 30 s (Mimir ingestion margin). Use `live_as_of` for the "updated N seconds ago" label. +- Poll cadence: list view every 30 to 60 s against the deployment list endpoint (`full_info=false`); detail view every 30 to 60 s against the individual deployment endpoint (`full_info=true`). +- Flags to surface in the UI: `complete: false` (show the value as a lower bound, for example a "since " note), `coverage_ratio < 1.0` (for example "78% of run time with data"), `degraded: true` (a small "data may be delayed" hint). +- `vo` values: `vo.ai4eosc.eu`, `vo.imagine-ai.eu`, `vo.ai4life.eu`, `kmd4eosc`, `tutorials`. The token must have a role for that VO. + +## Note on the test suite + +`tests/test_routes.py` requires `fastapi < 0.116` in the environment: from 0.116 `include_router` is lazy and the route assertions do not see the routes until the app starts. diff --git a/etc/main.yaml b/etc/main.yaml index c4f5ea5b..9fbcd2d5 100644 --- a/etc/main.yaml +++ b/etc/main.yaml @@ -85,3 +85,36 @@ mlflow: vo.ai4life.eu: https://mlflow.cloud.ai4eosc.eu/ kmd4eosc: https://mlflow.cloud.ai4eosc.eu/ tutorials: https://mlflow.cloud.ai4eosc.eu/ + +energy: # per-deployment energy consumption and environmental footprint stats + + enabled: true + sweep_seconds: 900 # background thread cadence + series_bucket_seconds: 900 # persisted time-series bucket size (matches Wattnet's 15-min cadence) + query_step_seconds: 30 # Mimir scrape interval + query_lag_seconds: 120 # ignore the most recent data (ingestion lag) + initial_lookback_hours: 24 # window covered on a deployment's first sweep + mimir_retention_days: 90 # tune to Mimir's actual retention + footprint_revision_hours: 24 # re-compute footprint for buckets newer than this + topup_cache_seconds: 25 # live top-up cache (< dashboard poll so the counter moves) + deployment_series_points: 1000 # downsample target for the per-deployment series in /deployments/* + user_series_points: 1000 # downsample target for the merged per-user series in /stats/user (full_info=true) + cluster_series_points: 500 # downsample target for the per-datacenter series in /stats/cluster (full_info=true) + + metrics_datacenters: # datacenters where power metrics exist + + - ifca-ai4eosc + - ifca-imagine + + # TUE (grid -> compute ratio, already covers CPU+GPU) is read per-datacenter + # from the `TUE` column of var/datacenters.csv; this is the fallback for + # datacenters that do not have it yet. + default_tue_factor: 2.73 + + data_quality: # drop anomalous samples from each source series before integrating + + enabled: true + min_power_w: 0 # drop samples below this (kills negatives) + max_power_w: 1000 # per-source ceiling (one allocation's CPU, or one GPU); 0 = off + spike_factor: 12 # drop a sample above spike_factor * rolling median (0 = off) + spike_window: 11 # neighbour samples used for the rolling median diff --git a/template.env b/template.env index 2df68a70..b7d1e90a 100644 --- a/template.env +++ b/template.env @@ -11,4 +11,7 @@ HARBOR_ROBOT_PASSWORD="*****************************" JENKINS_TOKEN="*************************************" PROVENANCE_TOKEN="**********************************" LITELLM_API_KEY="***********************************" +WATTNET_EMAIL="bot@example.org" WATTNET_PASSWORD="**********************************" +MIMIR_USER="platform-reader" +MIMIR_PASSWORD="**********************************" diff --git a/tests/main.py b/tests/main.py index eb226e64..39b50066 100644 --- a/tests/main.py +++ b/tests/main.py @@ -30,6 +30,12 @@ for module in modules: try: - __import__(module) + if module == "test_energy": # pytest-style, unlike the plain-script tests + import pytest + + if pytest.main(["-q", str(Path(__file__).parent / f"{module}.py")]): + raise RuntimeError("pytest reported failures") + else: + __import__(module) except Exception as e: print(f"🔴 Error testing {module}:\n{e}") diff --git a/tests/test_energy.py b/tests/test_energy.py new file mode 100644 index 00000000..27fd5bb9 --- /dev/null +++ b/tests/test_energy.py @@ -0,0 +1,1171 @@ +""" +Tests for the energy-accounting feature. + +Pure-function checks run everywhere. The sweep / read-API checks use a fake Mimir +and Nomad (no infra). The live Mimir check is skipped unless PAPI_TESTS_MIMIR and +MIMIR_USER/MIMIR_PASSWORD are set, so the suite stays runnable without the +monitoring stack. +""" + +import datetime +import json +import logging +import os + +import pytest + +from ai4papi import conf as papiconf +from ai4papi import energy, schemas +from ai4papi.clients import mimir +from ai4papi.energy import compute, config, live, ondemand, series, store, sweep + +_REAL_QUERY_RANGE = mimir.query_range + +_NOW = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0) +_NS, _UUID, _DC = "ai4eosc", "sweep-test-uuid", "ifca-ai4eosc" +# fixed, test-owned TUE: independent of whatever var/datacenters.csv says for +# `_DC` in production, so editing that file never breaks this test. +_TUE = 2.5 + + +def _series(values, scale_str=str): + """Build a Mimir matrix result with one series from [(ts, val)].""" + return [{"metric": {}, "values": [[t, scale_str(v)] for t, v in values]}] + + +def _fake_query_range(promql, start, end, step_s=30): + # 100 W constant (100e6 uW), one sample per step + t, out = start.timestamp(), [] + while t <= end.timestamp(): + out.append([t, "100000000" if "scaph" in promql else "0"]) + t += step_s + return [{"metric": {}, "values": out}] + + +@pytest.fixture(autouse=True) +def infra(monkeypatch, tmp_path): + """Fresh store + fake Mimir for every test; the feature counts as enabled.""" + monkeypatch.setenv("ACCOUNTING_PTH", str(tmp_path)) + monkeypatch.setattr(mimir, "MIMIR_USER", mimir.MIMIR_USER or "test") + monkeypatch.setattr(mimir, "MIMIR_PASSWORD", mimir.MIMIR_PASSWORD or "test") + monkeypatch.setitem( + papiconf.datacenters, _DC, {**papiconf.datacenters[_DC], "TUE": _TUE} + ) + monkeypatch.setattr(mimir, "_down_until", 0.0) + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + monkeypatch.setattr( + sweep.green_director, "metrics", {_DC: {"carbon": [], "water": []}} + ) + store._owner_idx.clear() + energy._read_doc.cache_clear() + energy.get_datacenter_power.cache_clear() + energy._cluster_archive_series.cache_clear() + for cache in ( + live._live_cache, + ondemand._live_only_doc_cache, + ondemand._bulk_sweep_cache, + ): + cache.clear() + + +@pytest.mark.skipif( + not (os.environ.get("PAPI_TESTS_MIMIR") and mimir.MIMIR_USER), + reason="set PAPI_TESTS_MIMIR + MIMIR_* creds", +) +def test_live_mimir_reachable(): + end = datetime.datetime.now(datetime.timezone.utc) + start = end - datetime.timedelta(hours=1) + assert isinstance(_REAL_QUERY_RANGE("up", start, end, step_s=60), list) + + +def test_integrate(): + # --- compute.integrate -------------------------------------------------------- + + # 1 W constant for 1h at 30s step -> 1 Wh at TUE 1.0, 2.03 Wh at TUE 2.03 + step = 30 + n = 3600 // step + 1 + cpu = _series([(1_000_000 + i * step, 1_000_000) for i in range(n)]) # 1e6 uW = 1 W + buckets = compute.integrate(cpu, [], step_s=step, bucket_s=3600, tue_factor=1.0) + total = sum(b.energy_wh for b in buckets) + assert abs(total - 1.0) < 1e-3, total + + buckets_tue = compute.integrate( + cpu, [], step_s=step, bucket_s=3600, tue_factor=2.03 + ) + assert abs(sum(b.energy_wh for b in buckets_tue) - 2.03) < 1e-2 + + # gap longer than gap_factor*step is not integrated across + gapped = _series( + [(0, 1_000_000), (30, 1_000_000), (600, 1_000_000), (630, 1_000_000)] + ) + gb = compute.integrate(gapped, [], step_s=step, bucket_s=3600, tue_factor=1.0) + # only the two 30s intervals contribute: 2 * (1W * 30s) = 60 Ws = 60/3600 Wh + assert abs(sum(b.energy_wh for b in gb) - 60 / 3600) < 1e-4 + + # a single sample -> no energy + assert compute.integrate(_series([(0, 1_000_000)]), [], 30, 3600) == [] + + +def test_sanitize(): + # --- compute.sanitize (data quality) --------------------------------------- + + dq = {"enabled": True, "min_power_w": 0, "max_power_w": 1000, "spike_factor": 5} + pw = [(0, 100.0), (30, -5.0), (60, 100.0), (90, 99999.0), (120, 100.0)] + clean = compute.sanitize(pw, dq) + assert (30, -5.0) not in clean # negative dropped + assert (90, 99999.0) not in clean # over ceiling dropped + assert len(clean) == 3 + + # spike detection against the rolling median + spiky = [(i * 30, 100.0) for i in range(20)] + spiky[10] = (300, 5000.0) + clean = compute.sanitize(spiky, {"spike_factor": 10, "spike_window": 7}) + assert (300, 5000.0) not in clean and len(clean) == 19 + + # disabled -> passthrough + assert compute.sanitize(pw, {"enabled": False}) == pw + assert compute.sanitize(pw, None) == pw + + # integrate drops the anomaly so it never inflates the accumulated energy + anom = _series( + [(t, 1_000_000) for t in range(0, 300, 30)] + + [(300, 900_000_000)] + + [(t, 1_000_000) for t in range(330, 600, 30)] + ) + raw = sum( + b.energy_wh + for b in compute.integrate(anom, [], step_s=30, bucket_s=3600, tue_factor=1.0) + ) + clean = sum( + b.energy_wh + for b in compute.integrate( + anom, + [], + step_s=30, + bucket_s=3600, + tue_factor=1.0, + data_quality={"max_power_w": 100}, + ) + ) + assert raw > 5 # the 900 W spike dominates the ~1 W baseline + assert clean < 0.3 # spike dropped -> only the ~1 W baseline remains + + # CPU + GPU add up + cpu2 = _series([(0, 2_000_000), (30, 2_000_000)]) # 2 W + gpu2 = _series([(0, 8), (30, 8)]) # 8 W + mb = compute.integrate(cpu2, gpu2, step_s=30, bucket_s=3600, tue_factor=1.0) + assert abs(sum(b.energy_wh for b in mb) - 10 * 30 / 3600) < 1e-4 + + # per-source data quality: a spike in one series does not discard the others + two_gpu = [ + {"metric": {"UUID": "g0"}, "values": [[0, "10"], [30, "10"], [60, "10"]]}, + {"metric": {"UUID": "g1"}, "values": [[0, "10"], [30, "9000"], [60, "10"]]}, + ] + mb = compute.integrate( + [], + two_gpu, + step_s=30, + bucket_s=3600, + tue_factor=1.0, + data_quality={"max_power_w": 100}, + ) + got = sum(b.energy_wh for b in mb) + # g0: 3 samples of 10 W over 60 s = 10*60/3600. g1: middle sample dropped, gap + # 0->60 s at 10 W = 10*60/3600. Total ~ 2 * 10 * 60/3600. + assert abs(got - 2 * 10 * 60 / 3600) < 1e-4, got + + +def test_apply_footprint(): + # --- compute.apply_footprint ------------------------------------------------- + + b = compute.Bucket("2026-01-01T00:00:00Z", 2000.0, 40.0, 0.0, 0.0) # 2 kWh + out = compute.apply_footprint([b]) # no series -> DEFAULTS + assert abs(out[0].carbon_g - 2.0 * compute.DEFAULT_CARBON) < 1e-6 + assert abs(out[0].water_l - 2.0 * compute.DEFAULT_WATER) < 1e-6 + + series = [["2026-01-01T00:00:00Z", 100.0], ["2026-01-01T00:15:00Z", 200.0]] + out = compute.apply_footprint([b], carbon_series=series, water_series=series) + assert abs(out[0].carbon_g - 2.0 * 100.0) < 1e-6 + + # bucket before the series -> DEFAULTS + early = compute.Bucket("2025-12-31T00:00:00Z", 1000.0, 20.0, 0.0, 0.0) + out = compute.apply_footprint([early], carbon_series=series) + assert abs(out[0].carbon_g - 1.0 * compute.DEFAULT_CARBON) < 1e-6 + + +def test_intensity_at(): + # --- compute.intensity_at --------------------------------------------------- + + ts_arr = [10.0, 20.0, 30.0] + val_arr = [1.0, 2.0, 3.0] + assert compute.intensity_at(ts_arr, val_arr, 5.0, -1) == -1 # before first + assert compute.intensity_at(ts_arr, val_arr, 20.0, -1) == 2.0 # exact + assert compute.intensity_at(ts_arr, val_arr, 25.0, -1) == 2.0 # between + assert compute.intensity_at(ts_arr, val_arr, 999.0, -1) == 3.0 # after last + assert compute.intensity_at([], [], 1.0, 7) == 7 # empty + + +def test_downsample(): + # --- compute.downsample --------------------------------------------------- + + many = [ + compute.Bucket(compute.iso(i * 300), 1.0, 10.0, 0.1, 0.01) for i in range(100) + ] + ds = compute.downsample(many, 10) + assert len(ds) == 10 + assert abs(sum(b.energy_wh for b in ds) - sum(b.energy_wh for b in many)) < 1e-6 + assert compute.downsample(many, 500) is many # nothing to do + + +def test_merge_buckets(): + # --- compute.merge_buckets ------------------------------------------------ + + b1 = compute.Bucket("t0", 2.0, 20.0, 0.5, 0.02) + b2 = compute.Bucket("t0", 3.0, 30.0, 0.7, 0.03) + merged = compute.merge_buckets([[b1], [b2]]) + assert len(merged) == 1 + assert merged[0].energy_wh == 5.0 and merged[0].power_w_avg == 50.0 + + +def test_running_seconds(): + # --- compute.running_seconds -------------------------------------------- + + W = compute.AllocWindow + assert compute.running_seconds([W("a", 0, 100)], 0, 200) == 100 + assert compute.running_seconds([W("a", 0, 100), W("b", 50, 150)], 0, 200) == 150 + assert compute.running_seconds([W("a", 100, None)], 0, 200) == 100 # still running + + +def test_sweep_pipeline(monkeypatch): + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-1"]}, + [compute.AllocWindow("alloc-1", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-1", "datacenter": _DC}], + False, + ), + ) + monkeypatch.setattr( + sweep.green_director, "metrics", {_DC: {"carbon": [], "water": []}} + ) + + _info = { + "uuid": _UUID, + "job_name": f"module-{_UUID}", + "namespace": _NS, + "owner": "user@egi.eu", + "submit_time": _NOW.timestamp() - 3 * 3600, + "status": "running", + } + + sweep.process_single(_info, _NOW) + doc = store.read_accum(_NS, _UUID) + acc = doc["accumulated"] + assert doc["metrics_available"] is True + assert abs(doc["tue_factor"] - _TUE) < 1e-9 + # ~3h of 100 W -> ~300 Wh raw, x _TUE (11 or 12 complete 15-min buckets + # depending on where "now" falls in the current bucket) + assert 270 * _TUE < acc["energy_wh"] < 305 * _TUE, acc + assert abs(acc["carbon_g"] - acc["energy_wh"] / 1000 * compute.DEFAULT_CARBON) < 1.0 + + # idempotent: same "now" -> byte-identical document + before = json.dumps(doc, sort_keys=True) + energy._read_doc.cache_clear() + sweep.process_single(_info, _NOW) + assert json.dumps(store.read_accum(_NS, _UUID), sort_keys=True) == before + + # monotonic on a later sweep + energy._read_doc.cache_clear() + sweep.process_single(_info, _NOW + datetime.timedelta(hours=1)) + assert store.read_accum(_NS, _UUID)["accumulated"]["energy_wh"] >= acc["energy_wh"] + + # buckets age out of the 24h tail into the persisted series + energy._read_doc.cache_clear() + sweep.process_single(_info, _NOW + datetime.timedelta(hours=30)) + assert len(store.read_series(_NS, _UUID, None, None)) > 0 + + # read API + pydantic validation + + energy._read_doc.cache_clear() + stats = energy.get_accumulated(_NS, _UUID, live=False) + schemas.EnergyStats(**stats) + series = energy.get_series(_NS, _UUID) + schemas.EnergyTimeSeries(**series) + assert series["series"] and all( + p["energy_wh"] is not None for p in series["series"] + ) + # per-bucket increments only; running total is cumsum(series[].energy_wh) + _pt = set(series["series"][-1]) + assert _pt == { + "ts", + "power_w", + "energy_wh", + "carbon_g", + "water_l", + "live", + }, _pt + _ps = energy.get_accumulated(_NS, _UUID, live=True) + assert _ps["power_w"] and abs(_ps["tue_factor"] - _TUE) < 1e-9 + + # per-user aggregate: sums the owner's deployments, EnergyStats-shaped + energy._read_doc.cache_clear() + _ue = energy.get_user_energy(_NS, "user@egi.eu", series=False) + schemas.EnergyStats(**_ue) + assert _ue["deployments"] >= 1 and _ue["series"] is None + assert _ue["energy_wh"] >= acc["energy_wh"] and _ue["since"] + _ues = energy.get_user_energy(_NS, "user@egi.eu", series=True) + schemas.EnergyStats(**_ues) + assert _ues["series"] and all(p["energy_wh"] is not None for p in _ues["series"]) + # the aggregate total must reconcile with the sum of the series (no rounding) + assert ( + abs(_ues["energy_wh"] - sum(p["energy_wh"] for p in _ues["series"])) < 1e-9 + ), _ues + assert abs(_ues["carbon_g"] - sum(p["carbon_g"] for p in _ues["series"])) < 1e-9 + # merge-then-downsample: uniform step across the whole persisted series + _uts = [ + datetime.datetime.fromisoformat(p["ts"].replace("Z", "+00:00")) + for p in _ues["series"] + if not p["live"] + ] + _steps = { + round((_uts[i + 1] - _uts[i]).total_seconds()) for i in range(len(_uts) - 1) + } + assert len(_steps) == 1, _steps + # hard cap on the persisted point count, same as the other levels + _capped = energy.get_user_energy(_NS, "user@egi.eu", series=True, target_points=5) + assert len([p for p in _capped["series"] if not p["live"]]) <= 5 + assert energy.get_user_energy(_NS, "nobody@egi.eu") is None + + # live tail: a running deployment's graph keeps growing past the stored series + _live_doc = { + "namespace": _NS, + "deployment_uuid": _UUID, + "metrics_available": True, + "pointer_ts": compute.iso( + datetime.datetime.now(datetime.timezone.utc).timestamp() - 600 + ), + "active_allocs": [{"alloc_id": "alloc-1", "datacenter": _DC}], + } + _lp = live.live_series(_live_doc, bucket_s=60) + assert _lp and all(p["live"] for p in _lp), _lp + assert all("energy_wh" in p and "carbon_g" in p for p in _lp) + + # the live top-up also carries the current power (100 W raw, x TUE) + _tu = live.topup(_live_doc) + _tue = config.tue(_DC) + assert _tu["power_w"] is not None + assert abs(_tu["power_w"] - 100.0 * _tue) < 5.0 * _tue, _tu + + # a running deployment's accumulated stats expose a power figure + energy._read_doc.cache_clear() + _st = energy.get_accumulated(_NS, _UUID, live=True) + assert _st["power_w"] is not None + + # the accumulated stats always carry a float coverage_ratio (1.0 = full) + assert isinstance(_st["coverage_ratio"], float) + + # GPU/CPU energy is clipped to the job's life: a finished allocation whose power + # metrics linger for a few minutes must not keep accumulating energy. + _ENDED_UUID = "sweep-ended-uuid" + _ended_end = _NOW.timestamp() - 2 * 3600 # allocation stopped 2h ago + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-e"]}, + [compute.AllocWindow("alloc-e", _NOW.timestamp() - 5 * 3600, _ended_end)], + [], # nothing running + False, + ), + ) + _ended_info = { + "uuid": _ENDED_UUID, + "job_name": f"module-{_ENDED_UUID}", + "namespace": _NS, + "owner": "user@egi.eu", + "submit_time": _NOW.timestamp() - 5 * 3600, + "status": "running", + } + energy._read_doc.cache_clear() + sweep.process_single(_ended_info, _NOW) + _edoc = store.read_accum(_NS, _ENDED_UUID) + _last_ts = max( + (b["ts"] for b in _edoc.get("tail", [])), + default=_edoc.get("settled_ts", ""), + ) + # fake Mimir returns 100 W for the whole [settled, now] window, but no bucket + # should land after the allocation ended (+ one bucket of slack). + assert compute.parse_iso(_last_ts) <= _ended_end + 300, _last_ts + + # a GPU-only monitored site (no scaphandre series at all) still accounts energy + # from the GPU power alone: a missing `scaph_*` metric comes back as an empty + # result, not an error, so the integrator just sums the GPU series. + _GPU_UUID = "sweep-gpu-only-uuid" + + def _fake_qr_gpu_only(promql, start, end, step_s=30): + if "scaph" in promql: + return [] + t, out = start.timestamp(), [] + while t <= end.timestamp(): + out.append([t, "150"]) # 150 W per GPU + t += step_s + return [{"metric": {}, "values": out}] + + monkeypatch.setattr(mimir, "query_range", _fake_qr_gpu_only) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_qr_gpu_only) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-g"]}, + [compute.AllocWindow("alloc-g", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-g", "datacenter": _DC}], + False, + ), + ) + _gpu_info = { + "uuid": _GPU_UUID, + "job_name": f"module-{_GPU_UUID}", + "namespace": _NS, + "owner": "gpu@egi.eu", + "submit_time": _NOW.timestamp() - 3 * 3600, + "status": "running", + } + energy._read_doc.cache_clear() + sweep.process_single(_gpu_info, _NOW) + _gdoc = store.read_accum(_NS, _GPU_UUID) + assert _gdoc["metrics_available"] is True, _gdoc + assert _gdoc["accumulated"]["energy_wh"] > 0, _gdoc + + +def test_on_demand_sweep(monkeypatch): + _OND_UUID = "sweep-on-demand-uuid" + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-o"]}, + [compute.AllocWindow("alloc-o", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-o", "datacenter": _DC}], + False, + ), + ) + + def _fake_get_job(id_, namespace): + if id_ != _OND_UUID: + raise KeyError(id_) + return { + "ID": _OND_UUID, + "Name": f"module-{_OND_UUID}", + "SubmitTime": int((_NOW.timestamp() - 3 * 3600) * 1e9), + "Meta": {"owner": "ondemand@egi.eu"}, + } + + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job) + + # never swept: no doc yet, the read API must trigger one sweep pass itself + assert store.read_accum(_NS, _OND_UUID) is None + energy._read_doc.cache_clear() + _ond_stats = energy.get_accumulated(_NS, _OND_UUID, live=False) + assert _ond_stats is not None and _ond_stats["energy_wh"] > 0, _ond_stats + assert store.read_accum(_NS, _OND_UUID) is not None + + # already swept once (on-demand or not): no further Nomad call needed + def _boom(id_, namespace): + raise AssertionError("ensure_swept should not touch Nomad once a doc exists") + + monkeypatch.setattr(sweep.Nomad.job, "get_job", _boom) + energy._read_doc.cache_clear() + assert energy.get_accumulated(_NS, _OND_UUID, live=False) is not None + + # job unknown to Nomad: on-demand sweep finds nothing, degrades to `None`, no crash + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job) + energy._read_doc.cache_clear() + assert energy.get_accumulated(_NS, "sweep-unknown-uuid", live=False) is None + energy._read_doc.cache_clear() + assert energy.get_series(_NS, "sweep-unknown-uuid") is None + + +def test_live_only_fallback(monkeypatch): + # `ensure_swept` correctly leaves such a deployment with no doc (nothing has + # settled into a full 15-min bucket yet); the read API must instead fall back + # to a pure live Mimir number -- but ONLY once Mimir actually has a sample for + # it, never a fabricated `0`. + + # "now" is frozen 10 minutes into a 15-min bucket and the deployment is + # submitted one minute into it: always younger than one bucket, whatever the + # wall clock says. A real "now - N minutes" would occasionally land in the + # previous bucket (making `process_single` write a real doc), and in the first + # minute of a bucket the live window would be shorter than one query step. + _frozen = datetime.datetime.fromtimestamp( + config.floor(_NOW.timestamp(), 900) + 600, datetime.timezone.utc + ) + monkeypatch.setattr(config, "now", lambda: _frozen) + _live_submit = config.floor(_frozen.timestamp(), 900) + 60 + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-live"]}, + [compute.AllocWindow("alloc-live", _live_submit, None)], + [{"alloc_id": "alloc-live", "datacenter": _DC}], + False, + ), + ) + + # Mimir has real (fake) samples for this young deployment -> populated, not null + _LIVE_UUID = "sweep-live-only-uuid" + + def _fake_get_job_live(id_, namespace): + if id_ != _LIVE_UUID: + raise KeyError(id_) + return { + "ID": _LIVE_UUID, + "Name": f"module-{_LIVE_UUID}", + "SubmitTime": int(_live_submit * 1e9), + "Meta": {"owner": "live@egi.eu"}, + } + + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job_live) + + assert store.read_accum(_NS, _LIVE_UUID) is None + energy._read_doc.cache_clear() + _live_stats = energy.get_accumulated(_NS, _LIVE_UUID, live=True) + assert _live_stats is not None and _live_stats["energy_wh"] > 0, _live_stats + assert _live_stats["power_w"] is not None + # `since` is pinned to the exact submit time (not clamped/started late), so + # `complete` must be `True` even though nothing is persisted yet + assert _live_stats["complete"] is True, _live_stats + # too young for a bucket: `process_single` still writes nothing, on purpose + assert store.read_accum(_NS, _LIVE_UUID) is None + + energy._read_doc.cache_clear() + _live_series = energy.get_series(_NS, _LIVE_UUID) + assert _live_series is not None and _live_series["accumulated"]["energy_wh"] > 0 + + # `power_w` must reflect the actual last Mimir sample even when it is stale + # (older than the `2 * query_step_seconds` freshness cushion `topup` itself + # enforces) -- there is nothing persisted here for a stale reading to mislead. + _STALE_UUID = "sweep-live-only-stale-uuid" + _stale_submit = (_NOW - datetime.timedelta(minutes=10)).timestamp() + + def _fake_get_job_stale(id_, namespace): + if id_ != _STALE_UUID: + raise KeyError(id_) + return { + "ID": _STALE_UUID, + "Name": f"module-{_STALE_UUID}", + "SubmitTime": int(_stale_submit * 1e9), + "Meta": {"owner": "live@egi.eu"}, + } + + def _fake_query_range_stale(promql, start, end, step_s=30): + # no sample within the last 5 minutes -- well past the 60s cushion + stale_end = end.timestamp() - 300 + t, out = start.timestamp(), [] + while t <= stale_end: + out.append([t, "100000000" if "scaph" in promql else "0"]) + t += step_s + return [{"metric": {}, "values": out}] + + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-stale"]}, + [compute.AllocWindow("alloc-stale", _stale_submit, None)], + [{"alloc_id": "alloc-stale", "datacenter": _DC}], + False, + ), + ) + monkeypatch.setattr(mimir, "query_range", _fake_query_range_stale) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range_stale) + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job_stale) + ondemand._live_only_doc_cache.clear() + live._live_cache.clear() + + # exercise `live_only_doc` / `topup` directly (not through `get_accumulated`'s + # `ensure_swept`): whether `process_single` would also consider this + # deployment "too young" depends on where in the current 15-min bucket the + # suite happens to run, which is irrelevant to what this checks. + _stale_doc = ondemand.live_only_doc(_NS, _STALE_UUID) + assert _stale_doc is not None, _stale_doc + _stale_topup = live.topup(_stale_doc) + assert _stale_topup is not None and _stale_topup["power_w"] is not None, ( + _stale_topup + ) + assert _stale_topup["power_w"] > 0, _stale_topup + + # Mimir has NOTHING for this alloc yet -> must stay `null`, not a fabricated `0` + live._live_cache.clear() # fresh top-up cache: don't reuse the populated result + + _EMPTY_UUID = "sweep-live-only-empty-uuid" + + def _fake_get_job_empty(id_, namespace): + if id_ != _EMPTY_UUID: + raise KeyError(id_) + return { + "ID": _EMPTY_UUID, + "Name": f"module-{_EMPTY_UUID}", + "SubmitTime": int(_live_submit * 1e9), + "Meta": {"owner": "live@egi.eu"}, + } + + def _fake_query_range_empty(promql, start, end, step_s=30): + return [] + + monkeypatch.setattr(mimir, "query_range", _fake_query_range_empty) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range_empty) + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job_empty) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-empty"]}, + [compute.AllocWindow("alloc-empty", _live_submit, None)], + [{"alloc_id": "alloc-empty", "datacenter": _DC}], + False, + ), + ) + + energy._read_doc.cache_clear() + assert energy.get_accumulated(_NS, _EMPTY_UUID, live=True) is None + energy._read_doc.cache_clear() + assert energy.get_series(_NS, _EMPTY_UUID) is None + assert store.read_accum(_NS, _EMPTY_UUID) is None + + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + + +def test_inconclusive_stub(monkeypatch): + # `process_single` persists a bare stub (no "accumulated" key, no + # `metrics_available` verdict) the first time it sees a job with no allocation + # resolved at all yet (eg. queued, alloc not placed by Nomad). That stub must + # not stop later reads from trying `live_only_doc` once Mimir has real data. + + _STUB_UUID = "sweep-inconclusive-stub-uuid" + _stub_submit = (_NOW - datetime.timedelta(minutes=1)).timestamp() + + def _fake_get_job_stub(id_, namespace): + if id_ != _STUB_UUID: + raise KeyError(id_) + return { + "ID": _STUB_UUID, + "Name": f"module-{_STUB_UUID}", + "SubmitTime": int(_stub_submit * 1e9), + "Meta": {"owner": "stub@egi.eu"}, + } + + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job_stub) + + # first on-demand attempt: no allocation resolved at all yet + monkeypatch.setattr( + compute, "resolve_alloc_ids", lambda job_id, ns: ({}, [], [], False) + ) + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + + assert store.read_accum(_NS, _STUB_UUID) is None + energy._read_doc.cache_clear() + assert energy.get_accumulated(_NS, _STUB_UUID, live=True) is None + _stub_doc = store.read_accum(_NS, _STUB_UUID) + assert _stub_doc is not None and "accumulated" not in _stub_doc, _stub_doc + assert ( + _stub_doc.get("metrics_available") is not False + ) # inconclusive, not a verdict + + # the allocation is now placed and Mimir has real samples: must populate, not + # stay `null` because of the stub written above + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-stub"]}, + [compute.AllocWindow("alloc-stub", _stub_submit, None)], + [{"alloc_id": "alloc-stub", "datacenter": _DC}], + False, + ), + ) + live._live_cache.clear() + ondemand._live_only_doc_cache.clear() # `live_only_doc` itself is cached now too + energy._read_doc.cache_clear() + _stub_stats = energy.get_accumulated(_NS, _STUB_UUID, live=True) + assert _stub_stats is not None and _stub_stats["energy_wh"] > 0, _stub_stats + + +def test_on_demand_bulk_sweep(monkeypatch): + _BULK_UUID = "sweep-bulk-uuid" + _BULK_OWNER = "bulk@egi.eu" + monkeypatch.setattr(mimir, "query_range", _fake_query_range) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_query_range) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-b"]}, + [compute.AllocWindow("alloc-b", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-b", "datacenter": _DC}], + False, + ), + ) + + def _fake_get_jobs(namespace, filter_=None): + return [{"ID": _BULK_UUID, "Name": f"module-{_BULK_UUID}"}] + + def _fake_get_job_bulk(id_, namespace): + if id_ != _BULK_UUID: + raise KeyError(id_) + return { + "ID": _BULK_UUID, + "Name": f"module-{_BULK_UUID}", + "SubmitTime": int((_NOW.timestamp() - 3 * 3600) * 1e9), + "Meta": {"owner": _BULK_OWNER}, + } + + monkeypatch.setattr(sweep.Nomad.jobs, "get_jobs", _fake_get_jobs) + monkeypatch.setattr(sweep.Nomad.job, "get_job", _fake_get_job_bulk) + + # never swept: the bulk/list read must sweep it on demand before serving + assert store.read_accum(_NS, _BULK_UUID) is None + _bulk = energy.get_accumulated_bulk(_NS, _BULK_OWNER, live=False) + assert _BULK_UUID in _bulk and _bulk[_BULK_UUID]["energy_wh"] > 0, _bulk + assert store.read_accum(_NS, _BULK_UUID) is not None + + _ue = energy.get_user_energy(_NS, _BULK_OWNER, series=False) + assert _ue is not None and _ue["deployments"] == 1, _ue + + # already swept: the listing is still cheap (Nomad.jobs.get_jobs), but no + # per-job fetch / re-sweep for a uuid that already has a doc + def _boom_get_job(id_, namespace): + raise AssertionError("ensure_swept_bulk should not re-fetch a swept job") + + monkeypatch.setattr(sweep.Nomad.job, "get_job", _boom_get_job) + _bulk2 = energy.get_accumulated_bulk(_NS, _BULK_OWNER, live=False) + assert _BULK_UUID in _bulk2 + + +def test_cluster_energy(monkeypatch): + _CNS = config.CLUSTER_NS + _ISO0 = "2020-01-01T00:00:00Z" + monkeypatch.setattr( + sweep.green_director, + "metrics", + {_DC: {"carbon": [[_ISO0, 300.0]], "water": [[_ISO0, 10.0]]}}, + ) + _tue_dc = config.tue(_DC) + + def _fake_query(promql, at=None, timeout=30): + if "scaph_host_power" in promql: + # 500 W reported in microwatts + return [ + { + "metric": {"datacenter": _DC}, + "value": [_NOW.timestamp(), "500000000"], + } + ] + if "DCGM" in promql: + return [ + {"metric": {"datacenter": _DC}, "value": [_NOW.timestamp(), "200"]} + ] # W + return [] + + monkeypatch.setattr(mimir, "query", _fake_query) + + # 1. instantaneous power: power_w = (host + gpu) * TUE, normalized only + energy.get_datacenter_power.cache_clear() + _cp = energy.get_datacenter_power() + assert _DC in _cp, _cp + assert abs(_cp[_DC]["power_w"] - (500.0 + 200.0) * _tue_dc) < 1.0, _cp + assert set(_cp[_DC]) == {"power_w", "tue_factor", "as_of"}, _cp[_DC] + + # 2. per-datacenter sweep: several raw host series -> sanitize each -> sum + def _fake_cluster_qr(promql, start, end, step_s=30): + def _rows(pairs): + out = [] + for label, val in pairs: + vals, t = [], start.timestamp() + while t <= end.timestamp(): + vals.append([t, val]) + t += step_s + out.append({"metric": {"node": label, "UUID": label}, "values": vals}) + return out + + if "scaph_host_power" in promql: # two hosts: 400 W + 600 W (in microwatts) + return _rows([("nodeA", "400000000"), ("nodeB", "600000000")]) + if "DCGM" in promql: # one GPU: 250 W + return _rows([("gpu0", "250")]) + return [] + + monkeypatch.setattr(mimir, "query_range", _fake_cluster_qr) + monkeypatch.setattr(mimir, "query_range_chunked", _fake_cluster_qr) + live._live_cache.clear() + + sweep.process_datacenter(_DC, _NOW) + cdoc = store.read_accum(_CNS, _DC) + assert cdoc["metrics_available"] is True + cacc = cdoc["accumulated"] + assert cdoc["tue_factor"] == _tue_dc + # window ~= initial_lookback_hours of (400 + 600 + 250) = 1250 W raw, x TUE + _expected = config.get()["initial_lookback_hours"] * 1250.0 * _tue_dc + assert 0.75 * _expected < cacc["energy_wh"] <= 1.05 * _expected, cacc + assert abs(cacc["carbon_g"] - cacc["energy_wh"] / 1000 * 300.0) < 1.0 + + # monotonic on a later sweep + sweep.process_datacenter(_DC, _NOW + datetime.timedelta(hours=1)) + assert store.read_accum(_CNS, _DC)["accumulated"]["energy_wh"] >= cacc["energy_wh"] + + # buckets age out of the 24h tail into the persisted series + sweep.process_datacenter(_DC, _NOW + datetime.timedelta(hours=30)) + assert len(store.read_series(_CNS, _DC, None, None)) > 0 + + # 3. read API: get_cluster_energy merges power + accumulator + series + energy.get_datacenter_power.cache_clear() + energy._cluster_archive_series.cache_clear() + live._live_cache.clear() + _ce = energy.get_cluster_energy(series=True) + assert _DC in _ce, _ce + _row = _ce[_DC] + assert _row["power_w"] is not None and _row["energy_wh"] is not None + assert _row["since"] and _row["as_of"] + assert _row["series"] and all(p["energy_wh"] is not None for p in _row["series"]) + for _p in _row["series"]: + schemas.EnergyTimeSeriesPoint(**_p) + schemas.EnergyStats(**_row) + assert _row["tue_factor"] == _tue_dc # only conversion factor exposed + # the accumulated total reconciles with the sum of the series (no rounding) + assert ( + abs(_row["energy_wh"] - sum(p["energy_wh"] for p in _row["series"])) < 1e-9 + ), _row + + # 4. datacenter live top-up carries the (TUE-normalized) energy deltas + live._live_cache.clear() + _ct = live.topup(cdoc) + assert _ct and _ct["energy_wh"] > 0 and _ct["live_as_of"] + assert abs(_ct["carbon_g"] - _ct["energy_wh"] / 1000 * 300.0) < 1.0, _ct + + # 5. a GPU-only monitored datacenter (in metrics_datacenters, no host power) is + # still reported: for a declared site, GPU power alone is a valid total. + def _fake_query_gpu_only(promql, at=None, timeout=30): + if "DCGM" in promql: + return [{"metric": {"datacenter": _DC}, "value": [_NOW.timestamp(), "200"]}] + return [] + + monkeypatch.setattr(mimir, "query", _fake_query_gpu_only) + energy.get_datacenter_power.cache_clear() + _gp = energy.get_datacenter_power() + assert _DC in _gp, _gp + assert abs(_gp[_DC]["power_w"] - 200.0 * _tue_dc) < 1.0, _gp + + # 6. a datacenter with no power series at all is absent from get_datacenter_power + monkeypatch.setattr(mimir, "query", lambda promql, at=None, timeout=30: []) + monkeypatch.setattr(mimir, "query_range", lambda *a, **k: []) + monkeypatch.setattr(mimir, "query_range_chunked", lambda *a, **k: []) + energy.get_datacenter_power.cache_clear() + assert energy.get_datacenter_power() == {} + + +def _failing_scaph(promql, start, end, step_s=30): + if "scaph" in promql: + raise mimir.MimirError("down") + return [] + + +def test_live_topup_is_degraded_when_mimir_fails(monkeypatch): + # a failing query must show up as `degraded`, never as a silent zero + monkeypatch.setattr(mimir, "query_range", _failing_scaph) + pointer = compute.iso(_NOW.timestamp() - 600) + cluster_doc = { + "namespace": config.CLUSTER_NS, + "deployment_uuid": _DC, + "metrics_available": True, + "pointer_ts": pointer, + } + dep_doc = { + "namespace": _NS, + "deployment_uuid": _UUID, + "metrics_available": True, + "pointer_ts": pointer, + "active_allocs": [{"alloc_id": "alloc-1", "datacenter": _DC}], + } + assert live.topup(cluster_doc) == {"degraded": True} + assert live.topup(dep_doc) == {"degraded": True} + assert live.live_series(dep_doc, bucket_s=60) == [] + + +def test_no_metrics_verdict_is_not_reported(monkeypatch): + # a deployment confirmed to have no usable data is `None`, never a zero + store.write_accum( + _NS, + _UUID, + {"owner": "u@egi.eu", "metrics_available": False, "accumulated": {}}, + ) + monkeypatch.setattr(sweep.Nomad.jobs, "get_jobs", lambda **kw: []) + assert energy.get_accumulated(_NS, _UUID, live=False) is None + assert energy.get_series(_NS, _UUID) is None + assert energy.get_accumulated_bulk(_NS, "u@egi.eu", live=False) == {} + assert energy.get_user_energy(_NS, "u@egi.eu") is None + + +def test_add_to_jobs(monkeypatch): + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-1"]}, + [compute.AllocWindow("alloc-1", _NOW.timestamp() - 3 * 3600, None)], + [{"alloc_id": "alloc-1", "datacenter": _DC}], + False, + ), + ) + monkeypatch.setattr(sweep.Nomad.jobs, "get_jobs", lambda **kw: []) + info = { + "uuid": _UUID, + "job_name": f"module-{_UUID}", + "namespace": _NS, + "owner": "u@egi.eu", + "submit_time": _NOW.timestamp() - 3 * 3600, + "status": "running", + } + sweep.process_single(info, _NOW) + + # accumulated stats by default, the extended series with `full_info` + jobs = [{"job_ID": _UUID}, {"job_ID": "not-accounted"}] + energy.add_to_jobs(jobs, _NS, "u@egi.eu", full_info=False) + schemas.EnergyStats(**jobs[0]["energy"]) + assert jobs[1]["energy"] is None + + job = {"job_ID": _UUID} + energy.add_to_job(job, _NS, full_info=True) + schemas.EnergyTimeSeries(**job["energy"]) + energy.add_to_job(job, _NS, full_info=False) + schemas.EnergyStats(**job["energy"]) + + # nothing to attach: no Nomad / store work at all + monkeypatch.setattr(energy, "get_accumulated_bulk", lambda *a, **k: 1 / 0) + energy.add_to_jobs([], _NS, "u@egi.eu", full_info=False) + + +def test_build_series_live_tail_only_after_persisted(): + def b(ts, wh): + return {"ts": ts, "energy_wh": wh, "power_w_avg": 1.0} + + persisted = [[b("2026-01-01T00:00:00Z", 1.0), b("2026-01-01T00:15:00Z", 2.0)]] + live = [[b("2026-01-01T00:15:00Z", 9.0), b("2026-01-01T00:30:00Z", 3.0)]] + points = series.build_series(persisted, live, target_points=100) + assert [(p["ts"], p["energy_wh"], p["live"]) for p in points] == [ + ("2026-01-01T00:00:00Z", 1.0, False), + ("2026-01-01T00:15:00Z", 2.0, False), + ("2026-01-01T00:30:00Z", 3.0, True), + ] + + # several entities (a user's deployments) are summed per timestamp + two = series.build_series([persisted[0], persisted[0]], [], target_points=100) + assert [p["energy_wh"] for p in two] == [2.0, 4.0] + + +def test_wattnet_failure_is_not_cached_nor_overwrites_good_data(monkeypatch, caplog): + from types import SimpleNamespace + + from ai4papi import wattnet + + ts = "2026-01-01T00:00:00Z" + fp = [{"footprint_type": "carbon", "series": [{"values": [[ts, 5.0]]}]}] + score = [{"footprint_type": "x", "series": [{"values": [[ts, 9.0]]}]}] + answers = {"ok": True} + calls = [] + + def fake_get(url, **kwargs): + calls.append(url) + assert kwargs["timeout"] # a stalled Wattnet must not block the threads + if not answers["ok"]: + return SimpleNamespace(ok=False, status_code=503) + body = score if "environmental" in url else fp + return SimpleNamespace(ok=True, status_code=200, json=lambda: body) + + monkeypatch.setattr(wattnet.session, "get", fake_get) + monkeypatch.setattr(papiconf, "datacenters", {"wn-test": {"lat": 1, "lon": 2}}) + g = wattnet.GreenDirector() + monkeypatch.setattr(g, "_retrieve_token", lambda: "t") + caplog.set_level(logging.INFO, logger="ai4papi.wattnet") + default_carbon = wattnet.GreenDirector.DEFAULTS["carbon"] + + # never had data and Wattnet is down: DEFAULTS, and nothing is cached + answers["ok"] = False + g.retrieve_footprints() + assert g.metrics["wn-test"]["carbon"][0][1] == default_carbon + assert len(calls) == 2 + assert "using default values" in caplog.text + + # still within the backoff: Wattnet is not asked again + g.retrieve_footprints() + assert len(calls) == 2 + + # backoff over and recovered: asks again, gets the real data, logs it once + g._backoff.clear() + answers["ok"] = True + g.retrieve_footprints() + assert len(calls) == 4 + assert g.metrics["wn-test"]["carbon"] == [[ts, 5.0]] + assert caplog.text.count("recovered") == 1 + + # blip after good data: the last good data is kept, not overwritten by DEFAULTS + wattnet.GreenDirector._fetch_footprint_data.cache_clear() + g._backoff.clear() + answers["ok"] = False + caplog.clear() + g.retrieve_footprints() + assert g.metrics["wn-test"]["carbon"] == [[ts, 5.0]] + assert f"keeping data up to {ts}" in caplog.text + + # a long outage logs once (on the change), not on every refresh + for _ in range(3): + g._backoff.clear() + g.retrieve_footprints() + assert caplog.text.count("refresh failed") == 1 + + +def test_store_series_dedupes_and_owner_index(monkeypatch): + # a bucket appended twice (crash between the series and the doc write) counts once + b = {"ts": "2026-01-01T00:00:00Z", "energy_wh": 1.0} + store.append_series(_NS, "dup", [b, {**b, "ts": "2026-01-01T00:15:00Z"}]) + store.append_series(_NS, "dup", [b]) + assert [x["ts"] for x in store.read_series(_NS, "dup", None, None)] == [ + "2026-01-01T00:00:00Z", + "2026-01-01T00:15:00Z", + ] + + # listing one owner scans the namespace once, and sees later writes at once + store.write_accum(_NS, "a1", {"owner": "a@egi.eu"}) + store.write_accum(_NS, "b1", {"owner": "b@egi.eu"}) + scans = [] + real_iter = store.iter_namespace + monkeypatch.setattr( + store, "iter_namespace", lambda ns: scans.append(ns) or real_iter(ns) + ) + assert [u for u, _ in store.iter_owner(_NS, "a@egi.eu")] == ["a1"] + store.write_accum(_NS, "a2", {"owner": "a@egi.eu"}) + assert [u for u, _ in store.iter_owner(_NS, "a@egi.eu")] == ["a1", "a2"] + assert [u for u, _ in store.iter_owner(_NS, "nobody@egi.eu")] == [] + assert len(scans) == 1 + + +def test_mimir_backoff_after_a_timeout(monkeypatch): + import requests + + calls = [] + + def hung(url, **kwargs): + calls.append(url) + raise requests.Timeout("no answer") + + monkeypatch.setattr(mimir.session, "get", hung) + start = datetime.datetime.now(datetime.timezone.utc) + + # the first call pays the timeout, the next ones fail fast without a request + with pytest.raises(requests.Timeout): + _REAL_QUERY_RANGE("up", start, start, step_s=30) + for call in ( + lambda: _REAL_QUERY_RANGE("up", start, start, step_s=30), + lambda: mimir.query("up"), + ): + with pytest.raises(mimir.MimirError): + call() + assert len(calls) == 1 + + # once the backoff is over Mimir is asked again + monkeypatch.setattr(mimir, "_down_until", 0.0) + with pytest.raises(requests.Timeout): + mimir.query("up") + assert len(calls) == 2 + + +def test_sweep_stops_the_pass_when_mimir_is_down(monkeypatch): + import requests + + uuids = [f"down-{i}" for i in range(3)] + submit = _NOW.timestamp() - 3 * 3600 + calls = [] + + def hung(url, **kwargs): + calls.append(url) + raise requests.Timeout("no answer") + + monkeypatch.setattr(mimir.session, "get", hung) + monkeypatch.setattr(mimir, "query_range", _REAL_QUERY_RANGE) + monkeypatch.setattr(mimir, "query_range_chunked", _REAL_QUERY_RANGE) + monkeypatch.setattr( + sweep.Nomad.jobs, + "get_jobs", + lambda **kw: [{"ID": u, "Name": f"module-{u}"} for u in uuids], + ) + fetched = [] + + def get_job(id_, namespace): + fetched.append(id_) + return { + "ID": id_, + "Name": f"module-{id_}", + "SubmitTime": int(submit * 1e9), + "Meta": {"owner": "u@egi.eu"}, + } + + monkeypatch.setattr(sweep.Nomad.job, "get_job", get_job) + monkeypatch.setattr( + compute, + "resolve_alloc_ids", + lambda job_id, ns: ( + {_DC: ["alloc-1"]}, + [compute.AllocWindow("alloc-1", submit, None)], + [{"alloc_id": "alloc-1", "datacenter": _DC}], + False, + ), + ) + + with pytest.raises(requests.Timeout): + sweep.process_single( + sweep.info_from_job(sweep.Nomad.job.get_job(uuids[0], _NS), _NS), _NOW + ) + calls.clear() + fetched.clear() + monkeypatch.setattr(mimir, "_down_until", 0.0) + + # three deployments, Mimir hung: the first pays the timeout, the second fails + # fast and stops the pass, the third is not even fetched from Nomad + sweep._sweep_namespace(_NS, _NOW) + assert len(calls) == 1 + assert fetched == uuids[:2] + assert all(store.read_accum(_NS, u) is None for u in uuids) + + +def test_store_ignores_ids_that_are_not_plain_names(tmp_path): + # an id used as a file name can never point outside the store + for bad in ("../evil", "a/b", "..", ".hidden", "", "x\n", "/etc/passwd"): + store.write_accum(_NS, bad, {"owner": "u@egi.eu"}) + store.append_series(_NS, bad, [{"ts": "2026-01-01T00:00:00Z"}]) + assert store.read_accum(_NS, bad) is None + assert store.read_series(_NS, bad, None, None) == [] + store.write_accum("../escape", "ok", {"owner": "u@egi.eu"}) + assert store.read_accum("../escape", "ok") is None + assert [p for p in tmp_path.rglob("*") if p.is_file()] == [] + assert not (tmp_path.parent / "evil.accum.json").exists() + + # real ids still work + for name in ( + "sweep-test-uuid", + "ifca-ai4eosc", + "a1b2c3d4-0000-1111-2222-333344445555", + ): + store.write_accum(_NS, name, {"owner": "u@egi.eu"}) + assert store.read_accum(_NS, name) == {"owner": "u@egi.eu"} + store.write_accum(config.CLUSTER_NS, "ifca-ai4eosc", {"owner": None}) + assert store.read_accum(config.CLUSTER_NS, "ifca-ai4eosc") is not None diff --git a/var/datacenters.csv b/var/datacenters.csv index e69c060f..fc3e5503 100644 --- a/var/datacenters.csv +++ b/var/datacenters.csv @@ -1,7 +1,7 @@ -name,country,lat,lon,PUE -ifca-ai4eosc,ES,43.471668,-3.802318,1.31 -ifca-imagine,ES,43.471706,-3.802240,1.31 -iisas-ai4eosc,SK,48.170095,17.070058,1.57 -incd-imagine,PT,38.759140,-9.142664,1.45 -tubitak-imagine,TR,40.785667,29.448592,1.35 -psnc-ai4eosc,PL,52.406889,16.953201,1.29 \ No newline at end of file +name,country,lat,lon,PUE,TUE +ifca-ai4eosc,ES,43.471668,-3.802318,1.31,2.73 +ifca-imagine,ES,43.471706,-3.802240,1.31,2.73 +iisas-ai4eosc,SK,48.170095,17.070058,1.57, +incd-imagine,PT,38.759140,-9.142664,1.45, +tubitak-imagine,TR,40.785667,29.448592,1.35, +psnc-ai4eosc,PL,52.406889,16.953201,1.29,