Add cache & freshness transparency (v0.4.1-alpha)

.env.example

@@ -1,6 +1,7 @@
 DATABASE_URL=sqlite:///./routeforge.db
 HTTP_TIMEOUT_SECONDS=10
 CACHE_TTL_SECONDS=900
+RIPESTAT_CACHE_TTL_SECONDS=900
 CORS_ORIGINS=http://localhost:3000,http://127.0.0.1:3000
 RIPESTAT_BASE_URL=https://stat.ripe.net/data
 ROUTEFORGE_DEMO_MODE=false

README.md

@@ -2,7 +2,7 @@
 
 RouteForge is a read-only routing preflight and explainability tool for BGP, RPKI, Registry/IRR and Routing Visibility checks.
 
-Current user-facing version: **v0.4.0-alpha**.
+Current user-facing version: **v0.4.1-alpha**.
 
 <!-- Screenshot gallery placeholder:
 - docs/screenshots/dashboard.png
@@ -32,7 +32,7 @@ Routing changes often require fast but traceable checks across multiple external
 
 ## Current Alpha Status
 
-RouteForge is a **functional alpha** release with production-like workflows for read-only validation and demo usage. Current release target: **v0.4.0-alpha**.
+RouteForge is a **functional alpha** release with production-like workflows for read-only validation and demo usage. Current release target: **v0.4.1-alpha**.
 
 ## Quickstart with Docker Compose
 
@@ -94,6 +94,17 @@ Preflight decision model:
 - `NO-GO`
 - `UNKNOWN`
 
+## Cache and Freshness
+
+RouteForge shows per-source diagnostics indicating whether data was fetched live or served from cache. Cache Age, TTL and Freshness help operators assess how current a result is.
+
+Freshness values:
+- `LIVE`: queried live
+- `FRESH`: cached and fresh
+- `EXPIRING_SOON`: cache close to expiry
+- `STALE`: older than TTL
+- `UNKNOWN`: cache age cannot be determined
+
 ## Export and Sharing
 
 RouteForge reports can be exported/shared as:

RELEASE_NOTES.md

@@ -1,5 +1,16 @@
 # Release Notes
 
+## v0.4.1-alpha
+
+### Highlights
+
+- Cache & Freshness Transparency
+- Cache age and TTL shown in diagnostics
+- Freshness classification LIVE/FRESH/EXPIRING_SOON/STALE/UNKNOWN
+- Force Refresh support prepared internally
+
+---
+
 ## v0.4.0-alpha
 
 **Product Demo & Release Readiness**

backend/app/api/routes_system.py

@@ -9,7 +9,7 @@
 def system_info():
     return {
         'name': 'RouteForge',
-        'version': 'v0.4.0-alpha',
+        'version': 'v0.4.1-alpha',
         'demo_mode': settings.demo_mode,
         'read_only': True,
         'data_sources': ['RIPEstat', 'RIPEstat Whois/Registry'],

backend/app/config.py

@@ -8,7 +8,7 @@ class Settings(BaseSettings):
     database_url: str = "sqlite:///./routeforge.db"
     ripestat_base_url: str = "https://stat.ripe.net/data"
     http_timeout_seconds: int = 10
-    cache_ttl_seconds: int = 900
+    cache_ttl_seconds: int = Field(default=900, validation_alias="RIPESTAT_CACHE_TTL_SECONDS")
     cors_origins: str = "http://192.168.58.167:3000,http://127.0.0.1:3000"
     demo_mode: bool = Field(default=False, validation_alias="ROUTEFORGE_DEMO_MODE")
 

backend/app/core/source_diagnostics.py

@@ -29,6 +29,34 @@
 }
 
 
+def make_cache_metadata(
+    cached: bool | None,
+    cache_age_seconds: int | None = None,
+    cache_ttl_seconds: int | None = None,
+    fetched_at: str | None = None,
+    expires_at: str | None = None,
+) -> dict[str, Any]:
+    freshness = "UNKNOWN"
+    if cached is False:
+        freshness = "LIVE"
+    elif cached is True and cache_age_seconds is not None and cache_ttl_seconds is not None:
+        if cache_age_seconds <= cache_ttl_seconds * 0.75:
+            freshness = "FRESH"
+        elif cache_age_seconds <= cache_ttl_seconds:
+            freshness = "EXPIRING_SOON"
+        else:
+            freshness = "STALE"
+
+    return {
+        "cached": cached,
+        "cache_age_seconds": cache_age_seconds,
+        "cache_ttl_seconds": cache_ttl_seconds,
+        "fetched_at": fetched_at,
+        "expires_at": expires_at,
+        "freshness": freshness,
+    }
+
+
 def make_source_diagnostic(
     name: str,
     endpoint: str,
@@ -37,6 +65,10 @@ def make_source_diagnostic(
     duration_ms: int | None = None,
     cached: bool | None = None,
     cache_age_seconds: int | None = None,
+    cache_ttl_seconds: int | None = None,
+    fetched_at: str | None = None,
+    expires_at: str | None = None,
+    freshness: str | None = None,
     http_status: int | None = None,
     error_type: str | None = None,
     details: dict[str, Any] | None = None,
@@ -49,6 +81,10 @@ def make_source_diagnostic(
         "duration_ms": duration_ms,
         "cached": cached,
         "cache_age_seconds": cache_age_seconds,
+        "cache_ttl_seconds": cache_ttl_seconds,
+        "fetched_at": fetched_at,
+        "expires_at": expires_at,
+        "freshness": freshness,
         "http_status": http_status,
         "error_type": error_type,
         "details": details or {},

backend/app/main.py

@@ -8,7 +8,7 @@
 from app.config import settings
 from app.database import Base, engine
 
-app = FastAPI(title="RouteForge", version="0.4.0")
+app = FastAPI(title="RouteForge", version="0.4.1")
 
 app.add_middleware(
     CORSMiddleware,

backend/app/services/ripe_stat_client.py

@@ -1,9 +1,10 @@
 import time
+from datetime import UTC, datetime, timedelta
 
 import httpx
 from sqlalchemy.orm import Session
 
-from app.config import settings
+import app.config as config
 from app.core.source_diagnostics import (
     EMPTY_RESPONSE,
     ERROR,
@@ -12,9 +13,10 @@
     PARSE_ERROR,
     RATE_LIMITED,
     TIMEOUT,
+    make_cache_metadata,
     make_source_diagnostic,
 )
-from app.services.cache import get_cached, set_cached
+from app.services.cache import _cache_key, get_cached, set_cached
 
 
 class RipeStatClient:
@@ -25,65 +27,86 @@ def get(self, endpoint: str, params: dict) -> dict:
         payload, _diagnostic = self.get_with_diagnostics(endpoint, params)
         return payload or {"error": "empty response", "endpoint": endpoint, "params": params}
 
-    def get_with_diagnostics(self, endpoint: str, params: dict) -> tuple[dict | None, dict]:
+    def get_with_diagnostics(self, endpoint: str, params: dict, force_refresh: bool = False) -> tuple[dict | None, dict]:
         started = time.perf_counter()
-        if settings.demo_mode:
+        cache_key = _cache_key(endpoint, params)
+        if config.settings.demo_mode:
             payload = self._get_demo_data(endpoint, params)
+            cache_meta = make_cache_metadata(cached=False)
             return payload, make_source_diagnostic(
                 name=f"RIPEstat {endpoint}",
                 endpoint=endpoint,
                 status=OK,
                 message="Demo data returned",
                 duration_ms=int((time.perf_counter() - started) * 1000),
-                cached=False,
-                details={"demo_mode": True},
+                **cache_meta,
+                details={"demo_mode": True, "force_refresh": force_refresh, "cache_key": cache_key},
             )
 
-        cached_payload = get_cached(self.db, endpoint, params)
-        if cached_payload:
-            return cached_payload, make_source_diagnostic(
-                name=f"RIPEstat {endpoint}",
-                endpoint=endpoint,
-                status=OK,
-                message="RIPEstat response returned from cache",
-                duration_ms=int((time.perf_counter() - started) * 1000),
-                cached=True,
-            )
+        if not force_refresh:
+            cached_payload = get_cached(self.db, endpoint, params)
+            if cached_payload:
+                payload = cached_payload if isinstance(cached_payload, dict) else {"payload": cached_payload}
+                # backward compatibility: older cache may store raw payload
+                if "payload" in payload and "fetched_at" in payload:
+                    wrapped = payload
+                else:
+                    wrapped = {"payload": payload, "fetched_at": datetime.now(UTC).isoformat(), "ttl_seconds": config.settings.cache_ttl_seconds, "cache_key": cache_key}
+
+                fetched_dt = datetime.fromisoformat(wrapped["fetched_at"]).replace(tzinfo=UTC)
+                age_seconds = max(0, int((datetime.now(UTC) - fetched_dt).total_seconds()))
+                ttl = int(wrapped.get("ttl_seconds") or config.settings.cache_ttl_seconds)
+                expires_at = (fetched_dt + timedelta(seconds=ttl)).isoformat()
+                cache_meta = make_cache_metadata(True, age_seconds, ttl, wrapped["fetched_at"], expires_at)
+                return wrapped.get("payload"), make_source_diagnostic(
+                    name=f"RIPEstat {endpoint}",
+                    endpoint=endpoint,
+                    status=OK,
+                    message="RIPEstat response served from cache",
+                    duration_ms=int((time.perf_counter() - started) * 1000),
+                    **cache_meta,
+                    details={"cache_key": cache_key, "fetched_at": wrapped["fetched_at"], "expires_at": expires_at, "force_refresh": False},
+                )
 
-        url = f"{settings.ripestat_base_url.rstrip('/')}/{endpoint}/data.json"
+        url = f"{config.settings.ripestat_base_url.rstrip('/')}/{endpoint}/data.json"
         try:
-            with httpx.Client(timeout=settings.http_timeout_seconds) as client:
+            with httpx.Client(timeout=config.settings.http_timeout_seconds) as client:
                 resp = client.get(url, params=params)
                 if resp.status_code == 429:
                     return {"error": "rate limited", "endpoint": endpoint, "params": params}, make_source_diagnostic(
-                        name=f"RIPEstat {endpoint}", endpoint=endpoint, status=RATE_LIMITED, message="RIPEstat rate limit reached", duration_ms=int((time.perf_counter() - started) * 1000), cached=False, http_status=resp.status_code, error_type="http_429"
+                        name=f"RIPEstat {endpoint}", endpoint=endpoint, status=RATE_LIMITED, message="RIPEstat rate limit reached", duration_ms=int((time.perf_counter() - started) * 1000), cached=False, freshness="LIVE", http_status=resp.status_code, error_type="http_429", details={"cache_key": cache_key, "force_refresh": force_refresh}
                     )
                 resp.raise_for_status()
                 data = resp.json()
         except httpx.TimeoutException as exc:
             return {"error": str(exc), "endpoint": endpoint, "params": params}, make_source_diagnostic(
-                name=f"RIPEstat {endpoint}", endpoint=endpoint, status=TIMEOUT, message="RIPEstat did not respond before timeout", duration_ms=int((time.perf_counter() - started) * 1000), cached=False, error_type=type(exc).__name__
+                name=f"RIPEstat {endpoint}", endpoint=endpoint, status=TIMEOUT, message="RIPEstat did not respond before timeout", duration_ms=int((time.perf_counter() - started) * 1000), cached=False, freshness="LIVE", error_type=type(exc).__name__, details={"cache_key": cache_key, "force_refresh": force_refresh}
             )
         except httpx.HTTPStatusError as exc:
             return {"error": str(exc), "endpoint": endpoint, "params": params}, make_source_diagnostic(
-                name=f"RIPEstat {endpoint}", endpoint=endpoint, status=HTTP_ERROR, message="RIPEstat responded with HTTP error", duration_ms=int((time.perf_counter() - started) * 1000), cached=False, http_status=exc.response.status_code, error_type=type(exc).__name__
+                name=f"RIPEstat {endpoint}", endpoint=endpoint, status=HTTP_ERROR, message="RIPEstat responded with HTTP error", duration_ms=int((time.perf_counter() - started) * 1000), cached=False, freshness="LIVE", http_status=exc.response.status_code, error_type=type(exc).__name__, details={"cache_key": cache_key, "force_refresh": force_refresh}
             )
         except ValueError as exc:
             return {"error": str(exc), "endpoint": endpoint, "params": params}, make_source_diagnostic(
-                name=f"RIPEstat {endpoint}", endpoint=endpoint, status=PARSE_ERROR, message="RIPEstat response could not be parsed as JSON", duration_ms=int((time.perf_counter() - started) * 1000), cached=False, error_type=type(exc).__name__
+                name=f"RIPEstat {endpoint}", endpoint=endpoint, status=PARSE_ERROR, message="RIPEstat response could not be parsed as JSON", duration_ms=int((time.perf_counter() - started) * 1000), cached=False, freshness="LIVE", error_type=type(exc).__name__, details={"cache_key": cache_key, "force_refresh": force_refresh}
             )
         except Exception as exc:
             return {"error": str(exc), "endpoint": endpoint, "params": params}, make_source_diagnostic(
-                name=f"RIPEstat {endpoint}", endpoint=endpoint, status=ERROR, message="RIPEstat request failed", duration_ms=int((time.perf_counter() - started) * 1000), cached=False, error_type=type(exc).__name__
+                name=f"RIPEstat {endpoint}", endpoint=endpoint, status=ERROR, message="RIPEstat request failed", duration_ms=int((time.perf_counter() - started) * 1000), cached=False, freshness="LIVE", error_type=type(exc).__name__, details={"cache_key": cache_key, "force_refresh": force_refresh}
             )
 
         if not data:
             return data, make_source_diagnostic(
-                name=f"RIPEstat {endpoint}", endpoint=endpoint, status=EMPTY_RESPONSE, message="RIPEstat returned an empty response", duration_ms=int((time.perf_counter() - started) * 1000), cached=False
+                name=f"RIPEstat {endpoint}", endpoint=endpoint, status=EMPTY_RESPONSE, message="RIPEstat returned an empty response", duration_ms=int((time.perf_counter() - started) * 1000), cached=False, freshness="LIVE", details={"cache_key": cache_key, "force_refresh": force_refresh}
             )
-        set_cached(self.db, endpoint, params, data)
+
+        fetched_at = datetime.now(UTC).isoformat()
+        ttl = config.settings.cache_ttl_seconds
+        expires_at = (datetime.now(UTC) + timedelta(seconds=ttl)).isoformat()
+        set_cached(self.db, endpoint, params, {"payload": data, "fetched_at": fetched_at, "ttl_seconds": ttl, "cache_key": cache_key})
+        cache_meta = make_cache_metadata(False, None, ttl, fetched_at, expires_at)
         return data, make_source_diagnostic(
-            name=f"RIPEstat {endpoint}", endpoint=endpoint, status=OK, message="RIPEstat response received", duration_ms=int((time.perf_counter() - started) * 1000), cached=False
+            name=f"RIPEstat {endpoint}", endpoint=endpoint, status=OK, message="RIPEstat response received", duration_ms=int((time.perf_counter() - started) * 1000), **cache_meta, details={"cache_key": cache_key, "fetched_at": fetched_at, "expires_at": expires_at, "force_refresh": force_refresh}
         )
 
     def _get_demo_data(self, endpoint: str, params: dict) -> dict:

backend/app/templates/report.html.j2

@@ -50,7 +50,7 @@
 </ul>
 
 <h2>Data Source Diagnostics</h2>
-{% if report.details and report.details.source_diagnostics %}<table border="1" cellpadding="6" cellspacing="0"><tr><th>Quelle</th><th>Endpoint</th><th>Status</th><th>Dauer</th><th>Cache</th><th>Message</th></tr>{% for d in report.details.source_diagnostics %}<tr><td>{{ d.name or '-' }}</td><td>{{ d.endpoint or '-' }}</td><td>{{ d.status or '-' }}</td><td>{{ d.duration_ms if d.duration_ms is not none else '-' }}</td><td>{{ 'HIT' if d.cached is sameas true else ('MISS' if d.cached is sameas false else '-') }}</td><td>{{ d.message or '-' }}{% if d.details %}<details><summary>Details</summary><pre>{{ d.details | tojson(indent=2) }}</pre></details>{% endif %}</td></tr>{% endfor %}</table>{% else %}<p>No source diagnostics available.</p>{% endif %}
+{% if report.details and report.details.source_diagnostics %}<table border="1" cellpadding="6" cellspacing="0"><tr><th>Quelle</th><th>Endpoint</th><th>Status</th><th>Dauer</th><th>Freshness</th><th>Cache</th><th>Cache Age</th><th>TTL</th><th>Message</th></tr>{% for d in report.details.source_diagnostics %}<tr><td>{{ d.name or '-' }}</td><td>{{ d.endpoint or '-' }}</td><td>{{ d.status or '-' }}</td><td>{{ d.duration_ms if d.duration_ms is not none else '-' }}</td><td>{{ d.freshness or 'UNKNOWN' }}</td><td>{{ 'HIT' if d.cached is sameas true else ('MISS' if d.cached is sameas false else '-') }}</td><td>{{ d.cache_age_seconds if d.cache_age_seconds is not none else 'Unknown' }}</td><td>{{ d.cache_ttl_seconds if d.cache_ttl_seconds is not none else 'Unknown' }}</td><td>{{ d.message or '-' }}{% if d.details %}<details><summary>Details</summary><pre>{{ d.details | tojson(indent=2) }}</pre></details>{% endif %}</td></tr>{% endfor %}</table>{% else %}<p>No source diagnostics available.</p>{% endif %}
 
 <h2>Rohdaten</h2>
 <h3>RPKI Rohdaten</h3>

backend/app/templates/report.md.j2

@@ -43,9 +43,9 @@
 
 ## Data Source Diagnostics
 {% if report.details and report.details.source_diagnostics %}
-| Quelle | Endpoint | Status | Dauer | Cache | Message |
+| Quelle | Endpoint | Status | Dauer | Freshness | Cache | Cache Age | TTL | Message |
 |---|---|---|---:|---|---|
-{% for d in report.details.source_diagnostics %}| {{ d.name or '-' }} | {{ d.endpoint or '-' }} | {{ d.status or '-' }} | {{ d.duration_ms if d.duration_ms is not none else '-' }} | {{ 'HIT' if d.cached is sameas true else ('MISS' if d.cached is sameas false else '-') }} | {{ d.message or '-' }} |
+{% for d in report.details.source_diagnostics %}| {{ d.name or '-' }} | {{ d.endpoint or '-' }} | {{ d.status or '-' }} | {{ d.duration_ms if d.duration_ms is not none else '-' }} | {{ d.freshness or 'UNKNOWN' }} | {{ 'HIT' if d.cached is sameas true else ('MISS' if d.cached is sameas false else '-') }} | {{ d.cache_age_seconds if d.cache_age_seconds is not none else 'Unknown' }} | {{ d.cache_ttl_seconds if d.cache_ttl_seconds is not none else 'Unknown' }} | {{ d.message or '-' }} |
 {% endfor %}
 {% else %}No source diagnostics available.
 {% endif %}

backend/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "routeforge-backend"
-version = "0.4.0"
+version = "0.4.1"
 description = "RouteForge backend"
 requires-python = ">=3.12"
 dependencies = [

backend/tests/test_api_smoke.py

@@ -34,6 +34,7 @@ def test_prefix_check_without_origin_as() -> None:
     assert payload.get('markdown')
     assert payload.get('html')
     assert payload.get('details', {}).get('resource_holder')
+    assert any('freshness' in d for d in payload.get('details', {}).get('source_diagnostics', []) if isinstance(d, dict))
 
 
 def test_asn_check() -> None:
@@ -47,6 +48,7 @@ def test_asn_check() -> None:
     assert 'extracted_prefixes' in details
     assert details.get('rpki_batch', {}).get('available') is True
     assert details.get('resource_holder')
+    assert any('freshness' in d for d in details.get('source_diagnostics', []) if isinstance(d, dict))
 
 
 def test_asn_check_without_prefixes_has_batch_reason() -> None:
@@ -112,6 +114,7 @@ def test_preflight_check() -> None:
     assert 'routing_visibility' in checks
     assert payload.get('details', {}).get('preflight_mode') is True
     assert isinstance(payload.get('details', {}).get('source_diagnostics'), list)
+    assert any('freshness' in d for d in payload.get('details', {}).get('source_diagnostics', []) if isinstance(d, dict))
     assert payload.get('details', {}).get('resource_holder')
     assert payload.get('details', {}).get('preflight_decision') in {'GO', 'CAUTION', 'NO-GO', 'UNKNOWN'}