chore: retain daily traffic metric snapshots

docs/evidence/README.md

@@ -14,6 +14,8 @@ the CV-safe impact record is
 lab and local performance record is
 [performance-baseline-2026-07-23.md](performance-baseline-2026-07-23.md), and
 collection rules are in [collection-method.md](collection-method.md).
+Future daily aggregate retention is described in
+[traffic-metric-retention.md](../operations/traffic-metric-retention.md).
 
 ## Status vocabulary
 

docs/operations/traffic-metric-retention.md

@@ -0,0 +1,84 @@
+# Traffic metric retention
+
+The existing traffic collector can be run once for historical analysis. The
+snapshot wrapper makes that evidence durable when Nginx later rotates or
+deletes raw logs. It writes one private, append-only aggregate JSON file per UTC
+day and never writes raw IP addresses, user agents, URLs, credentials, or log
+lines.
+
+This is server-side operational evidence, not a third-party user tracker.
+Requests, document-load candidates, session candidates, and estimated
+signatures remain separate measures. None may be described as verified people,
+valid visits, pageviews, or uptime.
+
+## Private server setup
+
+Keep the output, stable HMAC secret, and internal/test address exclusions
+outside the Git repository. Example placeholders:
+
+```bash
+install -d -m 700 /private/metrics/traffic
+install -d -m 700 /private/biphflora-metrics
+openssl rand -hex 32 > /private/biphflora-metrics/traffic-hmac
+chmod 600 /private/biphflora-metrics/traffic-hmac
+```
+
+Create an optional root-readable file containing one known developer,
+administrator, or synthetic-check address per line. Do not commit that file.
+The HMAC secret is read only in memory and is never included in output.
+
+Run a snapshot after the UTC day has closed, including the active access log
+and all retained rotated access logs. By default the wrapper names and cuts off
+the previous complete UTC day:
+
+```bash
+node scripts/snapshot-traffic-metrics.js \
+  --output-dir /private/metrics/traffic \
+  --secret-file /private/biphflora-metrics/traffic-hmac \
+  --exclude-ip-file /private/biphflora-metrics/internal-addresses \
+  --site-host biphflora.com \
+  /path/to/nginx/access.log.*.gz \
+  /path/to/nginx/access.log
+```
+
+The wrapper always enables the collector's strict known-product-route mode,
+sets directory/file modes to `0700`/`0600`, validates the secret file mode,
+resolves the real output path before refusing repository output, writes
+atomically, and will not replace an existing day's snapshot.
+
+## Scheduling and retention
+
+Use the existing 宝塔 scheduled-task facility or cron to run the command daily
+under a single-instance lock and alert on a non-zero exit. Keep the schedule,
+real paths, and secret-store location in the private operations record, not in
+the public repository. A representative cron shape is:
+
+```text
+25 00 * * * cd /path/to/biphflora && flock -n /run/lock/biphflora-traffic.lock node scripts/snapshot-traffic-metrics.js [private options and retained access logs]
+```
+
+Retain the small aggregate snapshots for the service lifetime unless a shorter
+approved policy applies. Raw Nginx logs keep their independent rotation policy.
+Do not delete a raw log until at least one successful snapshot covers its time
+range. Review snapshot creation monthly and record any collection gaps.
+
+Each daily file contains the exact definition, source range, collection time,
+30/90/365-day windows, monthly trend, peak request periods, filters,
+successful fixed endpoint-response categories, confidence limitations, and
+unavailable metrics. This preserves reproducible project evidence without
+moving raw identifiers off the server.
+
+## Optional product-use evidence
+
+`AUDIT_EVENTS_ENABLED` and `SEARCH_TELEMETRY_ENABLED` remain off by default.
+Enabling them changes production data collection and therefore requires a
+separate retention/privacy review and deployment approval. When enabled, run
+`npm run evidence:product` monthly and record the coverage start; earlier
+searches or operations cannot be backfilled.
+
+## Disable and recover
+
+Disable only the scheduled task; no Nginx reload, application restart,
+database migration, or media change is required. Existing aggregate snapshots
+remain readable JSON. A failed or missing day is recorded as a collection gap
+rather than reconstructed from assumptions.

package.json

@@ -21,6 +21,7 @@
     "evidence:product": "node scripts/collect-product-metrics.js",
     "evidence:search-performance": "node scripts/benchmark-search.js",
     "evidence:traffic": "node scripts/collect-traffic-metrics.js",
+    "evidence:traffic:snapshot": "node scripts/snapshot-traffic-metrics.js",
     "media:backfill": "node scripts/backfill-media-variants.js",
     "format": "prettier --write ."
   },

scripts/snapshot-traffic-metrics.js

@@ -0,0 +1,212 @@
+#!/usr/bin/env node
+
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+
+const { collectTrafficMetrics } = require("./collect-traffic-metrics");
+
+function usage() {
+  return [
+    "Usage: node scripts/snapshot-traffic-metrics.js [options] <oldest.log[.gz]> ... <newest.log>",
+    "",
+    "Options:",
+    "  --output-dir <absolute path>  Private directory for daily aggregate JSON",
+    "  --secret-file <path>         Root-readable HMAC secret file",
+    "  --exclude-ip-file <path>     Optional internal/test address list",
+    "  --as-of <ISO timestamp>      Cutoff within the snapshot UTC date",
+    "  --site-host <hostname>       Public hostname used for source grouping",
+    "  --snapshot-date <YYYY-MM-DD> Override the UTC snapshot filename date",
+  ].join("\n");
+}
+
+function parseArguments(argv) {
+  const result = { files: [] };
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    if (argument === "--output-dir") result.outputDir = argv[++index];
+    else if (argument === "--secret-file") result.secretFile = argv[++index];
+    else if (argument === "--exclude-ip-file")
+      result.excludeIpFile = argv[++index];
+    else if (argument === "--as-of") result.asOf = argv[++index];
+    else if (argument === "--site-host") result.siteHost = argv[++index];
+    else if (argument === "--snapshot-date")
+      result.snapshotDate = argv[++index];
+    else if (argument === "--help" || argument === "-h") result.help = true;
+    else if (argument.startsWith("--"))
+      throw new Error(`Unknown option: ${argument}`);
+    else result.files.push(argument);
+  }
+  return result;
+}
+
+function readNonEmptyLines(file) {
+  if (!file) return new Set();
+  return new Set(
+    fs
+      .readFileSync(file, "utf8")
+      .split(/\r?\n/)
+      .map((value) => value.trim())
+      .filter(Boolean),
+  );
+}
+
+function readSecret(secretFile) {
+  if (secretFile) {
+    const stat = fs.statSync(secretFile);
+    if (!stat.isFile()) {
+      throw new Error("--secret-file must be a regular file");
+    }
+    if (process.platform !== "win32" && (stat.mode & 0o077) !== 0) {
+      throw new Error("--secret-file must not be group- or world-readable");
+    }
+  }
+  const secret = secretFile
+    ? fs.readFileSync(secretFile, "utf8").trim()
+    : String(process.env.TRAFFIC_METRICS_HASH_SECRET || "").trim();
+  if (secret.length < 32) {
+    throw new Error(
+      "Provide a stable secret of at least 32 characters via --secret-file or TRAFFIC_METRICS_HASH_SECRET",
+    );
+  }
+  return secret;
+}
+
+function isInside(parent, child) {
+  const relative = path.relative(parent, child);
+  return (
+    relative === "" ||
+    (!relative.startsWith("..") && !path.isAbsolute(relative))
+  );
+}
+
+function canonicalProspectivePath(target) {
+  const missing = [];
+  let existing = target;
+  while (!fs.existsSync(existing)) {
+    const parent = path.dirname(existing);
+    if (parent === existing) break;
+    missing.unshift(path.basename(existing));
+    existing = parent;
+  }
+  return path.join(fs.realpathSync(existing), ...missing);
+}
+
+function writeAppendOnlySnapshot(outputDir, snapshotDate, payload) {
+  fs.mkdirSync(outputDir, { recursive: true, mode: 0o700 });
+  fs.chmodSync(outputDir, 0o700);
+
+  const destination = path.join(outputDir, `traffic-${snapshotDate}.json`);
+  if (fs.existsSync(destination)) {
+    return { destination, created: false };
+  }
+
+  const temporaryDirectory = fs.mkdtempSync(
+    path.join(outputDir, ".traffic-snapshot-"),
+  );
+  const temporaryFile = path.join(temporaryDirectory, "snapshot.json");
+  try {
+    fs.writeFileSync(temporaryFile, `${JSON.stringify(payload, null, 2)}\n`, {
+      mode: 0o600,
+    });
+    try {
+      fs.linkSync(temporaryFile, destination);
+      return { destination, created: true };
+    } catch (error) {
+      if (error.code === "EEXIST") return { destination, created: false };
+      throw error;
+    }
+  } finally {
+    fs.rmSync(temporaryDirectory, { recursive: true, force: true });
+  }
+}
+
+async function main() {
+  const options = parseArguments(process.argv.slice(2));
+  if (options.help) {
+    process.stdout.write(`${usage()}\n`);
+    return;
+  }
+  if (!options.outputDir || !path.isAbsolute(options.outputDir)) {
+    throw new Error("--output-dir must be an absolute path");
+  }
+  if (options.files.length === 0) {
+    throw new Error("At least one Nginx access log is required");
+  }
+
+  const outputDir = path.resolve(options.outputDir);
+  const repositoryRoot = path.resolve(__dirname, "..");
+  if (isInside(repositoryRoot, outputDir)) {
+    throw new Error("Traffic snapshots must be stored outside the repository");
+  }
+
+  const yesterday = new Date(Date.now() - 86_400_000)
+    .toISOString()
+    .slice(0, 10);
+  const snapshotDate = options.snapshotDate || yesterday;
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(snapshotDate)) {
+    throw new Error("--snapshot-date must use YYYY-MM-DD");
+  }
+  const asOf = options.asOf || `${snapshotDate}T23:59:59.999Z`;
+  const parsedCutoff = new Date(asOf);
+  if (
+    !Number.isFinite(parsedCutoff.getTime()) ||
+    parsedCutoff.toISOString().slice(0, 10) !== snapshotDate
+  ) {
+    throw new Error("--as-of must be a valid timestamp within --snapshot-date");
+  }
+
+  const canonicalOutputDir = canonicalProspectivePath(outputDir);
+  const canonicalRepositoryRoot = fs.realpathSync(repositoryRoot);
+  if (isInside(canonicalRepositoryRoot, canonicalOutputDir)) {
+    throw new Error("Traffic snapshots must be stored outside the repository");
+  }
+  const metrics = await collectTrafficMetrics({
+    files: options.files,
+    asOf,
+    secret: readSecret(options.secretFile),
+    excludedAddresses: readNonEmptyLines(options.excludeIpFile),
+    options: {
+      knownRoutesOnly: true,
+      siteHost: options.siteHost,
+    },
+  });
+  const snapshot = {
+    ...metrics,
+    definitions: {
+      ...metrics.definitions,
+      sessionCandidates:
+        "Included requests sharing a stable, server-secret HMAC of IP plus normalized user agent, split after more than 30 minutes of inactivity.",
+    },
+    privacy: {
+      ...metrics.privacy,
+      signatureSecret: "stable externally supplied; never emitted",
+    },
+    retentionSnapshot: {
+      snapshotDate,
+      reportingCutoff: parsedCutoff.toISOString(),
+      cadence: "daily",
+      appendOnly: true,
+      rawLogsRetainedAtSource: true,
+    },
+  };
+  const result = writeAppendOnlySnapshot(outputDir, snapshotDate, snapshot);
+  process.stdout.write(
+    `${result.created ? "Created" : "Already exists"} ${result.destination}${os.EOL}`,
+  );
+}
+
+if (require.main === module) {
+  main().catch((error) => {
+    process.stderr.write(`${error.message}\n`);
+    process.exitCode = 1;
+  });
+}
+
+module.exports = {
+  canonicalProspectivePath,
+  isInside,
+  parseArguments,
+  readSecret,
+  writeAppendOnlySnapshot,
+};