From 8263e9ed4f7946bba25dab6e7dd6c574beda22ce Mon Sep 17 00:00:00 2001 From: LQ458 Date: Thu, 23 Jul 2026 14:43:43 +0800 Subject: [PATCH] chore: retain daily traffic metric snapshots --- docs/evidence/README.md | 2 + docs/operations/traffic-metric-retention.md | 84 ++++++++ package.json | 1 + scripts/snapshot-traffic-metrics.js | 212 ++++++++++++++++++++ 4 files changed, 299 insertions(+) create mode 100644 docs/operations/traffic-metric-retention.md create mode 100644 scripts/snapshot-traffic-metrics.js diff --git a/docs/evidence/README.md b/docs/evidence/README.md index c4ed45a..47ffe98 100644 --- a/docs/evidence/README.md +++ b/docs/evidence/README.md @@ -14,6 +14,8 @@ the CV-safe impact record is lab and local performance record is [performance-baseline-2026-07-23.md](performance-baseline-2026-07-23.md), and collection rules are in [collection-method.md](collection-method.md). +Future daily aggregate retention is described in +[traffic-metric-retention.md](../operations/traffic-metric-retention.md). ## Status vocabulary diff --git a/docs/operations/traffic-metric-retention.md b/docs/operations/traffic-metric-retention.md new file mode 100644 index 0000000..e8f2918 --- /dev/null +++ b/docs/operations/traffic-metric-retention.md @@ -0,0 +1,84 @@ +# Traffic metric retention + +The existing traffic collector can be run once for historical analysis. The +snapshot wrapper makes that evidence durable when Nginx later rotates or +deletes raw logs. It writes one private, append-only aggregate JSON file per UTC +day and never writes raw IP addresses, user agents, URLs, credentials, or log +lines. + +This is server-side operational evidence, not a third-party user tracker. +Requests, document-load candidates, session candidates, and estimated +signatures remain separate measures. None may be described as verified people, +valid visits, pageviews, or uptime. + +## Private server setup + +Keep the output, stable HMAC secret, and internal/test address exclusions +outside the Git repository. Example placeholders: + +```bash +install -d -m 700 /private/metrics/traffic +install -d -m 700 /private/biphflora-metrics +openssl rand -hex 32 > /private/biphflora-metrics/traffic-hmac +chmod 600 /private/biphflora-metrics/traffic-hmac +``` + +Create an optional root-readable file containing one known developer, +administrator, or synthetic-check address per line. Do not commit that file. +The HMAC secret is read only in memory and is never included in output. + +Run a snapshot after the UTC day has closed, including the active access log +and all retained rotated access logs. By default the wrapper names and cuts off +the previous complete UTC day: + +```bash +node scripts/snapshot-traffic-metrics.js \ + --output-dir /private/metrics/traffic \ + --secret-file /private/biphflora-metrics/traffic-hmac \ + --exclude-ip-file /private/biphflora-metrics/internal-addresses \ + --site-host biphflora.com \ + /path/to/nginx/access.log.*.gz \ + /path/to/nginx/access.log +``` + +The wrapper always enables the collector's strict known-product-route mode, +sets directory/file modes to `0700`/`0600`, validates the secret file mode, +resolves the real output path before refusing repository output, writes +atomically, and will not replace an existing day's snapshot. + +## Scheduling and retention + +Use the existing 宝塔 scheduled-task facility or cron to run the command daily +under a single-instance lock and alert on a non-zero exit. Keep the schedule, +real paths, and secret-store location in the private operations record, not in +the public repository. A representative cron shape is: + +```text +25 00 * * * cd /path/to/biphflora && flock -n /run/lock/biphflora-traffic.lock node scripts/snapshot-traffic-metrics.js [private options and retained access logs] +``` + +Retain the small aggregate snapshots for the service lifetime unless a shorter +approved policy applies. Raw Nginx logs keep their independent rotation policy. +Do not delete a raw log until at least one successful snapshot covers its time +range. Review snapshot creation monthly and record any collection gaps. + +Each daily file contains the exact definition, source range, collection time, +30/90/365-day windows, monthly trend, peak request periods, filters, +successful fixed endpoint-response categories, confidence limitations, and +unavailable metrics. This preserves reproducible project evidence without +moving raw identifiers off the server. + +## Optional product-use evidence + +`AUDIT_EVENTS_ENABLED` and `SEARCH_TELEMETRY_ENABLED` remain off by default. +Enabling them changes production data collection and therefore requires a +separate retention/privacy review and deployment approval. When enabled, run +`npm run evidence:product` monthly and record the coverage start; earlier +searches or operations cannot be backfilled. + +## Disable and recover + +Disable only the scheduled task; no Nginx reload, application restart, +database migration, or media change is required. Existing aggregate snapshots +remain readable JSON. A failed or missing day is recorded as a collection gap +rather than reconstructed from assumptions. diff --git a/package.json b/package.json index 83f84d8..0c919ac 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "evidence:product": "node scripts/collect-product-metrics.js", "evidence:search-performance": "node scripts/benchmark-search.js", "evidence:traffic": "node scripts/collect-traffic-metrics.js", + "evidence:traffic:snapshot": "node scripts/snapshot-traffic-metrics.js", "media:backfill": "node scripts/backfill-media-variants.js", "format": "prettier --write ." }, diff --git a/scripts/snapshot-traffic-metrics.js b/scripts/snapshot-traffic-metrics.js new file mode 100644 index 0000000..55fa4bb --- /dev/null +++ b/scripts/snapshot-traffic-metrics.js @@ -0,0 +1,212 @@ +#!/usr/bin/env node + +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +const { collectTrafficMetrics } = require("./collect-traffic-metrics"); + +function usage() { + return [ + "Usage: node scripts/snapshot-traffic-metrics.js [options] ... ", + "", + "Options:", + " --output-dir Private directory for daily aggregate JSON", + " --secret-file Root-readable HMAC secret file", + " --exclude-ip-file Optional internal/test address list", + " --as-of Cutoff within the snapshot UTC date", + " --site-host Public hostname used for source grouping", + " --snapshot-date Override the UTC snapshot filename date", + ].join("\n"); +} + +function parseArguments(argv) { + const result = { files: [] }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === "--output-dir") result.outputDir = argv[++index]; + else if (argument === "--secret-file") result.secretFile = argv[++index]; + else if (argument === "--exclude-ip-file") + result.excludeIpFile = argv[++index]; + else if (argument === "--as-of") result.asOf = argv[++index]; + else if (argument === "--site-host") result.siteHost = argv[++index]; + else if (argument === "--snapshot-date") + result.snapshotDate = argv[++index]; + else if (argument === "--help" || argument === "-h") result.help = true; + else if (argument.startsWith("--")) + throw new Error(`Unknown option: ${argument}`); + else result.files.push(argument); + } + return result; +} + +function readNonEmptyLines(file) { + if (!file) return new Set(); + return new Set( + fs + .readFileSync(file, "utf8") + .split(/\r?\n/) + .map((value) => value.trim()) + .filter(Boolean), + ); +} + +function readSecret(secretFile) { + if (secretFile) { + const stat = fs.statSync(secretFile); + if (!stat.isFile()) { + throw new Error("--secret-file must be a regular file"); + } + if (process.platform !== "win32" && (stat.mode & 0o077) !== 0) { + throw new Error("--secret-file must not be group- or world-readable"); + } + } + const secret = secretFile + ? fs.readFileSync(secretFile, "utf8").trim() + : String(process.env.TRAFFIC_METRICS_HASH_SECRET || "").trim(); + if (secret.length < 32) { + throw new Error( + "Provide a stable secret of at least 32 characters via --secret-file or TRAFFIC_METRICS_HASH_SECRET", + ); + } + return secret; +} + +function isInside(parent, child) { + const relative = path.relative(parent, child); + return ( + relative === "" || + (!relative.startsWith("..") && !path.isAbsolute(relative)) + ); +} + +function canonicalProspectivePath(target) { + const missing = []; + let existing = target; + while (!fs.existsSync(existing)) { + const parent = path.dirname(existing); + if (parent === existing) break; + missing.unshift(path.basename(existing)); + existing = parent; + } + return path.join(fs.realpathSync(existing), ...missing); +} + +function writeAppendOnlySnapshot(outputDir, snapshotDate, payload) { + fs.mkdirSync(outputDir, { recursive: true, mode: 0o700 }); + fs.chmodSync(outputDir, 0o700); + + const destination = path.join(outputDir, `traffic-${snapshotDate}.json`); + if (fs.existsSync(destination)) { + return { destination, created: false }; + } + + const temporaryDirectory = fs.mkdtempSync( + path.join(outputDir, ".traffic-snapshot-"), + ); + const temporaryFile = path.join(temporaryDirectory, "snapshot.json"); + try { + fs.writeFileSync(temporaryFile, `${JSON.stringify(payload, null, 2)}\n`, { + mode: 0o600, + }); + try { + fs.linkSync(temporaryFile, destination); + return { destination, created: true }; + } catch (error) { + if (error.code === "EEXIST") return { destination, created: false }; + throw error; + } + } finally { + fs.rmSync(temporaryDirectory, { recursive: true, force: true }); + } +} + +async function main() { + const options = parseArguments(process.argv.slice(2)); + if (options.help) { + process.stdout.write(`${usage()}\n`); + return; + } + if (!options.outputDir || !path.isAbsolute(options.outputDir)) { + throw new Error("--output-dir must be an absolute path"); + } + if (options.files.length === 0) { + throw new Error("At least one Nginx access log is required"); + } + + const outputDir = path.resolve(options.outputDir); + const repositoryRoot = path.resolve(__dirname, ".."); + if (isInside(repositoryRoot, outputDir)) { + throw new Error("Traffic snapshots must be stored outside the repository"); + } + + const yesterday = new Date(Date.now() - 86_400_000) + .toISOString() + .slice(0, 10); + const snapshotDate = options.snapshotDate || yesterday; + if (!/^\d{4}-\d{2}-\d{2}$/.test(snapshotDate)) { + throw new Error("--snapshot-date must use YYYY-MM-DD"); + } + const asOf = options.asOf || `${snapshotDate}T23:59:59.999Z`; + const parsedCutoff = new Date(asOf); + if ( + !Number.isFinite(parsedCutoff.getTime()) || + parsedCutoff.toISOString().slice(0, 10) !== snapshotDate + ) { + throw new Error("--as-of must be a valid timestamp within --snapshot-date"); + } + + const canonicalOutputDir = canonicalProspectivePath(outputDir); + const canonicalRepositoryRoot = fs.realpathSync(repositoryRoot); + if (isInside(canonicalRepositoryRoot, canonicalOutputDir)) { + throw new Error("Traffic snapshots must be stored outside the repository"); + } + const metrics = await collectTrafficMetrics({ + files: options.files, + asOf, + secret: readSecret(options.secretFile), + excludedAddresses: readNonEmptyLines(options.excludeIpFile), + options: { + knownRoutesOnly: true, + siteHost: options.siteHost, + }, + }); + const snapshot = { + ...metrics, + definitions: { + ...metrics.definitions, + sessionCandidates: + "Included requests sharing a stable, server-secret HMAC of IP plus normalized user agent, split after more than 30 minutes of inactivity.", + }, + privacy: { + ...metrics.privacy, + signatureSecret: "stable externally supplied; never emitted", + }, + retentionSnapshot: { + snapshotDate, + reportingCutoff: parsedCutoff.toISOString(), + cadence: "daily", + appendOnly: true, + rawLogsRetainedAtSource: true, + }, + }; + const result = writeAppendOnlySnapshot(outputDir, snapshotDate, snapshot); + process.stdout.write( + `${result.created ? "Created" : "Already exists"} ${result.destination}${os.EOL}`, + ); +} + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + canonicalProspectivePath, + isInside, + parseArguments, + readSecret, + writeAppendOnlySnapshot, +};