From 8bd96426085a4bc1c9afab34d36431aaee9ae1a0 Mon Sep 17 00:00:00 2001 From: jamilahmadzai Date: Fri, 15 May 2026 10:44:36 +0200 Subject: [PATCH] tools: add S3 bucket posture audit helper --- README.md | 2 +- docs/security/s3-bucket-posture-audit.md | 83 ++++ tools/audit-s3-bucket-posture.mjs | 466 +++++++++++++++++++ tools/check-s3-bucket-posture-audit.mjs | 75 +++ tools/fixtures/s3-bucket-posture.sample.json | 111 +++++ 5 files changed, 736 insertions(+), 1 deletion(-) create mode 100644 docs/security/s3-bucket-posture-audit.md create mode 100755 tools/audit-s3-bucket-posture.mjs create mode 100755 tools/check-s3-bucket-posture-audit.mjs create mode 100644 tools/fixtures/s3-bucket-posture.sample.json diff --git a/README.md b/README.md index eb0a5ca6..720e7baf 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Detailed documentation is available in the `docs/` directory: - [Database Documentation](docs/database/README.md) - Database structure and diagrams - [Cron Jobs](docs/cron-jobs.md) - Scheduled tasks - [Analytics](docs/analytics.md) - Event tracking and analytics +- [S3 Bucket Posture Audit](docs/security/s3-bucket-posture-audit.md) - Offline bucket-hardening checklist for exported S3 settings ## Development @@ -204,4 +205,3 @@ END $$ DELIMITER; - diff --git a/docs/security/s3-bucket-posture-audit.md b/docs/security/s3-bucket-posture-audit.md new file mode 100644 index 00000000..2bef9391 --- /dev/null +++ b/docs/security/s3-bucket-posture-audit.md @@ -0,0 +1,83 @@ +# S3 Bucket Posture Audit + +This helper supports the Phase 9 bucket-hardening work from the AWS S3 remediation issue. It is intentionally offline-only: maintainers export S3 bucket posture data inside their trusted AWS environment, then run the local report without giving this repository live AWS credentials. + +## What It Checks + +The audit highlights posture gaps that matter for the `studenthub-*`, PlugN, and Wallet bucket incident class: + +- permanent buckets without versioning +- destructive lifecycle rules on permanent buckets +- broad public-access settings or public ACL grants +- CORS rules that allow wildcard origins for write methods +- object ownership that is not bucket-owner-enforced +- missing access logging evidence +- missing `owner`, `service`, or `environment` tags + +The report redacts account IDs and access-key-looking values before printing output. + +## Export Shape + +Create a private JSON file with one object per bucket. The script accepts either an array or an object with a `buckets` field: + +```json +{ + "buckets": [ + { + "name": "studenthub-uploads", + "versioning": { "Status": "Enabled" }, + "publicAccessBlock": { + "BlockPublicAcls": true, + "IgnorePublicAcls": true, + "BlockPublicPolicy": true, + "RestrictPublicBuckets": true + }, + "ownershipControls": { + "Rules": [{ "ObjectOwnership": "BucketOwnerEnforced" }] + }, + "logging": { "LoggingEnabled": { "TargetBucket": "studenthub-access-logs" } }, + "lifecycle": { "Rules": [] }, + "cors": { "CORSRules": [] }, + "tags": { + "owner": "platform", + "service": "studenthub", + "environment": "prod" + } + } + ] +} +``` + +Keep raw exports and generated reports out of public GitHub comments if they contain bucket ARNs, account IDs, internal hostnames, or incident notes. + +## Usage + +Markdown report: + +```bash +node tools/audit-s3-bucket-posture.mjs private/s3-buckets.json +``` + +The command exits with status `2` when high or critical findings are present so maintainers can use it as a guardrail in a private review workflow. + +CSV action plan: + +```bash +node tools/audit-s3-bucket-posture.mjs private/s3-buckets.json --format csv --output private/s3-bucket-posture.csv +``` + +Audit additional bucket families: + +```bash +node tools/audit-s3-bucket-posture.mjs private/s3-buckets.json --bucket-pattern bawes --bucket-pattern payroll +``` + +Run the synthetic regression check: + +```bash +node tools/check-s3-bucket-posture-audit.mjs +``` + +## Safety Boundary + +This tool does not call AWS, rotate keys, change bucket policy, enable versioning, modify lifecycle rules, or read candidate data. It only reads local JSON exports supplied by maintainers. diff --git a/tools/audit-s3-bucket-posture.mjs b/tools/audit-s3-bucket-posture.mjs new file mode 100755 index 00000000..138a1a2e --- /dev/null +++ b/tools/audit-s3-bucket-posture.mjs @@ -0,0 +1,466 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; + +const DEFAULT_BUCKET_PATTERNS = [ + "studenthub", + "studenthub-uploads", + "studenthub-public", + "plugn", + "wallet", +]; + +const WRITE_METHODS = new Set(["PUT", "POST", "DELETE"]); +const REQUIRED_TAGS = ["owner", "service", "environment"]; + +function parseArgs(argv) { + const options = { + format: "markdown", + includeAll: false, + output: null, + bucketPatterns: [...DEFAULT_BUCKET_PATTERNS], + }; + const files = []; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const readOptionValue = (name) => { + if (index + 1 >= argv.length) { + throw new Error(`${name} requires a value`); + } + index += 1; + return argv[index]; + }; + + if (arg === "--format") { + options.format = readOptionValue("--format"); + } else if (arg === "--output") { + options.output = readOptionValue("--output"); + } else if (arg === "--include-all") { + options.includeAll = true; + } else if (arg === "--bucket-pattern") { + options.bucketPatterns.push(readOptionValue("--bucket-pattern")); + } else if (arg === "--help" || arg === "-h") { + printHelp(); + process.exit(0); + } else if (arg.startsWith("-")) { + throw new Error(`Unknown option: ${arg}`); + } else { + files.push(arg); + } + } + + if (!["markdown", "csv"].includes(options.format)) { + throw new Error("--format must be markdown or csv"); + } + if (files.length === 0) { + throw new Error("Provide at least one JSON file or directory to audit"); + } + + return { files, options }; +} + +function printHelp() { + console.log(`Usage: node tools/audit-s3-bucket-posture.mjs [options] + +Options: + --format markdown|csv Output format. Default: markdown + --output Write report to a file + --include-all Audit buckets even when their names do not match defaults + --bucket-pattern Include bucket names containing this text. Can repeat + -h, --help Show this help + +Input can be a normalized array of bucket posture objects, an object with a +"buckets" field, or an AWS-style object with "Buckets".`); +} + +function listJsonFiles(target) { + const stats = fs.statSync(target); + if (stats.isFile()) { + return [target]; + } + if (!stats.isDirectory()) { + throw new Error(`Input path is neither a file nor directory: ${target}`); + } + + return fs + .readdirSync(target) + .flatMap((entry) => { + const child = path.join(target, entry); + const childStats = fs.statSync(child); + if (childStats.isDirectory()) { + return listJsonFiles(child); + } + return child.endsWith(".json") ? [child] : []; + }) + .sort(); +} + +function parseJson(filePath) { + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to parse JSON in ${filePath}: ${message}`, { cause: error }); + } +} + +function readBuckets(files) { + return files + .flatMap(listJsonFiles) + .flatMap((filePath) => { + const parsed = parseJson(filePath); + const records = Array.isArray(parsed) + ? parsed + : Array.isArray(parsed.buckets) + ? parsed.buckets + : Array.isArray(parsed.Buckets) + ? parsed.Buckets + : [parsed]; + + return records.map((record) => ({ ...record, sourceFile: filePath })); + }) + .filter((record) => bucketName(record)); +} + +function bucketName(bucket) { + return bucket.name ?? bucket.Name ?? bucket.Bucket ?? bucket.bucketName ?? bucket.bucket; +} + +function normalizeTagMap(tags) { + if (!tags) { + return {}; + } + const validTagEntries = (tagList) => + tagList.flatMap((tag) => { + const key = tag?.Key ?? tag?.key; + const value = tag?.Value ?? tag?.value; + if (key == null || value == null) { + return []; + } + return [[String(key), String(value)]]; + }); + + if (Array.isArray(tags)) { + return Object.fromEntries(validTagEntries(tags)); + } + if (Array.isArray(tags.TagSet)) { + return Object.fromEntries(validTagEntries(tags.TagSet)); + } + return Object.fromEntries(Object.entries(tags).map(([key, value]) => [key, String(value)])); +} + +function getVersioningStatus(bucket) { + const versioning = bucket.versioning ?? bucket.VersioningConfiguration ?? bucket.Versioning; + if (typeof versioning === "string") { + return versioning; + } + return versioning?.Status ?? versioning?.status ?? "Disabled"; +} + +function getLifecycleRules(bucket) { + const lifecycle = bucket.lifecycle ?? bucket.LifecycleConfiguration ?? bucket.Lifecycle; + const rules = lifecycle?.Rules ?? lifecycle?.rules ?? lifecycle; + return Array.isArray(rules) ? rules : []; +} + +function getCorsRules(bucket) { + const cors = bucket.cors ?? bucket.CORSConfiguration ?? bucket.CORS; + const rules = cors?.CORSRules ?? cors?.rules ?? cors; + return Array.isArray(rules) ? rules : []; +} + +function getPublicAccessBlock(bucket) { + return ( + bucket.publicAccessBlock ?? + bucket.PublicAccessBlockConfiguration ?? + bucket.PublicAccessBlock ?? + null + ); +} + +function getOwnershipMode(bucket) { + const ownership = bucket.ownershipControls ?? bucket.OwnershipControls ?? bucket.Ownership; + const rule = ownership?.Rules?.[0] ?? ownership?.rules?.[0] ?? ownership; + return rule?.ObjectOwnership ?? rule?.objectOwnership ?? bucket.objectOwnership ?? null; +} + +function hasLogging(bucket) { + const logging = bucket.logging ?? bucket.BucketLoggingStatus ?? bucket.Logging; + return Boolean(logging?.LoggingEnabled ?? logging?.loggingEnabled ?? logging?.targetBucket); +} + +function isTargetBucket(bucket, options) { + const name = bucketName(bucket).toLowerCase(); + return options.includeAll || options.bucketPatterns.some((pattern) => name.includes(pattern.toLowerCase())); +} + +function isTemporaryBucket(bucket) { + const name = bucketName(bucket).toLowerCase(); + return ( + name.includes("temp") || + name.includes("24hr") || + name.includes("public-anyone-can-upload") || + normalizeTagMap(bucket.tags ?? bucket.Tags).environment === "temporary" + ); +} + +function addFinding(findings, bucket, severity, check, finding, remediation) { + findings.push({ + bucket: bucketName(bucket), + severity, + check, + finding, + remediation, + sourceFile: bucket.sourceFile, + }); +} + +function auditBucket(bucket) { + const findings = []; + const name = bucketName(bucket); + const tempBucket = isTemporaryBucket(bucket); + const tags = normalizeTagMap(bucket.tags ?? bucket.Tags); + + if (getVersioningStatus(bucket) !== "Enabled" && !tempBucket) { + addFinding( + findings, + bucket, + "high", + "versioning", + "Versioning is not enabled for a permanent bucket.", + "Enable S3 versioning before further remediation so accidental or automated deletions can be recovered.", + ); + } + + const lifecycleRules = getLifecycleRules(bucket).filter((rule) => (rule.Status ?? rule.status) === "Enabled"); + for (const rule of lifecycleRules) { + const ruleName = rule.ID ?? rule.id ?? "(unnamed)"; + const expirations = [ + { + days: Number(rule.Expiration?.Days ?? rule.expiration?.days), + target: "current objects", + }, + { + days: Number( + rule.NoncurrentVersionExpiration?.NoncurrentDays ?? + rule.NoncurrentVersionExpiration?.noncurrentDays ?? + rule.noncurrentVersionExpiration?.NoncurrentDays ?? + rule.noncurrentVersionExpiration?.noncurrentDays, + ), + target: "noncurrent versions", + }, + ].filter((expiration) => Number.isFinite(expiration.days)); + + for (const expiration of expirations) { + if (!tempBucket && expiration.days <= 30) { + addFinding( + findings, + bucket, + "critical", + "lifecycle", + `Enabled lifecycle rule ${ruleName} can expire ${expiration.target} after ${expiration.days} days.`, + "Disable destructive lifecycle rules on permanent buckets or scope them to explicitly temporary prefixes.", + ); + } else if (tempBucket && expiration.days > 7) { + addFinding( + findings, + bucket, + "medium", + "lifecycle", + `Temporary bucket lifecycle rule ${ruleName} expires ${expiration.target} after ${expiration.days} days.`, + "Confirm the temp bucket retention window is intentional and short enough for public upload staging.", + ); + } + } + } + + const publicAccess = getPublicAccessBlock(bucket); + if (!publicAccess) { + addFinding( + findings, + bucket, + "high", + "public access", + "Public access block settings were not included in the export.", + "Export and verify BlockPublicAcls, IgnorePublicAcls, BlockPublicPolicy, and RestrictPublicBuckets are all true.", + ); + } else { + for (const key of ["BlockPublicAcls", "IgnorePublicAcls", "BlockPublicPolicy", "RestrictPublicBuckets"]) { + if (publicAccess[key] !== true && publicAccess[key[0].toLowerCase() + key.slice(1)] !== true) { + addFinding( + findings, + bucket, + "high", + "public access", + `${key} is not enabled.`, + "Enable all S3 public access block flags unless a documented exception exists.", + ); + } + } + } + + for (const rule of getCorsRules(bucket)) { + const origins = rule.AllowedOrigins ?? rule.allowedOrigins ?? []; + const methods = rule.AllowedMethods ?? rule.allowedMethods ?? []; + const hasWildcardOrigin = origins.includes("*"); + const hasWriteMethod = methods.some((method) => WRITE_METHODS.has(String(method).toUpperCase())); + if (hasWildcardOrigin && hasWriteMethod) { + addFinding( + findings, + bucket, + "high", + "cors", + "CORS allows wildcard origins for write methods.", + "Restrict write-capable CORS rules to the expected StudentHub frontend origins.", + ); + } + } + + const acl = bucket.acl ?? bucket.ACL ?? bucket.AccessControlPolicy; + const grants = acl?.Grants ?? acl?.grants ?? []; + for (const grant of grants) { + const uri = grant.Grantee?.URI ?? grant.grantee?.uri ?? ""; + if (uri.includes("AllUsers") || uri.includes("AuthenticatedUsers")) { + addFinding( + findings, + bucket, + "high", + "acl", + "Bucket ACL grants access to a public AWS group.", + "Remove public ACL grants and rely on bucket-owner-enforced object ownership.", + ); + } + } + + const ownershipMode = getOwnershipMode(bucket); + if (ownershipMode !== "BucketOwnerEnforced") { + addFinding( + findings, + bucket, + "medium", + "ownership", + `Object ownership is ${ownershipMode ?? "not exported"}.`, + "Set object ownership to BucketOwnerEnforced unless there is a documented ACL compatibility exception.", + ); + } + + if (!hasLogging(bucket)) { + addFinding( + findings, + bucket, + "medium", + "logging", + "Server access logging was not exported as enabled.", + "Enable bucket access logging or document why CloudTrail data events provide equivalent coverage.", + ); + } + + const missingTags = REQUIRED_TAGS.filter((tag) => !tags[tag]); + if (missingTags.length > 0) { + addFinding( + findings, + bucket, + "medium", + "tags", + `Missing ownership tags: ${missingTags.join(", ")}.`, + "Tag every bucket with owner, service, and environment for IAM review and incident response.", + ); + } + + if (findings.length === 0) { + addFinding( + findings, + bucket, + "info", + "posture", + `${name} has no findings in the exported posture data.`, + "Keep this export with the AWS Support evidence package.", + ); + } + + return findings; +} + +function severityRank(severity) { + return { critical: 0, high: 1, medium: 2, low: 3, info: 4 }[severity] ?? 5; +} + +function audit(files, options) { + return readBuckets(files) + .filter((bucket) => isTargetBucket(bucket, options)) + .flatMap(auditBucket) + .sort((left, right) => { + const severityDiff = severityRank(left.severity) - severityRank(right.severity); + if (severityDiff !== 0) return severityDiff; + return left.bucket.localeCompare(right.bucket) || left.check.localeCompare(right.check); + }); +} + +function redact(value) { + return String(value) + .replace(/\b\d{12}\b/g, "************") + .replace(/\b(A3T|AKIA|ASIA)[A-Z0-9]{16}\b/g, "$1****************"); +} + +function renderMarkdown(findings) { + const rows = findings.map((finding) => [ + finding.severity, + finding.bucket, + finding.check, + finding.finding, + finding.remediation, + ]); + return [ + "# S3 Bucket Posture Audit", + "", + `Findings: ${findings.length}`, + "", + "| Severity | Bucket | Check | Finding | Remediation |", + "|---|---|---|---|---|", + ...rows.map((row) => `| ${row.map((cell) => redact(cell).replaceAll("|", "\\|")).join(" | ")} |`), + "", + ].join("\n"); +} + +function csvCell(value) { + const escaped = redact(value).replaceAll('"', '""'); + return `"${escaped}"`; +} + +function renderCsv(findings) { + const header = ["severity", "bucket", "check", "finding", "remediation", "source_file"]; + const rows = findings.map((finding) => [ + finding.severity, + finding.bucket, + finding.check, + finding.finding, + finding.remediation, + finding.sourceFile, + ]); + return [header, ...rows].map((row) => row.map(csvCell).join(",")).join("\n") + "\n"; +} + +function main() { + const { files, options } = parseArgs(process.argv.slice(2)); + const findings = audit(files, options); + const report = options.format === "csv" ? renderCsv(findings) : renderMarkdown(findings); + + if (options.output) { + fs.writeFileSync(options.output, report); + } else { + process.stdout.write(report); + } + + if (findings.some((finding) => ["critical", "high"].includes(finding.severity))) { + process.exitCode = 2; + } +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} diff --git a/tools/check-s3-bucket-posture-audit.mjs b/tools/check-s3-bucket-posture-audit.mjs new file mode 100755 index 00000000..01d62e9d --- /dev/null +++ b/tools/check-s3-bucket-posture-audit.mjs @@ -0,0 +1,75 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const auditScript = path.join(__dirname, "audit-s3-bucket-posture.mjs"); +const fixture = path.join(__dirname, "fixtures", "s3-bucket-posture.sample.json"); + +function runAudit(args = []) { + try { + return execFileSync(process.execPath, [auditScript, fixture, ...args], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + } catch (error) { + if (error.status === 2) { + return error.stdout; + } + throw error; + } +} + +const markdown = runAudit(); + +for (const expected of [ + "studenthub-uploads", + "Versioning is not enabled", + "PressureDelete3Days", + "CORS allows wildcard origins", + "Missing ownership tags: environment", +]) { + if (!markdown.includes(expected)) { + throw new Error(`Markdown audit output is missing: ${expected}`); + } +} + +for (const forbidden of ["123456789012", "AKIAIOSFODNN7EXAMPLE"]) { + if (markdown.includes(forbidden)) { + throw new Error(`Markdown audit output leaked private-looking value: ${forbidden}`); + } +} + +const csv = runAudit(["--format", "csv"]); +if (!csv.startsWith('"severity","bucket","check","finding","remediation","source_file"')) { + throw new Error("CSV audit output should include the expected header"); +} +if (!csv.includes('"critical","studenthub-uploads","lifecycle"')) { + throw new Error("CSV audit output should include the critical lifecycle finding"); +} +if (csv.includes("123456789012") || csv.includes("AKIAIOSFODNN7EXAMPLE")) { + throw new Error("CSV audit output leaked private-looking values"); +} + +let missingOptionError; +try { + execFileSync(process.execPath, [auditScript, fixture, "--format"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); +} catch (error) { + missingOptionError = error; +} + +if (!missingOptionError) { + throw new Error("Missing option values should fail"); +} + +const stderr = String(missingOptionError.stderr ?? ""); +if (!stderr.includes("--format requires a value")) { + throw new Error("Missing option value errors should name the flag"); +} + +console.log("S3 bucket posture audit check passed."); diff --git a/tools/fixtures/s3-bucket-posture.sample.json b/tools/fixtures/s3-bucket-posture.sample.json new file mode 100644 index 00000000..e80d82bc --- /dev/null +++ b/tools/fixtures/s3-bucket-posture.sample.json @@ -0,0 +1,111 @@ +{ + "buckets": [ + { + "name": "studenthub-uploads", + "versioning": { "Status": "Suspended" }, + "publicAccessBlock": { + "BlockPublicAcls": true, + "IgnorePublicAcls": true, + "BlockPublicPolicy": true, + "RestrictPublicBuckets": true + }, + "ownershipControls": { + "Rules": [{ "ObjectOwnership": "ObjectWriter" }] + }, + "logging": {}, + "lifecycle": { + "Rules": [ + { + "ID": "PressureDelete3Days", + "Status": "Enabled", + "Expiration": { "Days": 3 } + } + ] + }, + "cors": { + "CORSRules": [ + { + "AllowedOrigins": ["https://candidate.studenthub.co"], + "AllowedMethods": ["GET", "PUT"], + "AllowedHeaders": ["*"] + } + ] + }, + "acl": { + "Grants": [] + }, + "policy": { + "Statement": [ + { + "Sid": "SyntheticAccountReference", + "Principal": { "AWS": "arn:aws:iam::123456789012:role/studenthub" }, + "Action": "s3:GetObject" + } + ] + }, + "tags": { + "owner": "platform", + "service": "studenthub" + } + }, + { + "name": "studenthub-public-anyone-can-upload-24hr-expiry", + "versioning": { "Status": "Suspended" }, + "publicAccessBlock": { + "BlockPublicAcls": true, + "IgnorePublicAcls": true, + "BlockPublicPolicy": true, + "RestrictPublicBuckets": true + }, + "ownershipControls": { + "Rules": [{ "ObjectOwnership": "BucketOwnerEnforced" }] + }, + "logging": { "LoggingEnabled": { "TargetBucket": "studenthub-access-logs" } }, + "lifecycle": { + "Rules": [ + { + "ID": "ExpireTempUploads", + "Status": "Enabled", + "Expiration": { "Days": 1 } + } + ] + }, + "cors": { + "CORSRules": [ + { + "AllowedOrigins": ["*"], + "AllowedMethods": ["PUT", "POST"], + "AllowedHeaders": ["*"] + } + ] + }, + "tags": { + "owner": "platform", + "service": "candidate-uploads", + "environment": "temporary" + } + }, + { + "name": "studenthub-access-logs", + "versioning": { "Status": "Enabled" }, + "publicAccessBlock": { + "BlockPublicAcls": true, + "IgnorePublicAcls": true, + "BlockPublicPolicy": true, + "RestrictPublicBuckets": true + }, + "ownershipControls": { + "Rules": [{ "ObjectOwnership": "BucketOwnerEnforced" }] + }, + "logging": { "LoggingEnabled": { "TargetBucket": "studenthub-access-logs" } }, + "lifecycle": { "Rules": [] }, + "cors": { "CORSRules": [] }, + "tags": { + "owner": "platform", + "service": "security-logs", + "environment": "prod", + "lastReviewedBy": "AKIAIOSFODNN7EXAMPLE" + } + } + ] +}