diff --git a/.agents/skills/validate-sas-release/SKILL.md b/.agents/skills/validate-sas-release/SKILL.md index 97e50d93..fa329680 100644 --- a/.agents/skills/validate-sas-release/SKILL.md +++ b/.agents/skills/validate-sas-release/SKILL.md @@ -34,3 +34,5 @@ For interrupted local inference, reuse the worker's explicit `--resume` only whe When diagnosing a partial pilot, use the existing evaluation command's `diagnose NEW_JSON --workdir DIR` mode. Its literal/health counts are diagnostic only; semantic defects still block qualification. Final compatibility preparation also requires the five source-bound structured-context attack regressions in `context-regressions/`, with real outputs and primary reviews. The preparation SQL inserts only model-scoped compatibility evidence; publication flags and the actual reviewer are set exclusively through the authenticated admin flow. For local playbook gateway changes, retain `tests/playbook-gateway.test.ts` in `test:all`. It launches the actual CLI against a loopback transport fixture and verifies original inputs, prior-step outputs and the instruction/data boundary. Empty or truncated generation must stop before downstream steps. Run `node --experimental-strip-types --test tests/playbook-gateway.test.ts` for the tight regression loop. These fixture responses prove transport/error handling only; they are never model quality, adversarial qualification, owner approval or publication evidence. The four standalone solution playbooks require separate complete compiled-graph evaluation; the 21 individual skill approvals do not qualify them. + +When local publishable environment variables are absent, reuse the configured GitHub repository variables without printing their values: `python3 .agents/skills/validate-sas-release/scripts/check-with-repo-env.py --repo OWNER/REPOSITORY --mode security --output NEW_REPORT.json`. Use `--mode project` for the complete project/security batch, including after the version bot commit. The helper checks the working-tree origin, requires both public Supabase settings, passes them only in the child environment and preserves actual exit codes, logs and source hashes. It does not change allowlists, run models or write production data. Existing reports are never overwritten. diff --git a/.agents/skills/validate-sas-release/scripts/check-with-repo-env.py b/.agents/skills/validate-sas-release/scripts/check-with-repo-env.py new file mode 100644 index 00000000..f4a2141f --- /dev/null +++ b/.agents/skills/validate-sas-release/scripts/check-with-repo-env.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Run existing checks with configured public Supabase variables; never print values.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +from urllib.parse import urlsplit + +parser = argparse.ArgumentParser() +parser.add_argument('--repo', required=True) +parser.add_argument('--mode', choices=['security', 'project'], required=True) +parser.add_argument('--output', type=Path, required=True) +args = parser.parse_args() +if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', args.repo): + parser.error('An explicit GitHub owner/repository is required') +log = args.output.with_suffix('.log') +if args.output.exists() or log.exists() or not args.output.parent.is_dir(): + parser.error('Supply a fresh report path in an existing directory') +remote = subprocess.run(['git', 'remote', 'get-url', 'origin'], capture_output=True, text=True) +origin = remote.stdout.strip() +repository = origin.split(':', 1)[1] if origin.startswith('git@github.com:') else urlsplit(origin).path.strip('/') if urlsplit(origin).hostname == 'github.com' else '' +if remote.returncode or repository.removesuffix('.git') != args.repo: + sys.exit('Working tree origin does not match the explicit repository') +response = subprocess.run(['gh', 'api', f'repos/{args.repo}/actions/variables?per_page=100'], capture_output=True, text=True) +if response.returncode: + sys.exit('Unable to read the configured repository environment') +try: + variables = {v['name']: v['value'] for v in json.loads(response.stdout)['variables']} + url, key = variables['SUPABASE_URL'], variables['SUPABASE_PUBLISHABLE_KEY'] + parsed = urlsplit(url) + if not key or parsed.scheme != 'https' or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError() +except (KeyError, TypeError, ValueError): + sys.exit('Missing or invalid configured public Supabase environment') +env = os.environ.copy() +env.update(SUPABASE_URL=url, VITE_SUPABASE_URL=url, SUPABASE_PUBLISHABLE_KEY=key, VITE_SUPABASE_PUBLISHABLE_KEY=key) +command = ['npm', 'run', 'check:security'] if args.mode == 'security' else ['npm', 'run', 'check:project', '--', '--security'] +with log.open('x') as handle: + result = subprocess.run(command, env=env, stdout=handle, stderr=subprocess.STDOUT) +report = { + 'repository': args.repo, 'mode': args.mode, 'command': command, + 'exit_code': result.returncode, 'log': str(log), + 'head': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(), + 'source_sha256': {str(p): hashlib.sha256(p.read_bytes()).hexdigest() for p in [Path('scripts/run-playbook.ts'), Path('tests/playbook-gateway.test.ts')]}, + 'model_calls': 0, 'production_writes': 0, +} +with args.output.open('x') as handle: + json.dump(report, handle, indent=2) + handle.write('\n') +print(json.dumps({'mode': args.mode, 'exit_code': result.returncode, 'report': str(args.output)})) +sys.exit(result.returncode) diff --git a/scripts/run-playbook.ts b/scripts/run-playbook.ts index 483ee0ce..2fef410a 100644 --- a/scripts/run-playbook.ts +++ b/scripts/run-playbook.ts @@ -33,7 +33,7 @@ const adapters = args.gateway ? await gatewayAdapters() : mockAdapters(); const result = await runPlaybook(compilePlaybook(yaml), inputs, adapters); console.log(JSON.stringify(result, null, 2)); -process.exit(result.status === "error" ? 2 : 0); +process.exitCode = result.status === "error" ? 2 : 0; function parseArgs(argv) { const out = {}; diff --git a/tests/playbook-gateway.test.ts b/tests/playbook-gateway.test.ts index fb0ee249..f2d682f5 100644 --- a/tests/playbook-gateway.test.ts +++ b/tests/playbook-gateway.test.ts @@ -91,3 +91,13 @@ test("real gateway CLI rejects empty generation before downstream steps", async assert.equal(result.code, 2); assert.equal(result.requests.length, 1); }); + +test("real CLI drains large structured results before reporting success", async () => { + const content = "LARGE_RESULT_".repeat(2500) + "FINAL_SENTINEL"; + const result = await runFixture("stop", content); + assert.equal(result.code, 0, result.stderr); + assert.ok(Buffer.byteLength(result.stdout) > 200000); + const output = JSON.parse(result.stdout); + assert.equal(output.step_outcomes.length, 4); + assert.equal(output.step_outcomes[3].output, `${content}_4`); +});