diff --git a/evals/cases/dependency-promotion-79/fixture-repo/Dockerfile b/evals/cases/dependency-promotion-79/fixture-repo/Dockerfile new file mode 100644 index 0000000..1ac21c1 --- /dev/null +++ b/evals/cases/dependency-promotion-79/fixture-repo/Dockerfile @@ -0,0 +1,6 @@ +FROM node:22-alpine +WORKDIR /app +COPY package.json ./ +RUN npm install +COPY . . +CMD ["npm", "start"] diff --git a/evals/cases/dependency-promotion-79/fixture-repo/README.md b/evals/cases/dependency-promotion-79/fixture-repo/README.md new file mode 100644 index 0000000..dfc6db6 --- /dev/null +++ b/evals/cases/dependency-promotion-79/fixture-repo/README.md @@ -0,0 +1,29 @@ +# Order API + +A small HTTP API server that accepts orders and depends on several external +systems it does not own, declared through a mix of formats - not just one: + +- **Postgres** (via **Prisma**) - persists orders. Local instance in + `docker-compose.yml`; access configured in `prisma/schema.prisma`. +- **Redis** - caches the most recent order per customer. Local instance in + `docker-compose.yml`. +- **Kafka** - publishes order-created events for downstream workers. Runs in + the cluster; see `k8s/kafka-statefulset.yaml`. +- **ClickHouse** - receives order analytics events. Runs in the cluster; see + `k8s/clickhouse-deployment.yaml`. + +## Deployment + +The service is containerized with **Docker** (see `Dockerfile`) and deployed +to **ECS** (see `deploy/ecs-task-definition.json`). + +## Running locally + +``` +docker compose up +npx prisma migrate deploy +npm start +``` + +Postgres, Kafka, Redis, and ClickHouse are all required at runtime - the +server fails to start if any of them is unreachable. diff --git a/evals/cases/dependency-promotion-79/fixture-repo/deploy/ecs-task-definition.json b/evals/cases/dependency-promotion-79/fixture-repo/deploy/ecs-task-definition.json new file mode 100644 index 0000000..8ebf112 --- /dev/null +++ b/evals/cases/dependency-promotion-79/fixture-repo/deploy/ecs-task-definition.json @@ -0,0 +1,20 @@ +{ + "family": "order-api", + "networkMode": "awsvpc", + "requiresCompatibilities": ["FARGATE"], + "cpu": "256", + "memory": "512", + "containerDefinitions": [ + { + "name": "order-api", + "image": "order-api:latest", + "portMappings": [{ "containerPort": 3000, "protocol": "tcp" }], + "environment": [ + { "name": "DATABASE_URL", "value": "postgresql://postgres:orders@postgres:5432/orders" }, + { "name": "KAFKA_BROKER", "value": "kafka:9092" }, + { "name": "REDIS_URL", "value": "redis://redis:6379" }, + { "name": "CLICKHOUSE_URL", "value": "http://clickhouse:8123" } + ] + } + ] +} diff --git a/evals/cases/dependency-promotion-79/fixture-repo/docker-compose.yml b/evals/cases/dependency-promotion-79/fixture-repo/docker-compose.yml new file mode 100644 index 0000000..9f3c265 --- /dev/null +++ b/evals/cases/dependency-promotion-79/fixture-repo/docker-compose.yml @@ -0,0 +1,21 @@ +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_PASSWORD: orders + ports: + - "5432:5432" + redis: + image: redis:7-alpine + ports: + - "6379:6379" + api: + build: . + depends_on: + - postgres + - redis + environment: + DATABASE_URL: postgresql://postgres:orders@postgres:5432/orders + REDIS_URL: redis://redis:6379 + KAFKA_BROKER: kafka.order-api.svc.cluster.local:9092 + CLICKHOUSE_URL: http://clickhouse.order-api.svc.cluster.local:8123 diff --git a/evals/cases/dependency-promotion-79/fixture-repo/k8s/clickhouse-deployment.yaml b/evals/cases/dependency-promotion-79/fixture-repo/k8s/clickhouse-deployment.yaml new file mode 100644 index 0000000..dfbbb6e --- /dev/null +++ b/evals/cases/dependency-promotion-79/fixture-repo/k8s/clickhouse-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clickhouse + namespace: order-api +spec: + replicas: 1 + selector: + matchLabels: + app: clickhouse + template: + metadata: + labels: + app: clickhouse + spec: + containers: + - name: clickhouse + image: clickhouse/clickhouse-server:24 + ports: + - containerPort: 8123 +--- +apiVersion: v1 +kind: Service +metadata: + name: clickhouse + namespace: order-api +spec: + selector: + app: clickhouse + ports: + - port: 8123 + targetPort: 8123 diff --git a/evals/cases/dependency-promotion-79/fixture-repo/k8s/kafka-statefulset.yaml b/evals/cases/dependency-promotion-79/fixture-repo/k8s/kafka-statefulset.yaml new file mode 100644 index 0000000..830a865 --- /dev/null +++ b/evals/cases/dependency-promotion-79/fixture-repo/k8s/kafka-statefulset.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka + namespace: order-api +spec: + serviceName: kafka + replicas: 1 + selector: + matchLabels: + app: kafka + template: + metadata: + labels: + app: kafka + spec: + containers: + - name: kafka + image: bitnami/kafka:3.7 + ports: + - containerPort: 9092 +--- +apiVersion: v1 +kind: Service +metadata: + name: kafka + namespace: order-api +spec: + selector: + app: kafka + ports: + - port: 9092 + targetPort: 9092 diff --git a/evals/cases/dependency-promotion-79/fixture-repo/package.json b/evals/cases/dependency-promotion-79/fixture-repo/package.json new file mode 100644 index 0000000..42153f3 --- /dev/null +++ b/evals/cases/dependency-promotion-79/fixture-repo/package.json @@ -0,0 +1,12 @@ +{ + "name": "order-api", + "version": "1.0.0", + "type": "module", + "scripts": { "start": "node src/server.js" }, + "dependencies": { + "@prisma/client": "^5.14.0", + "kafkajs": "^2.2.4", + "redis": "^4.6.0", + "@clickhouse/client": "^1.1.0" + } +} diff --git a/evals/cases/dependency-promotion-79/fixture-repo/prisma/schema.prisma b/evals/cases/dependency-promotion-79/fixture-repo/prisma/schema.prisma new file mode 100644 index 0000000..932b42d --- /dev/null +++ b/evals/cases/dependency-promotion-79/fixture-repo/prisma/schema.prisma @@ -0,0 +1,15 @@ +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") +} + +generator client { + provider = "prisma-client-js" +} + +model Order { + id String @id @default(uuid()) + customer String + status String + createdAt DateTime @default(now()) +} diff --git a/evals/cases/dependency-promotion-79/fixture-repo/src/analytics.js b/evals/cases/dependency-promotion-79/fixture-repo/src/analytics.js new file mode 100644 index 0000000..a794ca3 --- /dev/null +++ b/evals/cases/dependency-promotion-79/fixture-repo/src/analytics.js @@ -0,0 +1,13 @@ +import { createClient } from "@clickhouse/client"; + +// Ships order analytics events for reporting. ClickHouse is declared as a +// service in docker-compose.yml. +const clickhouse = createClient({ url: process.env.CLICKHOUSE_URL ?? "http://localhost:8123" }); + +export async function recordOrderAnalyticsEvent(order) { + await clickhouse.insert({ + table: "order_events", + values: [{ order_id: order.id, customer: order.customer, status: order.status }], + format: "JSONEachRow", + }); +} diff --git a/evals/cases/dependency-promotion-79/fixture-repo/src/routes.js b/evals/cases/dependency-promotion-79/fixture-repo/src/routes.js new file mode 100644 index 0000000..30d0f6e --- /dev/null +++ b/evals/cases/dependency-promotion-79/fixture-repo/src/routes.js @@ -0,0 +1,11 @@ +import { createOrder, prisma, publishOrderCreated, cacheLatestOrder } from "./server.js"; +import { recordOrderAnalyticsEvent } from "./analytics.js"; + +export async function handleCreateOrder(request, response) { + const order = createOrder(request.body); + await prisma.order.create({ data: order }); + await publishOrderCreated(order); + await cacheLatestOrder(order); + await recordOrderAnalyticsEvent(order); + response.status(201).json(order); +} diff --git a/evals/cases/dependency-promotion-79/fixture-repo/src/server.js b/evals/cases/dependency-promotion-79/fixture-repo/src/server.js new file mode 100644 index 0000000..7653789 --- /dev/null +++ b/evals/cases/dependency-promotion-79/fixture-repo/src/server.js @@ -0,0 +1,35 @@ +import { PrismaClient } from "@prisma/client"; +import { Kafka } from "kafkajs"; +import { createClient } from "redis"; +import { createClient as createClickHouseClient } from "@clickhouse/client"; + +// Persists orders. Postgres is declared in docker-compose.yml and its +// connection is configured in prisma/schema.prisma - the server cannot +// start without a reachable Postgres instance. +export const prisma = new PrismaClient(); + +// Publishes order-created events for downstream workers to consume. Kafka +// is declared as a service in docker-compose.yml. +const kafka = new Kafka({ clientId: "order-api", brokers: [process.env.KAFKA_BROKER ?? "localhost:9092"] }); +const kafkaProducer = kafka.producer(); +await kafkaProducer.connect(); + +export async function publishOrderCreated(order) { + await kafkaProducer.send({ + topic: "orders.created", + messages: [{ value: JSON.stringify(order) }], + }); +} + +// Caches the most recent order per customer for fast repeat lookups. Redis +// is declared as a service in docker-compose.yml. +const redisClient = createClient({ url: process.env.REDIS_URL ?? "redis://localhost:6379" }); +await redisClient.connect(); + +export async function cacheLatestOrder(order) { + await redisClient.set(`latest-order:${order.customer}`, JSON.stringify(order)); +} + +export function createOrder(body) { + return { id: crypto.randomUUID(), ...body, status: "created" }; +} diff --git a/evals/cases/dependency-promotion-79/rubric.json b/evals/cases/dependency-promotion-79/rubric.json new file mode 100644 index 0000000..df1e5fc --- /dev/null +++ b/evals/cases/dependency-promotion-79/rubric.json @@ -0,0 +1,127 @@ +{ + "case_id": "dependency-promotion-79", + "target_repo_url": "local-fixture:evals/cases/dependency-promotion-79/fixture-repo", + "target_commit": "generated-at-run-time", + "score": { + "pass_threshold": 80, + "node_coverage_points": 50, + "edge_coverage_points": 15, + "quality_points": 35, + "quality_penalties": { + "noisy_structure_node_points": 3, + "noisy_structure_node_max": 12, + "bad_claim_points": 4, + "bad_claim_max": 16, + "bad_anchor_points": 3, + "bad_anchor_max": 9, + "depth_violation_points": 8, + "weak_graph_linking_points": 8 + } + }, + "judge": { + "instructions": [ + "This eval tests one thing specifically: whether the bootstrap proposal promotes the fixture's external dependencies (Postgres, Kafka, Redis, ClickHouse, Docker, ECS) to their own first-class components instead of only mentioning them inside claim text or bundling them into an internal component's name/description (Greplica issue #79).", + "Judge whether the proposal is a good shallow bootstrap memory graph for this exact fixture repo.", + "Do not compute a numeric score. Only classify expected nodes, expected edges, and quality issues.", + "Judge semantic equivalence, not exact IDs or exact wording. A node can satisfy an expected item if it clearly represents the same concept under any reasonable id/name (e.g. component.kafka, component.message_broker, 'Kafka').", + "The critical check is component-level, per dependency: each of Postgres, Kafka, Redis, ClickHouse, Docker, and ECS must appear as its own component object in creates.components, not only as a word inside a claim's text, and not folded into a compound internal component name/description like 'Order API server (Postgres, Kafka, Redis)' that bundles several technologies together without giving any of them their own identity.", + "Compact relationship fields are graph edges: flow.touches creates Flow -> Component touches edges, claim.about creates Claim -> Component/Flow about edges, component.contains creates Component -> Component contains edges.", + "When judging expected edges, inspect compact relationship arrays directly. Do not require a separate creates.edges entry.", + "Each dependency component must be connected via `about` (from a claim) and/or `touches` (from a flow), not by being nested under the internal API component via `contains`. `contains` means ownership, and this repo does not own any of these external systems - nesting one of them under the API component via contains is a quality defect, note it under bad_claims with a reason mentioning contains misuse even though it is a component-level defect, since bad_claims is the closest available category.", + "It is fine, and arguably good, for the proposal to represent Prisma as the code_anchor/tooling detail on the Postgres component rather than as its own separate component, since Prisma is the ORM through which the repo talks to Postgres rather than a distinct runtime system. Do not penalize a proposal for treating Prisma this way; only flag it as missing if the proposal fails to represent Postgres as a component at all.", + "Judge claim code_anchors as navigation aids. For code_verified claims, a stable implementation symbol or a file-only anchor (e.g. docker-compose.yml, package.json, prisma/schema.prisma, deploy/ecs-task-definition.json) for config/manifest files without symbols is acceptable.", + "Do not penalize the proposal for keeping the fixture shallow - it is a small repo and a compact proposal with several components, a few flows, and a moderate number of claims is appropriate depth, not a depth violation.", + "The proposal should be useful to a future coding agent that needs to know this repo depends on these external systems and where each dependency is declared/used." + ], + "actual_repo_facts": [ + "This is a small Node.js HTTP API named order-api.", + "README.md states the server persists orders to Postgres (via Prisma), publishes order-created events to Kafka, caches the most recent order per customer in Redis, and ships order analytics events to ClickHouse.", + "README.md states the service is containerized with Docker (Dockerfile) and deployed to ECS (deploy/ecs-task-definition.json).", + "README.md states Postgres, Kafka, Redis, and ClickHouse are all required at runtime - the server fails to start if any is unreachable.", + "docker-compose.yml declares postgres and redis services for local development, plus an api service that depends_on both and receives DATABASE_URL, REDIS_URL, KAFKA_BROKER, and CLICKHOUSE_URL as environment variables.", + "k8s/kafka-statefulset.yaml is a Kubernetes StatefulSet and Service that declares Kafka in the order-api namespace.", + "k8s/clickhouse-deployment.yaml is a Kubernetes Deployment and Service that declares ClickHouse in the order-api namespace.", + "Dockerfile builds the api service's container image.", + "deploy/ecs-task-definition.json is a Fargate ECS task definition for the order-api container, listing the same environment variables as docker-compose.yml.", + "prisma/schema.prisma declares a postgresql datasource read from DATABASE_URL and an Order model (id, customer, status, createdAt).", + "package.json lists @prisma/client, kafkajs, redis, and @clickhouse/client as runtime dependencies.", + "src/server.js creates a PrismaClient (exported as `prisma`), a kafkajs Kafka producer connected at module load, and a Redis client connected at module load. It exports publishOrderCreated(order) (sends to the orders.created Kafka topic), cacheLatestOrder(order) (Redis SET keyed by customer), and createOrder(body) (pure order-object builder).", + "src/analytics.js creates a ClickHouse client and exports recordOrderAnalyticsEvent(order), which inserts a row into the order_events table.", + "src/routes.js exports handleCreateOrder(request, response), an HTTP handler that builds an order, writes it via prisma.order.create, publishes it to Kafka, caches it in Redis, records a ClickHouse analytics event, then responds 201 with the order." + ], + "expected_nodes": [ + { + "id": "component.order_api", + "kind": "component", + "description": "The internal Order API - server/route handling for creating orders (src/server.js, src/routes.js)." + }, + { + "id": "component.postgres", + "kind": "component", + "description": "Postgres, the relational datastore the repo does not own, declared in docker-compose.yml and accessed through prisma/schema.prisma. Part of the set of dependencies this eval exists to check for." + }, + { + "id": "component.kafka", + "kind": "component", + "description": "Kafka, the message broker the repo does not own, declared in k8s/kafka-statefulset.yaml and used to publish order-created events. Part of the set of dependencies this eval exists to check for." + }, + { + "id": "component.redis", + "kind": "component", + "description": "Redis, the cache the repo does not own, declared in docker-compose.yml and used to cache the latest order per customer. Part of the set of dependencies this eval exists to check for." + }, + { + "id": "component.clickhouse", + "kind": "component", + "description": "ClickHouse, the analytics datastore the repo does not own, declared in k8s/clickhouse-deployment.yaml and used to record order analytics events. Part of the set of dependencies this eval exists to check for." + }, + { + "id": "component.docker", + "kind": "component", + "description": "Docker, the container platform the repo is packaged with (Dockerfile). Part of the set of dependencies this eval exists to check for." + }, + { + "id": "component.ecs", + "kind": "component", + "description": "ECS, the deployment/orchestration platform the repo runs on (deploy/ecs-task-definition.json). Part of the set of dependencies this eval exists to check for." + }, + { + "id": "claim.persists_orders_to_postgres", + "kind": "claim", + "description": "A claim describing that the API persists orders to Postgres via Prisma." + }, + { + "id": "claim.publishes_order_created_to_kafka", + "kind": "claim", + "description": "A claim describing that the API publishes order-created events to Kafka." + } + ], + "expected_edges": [ + { + "id": "edge.claim_about_postgres", + "from": "the claim about persisting orders", + "to": "the Postgres component", + "description": "The claim connecting the API to Postgres must be `about` the Postgres component (not just the API component alone)." + }, + { + "id": "edge.claim_about_kafka", + "from": "the claim about publishing order-created events", + "to": "the Kafka component", + "description": "The claim connecting the API to Kafka must be `about` the Kafka component (not just the API component alone)." + }, + { + "id": "edge.claim_about_order_api", + "from": "important dependency claims", + "to": "the Order API component", + "description": "Claims describing a dependency interaction should also be `about` the internal Order API component, since the behavior is owned by that component." + } + ], + "quality_rules": { + "noisy_structure_nodes": "Components that fragment this fixture too finely (e.g. a separate component for every single file) or that are so generic they carry no information (e.g. a bare 'Dependencies' or 'Infrastructure' component listing several unrelated technologies together). One component per real dependency (Postgres, Kafka, Redis, ClickHouse, Docker, ECS) is exactly right, not noisy.", + "bad_claims": "Claims that are wrong, unsupported by the fixture, vague, or that nest a dependency under the Order API using `contains` (ownership) instead of `about`/`touches`. Also bad: a proposal that only mentions a dependency inside claim text, or folds it into an internal component's compound name, with no component of its own - that is the exact bug this eval exists to catch.", + "bad_anchors": "Fake paths, dependency/generated paths, or a vague repo-root anchor without reason. File-only anchors on docker-compose.yml, k8s/kafka-statefulset.yaml, k8s/clickhouse-deployment.yaml, prisma/schema.prisma, Dockerfile, or deploy/ecs-task-definition.json for the corresponding dependency component are good, not bad - those are the declaration points, and they intentionally span several different formats (compose, Kubernetes manifests, ORM schema, Dockerfile, ECS task definition), not just one.", + "depth_violation": "The proposal is clearly too deep for a bootstrap of this fixture, such as a claim per line of code or per environment variable.", + "weak_graph_linking": "One or more dependency components exist but are not linked to any claim or flow, making them orphan nodes with no relationships - this defeats the purpose of promoting them." + } + } +} diff --git a/evals/cases/dependency-promotion-79/run.ts b/evals/cases/dependency-promotion-79/run.ts new file mode 100644 index 0000000..42ae188 --- /dev/null +++ b/evals/cases/dependency-promotion-79/run.ts @@ -0,0 +1,591 @@ +import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { resolve } from "node:path"; +import { + type CommandResult, + findRepoRoot, + git, + readJson, + repoTree, + round, + run, + runOrThrow, + timestamp, + valueAfter, + writeJson, +} from "../../lib/common.js"; +import { runCodexAgent } from "../../../libs/agent-runner/codex.js"; +import type { AgentRunResult } from "../../../libs/agent-runner/types.js"; +import { loadRepoEnv } from "../../../libs/env/load-local-env.js"; + +// This eval case exists specifically to check issue #79: does the bootstrap +// skill actually make an agent promote external dependencies to their own +// first-class components, instead of only mentioning them inside claim text +// or folding them into an internal component's name/description? The +// bundled fixture is a small "Order API" that depends on six real external +// systems declared through several different formats (docker-compose, +// Kubernetes manifests, an ORM schema, a Dockerfile, an ECS task +// definition) - not just one - so the result generalizes across dependency +// shapes and declaration-point formats, not one narrow case. Unlike the +// other eval cases, the target is a small fixture repo bundled in this +// directory (not a live clone), git-initialized fresh on every run, so the +// result is deterministic and does not depend on a third-party repo +// staying reachable/unchanged. + +const caseId = "dependency-promotion-79"; + +// Component id tokens the fixture's dependencies should be promoted under. +// Matched against `id` only (not `name`) - see checkDependenciesPromoted. +const expectedDependencyTokens = ["postgres", "kafka", "redis", "clickhouse", "docker", "ecs"]; + +interface Args { + proposal?: string; + agent?: "codex"; + agentModel?: string; + judge?: "openai"; + judgeModel?: string; +} + +interface RunContext { + repoRoot: string; + runDir: string; + targetRepoDir: string; + fixtureDir: string; + greplicaHomeDir: string; + codexHomeDir: string; + proposalPath: string; + rubricPath: string; + greplicaCommand: string[]; +} + +interface EvalResult { + case_id: string; + target_repo_url: string; + target_commit: string; + run_dir: string; + target_repo_dir: string; + greplica_home_dir: string; + proposal_path: string; + success: boolean; + dependencies_promoted: Record; + commands: CommandResult[]; + generation?: AgentRunResult; + judge?: { + model: string; + judge_input_path: string; + judge_output_path: string; + score: ScoreResult; + }; +} + +interface Rubric { + case_id: string; + target_repo_url: string; + target_commit: string; + score: { + pass_threshold: number; + node_coverage_points: number; + edge_coverage_points: number; + quality_points: number; + quality_penalties: { + noisy_structure_node_points: number; + noisy_structure_node_max: number; + bad_claim_points: number; + bad_claim_max: number; + bad_anchor_points: number; + bad_anchor_max: number; + depth_violation_points: number; + weak_graph_linking_points: number; + }; + }; + judge: JudgeRubric; +} + +interface JudgeRubric { + instructions: string[]; + actual_repo_facts: string[]; + expected_nodes: Array<{ id: string; kind: string; description: string }>; + expected_edges: Array<{ id: string; description: string; from?: string; to?: string }>; + quality_rules: Record; +} + +interface JudgeInput { + task: string; + rubric: JudgeRubric; + repo_tree: string[]; + proposal: unknown; +} + +interface JudgeOutput { + nodes: Array<{ + expected_id: string; + present: boolean; + matched_ids: string[]; + reason: string; + }>; + edges: Array<{ + expected_id: string; + present: boolean; + matched: string[]; + reason: string; + }>; + quality: { + noisy_structure_nodes: Array<{ id: string; reason: string }>; + bad_claims: Array<{ id: string; reason: string }>; + bad_anchors: Array<{ id: string; anchor: string; reason: string }>; + depth_violation: { present: boolean; reason: string }; + weak_graph_linking: { present: boolean; reason: string }; + }; +} + +interface ScoreResult { + node_score: number; + edge_score: number; + quality_score: number; + final_score: number; + pass_threshold: number; + passed: boolean; +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + const context = prepareRun(); + const targetCommit = prepareTargetRepo(context); + prepareGreplicaHome(context); + const installCommand = runProductCommand(context, "install", "--platform", "codex", "--embedding", "local"); + const generation = await getProposal(context, args); + const commands = [ + installCommand, + ...runProductCommands(context), + ]; + const commandsSucceeded = commands.every((command) => command.exit_code === 0); + const dependenciesPromoted = commandsSucceeded + ? checkDependenciesPromoted(context) + : Object.fromEntries(expectedDependencyTokens.map((token) => [token, false])); + const allDependenciesPromoted = Object.values(dependenciesPromoted).every(Boolean); + const judge = commandsSucceeded && args.judge === "openai" ? await runOpenAiJudge(context, args) : undefined; + const success = commandsSucceeded && allDependenciesPromoted && (judge === undefined || judge.score.passed); + writeResult(context, targetCommit, commands, dependenciesPromoted, success, generation, judge); + + console.log(success ? "Dependency-promotion eval passed." : "Dependency-promotion eval failed."); + console.log(`Run directory: ${context.runDir}`); + for (const [token, promoted] of Object.entries(dependenciesPromoted)) { + console.log( + promoted + ? `${token}: promoted to its own component.` + : `${token}: NOT promoted to its own component (this is the failure mode issue #79 describes).`, + ); + } + if (judge) { + console.log(`Judge score: ${judge.score.final_score.toFixed(2)} / 100`); + } + process.exitCode = success ? 0 : 1; +} + +function prepareRun(): RunContext { + const repoRoot = findRepoRoot(import.meta.url); + loadRepoEnv(repoRoot); + const runDir = resolve(repoRoot, "eval-runs", timestamp(), caseId); + const targetRepoDir = resolve(runDir, "target-repo"); + const fixtureDir = resolve(repoRoot, "evals/cases/dependency-promotion-79/fixture-repo"); + const greplicaHomeDir = resolve(runDir, "greplica-home"); + const codexHomeDir = resolve(runDir, "codex-home"); + const proposalPath = resolve(runDir, "proposal.json"); + const rubricPath = resolve(repoRoot, "evals/cases/dependency-promotion-79/rubric.json"); + const greplicaCommand = ["node", resolve(repoRoot, "dist/apps/cli/main.js")]; + + mkdirSync(runDir, { recursive: true }); + + return { + repoRoot, + runDir, + targetRepoDir, + fixtureDir, + greplicaHomeDir, + codexHomeDir, + proposalPath, + rubricPath, + greplicaCommand, + }; +} + +function prepareTargetRepo(context: RunContext): string { + cpSync(context.fixtureDir, context.targetRepoDir, { recursive: true }); + runOrThrow(["git", "init", "-q"], context.targetRepoDir); + runOrThrow(["git", "config", "user.email", "eval@greplica.local"], context.targetRepoDir); + runOrThrow(["git", "config", "user.name", "Greplica Eval"], context.targetRepoDir); + runOrThrow(["git", "add", "-A"], context.targetRepoDir); + runOrThrow(["git", "commit", "-q", "-m", "Fixture: Order API with several external dependencies"], context.targetRepoDir); + return git(context.targetRepoDir, ["rev-parse", "HEAD"]); +} + +function prepareGreplicaHome(context: RunContext): void { + mkdirSync(context.greplicaHomeDir, { recursive: true }); + mkdirSync(context.codexHomeDir, { recursive: true }); + seedCodexRuntimeHome(context.codexHomeDir); +} + +function seedCodexRuntimeHome(codexHomeDir: string): void { + const sourceHome = resolve(homedir(), ".codex"); + for (const file of ["auth.json", "config.toml", "models_cache.json", ".codex-global-state.json", "installation_id"]) { + const source = resolve(sourceHome, file); + if (existsSync(source)) copyFileSync(source, resolve(codexHomeDir, file)); + } +} + +async function getProposal(context: RunContext, args: Args): Promise { + if (args.proposal) { + copyFileSync(resolve(args.proposal), context.proposalPath); + return undefined; + } + + if (args.agent === "codex") { + const model = args.agentModel; + const result = await runCodexAgent({ + cwd: context.targetRepoDir, + env: evalEnv(context), + model, + prompt: codexBootstrapPrompt(context), + transcriptPath: resolve(context.runDir, "agent-events.jsonl"), + finalMessagePath: resolve(context.runDir, "agent-final-message.txt"), + proposalPath: context.proposalPath, + }); + if (result.exit_code !== 0) { + throw new Error(`Codex agent failed with exit code ${String(result.exit_code)}.`); + } + if (!existsSync(context.proposalPath)) { + throw new Error(`Codex agent did not create proposal at ${context.proposalPath}.`); + } + return result; + } + + throw new Error("Expected either --proposal or --agent codex."); +} + +function runProductCommands(context: RunContext): CommandResult[] { + const commands = [ + [...context.greplicaCommand, "proposal", "validate", context.proposalPath], + [...context.greplicaCommand, "proposal", "apply", context.proposalPath], + ]; + + return commands.map((command) => runProductCommand(context, ...command.slice(context.greplicaCommand.length))); +} + +function runProductCommand(context: RunContext, ...args: string[]): CommandResult { + return run([...context.greplicaCommand, ...args], context.targetRepoDir, evalEnv(context), { stdio: "inherit" }); +} + +function evalEnv(context: RunContext): NodeJS.ProcessEnv { + return { + ...process.env, + CODEX_HOME: context.codexHomeDir, + GREPLICA_HOME: context.greplicaHomeDir, + }; +} + +// Deterministic, judge-independent check: for each expected dependency, is +// there a component that is SPECIFICALLY about it, not a compound/internal +// component whose name merely mentions it in passing (e.g. "Order API +// server and Redis publisher")? That compound-name shape is the exact bug +// issue #79 describes wearing a different disguise - the dependency still +// has no component of its own, no dedicated node, nothing `about`/`touches` +// it directly. Only the component `id` is checked: ids are structured, +// canonical identifiers an agent assigns specifically to a concept, unlike +// free-text `name` fields which can smuggle a mention of a dependency into +// an unrelated component's description without giving it its own identity. +function checkDependenciesPromoted(context: RunContext): Record { + const proposal = readJson<{ creates?: { components?: Array<{ id?: string }> } }>(context.proposalPath); + const components = proposal.creates?.components ?? []; + const componentIdTokens = components.map((component) => new Set((component.id ?? "").toLowerCase().split(/[^a-z0-9]+/))); + + const result: Record = {}; + for (const token of expectedDependencyTokens) { + result[token] = componentIdTokens.some((tokens) => tokens.has(token)); + } + return result; +} + +async function runOpenAiJudge( + context: RunContext, + args: Args, +): Promise> { + const apiKey = process.env.OPENAI_API_KEY; + if (!apiKey) throw new Error("OPENAI_API_KEY is required when using --judge openai."); + + const model = args.judgeModel ?? process.env.OPENAI_MODEL; + if (!model) throw new Error("Set OPENAI_MODEL or pass --judge-model when using --judge openai."); + + const rubric = readJson(context.rubricPath); + const proposal = readJson(context.proposalPath); + const judgeInput: JudgeInput = { + task: "Judge this Greplica bootstrap proposal for a fixture repo that exists to test whether external dependencies (Postgres, Kafka, Redis, ClickHouse, Docker, ECS) get promoted to first-class components. Return JSON classification only; do not compute numeric scores.", + rubric: rubric.judge, + repo_tree: repoTree(context.targetRepoDir), + proposal, + }; + const judgeInputPath = resolve(context.runDir, "judge-input.json"); + const judgeOutputPath = resolve(context.runDir, "judge-output.json"); + writeJson(judgeInputPath, judgeInput); + + const judgeOutput = await requestJudge(apiKey, model, judgeInput); + writeJson(judgeOutputPath, judgeOutput); + + return { + model, + judge_input_path: judgeInputPath, + judge_output_path: judgeOutputPath, + score: scoreJudgeOutput(rubric, judgeOutput), + }; +} + +function writeResult( + context: RunContext, + targetCommit: string, + commands: CommandResult[], + dependenciesPromoted: Record, + success: boolean, + generation: AgentRunResult | undefined, + judge: EvalResult["judge"], +): void { + const result: EvalResult = { + case_id: caseId, + target_repo_url: `local-fixture:${context.fixtureDir}`, + target_commit: targetCommit, + run_dir: context.runDir, + target_repo_dir: context.targetRepoDir, + greplica_home_dir: context.greplicaHomeDir, + proposal_path: context.proposalPath, + success, + dependencies_promoted: dependenciesPromoted, + commands, + generation, + judge, + }; + + writeJson(resolve(context.runDir, "result.json"), result); +} + +function parseArgs(args: string[]): Args { + const proposal = valueAfter(args, "--proposal"); + const agent = valueAfter(args, "--agent"); + if ((proposal === undefined && agent === undefined) || (proposal !== undefined && agent !== undefined)) { + throw new Error("Usage: npm run eval:dependency-promotion-79 -- (--proposal /path/to/proposal.json | --agent codex) [--agent-model model] [--judge openai] [--judge-model model]"); + } + if (agent !== undefined && agent !== "codex") throw new Error("Only --agent codex is supported."); + + const judge = valueAfter(args, "--judge"); + if (judge !== undefined && judge !== "openai") { + throw new Error("Only --judge openai is supported."); + } + + const agentModel = valueAfter(args, "--agent-model"); + const judgeModel = valueAfter(args, "--judge-model"); + + return { proposal, agent, agentModel, judge, judgeModel }; +} + +function codexBootstrapPrompt(context: RunContext): string { + const skill = readFileSync(resolve(context.repoRoot, "skills/greplica-bootstrap/SKILL.md"), "utf8"); + const greplica = context.greplicaCommand.join(" "); + + return `You are running a Greplica bootstrap workflow for this repository. + +Use this exact user-facing skill as the workflow contract: + + +${skill} + + +Runtime facts for this eval: +- Current working directory is the target repository root. +- GREPLICA_HOME is already set to an isolated eval directory. +- Use this greplica command exactly: ${greplica} +- Write the final proposal JSON exactly here: ${context.proposalPath} + +Task: +1. Run the skill workflow for bootstrap memory on this repo. +2. Inspect the repo shallowly. Prefer top-level components, flows, and durable claims. +3. Create a compact proposal JSON at ${context.proposalPath}. +4. Validate it with: ${greplica} proposal validate ${context.proposalPath} +5. Fix validation errors until valid. +6. Do not apply the proposal. + +The proposal should be useful to a future coding agent and should avoid deep implementation trivia.`; +} + +async function requestJudge(apiKey: string, model: string, input: JudgeInput): Promise { + const response = await fetch("https://api.openai.com/v1/responses", { + method: "POST", + headers: { + "Authorization": `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model, + input: [ + { + role: "system", + content: + "You are an evaluator for Greplica bootstrap proposals. Return JSON only. Classify expected nodes, expected edges, and quality issues. Do not calculate numeric scores.", + }, + { + role: "user", + content: JSON.stringify(input), + }, + ], + text: { + format: { + type: "json_schema", + name: "dependency_promotion_eval_judge", + strict: true, + schema: judgeOutputSchema(), + }, + }, + }), + }); + + const body = await response.json() as Record; + if (!response.ok) { + throw new Error(`OpenAI judge request failed: ${JSON.stringify(body)}`); + } + + const outputText = extractOutputText(body); + return JSON.parse(outputText) as JudgeOutput; +} + +function scoreJudgeOutput(rubric: Rubric, judge: JudgeOutput): ScoreResult { + const presentNodes = new Set(judge.nodes.filter((item) => item.present).map((item) => item.expected_id)); + const presentEdges = new Set(judge.edges.filter((item) => item.present).map((item) => item.expected_id)); + const nodeScore = (presentNodes.size / rubric.judge.expected_nodes.length) * rubric.score.node_coverage_points; + const edgeScore = (presentEdges.size / rubric.judge.expected_edges.length) * rubric.score.edge_coverage_points; + const penalties = rubric.score.quality_penalties; + const qualityPenalty = + Math.min(judge.quality.noisy_structure_nodes.length * penalties.noisy_structure_node_points, penalties.noisy_structure_node_max) + + Math.min(judge.quality.bad_claims.length * penalties.bad_claim_points, penalties.bad_claim_max) + + Math.min(judge.quality.bad_anchors.length * penalties.bad_anchor_points, penalties.bad_anchor_max) + + (judge.quality.depth_violation.present ? penalties.depth_violation_points : 0) + + (judge.quality.weak_graph_linking.present ? penalties.weak_graph_linking_points : 0); + const qualityScore = Math.max(0, rubric.score.quality_points - qualityPenalty); + const finalScore = nodeScore + edgeScore + qualityScore; + + return { + node_score: round(nodeScore, 2), + edge_score: round(edgeScore, 2), + quality_score: round(qualityScore, 2), + final_score: round(finalScore, 2), + pass_threshold: rubric.score.pass_threshold, + passed: finalScore >= rubric.score.pass_threshold, + }; +} + +function extractOutputText(body: Record): string { + if (typeof body.output_text === "string") return body.output_text; + + const output = body.output; + if (!Array.isArray(output)) throw new Error("OpenAI response did not include output text."); + + const texts: string[] = []; + for (const item of output) { + if (!isRecord(item) || !Array.isArray(item.content)) continue; + for (const content of item.content) { + if (isRecord(content) && typeof content.text === "string") texts.push(content.text); + } + } + + const text = texts.join(""); + if (text.length === 0) throw new Error("OpenAI response output text was empty."); + return text; +} + +function judgeOutputSchema(): Record { + const reasonedItem = { + type: "object", + additionalProperties: false, + properties: { + expected_id: { type: "string" }, + present: { type: "boolean" }, + matched_ids: { type: "array", items: { type: "string" } }, + reason: { type: "string" }, + }, + required: ["expected_id", "present", "matched_ids", "reason"], + }; + const edgeItem = { + type: "object", + additionalProperties: false, + properties: { + expected_id: { type: "string" }, + present: { type: "boolean" }, + matched: { type: "array", items: { type: "string" } }, + reason: { type: "string" }, + }, + required: ["expected_id", "present", "matched", "reason"], + }; + const issueItem = { + type: "object", + additionalProperties: false, + properties: { + id: { type: "string" }, + reason: { type: "string" }, + }, + required: ["id", "reason"], + }; + + return { + type: "object", + additionalProperties: false, + properties: { + nodes: { type: "array", items: reasonedItem }, + edges: { type: "array", items: edgeItem }, + quality: { + type: "object", + additionalProperties: false, + properties: { + noisy_structure_nodes: { type: "array", items: issueItem }, + bad_claims: { type: "array", items: issueItem }, + bad_anchors: { + type: "array", + items: { + type: "object", + additionalProperties: false, + properties: { + id: { type: "string" }, + anchor: { type: "string" }, + reason: { type: "string" }, + }, + required: ["id", "anchor", "reason"], + }, + }, + depth_violation: { + type: "object", + additionalProperties: false, + properties: { + present: { type: "boolean" }, + reason: { type: "string" }, + }, + required: ["present", "reason"], + }, + weak_graph_linking: { + type: "object", + additionalProperties: false, + properties: { + present: { type: "boolean" }, + reason: { type: "string" }, + }, + required: ["present", "reason"], + }, + }, + required: ["noisy_structure_nodes", "bad_claims", "bad_anchors", "depth_violation", "weak_graph_linking"], + }, + }, + required: ["nodes", "edges", "quality"], + }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/evals/cases/dependency-promotion-79/sample-good.proposal.json b/evals/cases/dependency-promotion-79/sample-good.proposal.json new file mode 100644 index 0000000..3f1fdb5 --- /dev/null +++ b/evals/cases/dependency-promotion-79/sample-good.proposal.json @@ -0,0 +1,118 @@ +{ + "title": "Dependency promotion sample: all six dependencies promoted", + "summary": "Sample proposal showing every external dependency promoted per the promotion guidance.", + "creates": { + "components": [ + { + "id": "component.order_api", + "name": "Order API", + "code_anchor": "src/server.js" + }, + { + "id": "component.postgres", + "name": "Postgres", + "code_anchor": "docker-compose.yml" + }, + { + "id": "component.kafka", + "name": "Kafka", + "code_anchor": "k8s/kafka-statefulset.yaml" + }, + { + "id": "component.redis", + "name": "Redis", + "code_anchor": "docker-compose.yml" + }, + { + "id": "component.clickhouse", + "name": "ClickHouse", + "code_anchor": "k8s/clickhouse-deployment.yaml" + }, + { + "id": "component.docker", + "name": "Docker", + "code_anchor": "Dockerfile" + }, + { + "id": "component.ecs", + "name": "ECS", + "code_anchor": "deploy/ecs-task-definition.json" + } + ], + "claims": [ + { + "id": "claim.persists_orders_to_postgres", + "kind": "fact", + "text": "The Order API persists orders to Postgres via Prisma (prisma/schema.prisma).", + "truth": "code_verified", + "intent": "intended", + "about": ["component.order_api", "component.postgres"], + "code_anchors": [ + { "file": "src/routes.js", "symbol": "handleCreateOrder" } + ] + }, + { + "id": "claim.publishes_order_created_to_kafka", + "kind": "fact", + "text": "The Order API publishes order-created events to Kafka via publishOrderCreated so downstream workers can process orders asynchronously.", + "truth": "code_verified", + "intent": "intended", + "about": ["component.order_api", "component.kafka"], + "code_anchors": [ + { "file": "src/server.js", "symbol": "publishOrderCreated" } + ] + }, + { + "id": "claim.caches_latest_order_in_redis", + "kind": "fact", + "text": "The Order API caches the most recent order per customer in Redis via cacheLatestOrder.", + "truth": "code_verified", + "intent": "intended", + "about": ["component.order_api", "component.redis"], + "code_anchors": [ + { "file": "src/server.js", "symbol": "cacheLatestOrder" } + ] + }, + { + "id": "claim.records_analytics_events_to_clickhouse", + "kind": "fact", + "text": "The Order API records order analytics events to ClickHouse via recordOrderAnalyticsEvent.", + "truth": "code_verified", + "intent": "intended", + "about": ["component.order_api", "component.clickhouse"], + "code_anchors": [ + { "file": "src/analytics.js", "symbol": "recordOrderAnalyticsEvent" } + ] + }, + { + "id": "claim.containerized_with_docker", + "kind": "fact", + "text": "The Order API is containerized with Docker for local builds and deployment images.", + "truth": "code_verified", + "intent": "intended", + "about": ["component.order_api", "component.docker"], + "code_anchors": [ + { "file": "Dockerfile" } + ] + }, + { + "id": "claim.deployed_on_ecs", + "kind": "fact", + "text": "The Order API is deployed to ECS as a Fargate task.", + "truth": "code_verified", + "intent": "intended", + "about": ["component.order_api", "component.ecs"], + "code_anchors": [ + { "file": "deploy/ecs-task-definition.json" } + ] + } + ], + "flows": [ + { + "id": "flow.create_order", + "name": "Create and process an order", + "touches": ["component.order_api", "component.postgres", "component.kafka", "component.redis", "component.clickhouse"] + } + ] + } +} diff --git a/package.json b/package.json index cccc5eb..e84198d 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "smoke:openhands": "npm run build && node scripts/smoke-openhands-install.mjs", "smoke:copilot": "npm run build && node scripts/smoke-copilot-install.mjs", "smoke:opencode": "npm run build && node scripts/smoke-opencode-install.mjs", - "test": "npm run build && node scripts/check-transcript-bundle.js && node scripts/check-repo-context.js && node scripts/check-install-options.js && node scripts/check-graph-view.js && node scripts/check-graph-view-offline-browser.js && node scripts/check-proposal-validate.js && node scripts/check-bm25-tokenizer.js && node scripts/check-anchor-drift.js", + "test": "npm run build && node scripts/check-transcript-bundle.js && node scripts/check-repo-context.js && node scripts/check-install-options.js && node scripts/check-graph-view.js && node scripts/check-graph-view-offline-browser.js && node scripts/check-proposal-validate.js && node scripts/check-dependency-promotion-guidance.js && node scripts/check-dependency-promotion-model.js && node scripts/check-bm25-tokenizer.js && node scripts/check-anchor-drift.js", "test:transcript-bundle": "npm run build && node scripts/check-transcript-bundle.js", "test:repo-context": "npm run build && node scripts/check-repo-context.js", "eval:bootstrap-current": "npm run build && node dist/evals/cases/bootstrap-current-repo-at-8038fe8/run.js", diff --git a/scripts/check-dependency-promotion-guidance.js b/scripts/check-dependency-promotion-guidance.js new file mode 100644 index 0000000..f4d1d80 --- /dev/null +++ b/scripts/check-dependency-promotion-guidance.js @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +// Regression guard for issue #79: the bootstrap/refresh prompts and skills +// must instruct the agent to promote externally-owned dependencies (a +// message broker, datastore, third-party API, build/deploy platform) to +// first-class components using the primitives that already exist +// (component + `about` + `touches`), and must do so generically - no fixed +// technology list, and no schema changes required for the fix to work. +// +// This does not (and cannot) prove an LLM will comply; see +// check-dependency-promotion-retrieval.js for proof that the mechanism the +// guidance teaches actually produces a retrievable first-class component +// when followed. + +const root = new URL("..", import.meta.url); +const readDoc = (relativePath) => readFileSync(fileURLToPath(new URL(relativePath, root)), "utf8"); + +// The whole point of this fix is that it is generic. If any of these +// literal technology names creep back into the guidance, the fix has +// regressed into exactly the hardcoded-list approach that was rejected. +const forbiddenTechnologyNames = [ + "kafka", + "docker", + " ecs", + "prisma", + "clickhouse", + "socket.io", + "redis", + "postgres", + "rabbitmq", +]; + +function assertContainsAll(label, text, phrases) { + const lower = text.toLowerCase(); + for (const phrase of phrases) { + assert.ok(lower.includes(phrase.toLowerCase()), `${label}: expected promotion guidance to mention "${phrase}"`); + } +} + +function assertNoHardcodedTechnologyNames(label, text) { + const lower = text.toLowerCase(); + for (const name of forbiddenTechnologyNames) { + assert.ok( + !lower.includes(name), + `${label}: guidance must stay generic - found hardcoded technology name "${name.trim()}"`, + ); + } +} + +// The two internal memory-build prompts share the same detailed criteria +// wording (load-bearing, multi-reference, queryable-by-name, generic +// judgment call) plus the connection mechanism and the contains anti-pattern. +function assertGuidancePresent(label, text) { + assertContainsAll(label, text, [ + "promote", + "externally owned", + "operationally load-bearing", + "graph context", + "not a fixed technology list", + "about", + "touches", + "code_anchor", + "contains", + ]); + assertNoHardcodedTechnologyNames(label, text); +} + +function assertSequentialHeadings(label, text) { + const headings = [...text.matchAll(/^### (\d+)\. /gm)].map((match) => Number(match[1])); + assert.ok(headings.length > 0, `${label}: expected numbered "### N. Title" workflow headings`); + const expected = headings.map((_, index) => index + 1); + assert.deepEqual( + headings, + expected, + `${label}: workflow headings must be sequential with no gaps or duplicates, got ${JSON.stringify(headings)}`, + ); +} + +// --- scripts/memory-build/prompts/deep-bootstrap.md --- +{ + const doc = readDoc("scripts/memory-build/prompts/deep-bootstrap.md"); + assert.match(doc, /### \d+\. Promote First-Class Dependencies/, "deep-bootstrap.md must have a promotion section"); + assertGuidancePresent("deep-bootstrap.md", doc); + assertSequentialHeadings("deep-bootstrap.md", doc); +} + +// --- scripts/memory-build/prompts/layered-deep-bootstrap.md --- +{ + const doc = readDoc("scripts/memory-build/prompts/layered-deep-bootstrap.md"); + assert.match( + doc, + /### \d+\. Promote First-Class Dependencies/, + "layered-deep-bootstrap.md must have a promotion section", + ); + assertGuidancePresent("layered-deep-bootstrap.md", doc); + assertSequentialHeadings("layered-deep-bootstrap.md", doc); + // The refresh variant is specifically about promoting from *existing* + // claims that already named the dependency, without discarding them. + assert.match( + doc.toLowerCase(), + /existing claims/, + "layered-deep-bootstrap.md must instruct scanning existing claims for unpromoted dependencies", + ); +} + +// --- skills/greplica-bootstrap/SKILL.md --- +// Condensed wording (not the detailed internal-prompt phrasing): still +// requires the same load-bearing/query-by-name criteria and connection +// mechanism, minus the internal prompts' "generic judgment call" sentence. +{ + const doc = readDoc("skills/greplica-bootstrap/SKILL.md"); + assertContainsAll("skills/greplica-bootstrap/SKILL.md", doc, [ + "does not own", + "operationally load-bearing", + "query it by name", + "about", + "code_anchor", + "contains", + ]); + assertNoHardcodedTechnologyNames("skills/greplica-bootstrap/SKILL.md", doc); +} + +// --- skills/greplica-fast-session-bootstrap/SKILL.md --- +// This skill only promotes dependencies the transcript bundle itself +// already raised - it must not encourage code-derived discovery. +{ + const doc = readDoc("skills/greplica-fast-session-bootstrap/SKILL.md"); + assertContainsAll("skills/greplica-fast-session-bootstrap/SKILL.md", doc, [ + "does not own", + "about", + ]); + assert.match( + doc.toLowerCase(), + /do not go looking for dependencies in code/, + "skills/greplica-fast-session-bootstrap/SKILL.md must scope promotion to bundle-raised dependencies, not code-derived discovery", + ); + assertNoHardcodedTechnologyNames("skills/greplica-fast-session-bootstrap/SKILL.md", doc); +} + +console.log("check-dependency-promotion-guidance: ok"); diff --git a/scripts/check-dependency-promotion-model.js b/scripts/check-dependency-promotion-model.js new file mode 100644 index 0000000..998127a --- /dev/null +++ b/scripts/check-dependency-promotion-model.js @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +// Regression test for issue #79's fix mechanism: promoting a dependency to +// a component and connecting it with `about`/`touches` (not `contains`) is +// what makes it a genuine graph object instead of text trapped inside a +// claim. This proves the structural half of the fix deterministically, +// without embeddings, across a spread of real dependency shapes named in +// the original issue (message broker, cache, relational store via an ORM, +// analytics store, deploy platform) so the proof isn't an artifact of one +// technology's name. See check-dependency-promotion-retrieval.js (run +// directly with `node scripts/check-dependency-promotion-retrieval.js`) for +// the slower embedding-backed proof that this structure is actually what +// `graph context ""` surfaces. + +const root = new URL("..", import.meta.url); +const { openDatabase } = await import(new URL("dist/libs/storage/sqlite/db.js", root)); +const { SqliteRepository } = await import(new URL("dist/libs/storage/sqlite/repository.js", root)); +const { KnowledgeGraphService } = await import(new URL("dist/libs/knowledge-graph/service.js", root)); +const { normalizeProposal } = await import(new URL("dist/libs/knowledge-graph/proposal.js", root)); + +const tmp = mkdtempSync(join(tmpdir(), "greplica-dependency-promotion-model-test-")); + +const dependencies = [ + { id: "component.kafka", name: "Kafka", anchor: "k8s/kafka-statefulset.yaml", relation: "publishes ingest events to" }, + { id: "component.redis", name: "Redis", anchor: "docker-compose.yml", relation: "caches session data in" }, + { id: "component.docker", name: "Docker", anchor: "Dockerfile", relation: "is containerized with" }, + { id: "component.ecs", name: "ECS", anchor: "deploy/ecs-task-definition.json", relation: "is deployed on" }, + { id: "component.prisma", name: "Prisma", anchor: "prisma/schema.prisma", relation: "accesses its database through" }, + { id: "component.clickhouse", name: "ClickHouse", anchor: "k8s/clickhouse-deployment.yaml", relation: "ships analytics events to" }, + { id: "component.postgres", name: "Postgres", anchor: "prisma/schema.prisma", relation: "persists orders in" }, +]; + +function setupRepo(name) { + const db = openDatabase(join(tmp, `${name}.db`)); + const repository = new SqliteRepository(db); + const service = new KnowledgeGraphService(repository); + const repo = { repo_root: join(tmp, name), repo_name: name, default_branch: "main" }; + const initialized = service.initRepo(repo); + const memoryCommit = repository.createMemoryCommit({ + scope_id: initialized.working_scope_id, + title: "seed", + }); + return { db, repository, service, repo, initialized, memoryCommit }; +} + +// --- Case A: the bug as reported - dependencies mentioned only in claim +// text, with no component of their own. --- +{ + const { db, service, repository, repo, initialized, memoryCommit } = setupRepo("bug-repro"); + try { + repository.createProposalRecords(initialized.working_scope_id, memoryCommit.id, normalizeProposal({ + title: "Bug repro: dependencies only in claim text", + creates: { + components: [{ id: "component.api", name: "API Server" }], + claims: dependencies.map((dependency) => ({ + id: `claim.integrates_with_${dependency.id.replace("component.", "")}`, + kind: "fact", + text: `API Server ${dependency.relation} ${dependency.name}.`, + truth: "unknown", + intent: "unknown", + about: ["component.api"], + })), + }, + })); + + const graph = service.readGraph(repo); + assert.equal(graph.components.length, 1, "only the internal component should exist - no dedicated node for any dependency"); + assert.equal( + graph.components[0].id, + "component.api", + "no dependency should have accidentally become a component in the buggy shape", + ); + for (const dependency of dependencies) { + assert.ok( + graph.claims.some((claim) => claim.text.includes(dependency.name)), + `${dependency.name} should only be reachable as text inside a claim, reproducing the reported bug`, + ); + } + } finally { + db.close(); + } +} + +// --- Case B: the fix as taught by the promotion guidance - a component per +// dependency, each connected via `about`/`touches`, never `contains`. --- +{ + const { db, service, repository, repo, initialized, memoryCommit } = setupRepo("fix-applied"); + try { + repository.createProposalRecords(initialized.working_scope_id, memoryCommit.id, normalizeProposal({ + title: "Fix applied: dependencies promoted to components", + creates: { + components: [ + { id: "component.api", name: "API Server", code_anchor: "src/api.ts" }, + ...dependencies.map((dependency) => ({ id: dependency.id, name: dependency.name, code_anchor: dependency.anchor })), + ], + flows: dependencies.map((dependency) => ({ + id: `flow.${dependency.id.replace("component.", "")}_usage`, + name: `API Server / ${dependency.name} interaction`, + touches: ["component.api", dependency.id], + })), + claims: dependencies.map((dependency) => ({ + id: `claim.api_uses_${dependency.id.replace("component.", "")}`, + kind: "fact", + text: `API Server ${dependency.relation} ${dependency.name}.`, + truth: "unknown", + intent: "unknown", + about: ["component.api", dependency.id], + })), + }, + })); + + const graph = service.readGraph(repo); + + for (const dependency of dependencies) { + const component = graph.components.find((candidate) => candidate.id === dependency.id); + assert.ok(component, `${dependency.name} must exist as its own component`); + assert.equal(component.name, dependency.name); + assert.equal(component.code_anchor, dependency.anchor, `${dependency.name} must be anchored at its declaration point`); + + const aboutEdges = graph.edges.filter((edge) => edge.kind === "about" && edge.to_id === dependency.id); + assert.equal(aboutEdges.length, 1, `${dependency.name} must be reachable through an \`about\` edge from a claim`); + + const touchesEdges = graph.edges.filter((edge) => edge.kind === "touches" && edge.to_id === dependency.id); + assert.equal(touchesEdges.length, 1, `${dependency.name} must participate in flow \`touches\` like any other component`); + + const containsEdges = graph.edges.filter( + (edge) => edge.kind === "contains" && (edge.to_id === dependency.id || edge.from_id === dependency.id), + ); + assert.equal( + containsEdges.length, + 0, + `${dependency.name} must not be nested under an internal component with \`contains\` - the repo does not own it`, + ); + } + } finally { + db.close(); + } +} + +console.log(`check-dependency-promotion-model: ok (${dependencies.length} dependency shapes verified: ${dependencies.map((d) => d.name).join(", ")})`); diff --git a/scripts/check-dependency-promotion-retrieval.js b/scripts/check-dependency-promotion-retrieval.js new file mode 100644 index 0000000..360dea7 --- /dev/null +++ b/scripts/check-dependency-promotion-retrieval.js @@ -0,0 +1,141 @@ +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +// Slow, embedding-backed proof for issue #79's fix: reproduces the exact +// symptom from the issue (`greplica graph context ""` returns claims +// instead of a dedicated node) and confirms it is fixed once the promotion +// guidance's recipe is followed. Covers a spread of real dependency shapes +// from the issue (message broker, cache, container platform, orchestrator, +// ORM, analytics store, relational store) so the proof isn't an artifact of +// one technology's name/word-shape. Uses the real local embedding model, +// which downloads/loads on first use (~10s+ for the first call, fast after +// that within the same process), so this is intentionally NOT part of +// `npm test`. Run explicitly with: +// +// node scripts/check-dependency-promotion-retrieval.js +// +// See check-dependency-promotion-model.js for the fast, embedding-free +// structural regression test that runs on every `npm test`. + +const root = new URL("..", import.meta.url); +const { openDatabase } = await import(new URL("dist/libs/storage/sqlite/db.js", root)); +const { SqliteRepository } = await import(new URL("dist/libs/storage/sqlite/repository.js", root)); +const { KnowledgeGraphService } = await import(new URL("dist/libs/knowledge-graph/service.js", root)); +const { normalizeProposal } = await import(new URL("dist/libs/knowledge-graph/proposal.js", root)); + +const tmp = mkdtempSync(join(tmpdir(), "greplica-dependency-promotion-retrieval-test-")); +process.env.GREPLICA_HOME = tmp; + +const dependencies = [ + { id: "component.kafka", name: "Kafka", anchor: "k8s/kafka-statefulset.yaml", relation: "publishes ingest events to" }, + { id: "component.redis", name: "Redis", anchor: "docker-compose.yml", relation: "caches session data in" }, + { id: "component.docker", name: "Docker", anchor: "Dockerfile", relation: "is containerized with" }, + { id: "component.ecs", name: "ECS", anchor: "deploy/ecs-task-definition.json", relation: "is deployed on" }, + { id: "component.prisma", name: "Prisma", anchor: "prisma/schema.prisma", relation: "accesses its database through" }, + { id: "component.clickhouse", name: "ClickHouse", anchor: "k8s/clickhouse-deployment.yaml", relation: "ships analytics events to" }, + { id: "component.postgres", name: "Postgres", anchor: "prisma/schema.prisma", relation: "persists orders in" }, +]; + +function setupRepo(name) { + const db = openDatabase(join(tmp, `${name}.db`)); + const repository = new SqliteRepository(db); + const service = new KnowledgeGraphService(repository); + const repo = { repo_root: join(tmp, name), repo_name: name, default_branch: "main" }; + const initialized = service.initRepo(repo); + const memoryCommit = repository.createMemoryCommit({ + scope_id: initialized.working_scope_id, + title: "seed", + }); + return { db, repository, service, repo, initialized, memoryCommit }; +} + +console.log("Loading local embedding model (first call is slow)..."); + +// --- Before: every dependency only mentioned in claim text - reproduces +// the issue for each technology. --- +const bugReproResults = {}; +{ + const { db, repository, service, repo, initialized, memoryCommit } = setupRepo("bug-repro"); + try { + repository.createProposalRecords(initialized.working_scope_id, memoryCommit.id, normalizeProposal({ + title: "Bug repro: dependencies only in claim text", + creates: { + components: [{ id: "component.api", name: "API Server" }], + claims: dependencies.map((dependency) => ({ + id: `claim.integrates_with_${dependency.id.replace("component.", "")}`, + kind: "fact", + text: `API Server ${dependency.relation} ${dependency.name}.`, + truth: "unknown", + intent: "unknown", + about: ["component.api"], + })), + }, + })); + + for (const dependency of dependencies) { + const result = await service.contextGraph(repo, dependency.name); + bugReproResults[dependency.name] = result.components.map((component) => component.object.name); + } + } finally { + db.close(); + } +} + +for (const dependency of dependencies) { + const components = bugReproResults[dependency.name]; + assert.ok( + !components.includes(dependency.name), + `expected no dedicated "${dependency.name}" component node when it is only mentioned in claim text (issue #79's reported bug), got: ${JSON.stringify(components)}`, + ); + console.log(`Before promotion: graph context "${dependency.name}" -> components: ${JSON.stringify(components)} (no dedicated node - bug reproduced)`); +} + +// --- After: every dependency promoted to a component per the guidance. --- +const fixedResults = {}; +{ + const { db, repository, service, repo, initialized, memoryCommit } = setupRepo("fix-applied"); + try { + repository.createProposalRecords(initialized.working_scope_id, memoryCommit.id, normalizeProposal({ + title: "Fix applied: dependencies promoted to components", + creates: { + components: [ + { id: "component.api", name: "API Server", code_anchor: "src/api.ts" }, + ...dependencies.map((dependency) => ({ id: dependency.id, name: dependency.name, code_anchor: dependency.anchor })), + ], + flows: dependencies.map((dependency) => ({ + id: `flow.${dependency.id.replace("component.", "")}_usage`, + name: `API Server / ${dependency.name} interaction`, + touches: ["component.api", dependency.id], + })), + claims: dependencies.map((dependency) => ({ + id: `claim.api_uses_${dependency.id.replace("component.", "")}`, + kind: "fact", + text: `API Server ${dependency.relation} ${dependency.name}.`, + truth: "unknown", + intent: "unknown", + about: ["component.api", dependency.id], + })), + }, + })); + + for (const dependency of dependencies) { + const result = await service.contextGraph(repo, dependency.name); + fixedResults[dependency.name] = result.components.map((component) => component.object.name); + } + } finally { + db.close(); + } +} + +for (const dependency of dependencies) { + const components = fixedResults[dependency.name]; + assert.ok( + components.includes(dependency.name), + `expected "${dependency.name}" to be returned as a dedicated ranked component after promotion, got: ${JSON.stringify(components)}`, + ); + console.log(`After promotion: graph context "${dependency.name}" -> components: ${JSON.stringify(components)} (fixed)`); +} + +console.log(`check-dependency-promotion-retrieval: ok (${dependencies.length} dependency shapes verified: ${dependencies.map((d) => d.name).join(", ")})`); diff --git a/scripts/memory-build/prompts/deep-bootstrap.md b/scripts/memory-build/prompts/deep-bootstrap.md index b488905..e259f6d 100644 --- a/scripts/memory-build/prompts/deep-bootstrap.md +++ b/scripts/memory-build/prompts/deep-bootstrap.md @@ -39,7 +39,17 @@ Treat cross-cutting utility layers as ownership boundaries when they encode dura For large repos, process one module group at a time. When your environment supports parallel workers and the user asked for parallelism, split independent module groups across workers, then merge proposals sequentially. -### 3. Inspect Deeply +### 3. Promote First-Class Dependencies + +Some things the repo depends on deserve their own component even though the repo does not own their code — a message broker, a datastore, a third-party API client, a build/deploy platform. Do not leave these buried inside claim text as an aside. Promote a dependency to a component when any of these hold: it is externally owned but operationally load-bearing (the system misbehaves without it); multiple claims or flows would reference it by name; a future agent would plausibly ask `greplica graph context ""`. Do not promote incidental utilities or dev-only tooling that no claim needs to reference. This is a generic judgment call, not a fixed technology list — apply the same criteria whether the dependency is infrastructure, a datastore, a library, or an external service. + +When you promote one: + +- Set `code_anchor` to the file where the repo declares or configures it: a deploy/compose file, schema file, client-initialization module, or dependency manifest. +- Connect it the same way as any component: write claims `about` both the dependency and the internal components that use it (with code anchors at the usage sites), and include it in flow `touches` where it participates in a runtime behavior. +- Never nest an external dependency under an internal component with `component.contains` — `contains` means ownership, and the repo does not own it. + +### 4. Inspect Deeply For each module group: @@ -54,7 +64,7 @@ Deep bootstrap should produce memory that lets a future agent jump near the righ Do not skip behavior-heavy helpers just because they are private or small. If a helper owns visible behavior such as truncation, column layout, selector parsing, retry logic, pagination, API query shaping, or config fallback, write a precise claim anchored to that helper or its smallest stable caller/test pair. -### 4. Write Proposals In Batches +### 5. Write Proposals In Batches Write one focused proposal per module group or cross-cutting flow. Validate and apply each proposal before moving to the next group so later groups can reuse existing components and supersede stale claims. @@ -129,7 +139,7 @@ Use compact relationship fields where possible: Do not create session sources for code inspection during bootstrap. Code-grounded bootstrap claims should usually be `code_verified`, source-free, and include `code_anchors`. Do not create broad code claims merely to cover a module. If a module has list, view, parse, render, truncate, width calculation, format, validate, and API-query behaviors, create separate claims for those behaviors with separate anchors. -### 5. Add Cross-Cutting Flows +### 6. Add Cross-Cutting Flows After module passes, add cross-cutting flows that future agents would search for: @@ -143,7 +153,7 @@ After module passes, add cross-cutting flows that future agents would search for Do not add a flow unless it touches at least two meaningful components. -### 6. Validate And Apply +### 7. Validate And Apply For each proposal: diff --git a/scripts/memory-build/prompts/layered-deep-bootstrap.md b/scripts/memory-build/prompts/layered-deep-bootstrap.md index db3cde9..4bf97a0 100644 --- a/scripts/memory-build/prompts/layered-deep-bootstrap.md +++ b/scripts/memory-build/prompts/layered-deep-bootstrap.md @@ -79,7 +79,18 @@ Create a short plan before inspecting deeply: - Treat cross-cutting utility layers as possible refresh targets when they own durable behavior: table/output rendering, terminal width and truncation, formatter helpers, parser/selector helpers, query builders, pagination, config bootstrap, and test harness utilities. - Do not use git history to infer what changed. If no changed-file list is supplied, use existing memory queries plus the current source tree to decide which areas need a partial rewalk. -### 3. Inspect Current Code +### 3. Promote First-Class Dependencies + +While building the refresh plan, scan existing claims for named systems that the repo depends on but that have no component of their own — a message broker, a datastore, a third-party API client, a build/deploy platform mentioned only in claim text. Promote one to a component when it is externally owned but operationally load-bearing, referenced by multiple claims or flows, or something a future agent would plausibly query by name (`greplica graph context ""`). This is a generic judgment call, not a fixed technology list. + +When you promote one: + +- Create the component with a `code_anchor` at its declaration point (deploy/compose file, schema file, client-init module, dependency manifest). +- Add `about` edges from the *existing* claims that already mention it — these edges may reference existing subjects directly; no `supersedes` is needed for the edge itself. Only supersede a claim if its text is now wrong or needs to change, not merely because it now also links to the new component. +- Include it in flow `touches` where it participates in a refreshed runtime behavior. +- Never nest it under an internal component with `contains` — that means ownership, and the repo does not own it. + +### 4. Inspect Current Code For each planned module group: @@ -91,7 +102,7 @@ For each planned module group: Do not write memory for every helper. Store facts that improve future navigation, correctness, or task planning. If a helper owns visible behavior such as truncation, column layout, selector parsing, API query shaping, pagination, config fallback, or error formatting, it is a valid memory target even when it is private. -### 4. Write Layered Proposals +### 5. Write Layered Proposals Write one proposal per changed module group or cross-cutting flow. Apply each proposal before moving to the next group so later proposals can reuse IDs already present in the parent graph or created earlier in this layered refresh. @@ -163,7 +174,7 @@ For claim `code_anchors`: - Use a private/helper symbol when that helper is the smallest accurate anchor for the claim and its behavior is externally visible or cross-cutting. - When refreshing a broad old claim, create narrower superseding claims instead of copying the old breadth with many anchors. -### 5. Validate, Apply, And Probe +### 6. Validate, Apply, And Probe For each proposal: @@ -177,7 +188,7 @@ After all proposals, run 3-5 retrieval probes matching likely future questions, Run `greplica graph audit anchors` when available. Treat missing anchors, missing files, missing symbols, ambiguous symbols, or unsupported languages on active `code_verified` claims as failures to fix before handing off to GitHub packet ingestion. -### 6. Write A Build Report +### 7. Write A Build Report When an output directory is available, write a small JSON or Markdown report with: diff --git a/skills/greplica-bootstrap/SKILL.md b/skills/greplica-bootstrap/SKILL.md index 8dc64db..19754c2 100644 --- a/skills/greplica-bootstrap/SKILL.md +++ b/skills/greplica-bootstrap/SKILL.md @@ -53,6 +53,8 @@ Use `component.repository` for whole-repo facts: Create narrower components or flows when they help navigation for public boundaries. Do not bury config, storage, validation, retrieval, protocol, or persistence boundaries inside a generic repo component when those boundaries have stable files. Flow `touches` should include the public boundary components that the flow claim depends on. +Promote a dependency the repo does not own — a datastore, message broker, third-party API, or build/deploy platform — to its own component when it is operationally load-bearing or a future agent would plausibly query it by name. Give it a `code_anchor` at its declaration point (compose/deploy file, schema, client-init module, manifest). Link it with claims `about` both it and the components that use it; never `contains`-nest it under an internal component. + ## Evidence And Anchors - Use `source_verified` for doc-derived claims and anchor them to the relevant doc/config file. diff --git a/skills/greplica-fast-session-bootstrap/SKILL.md b/skills/greplica-fast-session-bootstrap/SKILL.md index bc53c25..2bb0217 100644 --- a/skills/greplica-fast-session-bootstrap/SKILL.md +++ b/skills/greplica-fast-session-bootstrap/SKILL.md @@ -99,6 +99,8 @@ Reject mechanism trivia. Do not store internal helper names, command registry in For multi-session bundles, a complete proposal may need many claims. Do not stop early because the proposal feels long. If the claim count grows, review for code-detail sprawl, existing-memory leakage, duplicate implementation/decision pairs, and claims not directly raised by the bundle; keep the durable bundle-supported memories. +If the bundle repeatedly names a system the repo depends on but does not own — a datastore, message broker, third-party API, or build/deploy platform — and that name would otherwise only live inside claim text, give it its own component instead, anchored at its declaration point if the bundle names one, and linked via `about` edges to the components/claims that reference it. Only do this when the bundle itself raised the dependency as durable; do not go looking for dependencies in code that the bundle never mentioned. + Allowed values: - claim `kind`: `fact`, `requirement`, `decision`, `task`, `question`, `risk`