fix: calibrate agent precision benchmark gate

.github/workflows/external-benchmark.yml

@@ -0,0 +1,44 @@
+name: External repository precision benchmark
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '17 3 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Check out scanner
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Prepare pinned upstream repositories
+        shell: bash
+        run: |
+          set -euo pipefail
+          root="$RUNNER_TEMP/agent-control-plane-benchmark"
+          mkdir -p "$root"
+          jq -r '.cases[] | [.directory, .repository, .ref] | @tsv' testdata/benchmark/manifest.json |
+            while IFS=$'\t' read -r directory repository ref; do
+              mkdir -p "$root/$directory"
+              git -C "$root/$directory" init -q
+              git -C "$root/$directory" remote add origin "$repository"
+              git -C "$root/$directory" -c protocol.version=2 fetch --filter=blob:none --depth 1 origin "$ref"
+              git -C "$root/$directory" checkout --detach FETCH_HEAD
+              actual=$(git -C "$root/$directory" rev-parse HEAD)
+              test "$actual" = "$ref"
+            done
+          echo "BENCHMARK_ROOT=$root" >> "$GITHUB_ENV"
+
+      - name: Run precision benchmark
+        run: go run ./cmd/agentctl-benchmark --root "$BENCHMARK_ROOT"

README.md

@@ -65,6 +65,20 @@ The runtime fuzz targets are standalone safe parser seams. They classify in-memo
 
 `fuzzpayloadcheck` runs in strict mode by default. The `-allow-adversarial` option is only for the committed negative-case corpus: it permits expected malformed, duplicate-key, unsafe-URL, control-character, and inert command fixtures while keeping credential-like values, invalid UTF-8, resource-limit violations, unsafe paths, and filesystem errors blocking.
 
+Run the pinned external-repository precision benchmark locally after cloning the
+repositories listed in `testdata/benchmark/manifest.json`:
+
+```bash
+go run ./cmd/agentctl-benchmark --root /path/to/benchmark-root
+```
+
+The benchmark is a regression gate for candidate count, non-policy ACP-005
+findings and code-like metadata values. It reports only aggregate counts; it
+does not print source values. The weekly and manually triggered workflow
+`.github/workflows/external-benchmark.yml` checks out the exact manifest refs
+before running the gate. Thresholds are reviewed baselines, not a claim of
+ground-truth precision or recall.
+
 ## What it does
 
 The CLI scans an approved local directory in read-only mode and produces text or JSON inventory containing:
@@ -78,6 +92,11 @@ The CLI scans an approved local directory in read-only mode and produces text or
 - SARIF 2.1.0 output for GitHub Code Scanning and other security tooling;
 - safe `.mcp.json` and `server.json` metadata such as server name, transport and auth method.
 
+ACP-005 is emitted only for MCP policy sources such as `.mcp.json`,
+`server.json`, and explicitly named MCP/server/manifest JSON or YAML files.
+Source-code references to an MCP server are inventory evidence but do not create
+a policy finding by themselves.
+
 The default policy excludes common non-production paths such as tests, examples, samples, tutorials, fixtures, documentation and schemas. Agent definitions under tooling directories such as `.claude` and `.github`, Markdown agent files, and framework library layouts remain scannable when they contain strong agent signals. Use `.agentctl/config.yaml` to add workspace-specific exclusions and approved providers or MCP servers. `agentctl init` never overwrites an existing policy.
 
 ## Risk rules

cmd/agentctl-benchmark/main.go

@@ -0,0 +1,172 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/HlinorAI/agent-control-plane/internal/scan"
+)
+
+type manifest struct {
+	SchemaVersion string  `json:"schema_version"`
+	Cases         []bench `json:"cases"`
+}
+
+type bench struct {
+	ID         string `json:"id"`
+	Directory  string `json:"directory"`
+	Repository string `json:"repository"`
+	Ref        string `json:"ref"`
+	MinAgents  int    `json:"min_agents"`
+	MaxAgents  int    `json:"max_agents"`
+}
+
+type result struct {
+	ID                 string   `json:"id"`
+	Repository         string   `json:"repository"`
+	Ref                string   `json:"ref"`
+	Agents             int      `json:"agents"`
+	Findings           int      `json:"findings"`
+	SourceACP005       int      `json:"source_acp005"`
+	CodeExpressionHits int      `json:"code_expression_hits,omitempty"`
+	Pass               bool     `json:"pass"`
+	Errors             []string `json:"errors,omitempty"`
+}
+
+var benchmarkDottedIdentifierPattern = regexp.MustCompile(`^[a-z_][a-z0-9_]*(?:\.[a-z_][a-z0-9_]*)+$`)
+var benchmarkBracketExpressionPattern = regexp.MustCompile(`(?i)\b(required|schema)\s*\[`)
+
+func main() {
+	if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil {
+		fmt.Fprintln(os.Stderr, "agentctl-benchmark:", err)
+		os.Exit(1)
+	}
+}
+
+func run(args []string, stdout, stderr io.Writer) error {
+	fs := flag.NewFlagSet("agentctl-benchmark", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	root := fs.String("root", "", "directory containing the pinned benchmark repositories")
+	manifestPath := fs.String("manifest", "testdata/benchmark/manifest.json", "benchmark manifest")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *root == "" {
+		return errors.New("root is required")
+	}
+	data, err := os.ReadFile(*manifestPath)
+	if err != nil {
+		return fmt.Errorf("read manifest: %w", err)
+	}
+	var input manifest
+	if err := json.Unmarshal(data, &input); err != nil {
+		return fmt.Errorf("decode manifest: %w", err)
+	}
+	if input.SchemaVersion != "1" || len(input.Cases) == 0 {
+		return errors.New("unsupported or empty benchmark manifest")
+	}
+
+	results := make([]result, 0, len(input.Cases))
+	allPassed := true
+	for _, item := range input.Cases {
+		current := result{ID: item.ID, Repository: item.Repository, Ref: item.Ref}
+		path := filepath.Join(*root, item.Directory)
+		if info, err := os.Stat(path); err != nil || !info.IsDir() {
+			current.Errors = append(current.Errors, "benchmark repository directory is missing")
+			current.Pass = false
+			allPassed = false
+			results = append(results, current)
+			continue
+		}
+		report, err := scan.Run(path, scan.Options{})
+		if err != nil {
+			current.Errors = append(current.Errors, err.Error())
+			current.Pass = false
+			allPassed = false
+			results = append(results, current)
+			continue
+		}
+		current.Agents = len(report.Agents)
+		current.Findings = len(report.Findings)
+		for _, finding := range report.Findings {
+			if finding.RuleID == "ACP-005" && len(finding.Evidence) > 0 && !isMCPPolicyPath(finding.Evidence[0].Path) {
+				current.SourceACP005++
+			}
+		}
+		for _, agent := range report.Agents {
+			if codeExpression(agent.Name) {
+				current.CodeExpressionHits++
+			}
+			for _, model := range agent.Models {
+				if codeExpression(model) {
+					current.CodeExpressionHits++
+				}
+			}
+		}
+		if current.Agents < item.MinAgents {
+			current.Errors = append(current.Errors, fmt.Sprintf("agent count %d is below minimum %d", current.Agents, item.MinAgents))
+		}
+		if item.MaxAgents > 0 && current.Agents > item.MaxAgents {
+			current.Errors = append(current.Errors, fmt.Sprintf("agent count %d exceeds maximum %d", current.Agents, item.MaxAgents))
+		}
+		if current.SourceACP005 > 0 {
+			current.Errors = append(current.Errors, "ACP-005 was emitted for a non-policy source")
+		}
+		if current.CodeExpressionHits > 0 {
+			current.Errors = append(current.Errors, "code expressions were emitted as agent metadata")
+		}
+		current.Pass = len(current.Errors) == 0
+		if !current.Pass {
+			allPassed = false
+		}
+		results = append(results, current)
+	}
+	output := struct {
+		SchemaVersion string   `json:"schema_version"`
+		Passed        bool     `json:"passed"`
+		Results       []result `json:"results"`
+	}{SchemaVersion: "1", Passed: allPassed, Results: results}
+	encoder := json.NewEncoder(stdout)
+	encoder.SetIndent("", "  ")
+	if err := encoder.Encode(output); err != nil {
+		return fmt.Errorf("encode benchmark report: %w", err)
+	}
+	if !allPassed {
+		return errors.New("benchmark thresholds failed")
+	}
+	return nil
+}
+
+func isMCPPolicyPath(path string) bool {
+	base := strings.ToLower(filepath.Base(path))
+	if base == ".mcp.json" || base == "server.json" {
+		return true
+	}
+	extension := strings.ToLower(filepath.Ext(base))
+	return (extension == ".json" || extension == ".yaml" || extension == ".yml") && (strings.Contains(base, "mcp") || strings.Contains(base, "server") || strings.Contains(base, "manifest"))
+}
+
+func codeExpression(value string) bool {
+	lower := strings.ToLower(value)
+	for _, marker := range []string{"(", ")", "[", "]", "|", "select_choice", "removeprefix"} {
+		if strings.Contains(lower, marker) {
+			return true
+		}
+	}
+	if benchmarkBracketExpressionPattern.MatchString(value) || benchmarkDottedIdentifierPattern.MatchString(lower) {
+		return true
+	}
+	switch lower {
+	case "str", "model", "name", "bool", "int", "model_name":
+		return true
+	default:
+		return false
+	}
+}

cmd/agentctl-benchmark/main_test.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestRunBenchmarkPassesPinnedFixture(t *testing.T) {
+	root := t.TempDir()
+	repo := filepath.Join(root, "fixture")
+	if err := os.MkdirAll(filepath.Join(repo, "agents"), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(repo, "agents", "demo_agent.py"), []byte("name: demo-agent\nmodel: openai\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	manifestPath := filepath.Join(t.TempDir(), "manifest.json")
+	manifest := `{"schema_version":"1","cases":[{"id":"fixture","directory":"fixture","repository":"local","ref":"test","min_agents":1,"max_agents":1}]}`
+	if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	var output bytes.Buffer
+	if err := run([]string{"--root", root, "--manifest", manifestPath}, &output, &bytes.Buffer{}); err != nil {
+		t.Fatal(err)
+	}
+	var report struct {
+		Passed bool `json:"passed"`
+	}
+	if err := json.Unmarshal(output.Bytes(), &report); err != nil {
+		t.Fatal(err)
+	}
+	if !report.Passed {
+		t.Fatalf("benchmark did not pass: %s", output.String())
+	}
+}
+
+func TestRunBenchmarkRejectsMissingRepository(t *testing.T) {
+	manifestPath := filepath.Join(t.TempDir(), "manifest.json")
+	manifest := `{"schema_version":"1","cases":[{"id":"missing","directory":"missing","repository":"local","ref":"test","min_agents":1,"max_agents":1}]}`
+	if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	var output bytes.Buffer
+	if err := run([]string{"--root", t.TempDir(), "--manifest", manifestPath}, &output, &bytes.Buffer{}); err == nil {
+		t.Fatal("expected missing repository failure")
+	}
+	if !bytes.Contains(output.Bytes(), []byte(`"passed": false`)) {
+		t.Fatalf("benchmark report did not record failure: %s", output.String())
+	}
+}
+
+func TestCodeExpressionClassification(t *testing.T) {
+	for _, value := range []string{"self.model", "args.model", "str", "model_name", "Required[str]", "schema[\"name\"]"} {
+		if !codeExpression(value) {
+			t.Errorf("expected code expression: %q", value)
+		}
+	}
+	for _, value := range []string{"schema-sync-agent", "openai/gpt-4o", "claude-3-5-sonnet"} {
+		if codeExpression(value) {
+			t.Errorf("expected metadata value: %q", value)
+		}
+	}
+}