feat(monitoring): add production health checks with auto-issue filing

.claude/settings.json

@@ -40,7 +40,8 @@
       "mcp__hermit__run_requirements_sweep",
       "mcp__hermit__now",
       "mcp__hermit__get_loop_state",
-      "mcp__hermit__update_loop_state"
+      "mcp__hermit__update_loop_state",
+      "mcp__hermit__run_health_checks"
     ]
   }
 }

CLAUDE.md

@@ -43,15 +43,20 @@ Because this cycle now runs with the full tool access of the invoking context (n
 7. Run the requirements reconcile sweep roughly once an hour using `run_requirements_sweep`, tracking a separate "last requirements-sweep time" across passes the same way step 5 tracks its own PR-comment-check "since" timestamp (store the current time before calling):
    - Only call `run_requirements_sweep` when at least 3600 seconds have elapsed since the last recorded sweep time; otherwise skip this step for the current pass (do not call the tool early — it runs the configured `test_command` for every requirement and shouldn't be wasted on sub-hourly passes)
    - Update the stored last-sweep timestamp to now after calling
-8. If there are no Issues, end this pass — the recurring cron trigger starts the next pass
-9. For each Issue (up to 4 at a time):
-   a. Mark as in-progress with `assign_issue` (assignee: your own username)
-   b. Create a worktree with `create_worktree` (base_branch: default branch)
-10. **Spawn all Engineers for the Issues prepared in step 9 in parallel at once using the Agent tool** (`run_in_background: true`, so this inline pass can wait for them without blocking on each one sequentially)
+8. Run the production health-check sweep roughly every 5 minutes using `run_health_checks` (Issue #190), tracking a separate "last health-check time" across passes (`health_checks_since`) the same way step 7 tracks its own requirements-sweep timestamp:
+   - Only call `run_health_checks` when at least 300 seconds have elapsed since the last recorded health-check time; otherwise skip this step for the current pass
+   - `run_health_checks` itself runs each configured `[[health_checks]]` command, opens a deduped `production-incident`-labeled Issue for any newly-failing check, and posts a one-time "recovered at ..." comment on the corresponding Issue for any check that has returned to passing (never auto-closing it) — no further action is needed here beyond calling the tool and updating the timestamp
+   - Update the stored last-health-check timestamp to now after calling
+   - If no `[[health_checks]]` are configured for this project, the tool is a no-op; keep calling it on cadence anyway rather than special-casing it away, so a project that later adds health checks starts getting swept without a CLAUDE.md change
+9. If there are no Issues, end this pass — the recurring cron trigger starts the next pass
+10. For each Issue (up to 4 at a time):
+    a. Mark as in-progress with `assign_issue` (assignee: your own username)
+    b. Create a worktree with `create_worktree` (base_branch: default branch)
+11. **Spawn all Engineers for the Issues prepared in step 10 in parallel at once using the Agent tool** (`run_in_background: true`, so this inline pass can wait for them without blocking on each one sequentially)
     - Information to pass to each Engineer: Issue number, title, body, `worktree_path` and `branch` returned by `create_worktree`
     - If the parallel count exceeds 4, process the first 4 and defer the rest to the next pass
-11. Wait for all Engineers to complete
-12. Run `check_ci_status` on the PR for each Issue (including PRs from Engineers spawned on an earlier pass that are still awaiting evaluation — use `list_prs` to find open HERMIT PRs)
+12. Wait for all Engineers to complete
+13. Run `check_ci_status` on the PR for each Issue (including PRs from Engineers spawned on an earlier pass that are still awaiting evaluation — use `list_prs` to find open HERMIT PRs)
     - If CI is failing: the tool automatically posts an investigation comment listing the failing checks; skip merging and wait for fixes
     - If CI is passing: run `evaluate_risk`
       - LOW / MEDIUM: run `merge_pr` with `worktree_path` and `branch` so the worktree is cleaned up automatically after a successful merge
@@ -61,7 +66,7 @@ Because this cycle now runs with the full tool access of the invoking context (n
         - Check whether the branch is stale relative to the base branch in a way that could hide semantic conflicts, not just textual `mergeable` conflicts
         - Post your findings as a separate PR comment via `add_issue_comment`: a short summary of what changed, anything concerning, and an explicit recommendation (e.g. "looks safe to merge pending approval" vs. "found X, should be fixed first")
         - Skip merging and wait for a human decision
-13. End the pass with a short report of what was done, and return control to the prompt — do **not** loop back to step 1 yourself; the recurring cron job fires the next pass
+14. End the pass with a short report of what was done, and return control to the prompt — do **not** loop back to step 1 yourself; the recurring cron job fires the next pass
 
 ---
 

cmd/hermit/main.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/ytnobody/hermit/internal/git"
 	gh "github.com/ytnobody/hermit/internal/github"
+	"github.com/ytnobody/hermit/internal/healthcheck"
 	"github.com/ytnobody/hermit/internal/mcp"
 	"github.com/ytnobody/hermit/internal/notification"
 	"github.com/ytnobody/hermit/internal/permissions"
@@ -79,6 +80,17 @@ func (r RiskConfig) toRiskConfig() risk.Config {
 	}
 }
 
+// HealthCheckEntry is a single [[health_checks]] entry: a named production
+// health check HERMIT runs periodically via the run_health_checks MCP tool
+// (Issue #190). Only Type "command" (the default when empty) is supported
+// today; entries with any other Type are skipped by healthcheck.RunChecks
+// as a forward-compatible no-op.
+type HealthCheckEntry struct {
+	Name    string `toml:"name"`
+	Command string `toml:"command"`
+	Type    string `toml:"type"`
+}
+
 type Config struct {
 	GitHub struct {
 		Owner              string `toml:"owner"`
@@ -164,7 +176,14 @@ type Config struct {
 		// needing to opt in.
 		Paths []string `toml:"paths"`
 	} `toml:"requirements"`
-	Security struct {
+	// HealthChecks is the [[health_checks]] array of tables (Issue #190):
+	// production health-check commands run periodically by the
+	// run_health_checks MCP tool. An empty/omitted list means the project
+	// has not opted into production health checks, in which case
+	// run_health_checks is a no-op — existing projects see no change in
+	// behavior.
+	HealthChecks []HealthCheckEntry `toml:"health_checks"`
+	Security     struct {
 		// TrustedAuthorAssociations is the allowlist of GitHub
 		// "author_association" values (e.g. "OWNER", "MEMBER",
 		// "COLLABORATOR", "CONTRIBUTOR", "FIRST_TIME_CONTRIBUTOR", "NONE")
@@ -530,6 +549,8 @@ func cmdServe() {
 		TestCommand: cfg.Requirements.TestCommand,
 	}
 
+	healthChecks := resolveHealthChecks(cfg)
+
 	model := mcp.ModelConfig{
 		Superintendent:       cfg.Model.Superintendent,
 		Engineer:             cfg.Model.Engineer,
@@ -539,7 +560,7 @@ func cmdServe() {
 		AnalystEffort:        resolveAnalystEffort(cfg),
 	}
 
-	if err := mcp.Serve(client, cfg.GitHub.RateLimitThreshold, rootDir, prefix, cfg.Agent.LoopInterval, cfg.Notification.WebhookURL, cfg.Notification.Type, repos, cfg.Agent.TriggerComment, readinessCfg, defaultRiskCfg, repoRiskCfgs, model, requirementsCfg, cfg.Agent.MaxEngineers); err != nil {
+	if err := mcp.Serve(client, cfg.GitHub.RateLimitThreshold, rootDir, prefix, cfg.Agent.LoopInterval, cfg.Notification.WebhookURL, cfg.Notification.Type, repos, cfg.Agent.TriggerComment, readinessCfg, defaultRiskCfg, repoRiskCfgs, model, requirementsCfg, cfg.Agent.MaxEngineers, healthChecks); err != nil {
 		fatal(err.Error())
 	}
 }
@@ -705,6 +726,23 @@ func resolveRequirementsDoc(cfg Config) string {
 	return defaultRequirementsDoc
 }
 
+// resolveHealthChecks converts cfg's [[health_checks]] entries (TOML shape)
+// into []healthcheck.Check for the run_health_checks MCP tool (Issue #190).
+// Entries missing a name or command are skipped rather than passed through
+// as a broken check, since an empty Command would otherwise run `sh -c ""`
+// (a vacuous success) on every sweep.
+func resolveHealthChecks(cfg Config) []healthcheck.Check {
+	var checks []healthcheck.Check
+	for _, e := range cfg.HealthChecks {
+		if e.Name == "" || e.Command == "" {
+			log.Printf("health_checks: skipping entry with missing name or command: %+v", e)
+			continue
+		}
+		checks = append(checks, healthcheck.Check{Name: e.Name, Command: e.Command, Type: e.Type})
+	}
+	return checks
+}
+
 // runRequirementsSweep runs the requirements reconcile sweep (Issue #106) at
 // `hermit serve` startup: it parses the requirements document for "## REQ-xxx:"
 // blocks, runs each requirement's test via the configured test_command, and

cmd/hermit/serve_test.go

@@ -103,8 +103,8 @@ language      = "ja"
 				continue
 			}
 			tools, _ := msg.Result["tools"].([]any)
-			if len(tools) != 20 {
-				t.Errorf("expected 20 tools, got %d", len(tools))
+			if len(tools) != 21 {
+				t.Errorf("expected 21 tools, got %d", len(tools))
 			}
 			return
 		case <-deadline:
@@ -207,8 +207,8 @@ language      = "ja"
 				continue
 			}
 			tools, _ := msg.Result["tools"].([]any)
-			if len(tools) != 20 {
-				t.Errorf("expected 20 tools, got %d", len(tools))
+			if len(tools) != 21 {
+				t.Errorf("expected 21 tools, got %d", len(tools))
 			}
 			return
 		case <-deadline:
@@ -314,8 +314,8 @@ language      = "ja"
 				continue
 			}
 			tools, _ := msg.Result["tools"].([]any)
-			if len(tools) != 20 {
-				t.Errorf("expected 20 tools, got %d", len(tools))
+			if len(tools) != 21 {
+				t.Errorf("expected 21 tools, got %d", len(tools))
 			}
 			return
 		case <-deadline:

cmd/hermit/templates/CLAUDE.md.tmpl

@@ -24,27 +24,32 @@
    - Only call `run_requirements_sweep` when at least 3600 seconds have elapsed since the last recorded sweep time; otherwise skip this step for the current pass
    - Update the stored last-sweep timestamp to now after calling
    - Note this is distinct from step 5: step 5 is about a human dictating new/changed requirements via conversation; this step is about detecting when already-declared requirements have silently regressed or gone unimplemented in the code, by re-running the reconcile sweep against the current requirements document
-7. If there are no non-requirements Issues, end this pass — the recurring cron trigger starts the next pass
-8. For each remaining Issue (up to {{ .MaxEngineers }} at a time), perform a granularity check:
+7. Run the production health-check sweep roughly every 5 minutes using `run_health_checks` (Issue #190), tracking a "last health-check time" across passes (`health_checks_since`) the same way step 6 tracks its own requirements-sweep timestamp:
+   - Only call `run_health_checks` when at least 300 seconds have elapsed since the last recorded health-check time; otherwise skip this step for the current pass
+   - `run_health_checks` itself runs each configured `[[health_checks]]` command, opens a deduped `production-incident`-labeled Issue for any newly-failing check, and posts a one-time "recovered at ..." comment on the corresponding Issue for any check that has returned to passing (never auto-closing it) — no further action is needed here beyond calling the tool and updating the timestamp
+   - Update the stored last-health-check timestamp to now after calling
+   - If no `[[health_checks]]` are configured for this project, the tool is a no-op; keep calling it on cadence anyway rather than special-casing it away
+8. If there are no non-requirements Issues, end this pass — the recurring cron trigger starts the next pass
+9. For each remaining Issue (up to {{ .MaxEngineers }} at a time), perform a granularity check:
    - If the Issue body is empty, extremely short, or covers multiple independent features, judge it as **ambiguous or too large**
    - Ambiguous/oversized Issues: post a `[Clarification Needed]` comment via `add_issue_comment` and skip
    - For Issues of appropriate scope only:
      a. Mark as in-progress with `assign_issue` (assignee: your own username)
      b. Create a worktree with `create_worktree` (base_branch: default branch)
-9. **Spawn all Engineers for the Issues prepared in step 8 in parallel at once using the Agent tool** (`run_in_background: true`, so this inline pass can wait for them without blocking on each one sequentially)
-   - Information to pass to each Engineer: Issue number, title, body, `worktree_path` and `branch` returned by `create_worktree`
-   - Call the Agent tool with `model: "{{ .EngineerModel }}"`{{ if .EngineerEffort }} and `effort: "{{ .EngineerEffort }}"`{{ end }} so each Engineer runs with the model{{ if .EngineerEffort }}/effort{{ end }} configured in `harness.toml`'s `[model]` section
-   - If the parallel count exceeds {{ .MaxEngineers }}, process the first {{ .MaxEngineers }} and defer the rest to the next pass
-10. Wait for all Engineers to complete
-11. Run `evaluate_risk` on the PR for each Issue (including PRs opened by Engineers spawned on an earlier pass that are still awaiting evaluation — use `list_prs` to find open HERMIT PRs)
+10. **Spawn all Engineers for the Issues prepared in step 9 in parallel at once using the Agent tool** (`run_in_background: true`, so this inline pass can wait for them without blocking on each one sequentially)
+    - Information to pass to each Engineer: Issue number, title, body, `worktree_path` and `branch` returned by `create_worktree`
+    - Call the Agent tool with `model: "{{ .EngineerModel }}"`{{ if .EngineerEffort }} and `effort: "{{ .EngineerEffort }}"`{{ end }} so each Engineer runs with the model{{ if .EngineerEffort }}/effort{{ end }} configured in `harness.toml`'s `[model]` section
+    - If the parallel count exceeds {{ .MaxEngineers }}, process the first {{ .MaxEngineers }} and defer the rest to the next pass
+11. Wait for all Engineers to complete
+12. Run `evaluate_risk` on the PR for each Issue (including PRs opened by Engineers spawned on an earlier pass that are still awaiting evaluation — use `list_prs` to find open HERMIT PRs)
     - LOW / MEDIUM: run `merge_pr` with `worktree_path`/`branch` so the worktree is cleaned up automatically
     - HIGH: `evaluate_risk` auto-posts a generic risk comment (`⚠️ HERMIT: HIGH risk detected.\nReasons: [...]`) restating the `risk_reasons`. That comment is not a review — before skipping, perform a substantive review of the PR yourself:
       - Read the actual diff (not just the file paths / line counts in `risk_reasons`)
       - Assess correctness, test coverage of the changed behavior, and consistency with the linked Issue's requirements
       - Check whether the branch is stale relative to the base branch in a way that could hide semantic conflicts, not just textual `mergeable` conflicts
       - Post your findings as a separate PR comment via `add_issue_comment`: a short summary of what changed, anything concerning, and an explicit recommendation (e.g. "looks safe to merge pending approval" vs. "found X, should be fixed first")
       - Skip merging and wait for a human decision
-12. End the pass with a short report of what was done, and return control to the prompt — do **not** loop back to step 1 yourself; the recurring cron job fires the next pass
+13. End the pass with a short report of what was done, and return control to the prompt — do **not** loop back to step 1 yourself; the recurring cron job fires the next pass
 
 ---
 

cmd/hermit/templates/harness.toml.tmpl

@@ -79,3 +79,18 @@ analyst_effort         = "{{ .AnalystEffort }}"  # reasoning effort: low/medium/
 
 # [run]
 # failure_notify_threshold = 3  # `hermit run`: consecutive failed passes before a [notification] webhook fires (default: 3)
+
+# [[health_checks]]
+# # Production health checks run periodically by the run_health_checks MCP
+# # tool (Issue #190). Each entry's command is run with a 30s timeout; exit
+# # code 0 means healthy. A failing check with no existing open issue opens
+# # one labeled "production-incident" (deduped by an "[health-check: <name>]"
+# # title prefix); a check that recovers gets a one-time "recovered at ..."
+# # comment on that issue (it is never auto-closed). Omitting this section
+# # entirely (the default) means run_health_checks is a no-op.
+# name    = "api-health"
+# command = "curl -sf https://example.com/healthz"
+# # type: only "command" is supported today (default when omitted); other
+# # values are reserved for future check mechanisms (e.g. "http") and are
+# # skipped if set to anything else.
+# type    = "command"

cmd/hermit/unit_test.go

@@ -1165,3 +1165,84 @@ func TestResolveAnalystEffort_FallsBackToSuperintendentEffort(t *testing.T) {
 		t.Errorf("resolveAnalystEffort() = %q, want configured %q", got, "medium")
 	}
 }
+
+// --- health_checks (Issue #190) ---
+
+// TestLoadConfig_HealthChecks verifies that [[health_checks]] entries in
+// harness.toml are parsed into Config.HealthChecks.
+func TestLoadConfig_HealthChecks(t *testing.T) {
+	dir := t.TempDir()
+	content := `[github]
+owner = "owner"
+repo  = "repo"
+
+[[health_checks]]
+name    = "api-health"
+command = "curl -sf https://example.com/healthz"
+
+[[health_checks]]
+name    = "db-health"
+command = "pg_isready"
+type    = "command"
+`
+	if err := os.WriteFile(filepath.Join(dir, "harness.toml"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	prev, _ := os.Getwd()
+	os.Chdir(dir)
+	defer os.Chdir(prev)
+
+	cfg := loadConfig()
+	if len(cfg.HealthChecks) != 2 {
+		t.Fatalf("expected 2 health_checks entries, got %d: %+v", len(cfg.HealthChecks), cfg.HealthChecks)
+	}
+	if cfg.HealthChecks[0].Name != "api-health" || cfg.HealthChecks[0].Command != "curl -sf https://example.com/healthz" {
+		t.Errorf("unexpected first health check: %+v", cfg.HealthChecks[0])
+	}
+	if cfg.HealthChecks[1].Name != "db-health" || cfg.HealthChecks[1].Type != "command" {
+		t.Errorf("unexpected second health check: %+v", cfg.HealthChecks[1])
+	}
+}
+
+// TestLoadConfig_HealthChecks_Omitted verifies that a project with no
+// [[health_checks]] section at all parses to a nil/empty slice — the
+// "unconfigured is a no-op" contract from Issue #190.
+func TestLoadConfig_HealthChecks_Omitted(t *testing.T) {
+	dir := t.TempDir()
+	content := `[github]
+owner = "owner"
+repo  = "repo"
+`
+	if err := os.WriteFile(filepath.Join(dir, "harness.toml"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	prev, _ := os.Getwd()
+	os.Chdir(dir)
+	defer os.Chdir(prev)
+
+	cfg := loadConfig()
+	if len(cfg.HealthChecks) != 0 {
+		t.Errorf("expected no health_checks entries, got %+v", cfg.HealthChecks)
+	}
+	if got := resolveHealthChecks(cfg); len(got) != 0 {
+		t.Errorf("resolveHealthChecks() = %v, want empty", got)
+	}
+}
+
+// TestResolveHealthChecks_SkipsIncompleteEntries verifies that entries
+// missing a name or command are skipped rather than passed through as a
+// broken check (which would otherwise silently run `sh -c ""`, a vacuous
+// success, on every sweep).
+func TestResolveHealthChecks_SkipsIncompleteEntries(t *testing.T) {
+	cfg := Config{
+		HealthChecks: []HealthCheckEntry{
+			{Name: "good-check", Command: "exit 0"},
+			{Name: "", Command: "exit 0"},
+			{Name: "no-command", Command: ""},
+		},
+	}
+	got := resolveHealthChecks(cfg)
+	if len(got) != 1 || got[0].Name != "good-check" {
+		t.Errorf("resolveHealthChecks() = %+v, want only good-check", got)
+	}
+}