Fix PHP recovery leaf error parity

CHANGELOG.md

@@ -1044,17 +1044,19 @@ for tags and release notes while still in `0.x`.
   `docs/root-normalization-retirement.md` for current artifact hashes and
   reopening conditions.
 
-- Hardened the PHP issue #454 compact fallback guard at base commit
-  `c25686c882affd7408e5ef4a7d65e92cc8391fab`. The 140,287-byte edited
-  witness produced the pinned production and compact Go deep digest
-  `4456730ce6919a623dd6db2e6ae7f11933aeb454c7e337b7da5c08a8d9ba267c`.
-  The locked-C `gts-deep-tree-v1` digest is
+- Corrected the PHP issue #454 recovery-leaf flags at base commit
+  `55681868d3a23971d042f9f79083fd6d39c7e33b`. A recovery region now needs a
+  source-bearing parsed prefix. Each cleared leaf also needs positive internal
+  deterministic finite automaton (DFA) provenance.
+  The prefix proof rejects pending, missing, error, dirty, and invalid payloads.
+  It also rejects payloads that end after the current token starts.
+  Recovery rejects end-of-input, zero-width, generated, external, missing,
+  no-lookahead, and error-mode tokens. The raw, production, compact fallback,
+  and locked-C `gts-deep-tree-v1` digests now equal
   `1516308c38163089778464ad171875308c559af11af7c8c03ee17ae4eacd23c6`.
-  The digest covers byte and point spans, fields, extra and missing flags,
-  error flags, and root `HasError`. Both Go routes differ from locked C.
-  Compact recorded `routed=0` and `fallback=1`. Two one-CPU, 4 GiB Docker
-  runs passed with one test worker, a 20-minute timeout, `GOMAXPROCS=1`, and
-  `GOFLAGS=-p=1`. Keep issue #454 open. Ship no parser change. See
+  Compact still records `routed=0` and `fallback=1` on recovery. This change
+  does not graduate PHP compact admission. The incremental memory budget,
+  resident set size, and remaining issue #454 performance work stay open. See
   `docs/issue-454-compact-correctness-blocker.md`.
 
 - Reject the issue #454 generic recovery candidate from PR #793. The candidate

cgo_harness/php_issue454_parity_test.go

@@ -23,8 +23,9 @@ const issue454PHPGrammarBlobSHA256 = "15724627db479c27304b43fa3b5ef7d8d81f85e3b9
 const issue454PHPCArtifactSHA256 = "1daea60ac1ee31227b8e1ed3cbd76b841435fe693e95af65cc61dad447d27891"
 
 const (
-	issue454PHPProductionDeepDigest = "4456730ce6919a623dd6db2e6ae7f11933aeb454c7e337b7da5c08a8d9ba267c"
-	issue454PHPCompactDeepDigest    = "4456730ce6919a623dd6db2e6ae7f11933aeb454c7e337b7da5c08a8d9ba267c"
+	issue454PHPRawDeepDigest        = "1516308c38163089778464ad171875308c559af11af7c8c03ee17ae4eacd23c6"
+	issue454PHPProductionDeepDigest = "1516308c38163089778464ad171875308c559af11af7c8c03ee17ae4eacd23c6"
+	issue454PHPCompactDeepDigest    = "1516308c38163089778464ad171875308c559af11af7c8c03ee17ae4eacd23c6"
 	issue454PHPLockedCDeepDigest    = "1516308c38163089778464ad171875308c559af11af7c8c03ee17ae4eacd23c6"
 	issue454PHPRootHasError         = true
 )
@@ -44,6 +45,17 @@ func TestParityIssue454PHPWholeTreeFallback(t *testing.T) {
 	}
 
 	goLang := grammars.PhpLanguage()
+	rawParser := gotreesitter.NewParser(goLang)
+	rawParser.SetAdmissionCandidateRoute(false)
+	rawTree, err := rawParser.ParseNoResultCompatibilityBenchmarkOnly(edited)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer releaseGoTree(rawTree)
+	if rawTree.ParseStoppedEarly() {
+		t.Fatalf("raw Go parse stopped early: %s", rawTree.ParseRuntime().Summary())
+	}
+
 	goParser := gotreesitter.NewParser(goLang)
 	goParser.SetAdmissionCandidateRoute(false)
 	goTree, err := goParser.Parse(edited)
@@ -87,6 +99,10 @@ func TestParityIssue454PHPWholeTreeFallback(t *testing.T) {
 		t.Fatal("C reference parser returned nil tree")
 	}
 	defer cTree.Close()
+	rawInspection, err := benchfixtures.InspectGoTree(rawTree.RootNode(), goLang)
+	if err != nil {
+		t.Fatal(err)
+	}
 	productionInspection, err := benchfixtures.InspectGoTree(goTree.RootNode(), goLang)
 	if err != nil {
 		t.Fatal(err)
@@ -95,22 +111,39 @@ func TestParityIssue454PHPWholeTreeFallback(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	if rawInspection.SHA256 != issue454PHPRawDeepDigest {
+		t.Fatalf("PHP issue #454 raw gts-deep-tree-v1 digest=%s, want %s", rawInspection.SHA256, issue454PHPRawDeepDigest)
+	}
 	if productionInspection.SHA256 != issue454PHPProductionDeepDigest {
 		t.Fatalf("PHP issue #454 production gts-deep-tree-v1 digest=%s, want %s", productionInspection.SHA256, issue454PHPProductionDeepDigest)
 	}
 	if cDeepDigest != issue454PHPLockedCDeepDigest {
 		t.Fatalf("PHP issue #454 locked-C gts-deep-tree-v1 digest=%s, want %s", cDeepDigest, issue454PHPLockedCDeepDigest)
 	}
+	if rawInspection.SHA256 != productionInspection.SHA256 || productionInspection.SHA256 != cDeepDigest {
+		t.Fatalf("PHP issue #454 digests differ: raw=%s production=%s locked-C=%s", rawInspection.SHA256, productionInspection.SHA256, cDeepDigest)
+	}
+	if got := rawTree.RootNode().HasError(); got != issue454PHPRootHasError {
+		t.Fatalf("PHP issue #454 raw root HasError=%t, want %t", got, issue454PHPRootHasError)
+	}
 	if got := goTree.RootNode().HasError(); got != issue454PHPRootHasError {
 		t.Fatalf("PHP issue #454 production root HasError=%t, want %t", got, issue454PHPRootHasError)
 	}
 	if got := cTree.RootNode().HasError(); got != issue454PHPRootHasError {
 		t.Fatalf("PHP issue #454 locked-C root HasError=%t, want %t", got, issue454PHPRootHasError)
 	}
-	if productionInspection.SHA256 == cDeepDigest {
-		t.Fatal("PHP issue #454 production deep digest unexpectedly matches locked C; review the NO-GO receipt before changing route policy")
+	if diff := FirstDivergenceDumpV1(rawTree.RootNode(), goLang, cTree.RootNode()); diff != nil {
+		t.Fatalf("PHP issue #454 raw tree differs from the pinned C oracle: %+v", *diff)
+	}
+	if diff := FirstDivergenceDumpV1(goTree.RootNode(), goLang, cTree.RootNode()); diff != nil {
+		t.Fatalf("PHP issue #454 production tree differs from the pinned C oracle: %+v", *diff)
 	}
 
+	var rawErrs []string
+	compareNodes(rawTree.RootNode(), goLang, cTree.RootNode(), "root", &rawErrs)
+	if len(rawErrs) > 0 {
+		t.Fatalf("PHP issue #454 raw tree differs from the pinned C oracle: %s", rawErrs[0])
+	}
 	var errs []string
 	compareNodes(goTree.RootNode(), goLang, cTree.RootNode(), "root", &errs)
 	if len(errs) > 0 {
@@ -153,28 +186,29 @@ func TestParityIssue454PHPWholeTreeFallback(t *testing.T) {
 	if got := compactTree.RootNode().HasError(); got != issue454PHPRootHasError {
 		t.Fatalf("PHP issue #454 compact root HasError=%t, want %t", got, issue454PHPRootHasError)
 	}
-	if compactInspection.SHA256 == cDeepDigest {
-		t.Fatal("PHP issue #454 compact deep digest unexpectedly matches locked C; review the NO-GO receipt before changing route policy")
+	if compactInspection.SHA256 != cDeepDigest {
+		t.Fatalf("PHP issue #454 compact and locked-C deep digests differ: compact=%s locked-C=%s", compactInspection.SHA256, cDeepDigest)
 	}
-	t.Logf("deep_digest format=%s production=%s compact=%s locked_c=%s production_root_has_error=%t compact_root_has_error=%t locked_c_root_has_error=%t exact_locked_c=false fields=type+named,field,byte+point,extra+missing+error+has_error,child_order", benchfixtures.DeepTreeDigestVersion, productionInspection.SHA256, compactInspection.SHA256, cDeepDigest, goTree.RootNode().HasError(), compactTree.RootNode().HasError(), cTree.RootNode().HasError())
+	t.Logf("deep_digest format=%s raw=%s production=%s compact=%s locked_c=%s raw_root_has_error=%t production_root_has_error=%t compact_root_has_error=%t locked_c_root_has_error=%t exact_locked_c=true fields=type+named,field,byte+point,extra+missing+error+has_error,child_order", benchfixtures.DeepTreeDigestVersion, rawInspection.SHA256, productionInspection.SHA256, compactInspection.SHA256, cDeepDigest, rawTree.RootNode().HasError(), goTree.RootNode().HasError(), compactTree.RootNode().HasError(), cTree.RootNode().HasError())
 	t.Logf("PHP issue #454 compact route source_sha256=%x bytes=%d routed=%d fallback=%d reason=%q", sha256.Sum256(edited), len(edited), routedAfter-routedBefore, fallbackAfter-fallbackBefore, gotreesitter.AdmissionCandidateLastFallbackReason())
 	document, err := os.ReadFile("../docs/issue-454-compact-correctness-blocker.md")
 	if err != nil {
 		t.Fatalf("read issue #454 compact receipt: %v", err)
 	}
 	documentText := strings.Join(strings.Fields(string(document)), " ")
 	for _, marker := range []string{
-		"## 2026-08-24 PHP compact fallback guard",
-		"Publication base: `c25686c882affd7408e5ef4a7d65e92cc8391fab`.",
+		"## 2026-08-24 PHP recovery-leaf correction",
+		"Candidate base: `55681868d3a23971d042f9f79083fd6d39c7e33b`.",
 		"The locked-C artifact SHA-256 is `1daea60ac1ee31227b8e1ed3cbd76b841435fe693e95af65cc61dad447d27891`.",
+		issue454PHPRawDeepDigest,
 		issue454PHPProductionDeepDigest,
 		issue454PHPCompactDeepDigest,
 		issue454PHPLockedCDeepDigest,
 		"The `gts-deep-tree-v1` stream covers type and named identity, incoming fields, byte and point spans, and child order. It also covers extra and missing flags, error flags, and the `HasError` flag.",
-		"The production and compact deep digests differ from the locked-C digest.",
-		"All three roots report `HasError=true`.",
+		"The raw, production, compact fallback, and locked-C deep digests are equal.",
+		"All four roots report `HasError=true`.",
 		"The compact route recorded `routed=0` and `fallback=1`.",
-		"This guard does not graduate PHP compact admission.",
+		"This correction does not graduate PHP compact admission.",
 	} {
 		marker = strings.Join(strings.Fields(marker), " ")
 		if !strings.Contains(documentText, marker) {

docs/issue-454-compact-correctness-blocker.md

@@ -1,6 +1,7 @@
 # Issue #454 compact-parser correctness blocker
 
-Status: **NO-GO**. Ship no parser change from this investigation. Keep issue
+Status: **KEEP LIVE**. The PHP recovery-leaf candidate restores locked-C
+correctness. Compact admission and performance work remain open. Keep issue
 [#454](https://github.com/odvcencio/gotreesitter/issues/454) open.
 
 ## Scope
@@ -205,14 +206,23 @@ is not safe without wider recovery validation.
 The 1 KiB known-divergence ratchet passes in
 `/tmp/gts-issue454-artifacts-rebase/20260822T230037Z-issue454-c-1k-ratchet-20260822`.
 
-## 2026-08-24 PHP compact fallback guard
+## 2026-08-24 PHP recovery-leaf correction
 
-Publication base: `c25686c882affd7408e5ef4a7d65e92cc8391fab`.
+Candidate base: `55681868d3a23971d042f9f79083fd6d39c7e33b`.
 
-Status: **KEEP LIVE / NO-GO**. Keep issue #454 open. Ship no parser change.
+Status: **CORRECTNESS FIX / KEEP LIVE**. Keep issue #454 open.
 
-The focused guard extends the existing PHP issue #454 parity test. It checks
-the compact candidate route against the production and locked-C trees.
+The correction adds positive internal deterministic finite automaton (DFA)
+provenance to tokens. Recovery requires a source-bearing parsed stack prefix.
+It then records a region proof only for a direct, visible, named DFA token.
+The prefix proof rejects pending, missing, error, dirty, and invalid payloads.
+It rejects state mismatches and payloads that end after the current token starts.
+It rejects the end-of-input symbol and zero-width tokens. It also rejects
+generated, external, missing, no-lookahead, and error-mode tokens.
+
+Each later absorbed token must carry its own positive DFA proof. A region
+proof cannot clear a later token by itself. A skipped-prefix token can qualify
+only when the internal DFA produced that token directly.
 
 The edited PHP source has 140,287 bytes. Its SHA-256 is
 `cbf52f81ea212353a3bf04d7c9b37668b5cdfb6cd428c2d0cb3799a8e13ae82f`.
@@ -222,13 +232,6 @@ The embedded PHP grammar blob SHA-256 is
 The locked-C artifact SHA-256 is
 `1daea60ac1ee31227b8e1ed3cbd76b841435fe693e95af65cc61dad447d27891`.
 
-The compact route recorded `routed=0` and `fallback=1`.
-Its fallback reason was:
-
-```text
-compact route declined at recovery [mechanism=recovery-entered]: did not accept EOF: generic scheduler has no table action for the elected token
-```
-
 The `gts-deep-tree-v1` stream covers type and named identity, incoming fields,
 byte and point spans, and child order. It also covers extra and missing flags,
 error flags, and the `HasError` flag.
@@ -237,25 +240,18 @@ The pinned deep digests are:
 
 | Route | Deep digest | Root `HasError` |
 | --- | --- | --- |
-| Production Go | `4456730ce6919a623dd6db2e6ae7f11933aeb454c7e337b7da5c08a8d9ba267c` | `true` |
-| Compact fallback Go | `4456730ce6919a623dd6db2e6ae7f11933aeb454c7e337b7da5c08a8d9ba267c` | `true` |
+| Raw Go | `1516308c38163089778464ad171875308c559af11af7c8c03ee17ae4eacd23c6` | `true` |
+| Production Go | `1516308c38163089778464ad171875308c559af11af7c8c03ee17ae4eacd23c6` | `true` |
+| Compact fallback Go | `1516308c38163089778464ad171875308c559af11af7c8c03ee17ae4eacd23c6` | `true` |
 | Locked C | `1516308c38163089778464ad171875308c559af11af7c8c03ee17ae4eacd23c6` | `true` |
 
-All three roots report `HasError=true`.
-
-The compact tree equals the production tree. The production and compact deep
-digests differ from the locked-C digest. This full comparison keeps the PHP
-route at **NO-GO**.
-
-The guard passed twice with one CPU, 4 GiB, one test worker, a 20-minute
-timeout, `GOMAXPROCS=1`, and `GOFLAGS=-p=1`. Both runs had no out-of-memory
-kill and no wall timeout.
-
-The final artifacts are:
+The raw, production, compact fallback, and locked-C deep digests are equal.
+All four roots report `HasError=true`.
 
-- `/tmp/gotreesitter-php454-deep-guard-refresh-artifacts/20260824T124547Z-php454-refresh-1`
-- `/tmp/gotreesitter-php454-deep-guard-refresh-artifacts/20260824T124620Z-php454-refresh-2`
+The compact route recorded `routed=0` and `fallback=1`. Its fallback reason
+still reports `mechanism=recovery-entered`. The compact parser does not accept
+this recovery path yet.
 
-This guard does not graduate PHP compact admission.
-Reopen the route only after a generic recovery proof removes the fallback,
-matches both Go route digests to locked C, and preserves all deep-digest fields.
+This correction does not graduate PHP compact admission. It does not close
+the remaining issue #454 performance work. The incremental memory-budget
+fallback and the large-file resident-set-size target also remain open.

lexer.go

@@ -38,6 +38,9 @@ type Token struct {
 	// lexerErrorModeLexed proves that the active DFA source produced this
 	// recovery token while parser state zero selected the error lex mode.
 	lexerErrorModeLexed bool
+	// lexerInternalDFALexed proves that Lexer.scan accepted this token from
+	// the internal DFA. External, generated, missing, and EOF tokens omit it.
+	lexerInternalDFALexed bool
 }
 
 func bytesToStringNoCopy(b []byte) string {
@@ -418,6 +421,7 @@ func (l *Lexer) scan(startState uint32, startPos int, startRow, startCol uint32)
 		EndPoint:                Point{Row: acceptRow, Column: acceptCol},
 		lexerSkippedPrefix:      skippedPrefix,
 		lexerSkippedPrefixStart: uint32(startPos),
+		lexerInternalDFALexed:   true,
 	}, true
 }
 

parser_dfa_token_source_test.go

@@ -1628,6 +1628,9 @@ func TestNextDFATokenPrefersParserValidZeroWidthStartAccept(t *testing.T) {
 	if got, want := tok.EndByte, uint32(0); got != want {
 		t.Fatalf("token end = %d, want %d", got, want)
 	}
+	if tok.lexerInternalDFALexed {
+		t.Fatal("synthetic zero-width start token has internal-DFA provenance")
+	}
 }
 
 func TestNextDFATokenSynthesizesGeneratedNULSentinelLookahead(t *testing.T) {
@@ -1675,6 +1678,9 @@ func TestNextDFATokenSynthesizesGeneratedNULSentinelLookahead(t *testing.T) {
 	if tok.StartByte != 0 || tok.EndByte != 0 {
 		t.Fatalf("token span = %d..%d, want zero-width at 0", tok.StartByte, tok.EndByte)
 	}
+	if tok.lexerInternalDFALexed {
+		t.Fatal("generated NUL token has internal-DFA provenance")
+	}
 }
 
 func TestNextDFATokenDoesNotSynthesizeGeneratedNULSentinelOverValidToken(t *testing.T) {