Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
fbc4850
docs: diagnose v0.7.2 summary failures and convergence limits
ivgtr Oct 1, 2026
d20829a
docs: approve summary convergence implementation plan
ivgtr Oct 1, 2026
253a53c
fix: unify summary source evidence and bounded repair
ivgtr Oct 1, 2026
92e2797
feat: retain and export summary generation diagnostics
ivgtr Oct 1, 2026
b5fe1b1
test: verify summary convergence and document remaining limits
ivgtr Oct 1, 2026
e22440f
fix: preserve selected coverage and required repair capacity
ivgtr Oct 1, 2026
93372b4
fix: match exported summary diagnostics to the current result
ivgtr Oct 1, 2026
7a2b7c7
fix: reject ambiguous prose quantity and period mappings
ivgtr Oct 1, 2026
8f56ccc
fix: restore summary controls after settings change and record self-r…
ivgtr Oct 1, 2026
93cf139
fix: validate reporting attributes and quoted quantity bounds
ivgtr Oct 1, 2026
e826cda
fix: retain reporting attributes from Q-style earnings captions
ivgtr Oct 1, 2026
675d564
fix: align supported reporting caption attributes
ivgtr Oct 1, 2026
352943b
fix: verify prose quantities and unify reporting context
ivgtr Oct 1, 2026
e81d816
fix: address summary semantics and repair review findings
ivgtr Oct 1, 2026
428b325
fix: align assertion polarity and financial reporting context
ivgtr Oct 1, 2026
8cabd2f
fix: bound cover reporting evidence and negative contrasts
ivgtr Oct 2, 2026
9a55639
fix: preserve assertion identity and signed reporting values
ivgtr Oct 2, 2026
17f5ca9
fix: align reporting labels and negative assertions
ivgtr Oct 2, 2026
739bc75
fix: require explicit net profit and accept passive forecasts
ivgtr Oct 2, 2026
979e6eb
fix: accept bounded passive forecast endings
ivgtr Oct 2, 2026
4a2cd9a
fix: preserve event polarity and unrealized action states
ivgtr Oct 2, 2026
af9738e
fix: align prose quantities, margins, and forecast evidence
ivgtr Oct 2, 2026
eef6961
fix: verify inherited periods, outlook negation, and yen scales
ivgtr Oct 2, 2026
02733e4
fix: align period coverage and prose quantity semantics
ivgtr Oct 2, 2026
27e41ff
fix: prove units and share reporting period semantics
ivgtr Oct 2, 2026
22fec63
fix: 原数量と期間の意味照合を統一し保存根拠を厳密化
ivgtr Oct 2, 2026
f68e3a4
fix: 報告期間と節の判定を共有し必須事実の修復枠を確保
ivgtr Oct 2, 2026
89ba53e
fix: 主張の述語と期間の対応を共有し複合表記を整合
ivgtr Oct 2, 2026
66928db
fix: 期間の限定と計上区間を原文へ対応づけ予想参照を除外
ivgtr Oct 2, 2026
0c07142
fix: preserve predicate and declaration ownership in summary facts
ivgtr Oct 3, 2026
926115d
docs: clarify source of the replay diagnostic difference
ivgtr Oct 3, 2026
2ba1477
fix: align forecast ownership, schedule semantics and repair diagnostics
ivgtr Oct 3, 2026
4b190ad
fix: preserve scoped inheritance and repair key promotion
ivgtr Oct 3, 2026
bfd83e8
fix: bind coverage and diagnostics to their owners
ivgtr Oct 3, 2026
5ba3d3f
fix: 必須事実の原文所有者と生成前診断を統一
ivgtr Oct 3, 2026
cfc552a
fix: M&Aの限定付き期間と拒否URL診断を保持
ivgtr Oct 3, 2026
f43443f
fix: prove representable periods before creating numeric obligations
ivgtr Oct 3, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ npm test
npm run format
```

## テスト方針

テストの追加・整理・再実行は [開発ガイドのテスト戦略](docs/development.md#テスト戦略) に従う。語彙の網羅は部品、責務の接続は代表例、実PDFは固定コーパスで証明する。同じ反例を全経路や全旧指紋へ複製しない。追加時には新しい分岐・境界と担当テストを示し、重複例を置き換えられるか確認する。実API・画面検証の実施条件と停止境界は [評価ガイド](evaluation/README.md#検証の選択と停止条件) を正本とする。

## ファイル構成

```
Expand Down
621 changes: 621 additions & 0 deletions docs/development.md

Large diffs are not rendered by default.

893 changes: 893 additions & 0 deletions evaluation/README.md

Large diffs are not rendered by default.

935 changes: 935 additions & 0 deletions evaluation/scripts/additional-review-fixture.ts

Large diffs are not rendered by default.

676 changes: 623 additions & 53 deletions evaluation/scripts/check-extension.ts

Large diffs are not rendered by default.

32 changes: 26 additions & 6 deletions evaluation/scripts/regrade-fact-summary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ import { expectedErrors, type Case } from './fact-summary-expectations';
const cases: Case[] = JSON.parse(
await readFile('evaluation/fixtures/fact-summary-cases.json', 'utf8')
);
for (const file of process.argv.slice(2)) {
const diagnosticRenumber = process.argv.includes('--diagnostic-renumber');
for (const file of process.argv.slice(2).filter((a) => a !== '--diagnostic-renumber')) {
if (
path.dirname(file) !== 'evaluation/results/local' ||
!cases.some((item) => path.basename(file).startsWith(`${item.id}-`)) ||
Expand All @@ -34,12 +35,21 @@ for (const file of process.argv.slice(2)) {
}
await pdf.destroy();
const errors: string[] = [];
let acceptedFacts = 0,
rejectedFacts = 0;
const original = saved.result ?? saved.facts;
if (!original) throw new Error('保存された確定結果がありません');
const input = diagnosticRenumber
? {
...original,
facts: original.facts.map((f: object, i: number) => ({ ...f, id: `f${i + 1}` })),
}
: original;
try {
const result = parseFactSummary(
JSON.stringify(saved.result ?? saved.facts),
item.documentType,
pages
);
const partial = parseFactSummary(JSON.stringify(input), item.documentType, pages, false);
acceptedFacts = partial.facts.length;
rejectedFacts = original.facts.length - acceptedFacts;
const result = parseFactSummary(JSON.stringify(input), item.documentType, pages);
errors.push(...expectedErrors(item, result));
if (result.unverified.length !== (saved.result ?? saved.facts).unverified.length)
errors.push('現行検証で確定事実の拒否が増えました');
Expand All @@ -49,13 +59,20 @@ for (const file of process.argv.slice(2)) {
const checker = createHash('sha256');
for (const name of [
'src/lib/fact-contract.ts',
'src/lib/document-context.ts',
'src/lib/source-mappings.ts',
'src/lib/source-periods.ts',
'src/lib/fact-candidates.ts',
'src/lib/assertion-semantics.ts',
'src/lib/quantity.ts',
'src/lib/period-semantics.ts',
'src/lib/document-structure.ts',
'src/lib/document-links.ts',
'src/lib/pdf-layout.ts',
'src/lib/fact-summary.ts',
'src/lib/fact-validation.ts',
'src/lib/fact-coverage.ts',
'src/lib/metric-semantics.ts',
'src/lib/numeric-evidence.ts',
'evaluation/scripts/fact-summary-expectations.ts',
'evaluation/fixtures/fact-summary-cases.json',
Expand All @@ -71,6 +88,9 @@ for (const file of process.argv.slice(2)) {
checkerDigest: checker.digest('hex'),
regradedAt: new Date().toISOString(),
modelCalls: 0,
mode: diagnosticRenumber ? 'explicit-ID-renumber-diagnostic-only' : 'strict-saved-contract',
acceptedFacts,
rejectedFacts,
success: errors.length === 0,
errors,
};
Expand Down
17 changes: 13 additions & 4 deletions evaluation/scripts/run-fact-summary.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { readFile, mkdir, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { serializeLayout } from '../../src/lib/pdf-layout';
import { serializeCandidateSource } from '../../src/lib/fact-candidates';
import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
import { extractPageLayout } from '../../src/lib/pdf-layout';
import { serializePagesForAnalysis } from '../../src/lib/page-text';
Expand Down Expand Up @@ -37,7 +37,13 @@ if (process.argv.includes('--browser')) {
await mkdir('evaluation/results/local', { recursive: true });
const implementationFiles = [
'src/lib/fact-contract.ts',
'src/lib/document-context.ts',
'src/lib/source-mappings.ts',
'src/lib/source-periods.ts',
'src/lib/fact-candidates.ts',
'src/lib/assertion-semantics.ts',
'src/lib/quantity.ts',
'src/lib/period-semantics.ts',
'src/lib/document-structure.ts',
'src/lib/document-links.ts',
'src/lib/pdf-layout.ts',
Expand Down Expand Up @@ -81,6 +87,7 @@ for (const item of selected) {
{
provider,
model,
signal: AbortSignal.timeout(300_000),
apiKey,
baseUrl: process.env.TDNET_DIGEST_BASE_URL || undefined,
onUsage: (item) => usage.push(item),
Expand All @@ -105,14 +112,16 @@ for (const item of selected) {
implementationDigest,
requestLimits: factSummaryRequestLimits({ provider, model }),
sourceHash,
inputHash: createHash('sha256').update(serializeLayout(pages)).digest('hex'),
inputChars: serializeLayout(pages).length,
inputHash: createHash('sha256')
.update(serializeCandidateSource(pages, undefined, item.documentType))
.digest('hex'),
inputChars: serializeCandidateSource(pages, undefined, item.documentType).length,
usage,
provider,
model,
elapsedSeconds,
pages: pages.length,
repairAttempted: attempt?.repairAttempted ?? null,
repairAttempted: attempt?.repairAttempted ?? attempts.some((a) => a.phase === 'repair'),
success: errors.length === 0,
errors,
result,
Expand Down
Loading
Loading