From 9a717d09a30f66320c360405acd6463eb9b6b155 Mon Sep 17 00:00:00 2001 From: Ali Farrokhnejad Date: Sun, 6 Sep 2026 07:29:31 +0300 Subject: [PATCH 1/2] Harden evidence privacy and corpus boundaries --- .github/workflows/ci.yml | 71 ++--- AGENTS.md | 38 ++- BLUEPRINT.md | 114 ++++++++ DEV_LOG.md | 53 ++++ DEV_STATE.md | 17 ++ PUBLICATION.md | 35 ++- QA_REPORT.md | 71 +++++ README.md | 253 +++--------------- RISK_REGISTER.md | 35 +++ SECURITY.md | 41 +-- docs/ARCHITECTURE.md | 36 +++ docs/BOARD_DEMO_READINESS.md | 35 +-- docs/BOOTSTRAP_AUDIT.md | 20 ++ docs/CLAIM_REGISTER.md | 14 + docs/DATA_MANIFEST.md | 47 ++++ docs/DATA_PROVENANCE.md | 13 + docs/DEMO_METRICS_SNAPSHOT.md | 111 +------- docs/DEMO_STORYBOARD.md | 69 ++--- docs/DEPENDENCY_POLICY.md | 21 ++ docs/EMUAdvisor Full Analysis.md | 3 + docs/EVIDENCE_MAP.md | 13 + docs/EXPERIMENT_PROTOCOL.md | 29 ++ docs/MIGRATION_BACKLOG.md | 11 +- docs/MODEL_EVALUATION.md | 33 +++ docs/PROJECT_STATE.md | 173 ++++-------- docs/PROJECT_STATUS_PROGRESS_PLAN.md | 213 ++------------- docs/PUBLICATION_CHECKLIST.md | 47 ++-- docs/RELEASE_CANDIDATE.md | 84 +----- docs/REPO_MAP.md | 275 +++---------------- docs/REPO_PROFILE.json | 95 +++++++ docs/REPO_PROFILE.md | 82 ++++++ docs/REPRODUCIBILITY.md | 14 + docs/RUN_PROTOCOL.md | 275 +++---------------- docs/SPRINT_PLAN.md | 2 + docs/SPRINT_STATUS.md | 76 +----- docs/TEST_STRATEGY.md | 33 +++ docs/VERSION_LOG.md | 14 + docs/eval_spec.md | 73 ++---- emu_advisor/audit_log.py | 24 +- emu_advisor/conversation_store.py | 292 +++++++++++---------- emu_advisor/corpus.py | 63 ++++- emu_advisor/evaluation.py | 78 +++++- emu_advisor/html_ingest.py | 4 +- emu_advisor/metrics.py | 233 +++++++++------- emu_advisor/pdf_ingest.py | 4 +- emu_advisor/pipeline.py | 97 +++++-- emu_advisor/readiness.py | 33 ++- emu_advisor/server.py | 244 +++++++++++------ eval_sets/v1_gold.jsonl | 120 ++++----- requirements-audit.txt | 2 + requirements-lock.txt | 6 +- requirements.txt | 2 +- shared/audit.md | 9 + shared/context.md | 8 + shared/errors.md | 4 + shared/history.md | 7 + shared/locks.json | 4 + shared/messages.md | 3 + shared/status.md | 14 + static/admin-diagnostics.js | 8 +- static/admin.html | 21 +- static/landing.html | 3 + static/shared.js | 9 + static/style.css | 13 + static/user-chat.js | 121 ++++----- tests/test_board_readiness_tools.py | 2 +- tests/test_ingestion_routing_evaluation.py | 5 +- tests/test_pipeline_metrics_generation.py | 15 +- tests/test_security_provenance.py | 176 +++++++++++++ tests/test_server_logging_load.py | 57 +++- tools/browser_smoke.py | 14 +- tools/publication_guard.py | 47 +++- tools/syntax_check.py | 28 ++ 73 files changed, 2348 insertions(+), 2061 deletions(-) create mode 100644 BLUEPRINT.md create mode 100644 DEV_LOG.md create mode 100644 DEV_STATE.md create mode 100644 QA_REPORT.md create mode 100644 RISK_REGISTER.md create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/BOOTSTRAP_AUDIT.md create mode 100644 docs/CLAIM_REGISTER.md create mode 100644 docs/DATA_MANIFEST.md create mode 100644 docs/DATA_PROVENANCE.md create mode 100644 docs/DEPENDENCY_POLICY.md create mode 100644 docs/EVIDENCE_MAP.md create mode 100644 docs/EXPERIMENT_PROTOCOL.md create mode 100644 docs/MODEL_EVALUATION.md create mode 100644 docs/REPO_PROFILE.json create mode 100644 docs/REPO_PROFILE.md create mode 100644 docs/REPRODUCIBILITY.md create mode 100644 docs/TEST_STRATEGY.md create mode 100644 requirements-audit.txt create mode 100644 shared/audit.md create mode 100644 shared/context.md create mode 100644 shared/errors.md create mode 100644 shared/history.md create mode 100644 shared/locks.json create mode 100644 shared/messages.md create mode 100644 shared/status.md create mode 100644 tests/test_security_provenance.py create mode 100644 tools/syntax_check.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f36fbed..7995612 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,46 +4,43 @@ on: push: pull_request: +permissions: + contents: read + +env: + EMU_ADVISOR_PROFILE: test + EMU_ADVISOR_CORPUS_MODE: fixture + EMU_ADVISOR_QUERY_REWRITE: deterministic + EMU_ADVISOR_ENABLE_CHAT_PERSISTENCE: "0" + EMU_ADVISOR_ENABLE_AUDIT_LOGGING: "0" + EMU_ADVISOR_LOG_RAW_QUERY: "0" + jobs: core: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + # actions/checkout v6.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 + # actions/setup-python v6.3.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 with: python-version: "3.12" cache: pip - cache-dependency-path: requirements-lock.txt + cache-dependency-path: | + requirements-lock.txt + requirements-audit.txt - name: Install pinned dependencies run: | python -m pip install --upgrade pip python -m pip install -r requirements-lock.txt - - name: Publication guard - run: python tools/publication_guard.py - - name: Dependency audit - run: | - python -m pip install pip-audit - pip-audit -r requirements-lock.txt + python -m pip install -r requirements-audit.txt + python -m pip check - name: Syntax scan - run: | - python - <<'PY' - import ast - from pathlib import Path - failed = [] - checked = 0 - for base in [Path("emu_advisor"), Path("tests"), Path("tools")]: - for path in base.rglob("*.py"): - checked += 1 - try: - ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except Exception as exc: - failed.append((str(path), repr(exc))) - if failed: - for path, exc in failed: - print(path, exc) - raise SystemExit(1) - print(f"syntax ok: {checked} files") - PY + run: python tools/syntax_check.py - name: Unit tests run: python -m unittest discover -s tests - name: Evaluation set validation @@ -51,16 +48,20 @@ jobs: python -m emu_advisor.evaluation eval_sets/v1_gold.jsonl python -m emu_advisor.evaluation eval_sets/v1_hard.jsonl python -m emu_advisor.evaluation eval_sets/emu_gold_seed.jsonl - - name: Verified gold review status - run: python -m emu_advisor.eval_review status eval_sets/v1_gold.jsonl - - name: Auxiliary evaluation status - run: python -m emu_advisor.eval_review status eval_sets/v1_hard.jsonl eval_sets/emu_gold_seed.jsonl + - name: Evaluation review status + run: python -m emu_advisor.eval_review status eval_sets/v1_gold.jsonl eval_sets/v1_hard.jsonl eval_sets/emu_gold_seed.jsonl + - name: Publication guard + run: python tools/publication_guard.py + - name: Dependency audit + run: python -m pip_audit -r requirements-lock.txt browser-smoke: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + # actions/checkout v6.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 + # actions/setup-python v6.3.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 with: python-version: "3.12" cache: pip diff --git a/AGENTS.md b/AGENTS.md index 1452adb..b37dbee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,10 +4,10 @@ - This workspace is the EMUAdvisor project for a local-only EMU Regulation Assistant. - The root folder is the primary Git repository for future `emu-advisor` work. -- `.old/` is an ignored local archive of the previous NLPCrawler demo; treat it as legacy/prototype reference code unless the user explicitly promotes or imports it. +- `.old/` is absent in this checkout. If historical prototype material is recovered from Git history, treat it as non-authoritative unless the user explicitly promotes it. - The project is a research/prototype-to-product workspace, not a finished production app. -## Source Of Truth +## Authority order Use these sources in order: @@ -17,10 +17,13 @@ Use these sources in order: 4. `docs/RUN_PROTOCOL.md` for local setup and verification. 5. `.old/README.md` and `.old/backend/README.md` only for old-demo mechanics and commands. +`DEV_STATE.md` is the active workflow-state authority and the **Active batch** section of +`BLUEPRINT.md` is the accepted task/acceptance authority once a batch is planned. + ## Product Boundaries - V1 is a staff-facing demo for answering questions about official EMU rules and regulations. -- V1 source scope is `mevzuat.emu.edu.tr` plus official PDFs linked from or belonging to that regulation source set. +- V1 source scope is exact-host HTTPS on `mevzuat.emu.edu.tr`; every redirect and final URL must remain in that boundary. File inputs are explicit test fixtures and never official-source evidence. - Runtime behavior must remain local-only; do not add external API dependencies for answering, retrieval, embeddings, reranking, or generation. - English and Turkish regulation corpora must remain separate; do not provide cross-corpus EN/TR search or comparison in V1 unless the project scope is explicitly changed. - Answers must be grounded in retrieved evidence with citations and must refuse, clarify, or state uncertainty when support is insufficient. @@ -39,12 +42,39 @@ Use these sources in order: ## Verification Rules -- This repo has no single standard test command yet. Follow `docs/RUN_PROTOCOL.md`. +- The standard core command is `python -m unittest discover -s tests`; follow `docs/RUN_PROTOCOL.md` for the required explicit safe environment and broader checks. - For documentation-only changes, verify file creation and internal consistency. - For Python code changes, start with syntax checks that do not write bytecode, then run targeted imports or CLI commands as dependencies allow. - For retrieval or answer changes, use a built index plus evaluation queries; do not substitute syntax checks for retrieval validation. - For backend/UI changes, verify FastAPI startup and perform at least one local browser/API smoke check when dependencies and index artifacts exist. +## Protected paths + +- `eval_sets/**`: evaluation inputs and review metadata; changes require explicit provenance and claim reconciliation. +- `artifacts/**`, `logs/**`, local Qdrant/index/corpus outputs, and chat transcripts: generated or potentially sensitive; do not commit them by default. +- `.env*`, tokens, local service configuration, reviewer identities, and any non-public university material. +- `EMU_RAG_Current_System_Specs.md`, `LICENSE`, Git history, remote `main`, tags, releases, and deployments. +- `.old/**`: ignored historical prototype evidence; do not promote or rewrite it without explicit scope. + +## Required commands + +Use the active environment and the narrowest applicable subset from `docs/RUN_PROTOCOL.md`. +Before closing a Python/security/evaluation batch, run at minimum: + +```powershell +python tools\syntax_check.py +python -m unittest discover -s tests +python -m emu_advisor.evaluation eval_sets\v1_gold.jsonl +python -m emu_advisor.evaluation eval_sets\v1_hard.jsonl +python -m emu_advisor.evaluation eval_sets\emu_gold_seed.jsonl +python -m emu_advisor.eval_review status eval_sets\v1_gold.jsonl eval_sets\v1_hard.jsonl eval_sets\emu_gold_seed.jsonl +python tools\publication_guard.py +python tools\browser_smoke.py --start-server +``` + +Also run the dependency audit and platform-specific installation checks defined by the active +batch. Full live crawling, Qdrant, or Ollama work is not implicit and must be separately bounded. + ## Done Criteria End implementation tasks with: diff --git a/BLUEPRINT.md b/BLUEPRINT.md new file mode 100644 index 0000000..2a79104 --- /dev/null +++ b/BLUEPRINT.md @@ -0,0 +1,114 @@ +# Blueprint + +Workflow schema: `agentic-workflow/v2` +Project: EMUAdvisor +Repository profile: mixed +Initialized: 2026-09-05 + +## Product objective + +Maintain a local-only English/Turkish EMU regulation assistant whose source provenance, automated evidence, privacy boundary, and supported development environments are stated and verified honestly. + +## Active batch + +Status: READY +Batch ID: EMU-B001 + +### Objective + +Correct unsupported evaluation-review claims; label automated measurements as regression/evidence proxies; require expected-evidence citation matches; isolate transcripts with server-issued per-session capabilities; make transcript persistence and raw-query logging opt-in; require an explicit fixture or artifact corpus mode; enforce exact HTTPS official-source scope through redirects; and make the pinned Python 3.12 environment verifiable on Windows and Ubuntu. + +### Intended files + +- Evaluation/evidence: `eval_sets/v1_gold.jsonl`, `emu_advisor/evaluation.py`, `emu_advisor/metrics.py`, `emu_advisor/eval_review.py`, `emu_advisor/readiness.py`, `tools/publication_guard.py`, and focused tests. +- Transcript/privacy: `emu_advisor/conversation_store.py`, `emu_advisor/audit_log.py`, `emu_advisor/server.py`, relevant static UI files, browser smoke, and focused tests. +- Corpus/source provenance: `emu_advisor/corpus.py`, `emu_advisor/pipeline.py`, ingestion/schema files only as required, and synthetic fixture tests. +- Portability/CI: dependency declarations, `.github/workflows/ci.yml`, and narrowly scoped verification tooling. +- Current claim/governance surfaces: README, publication/security/readiness/state/run/evaluation documentation, this workflow pack, and authorized `shared/**` records. + +### Allowed adjacent files + +- A small provenance helper or focused security/provenance test module. +- A pinned audit requirements file or narrowly scoped CI-maintenance configuration. +- `static/landing.html`, `static/admin-diagnostics.js`, or `docs/ARCHITECTURE.md` only when needed to expose the accepted corpus/session boundary accurately. + +### Out of scope + +- Human semantic grading, institutional attestation, or invented reviewer evidence. +- Live crawl, Qdrant/Ollama benchmark, corpus/index/metric publication, or transcript/log inspection. +- Retrieval/model-quality improvements, full identity architecture, manifest signing/implementation, or licensing conclusions. +- Repository/history/name changes, presentation media, tag, release, deployment, merge, or production-readiness claim. +- Any change to `EMU_RAG_Current_System_Specs.md`, `eval_sets/v1_hard.jsonl`, `eval_sets/emu_gold_seed.jsonl`, `.old/**`, `artifacts/**`, or `logs/**`. + +### Preconditions + +- Record the baseline branch/HEAD/status and pre-existing bootstrap changes. +- Acquire one cooperative root lock before source edits; keep root as canonical writer. +- Preserve evaluation identities/content/order; only provenance/adjudication fields in `v1_gold.jsonl` are authorized. +- Use synthetic fixtures and temporary paths only. Do not read or alter existing ignored transcripts/logs. +- Resolve official GitHub Action releases to real immutable commit SHAs before pinning them. + +### Implementation plan + +1. Relabel all 60 `v1_gold` rows as assistant-curated and pending independent review; clear unsupported correctness/citation judgments while preserving every protected content field and row order. +2. Require durable evidence references for any future human-reviewed status. Version new metric output as automated proxy evidence and replace semantic-sounding fields with precise retrieval, behavior, citation-presence, expected-evidence citation, format, and latency labels. +3. Match citations to expected chunk evidence, falling back to exact normalized source only when no expected chunks exist; conflict cases require the distinct expected evidence sides. +4. Issue a random session ID and separate random capability server-side. Store only a capability hash; require the capability for continuation/read/export/clear; make enumeration admin-only and omit raw last-message text by default. +5. Disable transcript persistence and raw-query logging by default. Keep opt-ins explicit, bounded, pruned, and tested using temporary paths. +6. Require explicit `fixture` or `artifact` corpus mode. Permit fixture mode only in development/test, fail closed for missing/invalid artifacts and unsafe profiles, expose the mode through status endpoints, and show an unavoidable fixture warning in the user UI. +7. Validate HTTPS, exact host, credentials, port, every redirect target, and final URL before attribution. File fixtures require explicit fixture mode and retain non-laundered, path-safe fixture provenance. +8. Make the lock platform-aware, add Windows and Ubuntu Python 3.12 core CI, pin official actions to verified full SHAs, and pin audit tooling. +9. Add focused negative/regression tests, reconcile current public claims, and define the design-only immutable run-manifest contract without claiming historical reproducibility. +10. Run focused then full verification, freeze the diff, obtain an independent tester verdict, and reconcile QA/state/risk records without removing residual blockers. + +### Acceptance criteria + +- `v1_gold.jsonl` remains 60 rows with protected fields/order unchanged; all rows are assistant-curated/pending independent review with `is_correct` and `citation_ok` null and no human/university-review claim. +- Validation/review status reports 60 pending cases; a self-asserted verified row without the evidence contract fails publication checks. +- New reports use a versioned automated-proxy schema and do not present response accuracy, answer correctness, citation precision/coverage, groundedness, or presentability as semantic evidence. +- Wrong citations fail expected-evidence matching; matching chunks pass; source fallback applies only without expected chunks; conflict cases require both evidence sides. +- Legacy metrics are labeled legacy/unverified and are not rendered as current semantic claims. +- Public session creation produces a distinct opaque ID and high-entropy capability; only the hash is stored. Continuation/read/export/clear require the matching capability and fail uniformly otherwise. +- Public enumeration is unavailable; admin enumeration requires a configured valid admin token and omits raw last-user-message text by default. +- With no opt-in, no transcript file is loaded or written and audit output contains no raw question, session ID, capability, token, or direct identifier. Explicit opt-ins work only against temporary test paths. +- Corpus mode is explicit. Fixture starts only in dev/test and is visibly reported; artifact mode requires a valid nonempty artifact; unsafe unset/unknown/production fixture configurations fail closed. +- Real requests stay on exact-host HTTPS through every redirect/final URL. File fixtures cannot be attributed to an official EMU URL or expose an absolute local path. +- The lock installs under clean Windows and Ubuntu CPython 3.12; core tests run on both CI platforms, browser smoke remains green on its supported runner, and official action refs are verified immutable SHAs. +- Public/governance docs retain the benchmark/presentation/release block and accurately label historical metrics as legacy automated proxies. +- Protected inputs remain unchanged; no live services, generated private artifacts, tag, release, deployment, or presentation action occurs. +- Full verification passes and the independent tester returns `PASS` or `PASS_WITH_RISKS`. + +### Verification + +- Baseline/final: `git status --short --branch`, `git rev-parse HEAD`, `git diff --check`, changed-file inventory, and protected-path diff. +- Python 3.12 with explicit test profile/fixture mode: focused tests, full unittest discovery, all three evaluation validations, review status, publication guard, and browser smoke. +- Focused tests for evaluation transformation, proxy schema/citations, capability isolation, admin enumeration, privacy defaults, corpus startup matrix, redirect prevalidation, and fixture provenance. +- Clean temporary Windows Python 3.12 environment: install pinned lock, imports, full core checks, and pinned dependency audit. +- CI structure check: Windows/Ubuntu Python 3.12, explicit safe environment, and 40-character official-action SHAs with release comments. +- Claim scan and independent tester review against a frozen final diff. + +### Protected inputs + +- `EMU_RAG_Current_System_Specs.md`; all evaluation content except authorized `v1_gold` provenance/adjudication fields; all generated/private `artifacts/**` and `logs/**`; environment files, credentials, identities, private attestations, `.old/**`, `LICENSE`, Git history, remote `main`, tags, releases, and deployments. + +### Risks + +- The capability boundary is local session isolation, not full Internet identity/authentication. +- Legacy persisted sessions intentionally become unavailable to public callers and remain untouched. +- Expected-source fallback is weaker than chunk matching and must be reported as such. +- Windows resolution may expose further platform incompatibilities; stop rather than loosen reproducibility. +- Source-scope hardening may reject legitimate but out-of-policy redirects; record rather than broaden silently. +- The immutable-manifest work is design-only. Human review, live corpus rerun, corpus rights, deployment, and presentation remain open. + +### Rollback + +- Use targeted reverts of EMU-B001 commits; never reset or overwrite bootstrap/user work. +- Do not restore unsupported review/semantic claims for compatibility. +- Do not migrate, delete, or rewrite existing private artifacts. Retain fail-closed source/corpus behavior if a legitimate redirect or environment is unsupported, and record `STOP_NEEDS_HUMAN` when a portable lock cannot be proven. + +### Evidence required for done + +- Matching accepted plan/state, passing bootstrap audit, released-at-close lock, and pre/post file inventory. +- Machine-readable proof that only authorized `v1_gold` fields changed. +- Passing provenance/guard, proxy/citation, transcript/privacy, corpus-mode/source-scope, clean Windows install/audit, full suite, browser smoke, CI, and diff-integrity checks. +- Green independent tester verdict and docs-QA reconciliation that preserves all residual risks and the presentation/release block. diff --git a/DEV_LOG.md b/DEV_LOG.md new file mode 100644 index 0000000..069243c --- /dev/null +++ b/DEV_LOG.md @@ -0,0 +1,53 @@ +# Development Log + +Workflow schema: `agentic-workflow/v2` +Project: EMUAdvisor +Repository profile: mixed +Initialized: 2026-09-05 + +## 2026-09-05 - Repository bootstrap + +- Classified as `mixed` with traits: software, data-ml. +- Created missing governance files without modifying product/source artifacts. +- Classification evidence is recorded in `docs/REPO_PROFILE.md`. + +## 2026-09-05 - EMU-B001 plan accepted + +- Independent repository mapping and risk audit found High evidence, transcript, source-provenance, fixture-mode, and Windows-portability risks. +- Accepted one bounded hardening batch in `BLUEPRINT.md`. +- Benchmark, presentation, release, deployment, live crawl, and human-review claims remain explicitly out of scope. + +## 2026-09-05 - EMU-B001 implementation baseline + +- Branch: `remediation/evidence-security-portability`. +- Base HEAD: `0eb111abe5310c2f60382366eaf26fdceb24dbb7`. +- Pre-implementation tracked diff SHA-256: `4bec1a7d15745442b5b145682397c819a084c3b335f379745660fd1384ca61ad` (`AGENTS.md` bootstrap repair only). +- Pre-implementation untracked inventory: 25 governance/bootstrap files; sorted-name SHA-256 `9455fd43a12511a41ac2e2bf43d7438fbcfcbe8ab045299af46477d07a702d04`. +- Repository bootstrap audit: `PASS`, no errors or warnings. +- Root acquired the cooperative EMU-B001 writer lock before product changes. + +## 2026-09-06 - EMU-B001 executor verification + +- Machine comparison passed for all 60 `v1_gold` rows: only authorized provenance/adjudication fields changed; protected content and ordering are identical to base. +- Focused implementation checkpoint passed 53 tests; expanded suite later passed 81 tests. +- All three evaluation sets validate; review status reports all 60 primary cases pending independent review; publication guard passes. +- Browser smoke passes with visible fixture warning and same-tab capability handling. +- Clean Windows CPython 3.12.10 environment installed the platform-aware lock, passed imports, syntax, 79 tests at that checkpoint, all evaluation/guard checks, `pip check`, and dependency audit. +- Upgraded `pypdf` from 6.15.0 to 6.17.0 after the audit identified three fixed 2026 CVEs; the current lock audit reports no known vulnerabilities. +- Official action pins were resolved from the upstream repositories: checkout v6.0.0 and setup-python v6.3.0. +- Presentation, release, deployment, live crawl/services, corpus publication, and semantic human-review claims remain blocked. + +## 2026-09-06 - EMU-B001 independent TEST and repair + +- The initial frozen snapshot `e195f40ca72ebe3bb1491814dc5121066ae58bb737bfbe5c6b9a824d4854a02f` received `FAIL` because three historical documents placed legacy/unverified caveats after current-tense official-source, board-readiness, or semantic-metric claims. +- The bounded repair moved and expanded the caveats near the top of `docs/EMUAdvisor Full Analysis.md`, `docs/SPRINT_PLAN.md`, and `docs/VERSION_LOG.md`; no product behavior or protected input changed in the repair. +- Fresh independent TEST on repaired snapshot `d56a5dcdd63dbe2a1be14d42b3e16156250ab0394a849757c4269bac4ec11b26` returned `PASS_WITH_RISKS`. +- The repaired snapshot hash matched at docs-QA entry, demonstrating no unexpected product or test side effects between the verdict and reconciliation. + +## 2026-09-06 - EMU-B001 docs-QA closure + +- Acceptance evidence maps to the bounded EMU-B001 criteria: protected evaluation content/order preserved, evidence claims corrected, proxy/citation checks passed, transcript/corpus/source controls tested, and Windows portability verified locally. +- Recorded unavailable or deferred verification as residual risks: remote Ubuntu CI, live official-host crawl/redirect behavior, artifact-backed corpus/Qdrant/Ollama, history-aware secret scan, human semantic review/institutional attestation, corpus rights, and Internet-grade identity/public deployment security. +- Preserved the transient browser-smoke timeout and Starlette/httpx deprecation warning as maintenance risks; two subsequent browser-smoke passes provide bounded local evidence, not a reliability guarantee. +- Closed the cycle as `COMPLETE_WITH_RISKS`; benchmark, presentation, publication, release, deployment, merge, and production-readiness actions remain blocked. +- The first closure audit returned `FAIL` because `DEV_STATE.md` replaced the canonical batch ID with a descriptive closed value while `BLUEPRINT.md` retained `EMU-B001`; docs-QA restored the exact batch ID without changing the closed cycle status, and the rerun returned `PASS` with no errors or warnings. diff --git a/DEV_STATE.md b/DEV_STATE.md new file mode 100644 index 0000000..2e01c3c --- /dev/null +++ b/DEV_STATE.md @@ -0,0 +1,17 @@ +# Development State + +Workflow schema: `agentic-workflow/v2` +Project: EMUAdvisor +Repository profile: mixed +Initialized: 2026-09-05 + +- Phase: CLOSE +- Cycle status: COMPLETE_WITH_RISKS +- Active task: EMU-B001 evidence, privacy, source-scope, fixture, and Windows hardening completed +- Active batch: EMU-B001 +- Owner: root +- Blockers: Benchmark/presentation/publication/release/deployment remain blocked by missing independent human adjudication, reproducible artifact-backed evidence, corpus-rights decision, live-service validation, history-aware secret review, and Internet-grade identity/security review +- Current risks: No Critical risk is open; residual High and Medium gates remain in `RISK_REGISTER.md` +- Tester verdict: PASS_WITH_RISKS on repaired snapshot `d56a5dcdd63dbe2a1be14d42b3e16156250ab0394a849757c4269bac4ec11b26` +- Next action: Root performs the final integrity review, commits the closed batch, pushes the remediation branch, and opens a review PR without merging or releasing +- Last updated: 2026-09-06 diff --git a/PUBLICATION.md b/PUBLICATION.md index 0ee9f69..7fc6d92 100644 --- a/PUBLICATION.md +++ b/PUBLICATION.md @@ -1,28 +1,25 @@ -# Publication Readiness +# Publication Policy -EMUAdvisor may be published by the repository owner. The source code is released under the MIT License. The project remains independent research/software work and is not an official Eastern Mediterranean University administrative service. +Publication, release, and portfolio-presentation readiness are currently blocked. -## Release checklist +## What may be published now -- [x] Publishing rights confirmed by the repository owner. -- [x] MIT license added. -- [x] `eval_sets/v1_gold.jsonl` has been reviewed and verified by the project author and university staff. -- [x] Browser/admin authentication no longer accepts credentials from URL query parameters; protected requests use headers and the browser UI stores credentials only in tab-scoped session storage. -- [x] `eval_sets/emu_gold_seed.jsonl` remains explicitly described as provisional unless it completes the same review process. -- [x] Generated crawl, index, audit, review, and conversation artifacts remain outside the committed public-release workflow. -- [x] Pinned Python dependencies and a dependency audit are part of CI. -- [ ] Confirm no secrets or non-public university material exist in Git history before changing repository visibility. -- [ ] Run the required GitHub Actions core and browser-smoke jobs successfully on the final publication branch. -- [x] Keep the README disclaimer that this is independent research software, not an official EMU administrative service. +- Source code, tests, schemas, and current documentation under the repository license. +- The tracked assistant-curated evaluation questions and labels, with their pending independent-review status stated accurately. +- Automated test results described only as software/regression evidence for the named commit and environment. -## Evaluation terminology +## What must not be claimed -`eval_sets/v1_gold.jsonl` is the verified human-reviewed gold evaluation set. Its cases were reviewed by the project author and university staff. Public metrics reported for `v1_gold` should still be described as local evaluation results for this fixed corpus and implementation, not production-service guarantees or evidence of universal model quality. +- That `v1_gold.jsonl` is verified gold or that university personnel reviewed it. +- That automated proxy values establish semantic answer correctness, citation precision, groundedness, legal reliability, or production quality. +- That fixture-mode output represents the official corpus. +- That historical ignored corpus/metric artifacts are reproducible or immutable without their required manifests and hashes. +- That the application is an official EMU service or ready for public Internet exposure. -`eval_sets/v1_hard.jsonl` is a hard regression suite focused on difficult table, grouped-query, and refusal behavior. It is not presented as a second gold benchmark unless separately reviewed and documented as such. +## Excluded artifacts -`eval_sets/emu_gold_seed.jsonl` remains a provisional seed set until its source bindings and labels complete the same review process. The historical filename is retained for compatibility; public documentation should call it a provisional evaluation seed rather than a verified gold benchmark. +Do not commit or publish corpora, crawled pages, indexes, Qdrant state, generated metrics, review working files, transcripts, audit logs, credentials, private attestations, or machine-local paths. The code license does not determine rights in external source content. -## Public-release principle +## Gate to revisit publication -The public repository should demonstrate retrieval, citation, refusal, evaluation, local-generation, and operational safeguards without implying institutional endorsement or production deployment. Claims should distinguish verified benchmark results from regression results, provisional evaluation material, and runtime/deployment limitations. +Publication needs, at minimum, independent evaluation review with a privacy-safe durable evidence reference; an implemented immutable run manifest; a lawful corpus-distribution decision; reproducible artifact-backed proxy results; live service/deployment validation; and an independent security/QA verdict. EMU-B001 does not satisfy those later gates by itself. diff --git a/QA_REPORT.md b/QA_REPORT.md new file mode 100644 index 0000000..dd43074 --- /dev/null +++ b/QA_REPORT.md @@ -0,0 +1,71 @@ +# QA Report + +Workflow schema: `agentic-workflow/v2` +Project: EMUAdvisor +Repository profile: mixed +Initialized: 2026-09-05 + +## Current cycle + +- Batch: EMU-B001 +- Verdict: PASS_WITH_RISKS +- Closure: COMPLETE_WITH_RISKS +- Tested snapshot: `d56a5dcdd63dbe2a1be14d42b3e16156250ab0394a849757c4269bac4ec11b26` +- Evidence: Fresh independent TEST passed the repaired frozen diff. Residual external, human, live-service, reproducibility, rights, identity, and maintenance checks remain explicit below. + +## Verdict history + +1. Initial snapshot `e195f40ca72ebe3bb1491814dc5121066ae58bb737bfbe5c6b9a824d4854a02f`: `FAIL`. + - `docs/EMUAdvisor Full Analysis.md` presented current-tense official-source/board-readiness language before its historical disclaimer. + - `docs/SPRINT_PLAN.md` placed its disclaimer after historical readiness content. + - `docs/VERSION_LOG.md` placed its legacy-metric caveat after historical semantic metric claims. +2. Bounded documentation repair: caveats moved and expanded ahead of the claims in those three files; no product behavior or protected evaluation content changed. +3. Repaired snapshot `d56a5dcdd63dbe2a1be14d42b3e16156250ab0394a849757c4269bac4ec11b26`: fresh `PASS_WITH_RISKS`. +4. Docs-QA recomputed the same repaired aggregate snapshot before reconciliation; no tester-introduced source/product side effect was present. + +## Executor evidence + +- Syntax: 42 Python files passed. +- Focused tests: 11/11 passed independently. +- Full tests: 81/81 passed twice, including a fresh Windows CPython 3.12 environment. +- Evaluation validation: 60 primary, 50 hard-regression, and 50 provisional-seed cases passed schema checks. +- Review status: 60/60 primary cases and 160/160 combined cases are pending/non-verified. +- Publication guard: passed, including rejection of unsupported human-review evidence. +- Browser smoke: passed twice in explicit fixture mode after one transient timeout. +- Windows clean install: Python 3.12.10 lock/install/import/core checks and `pip check` passed. +- Dependency audit: no known vulnerabilities after `pypdf==6.17.0` update. +- Protected-input comparison: all 60 `v1_gold` records preserved content/order and changed only the four authorized provenance/adjudication fields; other protected paths have no diff. +- CI/source checks: Windows/Ubuntu Python 3.12 matrix and immutable official Action SHAs verified; tracked-tree credential scan found no matches. + +## Bootstrap validation + +- First closure audit: `FAIL` on an exact active-batch mismatch between `DEV_STATE.md` and `BLUEPRINT.md`; docs-QA corrected the state field without changing the closure verdict. +- `docs/BOOTSTRAP_AUDIT.md` rerun: `PASS`, no errors or warnings. + +## Acceptance mapping + +| Acceptance area | Evidence | Result | +| --- | --- | --- | +| Evaluation provenance | 60-row machine comparison; only four authorized fields changed; all primary judgments null/pending | PASS | +| Automated evidence honesty | Proxy-schema tests, evaluation validation, publication guard, and caveat-position retest | PASS | +| Expected-evidence citations | Focused wrong/matching/fallback/conflict tests | PASS | +| Transcript isolation/privacy defaults | Focused capability, cross-session, admin, persistence, legacy-file, and audit-minimization tests | PASS | +| Corpus/source provenance | Explicit-mode startup matrix, fixture provenance, and redirect-prevalidation tests | PASS_WITH_RISKS — live official-host crawl not run | +| Windows/Ubuntu portability | Fresh Windows 3.12 install/full suite; CI matrix and immutable refs inspected | PASS_WITH_RISKS — remote Ubuntu CI pending | +| Protected boundaries | Diff comparison, tracked-tree credential scan, and no live/generated/private artifact action | PASS_WITH_RISKS — history-aware scan and owner cleanup remain separate | +| Full verification | Syntax 42 files, full suite 81/81, three evaluation validations, review status, guard, browser smoke, dependency audit | PASS_WITH_RISKS — one transient browser timeout and deprecation warning retained | + +## Unavailable or deferred checks + +- Remote Ubuntu CI: run and retain the GitHub Actions result after push. +- Live exact-host crawl and redirect chain: perform only in a separately authorized, controlled live-ingestion batch. +- Artifact-backed corpus, Qdrant, Ollama, target hardware, and deployment: reproduce with an implemented immutable run manifest before claims. +- Human semantic review/institutional attestation: obtain privacy-safe durable reviewer evidence; automated checks cannot substitute. +- Corpus redistribution rights: obtain an owner/legal decision before publishing derived corpus content. +- Internet-grade identity and public deployment threat model: complete security review before any network-exposed demo. +- History-aware secret/non-public-material scan: run before presentation, release, or merge to a public successor. +- Maintenance: investigate the transient browser-smoke timeout if it recurs and resolve the Starlette/httpx deprecation before it becomes incompatible. + +## QA conclusion + +The bounded EMU-B001 implementation meets its acceptance criteria and the independent tester verdict is preserved as `PASS_WITH_RISKS`. No Critical risk is open. This closes the development cycle, not the presentation/publication/release gates. diff --git a/README.md b/README.md index eabe272..6cf5be9 100644 --- a/README.md +++ b/README.md @@ -1,246 +1,61 @@ # EMUAdvisor -Local RAG assistant for answering Eastern Mediterranean University regulation questions from official cited sources. - -EMUAdvisor crawls official EMU regulation pages and linked PDFs, builds language-separated retrieval corpora, retrieves and reranks supporting evidence, and produces cited answers with refusal and clarification behavior when evidence is insufficient or the question falls outside the indexed regulation scope. - -> **Independent project:** EMUAdvisor is research/software work and is not an official Eastern Mediterranean University administrative service. Its answers are informational and must not be treated as final university decisions. - -## What this project demonstrates - -- end-to-end RAG ingestion over official HTML and PDF sources; -- English/Turkish corpus routing and query understanding; -- hybrid retrieval with structured table evidence and optional Qdrant storage; -- extractive answers, optional local Ollama generation, and fallback behavior; -- citations, refusal/clarification policies, and conflict-aware responses; -- fixed-set evaluation, human-reviewed gold data, regression suites, and review tooling; -- FastAPI APIs, local diagnostics, browser smoke testing, CI, and reproducible dependency snapshots. - -## Architecture - -```text -Official EMU HTML/PDF sources - | - v - crawler -> parser/table extraction -> normalized chunks - | | - | v - | local or Qdrant index - | | - +-----------------------------+ - v -question -> language/query understanding -> hybrid retrieval - | - v - evidence + answerability decision - | | - v v - extractive answer refuse/clarify - | - +---- optional local Ollama generation - | - v - cited user response -``` - -## Evaluation snapshot - -The primary benchmark is `eval_sets/v1_gold.jsonl`, a **60-case human-reviewed and verified gold set** reviewed by the project author and university staff. - -Recorded extractive results on the fixed corpus snapshot: - -| Metric | `v1_gold` | -|---|---:| -| Cases | 60 | -| Retrieval top-1 | 92.31% | -| Retrieval top-3 | 98.08% | -| Retrieval top-5 | 100.00% | -| Response accuracy | 100.00% | -| Rejection accuracy | 100.00% | -| Clarification accuracy | 100.00% | -| Citation coverage | 100.00% | -| Extractive latency p50 | 674 ms | -| Extractive latency p95 | 1,267 ms | - -These are local results on a fixed benchmark and corpus snapshot, **not production-service guarantees or universal model-quality claims**. - -Additional evaluation material is intentionally separated: - -- `eval_sets/v1_hard.jsonl`: 50-case hard regression suite focused on table/broad-query/refusal behavior; -- `eval_sets/emu_gold_seed.jsonl`: provisional evaluation seed retained under its historical filename until it completes the same review process. - -See `docs/DEMO_METRICS_SNAPSHOT.md` for the full recorded snapshot and `PUBLICATION.md` for evaluation/publication terminology. - -## Corpus snapshot - -The recorded demo corpus contains: - -- 8,714 chunks from 119 official sources; -- 22 linked official PDFs; -- 3,878 English and 4,836 Turkish chunks; -- structured table summaries, row-level table chunks, and derived table facts. - -Generated crawl/index artifacts are intentionally ignored and are not committed to the repository. - -## Setup - -Python 3.12 is the CI reference environment. +EMUAdvisor is an independent, local-only English/Turkish retrieval assistant for public Eastern Mediterranean University regulations. It is a research/demo project, not an official university service, decision-maker, or production deployment. -For the reproducible portfolio/demo environment: +## Current status -```powershell -python -m venv .venv -.\.venv\Scripts\Activate.ps1 -python -m pip install --upgrade pip -pip install -r requirements-lock.txt -``` - -`requirements.txt` and `requirements-dev.txt` retain the broader dependency declarations; `requirements-lock.txt` records the exact tested snapshot used by CI. - -Optional local services: +The software path is active, but benchmark, board-demo, publication, release, and production-readiness claims are blocked. The tracked 60-case `eval_sets/v1_gold.jsonl` file is an assistant-curated regression set pending independent human review. Its historical filename is retained for compatibility; it is not verified gold evidence. -- Ollama generation model: `qwen3:8b`; -- Ollama embedding model: `qwen3-embedding:4b`; -- Qdrant for service-backed vector storage. +Historical percentages in older project records were produced by automated retrieval/behavior proxies. They did not semantically grade answer correctness, citation precision, or claim-level grounding. New metric output uses schema `emu-advisor-automated-proxy/v2` and labels those observations explicitly. -The answering/retrieval path does not require external hosted LLM APIs. +## Trust boundary -## Build a local corpus +- Runtime answering and optional generation remain local. No external answering, retrieval, or model API is used. +- Real corpus ingestion accepts only HTTPS URLs on exactly `mevzuat.emu.edu.tr`, validates redirect targets before requesting them, and records the validated final URL. +- Fixture mode is explicit, limited to development/test, visibly labeled, and never attributed to the official source domain. +- Chat sessions use a server-issued opaque ID plus a separate per-session capability. Public callers cannot enumerate sessions. +- Transcript persistence, audit logging, and raw-query logging are disabled by default and require separate positive opt-ins. +- Generated corpora, indexes, metrics, reviews, transcripts, and logs remain outside Git. -```powershell -python -m emu_advisor.pipeline build ` - --seed https://mevzuat.emu.edu.tr/content.htm ` - --seed https://mevzuat.emu.edu.tr/Content-en.htm ` - --out artifacts\demo_corpus\latest ` - --max-pages 1000 ` - --include-pdfs - -python -m emu_advisor.validate_jsonl ` - artifacts\demo_corpus\latest\chunks.jsonl ` - --kind chunk -``` +## Quick start (explicit fixture mode) -The application loads `artifacts/demo_corpus/latest/chunks.jsonl` when present and otherwise falls back to a small fixture corpus for development/testing. - -## Run the application +PowerShell: ```powershell +$env:EMU_ADVISOR_PROFILE = "dev" +$env:EMU_ADVISOR_CORPUS_MODE = "fixture" +$env:EMU_ADVISOR_QUERY_REWRITE = "deterministic" python -m uvicorn emu_advisor.server:app --host 127.0.0.1 --port 8000 ``` -Then open: - -- `http://127.0.0.1:8000` for the landing page; -- `http://127.0.0.1:8000/admin?view=user` for the staff-facing chat; -- `http://127.0.0.1:8000/admin?view=diagnostics` for local diagnostics. +Fixture mode uses four synthetic records and displays a warning. It is for software verification only. -### Admin authentication - -Protected profiles use `EMU_ADVISOR_ADMIN_TOKEN`. - -```powershell -$env:EMU_ADVISOR_PROFILE="production" -$env:EMU_ADVISOR_ADMIN_TOKEN="" -``` - -API clients send the token through either: - -```text -Authorization: Bearer -``` - -or: - -```text -X-EMU-Admin-Token: -``` - -The browser diagnostics view provides a password-style token field. The value is stored only in the current tab's `sessionStorage` and sent through the `Authorization` header. **Admin tokens are not accepted from URL query parameters.** - -See `SECURITY.md` for the security boundary and deployment cautions. - -## API surface - -Public/user path: - -- `GET /health` -- `GET /whoami` -- `POST /chat` -- `POST /chat/stream` - -Diagnostics/admin path: - -- `GET /metrics` -- `GET /metrics/modes` -- `GET /analytics` -- `GET /corpus/status` -- `GET /llm/status` -- `POST /ask` -- `POST /ask/stream` - -`POST /chat` returns the sanitized user-facing response. `POST /ask` exposes richer retrieval/generation diagnostics for the local admin console. - -## Qdrant backend - -Development defaults to the local in-memory store. A production profile requires Qdrant. - -```powershell -$env:EMU_ADVISOR_VECTOR_BACKEND="qdrant" -$env:EMU_ADVISOR_QDRANT_URL="http://localhost:6333" -$env:EMU_ADVISOR_QDRANT_COLLECTION="emu_regulations" - -python -m emu_advisor.index build ` - --chunks artifacts\demo_corpus\latest\chunks.jsonl ` - --backend qdrant ` - --collection emu_regulations -``` - -Embedded local Qdrant can also be used for development when a service is unavailable. - -## Evaluation and review - -Run the verified gold set: - -```powershell -python -m emu_advisor.evaluation eval_sets\v1_gold.jsonl -python -m emu_advisor.eval_review status eval_sets\v1_gold.jsonl -``` - -Run auxiliary regression/evaluation material: - -```powershell -python -m emu_advisor.evaluation eval_sets\v1_hard.jsonl -python -m emu_advisor.evaluation eval_sets\emu_gold_seed.jsonl -python -m emu_advisor.eval_review status eval_sets\v1_hard.jsonl eval_sets\emu_gold_seed.jsonl -``` - -Metrics runs write ignored artifacts such as `metrics.json`, `per_case.csv`, and review CSVs under `artifacts/`. +For an artifact-backed run, set `EMU_ADVISOR_CORPUS_MODE=artifact` and provide a valid nonempty `artifacts/demo_corpus/latest/chunks.jsonl`. Missing or invalid artifacts fail closed. Production also requires an admin token and service-backed Qdrant; those paths are not currently release-validated. ## Verification -The GitHub Actions pipeline uses the pinned dependency snapshot and requires both core verification and the browser smoke test. - -Equivalent local checks: - ```powershell -python tools\publication_guard.py +$env:EMU_ADVISOR_PROFILE = "test" +$env:EMU_ADVISOR_CORPUS_MODE = "fixture" +$env:EMU_ADVISOR_QUERY_REWRITE = "deterministic" +$env:EMU_ADVISOR_ENABLE_CHAT_PERSISTENCE = "0" +$env:EMU_ADVISOR_ENABLE_AUDIT_LOGGING = "0" +python tools/syntax_check.py python -m unittest discover -s tests -python -m emu_advisor.evaluation eval_sets\v1_gold.jsonl -python -m emu_advisor.evaluation eval_sets\v1_hard.jsonl -python -m emu_advisor.evaluation eval_sets\emu_gold_seed.jsonl -python tools\browser_smoke.py --start-server +python -m emu_advisor.evaluation eval_sets/v1_gold.jsonl +python -m emu_advisor.evaluation eval_sets/v1_hard.jsonl +python -m emu_advisor.evaluation eval_sets/emu_gold_seed.jsonl +python -m emu_advisor.eval_review status eval_sets/v1_gold.jsonl eval_sets/v1_hard.jsonl eval_sets/emu_gold_seed.jsonl +python tools/publication_guard.py +python tools/browser_smoke.py --start-server ``` -CI additionally audits the pinned Python dependencies with `pip-audit`. +See `docs/RUN_PROTOCOL.md` for clean-environment and dependency-audit commands. -## Known limits +## Evidence limits -- The recorded metrics describe a fixed local evaluation/corpus snapshot, not production availability. -- `v1_gold` is verified; `v1_hard` and `emu_gold_seed` retain separate review status and should not be silently merged into the gold claim. -- Generated answers are optional and fallback-safe; the recorded bounded `qwen3:8b` smoke run timed out at a 2-second generation limit. -- A service-backed Qdrant production deployment and target production hardware have not yet been validated in the recorded environment. -- The repository is intended for local research, evaluation, and demonstration; public Internet deployment requires an independent security/deployment review. +Passing automated tests establishes software behavior against synthetic/tracked inputs. It does not establish semantic answer quality, institutional approval, corpus redistribution rights, live-source reproducibility, public-Internet security, or production fitness. See `docs/CLAIM_REGISTER.md`, `docs/DATA_MANIFEST.md`, and `RISK_REGISTER.md`. ## License -MIT. See `LICENSE`. +Code is licensed under `LICENSE`. That license does not grant rights to redistribute source-site content or generated corpus artifacts. diff --git a/RISK_REGISTER.md b/RISK_REGISTER.md new file mode 100644 index 0000000..05f0feb --- /dev/null +++ b/RISK_REGISTER.md @@ -0,0 +1,35 @@ +# Risk Register + +Workflow schema: `agentic-workflow/v2` +Project: EMUAdvisor +Repository profile: mixed software/data-ML/research +Initialized: 2026-09-05 + +## Active risks + +| ID | Severity | Status | Risk and evidence | Required control | +| --- | --- | --- | --- | --- | +| EMU-R01 | High | Mitigated in EMU-B001; residual gate open | Unsupported verified/human/university-review labels were removed from all 60 primary rows; the publication guard enforces a durable evidence contract. No independent semantic review or institutional attestation exists. | Keep benchmark/presentation/release blocked until privacy-safe durable human-review evidence is supplied. | +| EMU-R02 | High | Mitigated in EMU-B001; residual gate open | New reports use automated-proxy v2 labels and expected-evidence citation matching; historical semantic percentages are caveated before claims. Automated proxies still do not prove answer correctness or legal reliability. | Permit semantic claims only after independent adjudication and reproducible artifact-backed evaluation. | +| EMU-R03 | High | Mitigated for local use; public deployment gate open | Public session operations now require a server-issued capability and enumeration is minimized/admin-only; negative cross-session tests pass. This is session isolation, not Internet-grade identity. | Complete identity, abuse, and deployment threat-model review before any network-exposed demo. | +| EMU-R04 | High | Mitigated by default; owner-data action open | Persistence, audit logging, and raw-query logging are separate positive opt-ins and privacy tests pass. Existing ignored transcript/log files were intentionally neither read nor deleted. | Keep ignored files protected; owner decides cleanup/migration separately and validates opt-in retention in any deployment. | +| EMU-R05 | High | Mitigated synthetically; live verification open | Exact-host HTTPS, redirect-hop/final validation, credential/port rejection, and non-laundered fixture provenance are covered by tests. No live official-host crawl was run. | Run a separately authorized controlled live crawl and retain redirect/provenance evidence before source claims. | +| EMU-R06 | High | Mitigated in configuration; artifact gate open | Corpus mode is explicit, fixture is dev/test-only and visible, and invalid/unset/production-fixture configurations fail closed. Artifact-backed corpus startup was not exercised. | Validate a hashed nonempty artifact and target-service path before corpus-backed demonstration claims. | +| EMU-R07 | High | Mitigated locally; remote CI gate open | Platform markers, Windows dependencies, Windows/Ubuntu Python 3.12 CI, and immutable Action refs are present; a fresh Windows 3.12 install/full suite passed. Remote Ubuntu CI has not run on this branch. | Require green remote matrix checks after push before merge or release consideration. | +| EMU-R08 | Medium | Partially mitigated — later | Audit tooling and official Actions are pinned, but lock generation inputs/hashes remain incompletely documented. | Record reproducible lock-generation inputs and integrity hashes in a later dependency-maintenance batch. | +| EMU-R09 | High | Open — later | Published benchmark inputs/results lack immutable corpus/eval/commit/environment hashes and the public set has been iterated against implementation. | Add immutable run manifests and a held-out/adjudicated evaluation protocol before publishing benchmark claims. | +| EMU-R10 | Medium | Open — later | MIT covers software, not necessarily redistribution of regulation-derived corpus/evaluation content. | Keep corpus out of Git, publish minimal/synthetic fixtures and source/hash manifests, and obtain rights confirmation before substantial redistribution. | +| EMU-R11 | Medium | Open — later | Corpus/metric writers can overwrite canonical targets without a common staged atomic promotion policy. | Write unique run directories, validate, atomically promote pointers, and retain rollback metadata. | +| EMU-R12 | High | Mitigated — EMU-B001 | Bootstrap audit passed; scope, protected inputs, baseline, one writer, rollback, independent TEST, and docs-QA evidence are recorded. | Preserve the workflow evidence and reacquire a scoped lock for any future batch. | +| EMU-R13 | Medium | Open — later | Current-tree common-secret scan is clean, but a dedicated redacted history-aware scan is not recorded in the publication gate. | Run a history-aware secret/non-public-material review before presentation or release. | +| EMU-R14 | Medium | Open — maintenance | Browser smoke passed twice after one transient timeout; tests also emit a Starlette/httpx deprecation warning. Current evidence supports the batch but not long-term reliability. | Observe remote CI, investigate recurrence, and update the compatible dependency/test-client path before deprecation becomes failure. | + +## Release rule + +No benchmark, presentation, publication, network-exposed demo, tag, release, deployment, or +production-readiness claim may proceed while the residual gates in EMU-R01 through EMU-R07, +EMU-R09, EMU-R10, and EMU-R13 remain unresolved. A green remote Windows/Ubuntu CI result is also +required before merge consideration. Local implementation/testing may use synthetic fixtures without +reading or publishing ignored user artifacts. Full live crawling, Qdrant/Ollama validation, corpus +redistribution, and deletion of existing local transcripts/logs require separate explicit scope or +owner action. diff --git a/SECURITY.md b/SECURITY.md index 6ee60af..3943b6c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,40 +1,27 @@ -# Security Policy +# Security and Privacy -EMUAdvisor is a local research/demo application and is not an official Eastern Mediterranean University service. +EMUAdvisor is designed for local use and is not approved for public Internet deployment. -## Supported use +## Session isolation -The repository is intended for local development, evaluation, and demonstration. It should not be exposed directly to the public Internet without an independent deployment/security review, service hardening, and validation of the target Qdrant/runtime environment. +Public chat creation returns a server-issued opaque session ID and a separate high-entropy capability. The browser keeps both in same-tab `sessionStorage` and sends the capability only in the `X-EMU-Session-Capability` header. Continuation, transcript read, export, and clear require the matching capability. The server stores only its SHA-256 hash. Public session enumeration is unavailable; administrative enumeration requires a configured admin token and omits message text. -## Admin authentication +This is capability-based local isolation, not full user authentication. Anyone who can read the browser tab or steal the capability can act on that session. -When `EMU_ADVISOR_PROFILE=production`, `EMU_ADVISOR_ADMIN_TOKEN` is required for protected diagnostic/data APIs. Admin credentials must be supplied through request headers. +## Data minimization defaults -The static `/admin` application shell remains reachable so a user can enter the credential locally; loading that HTML does not grant access to protected metrics, retrieval diagnostics, analytics, corpus status, or `/ask` responses. +- Transcript persistence: off unless `EMU_ADVISOR_ENABLE_CHAT_PERSISTENCE=1`. +- Audit logging: off unless `EMU_ADVISOR_ENABLE_AUDIT_LOGGING=1`. +- Raw questions in audit records: off unless `EMU_ADVISOR_LOG_RAW_QUERY=1`. -Tokens must **not** be placed in URLs, query parameters, source files, screenshots, logs, or committed environment files. Query parameters are not an authentication mechanism. +When persistence is enabled, plaintext transcript content is written to the configured local path with bounded message count and TTL pruning; capability cleartext is never stored. Protect the OS account and file permissions. Existing legacy local transcript/log files are not loaded under default settings and require an explicit owner-led cleanup/migration decision. -The browser diagnostics client accepts a token through an explicit password-style field, stores it in `sessionStorage` for the current browser tab only, and sends it as an `Authorization: Bearer` header. Clearing or closing the tab removes that browser-session credential. +## Source and runtime controls -## Secrets and local artifacts +Official ingestion is restricted to HTTPS on exactly `mevzuat.emu.edu.tr`, without credentials or nonstandard ports. Each redirect target is validated before request and the final response URL is revalidated. Local files are accepted only through explicit fixture ingestion and retain non-official fixture provenance. -Do not commit: - -- `.env` files or service credentials; -- local Qdrant data; -- generated crawl/index artifacts; -- audit logs or chat histories; -- human-review working files containing information not intended for publication; -- private university material not already publicly available from official sources. - -## Dependency security - -`requirements-lock.txt` is the pinned Python environment used by CI. CI installs that snapshot and runs `pip-audit` before the test/evaluation stages. Dependency updates should regenerate the lock file and rerun the complete core and browser verification jobs. - -## Publication safeguards - -`tools/publication_guard.py` enforces key release invariants, including verified-gold metadata, required publication files, and the absence of URL-based admin-token examples in the public tree. +Fixture corpus mode is limited to development/test and visibly labeled. Artifact mode requires a valid nonempty corpus. Production refuses fixture mode, requires an admin token, and retains the documented Qdrant requirement. ## Reporting -If you find a security issue, report it privately to the repository owner rather than opening a public issue containing exploit details, credentials, or sensitive data. +Do not place secrets, private transcript content, corpus data, or security-sensitive reproduction material in a public issue. Use the repository owner’s private contact channel. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..11aabe8 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,36 @@ +# Architecture + +Workflow schema: `agentic-workflow/v2` + +## System boundary + +EMUAdvisor is a local-only FastAPI/browser research demo over public regulation HTML/PDF from the +exact host `mevzuat.emu.edu.tr`. English and Turkish remain separate corpora. External answering, +embedding, reranking, or generation APIs are out of scope; Ollama and Qdrant are optional local +services. The application is independent and unofficial. + +## Components and flow + +```text +official HTML/PDF -> pipeline/html_ingest/pdf_ingest -> canonical ignored chunks + -> local or Qdrant hybrid retrieval -> answerability/extractive answer + -> optional local Ollama generation -> FastAPI -> static user/admin UI +tracked evaluation cases + ignored corpus -> metrics -> ignored run artifacts -> reviewed summaries +``` + +- `emu_advisor/schema.py` is the canonical document/chunk contract. +- `emu_advisor/server.py` is the API/UI composition boundary. +- `emu_advisor/conversation_store.py` issues opaque session IDs/capabilities and stores capability + hashes; persistence is opt-in. `audit_log.py` emits minimized optional records. +- `emu_advisor/evaluation.py`, `metrics.py`, and `eval_review.py` define evaluation behavior. +- `artifacts/**` and `logs/**` are generated/protected, not publication inputs by default. + +## Current trust controls and debt + +Corpus startup is explicit fixture/artifact mode; fixture records are visibly synthetic and carry +non-official provenance. Real ingestion validates exact-host HTTPS before every request and after the +final response. The browser keeps each cleartext session capability in same-tab storage only. + +Remaining debt includes full Internet identity/authentication, opt-in transcript encryption, +implemented immutable run manifests, independent human evaluation, live artifact/service validation, +corpus-rights decisions, and deployment review. Large-module refactoring remains deferred. diff --git a/docs/BOARD_DEMO_READINESS.md b/docs/BOARD_DEMO_READINESS.md index ae86917..464bf83 100644 --- a/docs/BOARD_DEMO_READINESS.md +++ b/docs/BOARD_DEMO_READINESS.md @@ -1,30 +1,19 @@ # Board Demo Readiness -Status: `partial` +Status: `blocked` -The local board demo is presentable with named deployment gaps that must not be described as production-ready. +The repository is not currently approved for board-demo, portfolio-presentation, release, deployment, or production claims. -## Checks +## Evidence status -| Check | Status | Detail | +| Area | Status | Meaning | |---|---|---| -| current product spec | `pass` | EMU_RAG_Current_System_Specs.md | -| full analysis | `pass` | docs/EMUAdvisor Full Analysis.md | -| demo storyboard | `pass` | docs/DEMO_STORYBOARD.md | -| publication checklist | `pass` | docs/PUBLICATION_CHECKLIST.md | -| verified gold evaluation set | `pass` | eval_sets/v1_gold.jsonl | -| hard regression set | `pass` | eval_sets/v1_hard.jsonl | -| latest metrics artifact | `pass` | top5=1.0 citation=1.0 | -| verified gold review status | `pass` | 60 cases; 0 pending | -| auxiliary evaluation review status | `partial` | 100 hard/seed cases retain separate review status | -| production admin token | `partial` | not set for this local check | -| live Qdrant service | `blocked` | service-backed Qdrant not documented in environment | -| local analytics log | `pass` | 193 audit events in recorded snapshot | +| Software regression suite | In progress | Automated checks can verify implementation behavior only. | +| `v1_gold.jsonl` | Blocked | 60 assistant-curated cases pending independent human review. | +| Historical percentages | Legacy/unverified | Automated proxy outputs, not semantic answer-quality evidence. | +| Corpus artifacts | Blocked | Not present in Git; no immutable reproducibility manifest for historical runs. | +| Fixture mode | Test-only | Synthetic data, visibly labeled, never official-corpus evidence. | +| Production services | Blocked | Live Qdrant/Ollama/target hardware/deployment are unverified. | +| Publication rights | Blocked | Code license does not resolve corpus redistribution rights. | -## Non-Negotiable Limits - -- This is a board-demo readiness report, not a production approval. -- `v1_gold` is human-reviewed and verified by the project author and university staff. -- `v1_hard` is a regression suite and `emu_gold_seed` is provisional unless separately reviewed and documented. -- Production-style deployment remains blocked until service-backed Qdrant and target hardware are validated. -- Generated mode remains extractive-first unless local model latency and answer quality are characterized. +Re-run `python -m emu_advisor.readiness --out docs/BOARD_DEMO_READINESS.md` only in a controlled workspace. The generated report must preserve these blockers while independent review is pending. diff --git a/docs/BOOTSTRAP_AUDIT.md b/docs/BOOTSTRAP_AUDIT.md new file mode 100644 index 0000000..9ca2d25 --- /dev/null +++ b/docs/BOOTSTRAP_AUDIT.md @@ -0,0 +1,20 @@ +# Bootstrap Audit + +Date: 2026-09-06 +Verdict: PASS + +## Errors + +- None + +## Warnings + +- None + +## Notes + +- Repository profile: mixed; traits: ['software', 'data-ml'] + +## Handoff + +- READY_FOR_DEV_LOOP diff --git a/docs/CLAIM_REGISTER.md b/docs/CLAIM_REGISTER.md new file mode 100644 index 0000000..18958a5 --- /dev/null +++ b/docs/CLAIM_REGISTER.md @@ -0,0 +1,14 @@ +# Claim Register + +| ID | Public claim | Status | Required evidence/action | +| --- | --- | --- | --- | +| EMU-C01 | Local unofficial assistant over public EMU regulations | Supported | Keep scope/disclaimer prominent. | +| EMU-C02 | Separate English/Turkish retrieval | Supported in code/tests | Retain regression coverage. | +| EMU-C03 | 8,714 chunks, 119 sources, 22 PDFs | Dated historical/current-site observation | Bind any future use to a manifest and crawl date. | +| EMU-C04 | 60-case human-reviewed/verified gold set reviewed by university staff | Unsupported by durable repository evidence | Relabel assistant-curated/pending independent review unless attestation is supplied. | +| EMU-C05 | 100% response accuracy | Unsupported semantic label | Rename to observable proxy behavior and do not claim answer correctness. | +| EMU-C06 | 100% citation precision | Unsupported; implementation measures presence | Report citation presence and expected-evidence matching separately. | +| EMU-C07 | 100% groundedness | Unsupported; no claim/evidence semantic check | Remove until independently adjudicated. | +| EMU-C08 | Windows reproducible locked setup | False before EMU-B001 | Add platform marker and Windows CI evidence. | +| EMU-C09 | Production profile protects conversation data | False before EMU-B001 | Add transcript ownership/admin boundary and privacy tests. | +| EMU-C10 | Browser smoke validates real product corpus | False when fallback fixture is used | Make fixture mode explicit and fail closed for real/presentation profiles. | diff --git a/docs/DATA_MANIFEST.md b/docs/DATA_MANIFEST.md new file mode 100644 index 0000000..46bd46d --- /dev/null +++ b/docs/DATA_MANIFEST.md @@ -0,0 +1,47 @@ +# Data Manifest + +Workflow schema: `agentic-workflow/v2` + +## Sources and rights boundary + +- Runtime source scope: public HTML/PDF from `https://mevzuat.emu.edu.tr` only. +- Full crawl/corpus/index/metrics/review artifacts remain ignored and are not licensed by the MIT + software license. Public availability does not establish redistribution permission. +- Tracked test fixtures should be synthetic/minimal. Substantial third-party source text requires + separate rights confirmation. + +## Tracked evaluation inputs + +- `eval_sets/v1_gold.jsonl`: 60 assistant-curated bilingual cases pending independent human review; + external review is not evidenced in the repository. +- `eval_sets/v1_hard.jsonl`: 50 assistant-curated hard regression cases. +- `eval_sets/emu_gold_seed.jsonl`: 50 provisional source-binding/human-review-pending seed cases. + +## Generated/protected data + +- `artifacts/demo_corpus/**`: canonical chunks and crawl manifests. +- `artifacts/qdrant/**`: local vector state. +- `artifacts/metrics/**`: per-run metrics/results. +- `artifacts/review/**`: review working data, potentially sensitive. +- `artifacts/chat_sessions.json` and `logs/audit.jsonl`: local transcript/query data; never publish. + +## Immutable run-manifest design + +This is a design contract, not an implemented claim about historical runs. Each future run directory +must be append-only and named by a unique run ID. A canonical JSON manifest must contain: + +- manifest schema version, run ID, creation time, and explicit fixture flag; +- requested source URLs, validated final URLs, and complete redirect chains; +- corpus, evaluation-set, dependency-lock, code-commit, configuration, and environment hashes; +- Python/platform/tool versions and all material parameters; +- output filenames, media types, sizes, and SHA-256 hashes; +- evaluation review status plus a privacy-safe evidence reference when independently reviewed; +- the exact command/entry point and deterministic/random seed settings. + +Canonical serialization must use UTF-8 JSON, sorted object keys, stable separators, and no absolute +machine paths. Output hashes are computed before the manifest is finalized. A mutable `latest` +pointer may reference an immutable run ID but is never evidence itself. + +Signing, transparency logs, a manifest writer, corpus publication, and retroactive historical +validation are out of scope for EMU-B001. Until the design is implemented and independently rerun, +historical numbers remain legacy local automated proxy observations, not reproducible benchmarks. diff --git a/docs/DATA_PROVENANCE.md b/docs/DATA_PROVENANCE.md new file mode 100644 index 0000000..6462381 --- /dev/null +++ b/docs/DATA_PROVENANCE.md @@ -0,0 +1,13 @@ +# Data Provenance + +Official-source scope is limited to public `mevzuat.emu.edu.tr` HTML/PDF. Ingestion must validate +the initial URL, redirect chain, and final URL and record the actual final source; local fixtures +must remain labeled as fixtures and may not be laundered into an official URL. + +Evaluation questions/expected labels are authored project data but may contain regulation-derived +terms or spans. The repository currently does not evidence the claimed external reviewer identity, +date, method, or per-case decisions for `v1_gold`. Until such an artifact is supplied and verified, +the set is assistant-curated and pending independent review. + +The MIT license applies to software, not automatically to third-party source content. Full crawl +artifacts remain ignored; substantial redistribution requires separate rights confirmation. diff --git a/docs/DEMO_METRICS_SNAPSHOT.md b/docs/DEMO_METRICS_SNAPSHOT.md index a775164..a98a302 100644 --- a/docs/DEMO_METRICS_SNAPSHOT.md +++ b/docs/DEMO_METRICS_SNAPSHOT.md @@ -1,105 +1,16 @@ -# Demo Metrics Snapshot +# Historical Metrics Record -Last updated: 2026-08-08 +The prior metric snapshots are retained only as historical automated proxy observations. They are not current benchmark evidence and do not establish semantic answer correctness, citation precision, groundedness, institutional review, or production fitness. -## Corpus Snapshot +Legacy runs reported perfect percentages on several tracked sets and lower values on earlier seed runs. Those labels were too broad: the implementation primarily checked expected-evidence retrieval, answer/refusal/clarification mode, citation presence, format, and latency. The ignored corpus/metric artifacts and immutable input hashes required to reproduce the historical runs are absent from this checkout. -- Source scope: official `mevzuat.emu.edu.tr` HTML plus linked official PDFs. -- Crawl seeds: `https://mevzuat.emu.edu.tr/content.htm`, `https://mevzuat.emu.edu.tr/Content-en.htm`. -- Crawl size: 123 pages, 22 PDFs, 4 crawl errors. -- Active chunks: 8,714 total; 3,878 English and 4,836 Turkish. -- Source types: 8,599 HTML chunks and 115 PDF chunks. -- Structured evidence: 493 table summaries, 7,601 table rows, 8 derived salary facts, 497 text chunks, 115 PDF chunks without table metadata. -- Active artifact: `artifacts/demo_corpus/latest/chunks.jsonl` (ignored). -- Embedded Qdrant index: `artifacts/qdrant/latest` (ignored), collection `emu_regulations`. +New runs use schema `emu-advisor-automated-proxy/v2` and report: -## Verified Gold Evaluation +- expected-evidence retrieval match rates; +- answer-mode plus expected-evidence proxy rate; +- refusal and clarification behavior-match rates; +- citation presence separately from expected-evidence citation match; +- nonempty-format and latency proxies; +- explicit `verified: false` and automated-evidence classification. -Evaluation set: `eval_sets/v1_gold.jsonl` - -Review status: `human_reviewed_verified` - -The 60-case gold set has been reviewed and verified by the project author and university staff. The metrics below are results on this fixed local evaluation set and corpus snapshot; they are not production-service guarantees. - -| Metric | Value | -|---|---:| -| Cases | 60 | -| Answerable/cross-source cases | 52 | -| Refusal cases | 4 | -| Clarification cases | 4 | -| Retrieval top-1 | 92.31% | -| Retrieval top-3 | 98.08% | -| Retrieval top-5 | 100.00% | -| Response accuracy | 100.00% | -| Rejection accuracy | 100.00% | -| Clarification accuracy | 100.00% | -| Citation coverage | 100.00% | -| Extractive latency p50 | 674 ms | -| Extractive latency p95 | 1,267 ms | -| Failed cases | 0 | - -## Hard Regression Suite - -Evaluation set: `eval_sets/v1_hard.jsonl` - -This 50-case suite targets difficult table/broad-query behavior and regressions. It is kept separate from the verified gold benchmark. - -| Metric | Value | -|---|---:| -| Cases | 50 | -| Salary-table cases | 24 | -| Scholarship-bundle cases | 24 | -| Refusal cases | 2 | -| Retrieval top-1 | 97.92% | -| Retrieval top-3 | 100.00% | -| Retrieval top-5 | 100.00% | -| Response accuracy | 100.00% | -| Rejection accuracy | 100.00% | -| Citation coverage | 100.00% | -| Extractive latency p50 | 468 ms | -| Extractive latency p95 | 2,867 ms | -| Failed cases | 0 | - -## Generated Mode - -The local Ollama service and `qwen3:8b` model were detected, but the bounded smoke generation with `--ollama-timeout-s 2` timed out on the recorded machine. Generated metrics are therefore unavailable in that snapshot; extractive and grouped answers remain the validated continuity path. - -## Failure Analysis - -- Verified-gold failed cases: 0. -- Hard-regression failed cases: 0. -- Expected source missing from top-5: 0 in both recorded sets. -- Missing citation: 0 in both recorded sets. -- False refusal: 0 in both recorded sets. -- False answer on refusal cases: 0 in both recorded sets. -- Missing clarification: 0 in the verified gold set. - -## Sample Outputs - -Question: `What is the salary range of a professor compared to assistant professor?` - -Mode: `answer`, answer type `table`. The answer uses a derived salary fact and preserves the numeric ranges: Professor uses scale 7, steps 1-14, from 159,600.00 to 188,200.00; Assistant Professor uses scale 5, steps 1-14, from 107,900.00 to 145,600.00. - -Question: `How to get a scholarship?` - -Mode: `answer`, answer type `topic_bundle`. The answer returns grouped cited evidence for entrance/incentive scholarships, international discounts, high-honour awards, sports grants, research assistant/postgraduate scholarships, and disability scholarships, then asks which type to expand. - -Question: `Lisansüstü burslar hangi oranlarda verilir?` - -Mode: `answer`, answer type `table`. The top evidence comes from the Turkish scholarship/discount regulation and cites the postgraduate scholarship rows. - -Question: `Araştırma görevlisi burs kuralları farklı veya çelişkili mi?` - -Mode: `show_conflict`, answer type `direct`. Retrieval remains within the detected-language corpus and cites the research-assistant rules rather than only the general scholarship table. - -Question: `Bugün kampüste hangi burs etkinlikleri var?` - -Mode: `refuse`. The answer refuses because campus events are outside the V1 official-regulation scope, even though the query contains the word `burs`. - -## Known Limits - -- The verified 60-case `v1_gold` benchmark is human-reviewed; `v1_hard` remains a regression suite and `emu_gold_seed` remains provisional unless separately documented as reviewed. -- These are fixed local evaluation results, not production availability or universal-quality claims. -- Broad scholarship prompts run several deterministic subqueries; p50 remains under 1 second, but p95 is higher than direct extractive queries. -- Generated mode is implemented and fallback-safe, but `qwen3:8b` timed out under the recorded 2-second smoke limit. -- Qdrant has unit coverage, CLI support, and a validated embedded local Qdrant index; a Docker/live Qdrant service deployment is still not validated in the recorded environment. +No new artifact-backed run was performed in EMU-B001. Presentation remains blocked. diff --git a/docs/DEMO_STORYBOARD.md b/docs/DEMO_STORYBOARD.md index 17b6f34..d880cf9 100644 --- a/docs/DEMO_STORYBOARD.md +++ b/docs/DEMO_STORYBOARD.md @@ -1,59 +1,22 @@ -# Board Demo Storyboard +# Controlled Demo Storyboard (Deferred) -Status: board-demo script, not production approval. +This storyboard is a future controlled-demo outline, not approval to present the repository. -## Slide 1 - Purpose +## Entry conditions -- EMU Regulation Assistant answers staff-facing questions about indexed EMU regulations. -- Scope is official `mevzuat.emu.edu.tr` HTML plus linked official PDFs. -- The demo is local-only and informational. +- Independent human review is durably evidenced. +- An artifact-backed corpus and proxy run have immutable manifests and hashes. +- Corpus-use/publication rights are resolved. +- Target hardware, local services, privacy settings, and deployment boundaries are validated. +- Independent QA explicitly clears the demo gate. -## Slide 2 - Trust Model +## Future sequence -- English and Turkish corpora remain separate in V1. -- Every substantive answer carries citations. -- Weak, ambiguous, or out-of-scope questions are clarified or refused. +1. State that the tool is independent and informational. +2. Show the runtime profile and corpus mode before any question. +3. Demonstrate English and Turkish routing with citations. +4. Demonstrate refusal/clarification on weak or out-of-scope evidence. +5. Show precise automated proxy names and their input manifest; do not translate them into semantic quality claims. +6. Close with known limits, privacy defaults, and escalation to the official regulation source. -## Slide 3 - Architecture - -- Official sources are crawled into canonical chunks. -- Hybrid retrieval selects cited evidence. -- Deterministic answerability gates run before optional local generation. -- Local audit logs and metrics support review without external telemetry. - -## Slide 4 - Live Question - -Use: `What is the attendance requirement?` - -Expected demo behavior: - -- Answer from indexed regulation evidence. -- Show source language and bottom citations. -- Keep `/chat` output public-safe without raw hit diagnostics (User mode at `/admin?view=user`). - -## Slide 5 - Edge Cases - -Use: - -- `What about graduation?` -- `Bugun kampuste hangi burs etkinlikleri var?` -- `Arastirma gorevlisi burs kurallari farkli veya celiskili mi?` - -Expected demo behavior: - -- Clarify vague questions. -- Refuse event/general-campus questions. -- Keep the answer within the detected-language regulation corpus. - -## Slide 6 - Evidence And Metrics - -- Candidate and hard-regression metrics are useful regression evidence. -- They remain assistant-curated until human review is complete. -- Present extractive latency, top-5 retrieval, citation coverage, refusal behavior, and known generated-mode limits. - -## Slide 7 - Roadmap Ask - -- Complete human review of the evaluation sets. -- Validate live Qdrant service and target campus hardware. -- Decide whether generated mode remains opt-in or becomes part of the demo. -- Approve pilot constraints before any production-like deployment. +Until the entry conditions pass, use fixture mode only for software verification and do not capture presentation media. diff --git a/docs/DEPENDENCY_POLICY.md b/docs/DEPENDENCY_POLICY.md new file mode 100644 index 0000000..8d54bf8 --- /dev/null +++ b/docs/DEPENDENCY_POLICY.md @@ -0,0 +1,21 @@ +# Dependency Policy + +Workflow schema: `agentic-workflow/v2` + +- `requirements.txt` and `requirements-dev.txt` express direct compatible ranges. +- `requirements-lock.txt` is the exact CI/demo snapshot and must install on every declared platform. +- Platform-specific packages require environment markers; Linux-only `uvloop` must not block Windows. +- Use Python 3.12 for the current verification baseline and test both Ubuntu and Windows in CI. +- Regenerate the lock deliberately; do not bulk-upgrade majors during an unrelated batch. +- Run `python -m pip check` and a dependency audit after installation. +- Record the generator command, Python/platform inputs, and audit tool version before release. +- Optional model/Qdrant/CUDA dependencies must remain explicit and may not be implied by core CI. + +## Lock maintenance + +Resolve from `requirements-dev.txt` in a clean Python 3.12 environment, capture the resolved set with +`python -m pip freeze`, then review platform-specific packages before replacing the lock. Preserve +`uvloop; sys_platform != "win32"`, install the candidate lock on both Windows and Ubuntu, run +`python -m pip check`, and run the separately pinned `pip-audit==2.10.1`. Record the Python, pip, +platform, direct-input hashes, and resulting lock hash. A freeze from one platform is not accepted +until the other declared platform installs and tests it successfully. diff --git a/docs/EMUAdvisor Full Analysis.md b/docs/EMUAdvisor Full Analysis.md index 8514fb6..764ed93 100644 --- a/docs/EMUAdvisor Full Analysis.md +++ b/docs/EMUAdvisor Full Analysis.md @@ -1,4 +1,7 @@ # Executive Summary + +> **Historical planning analysis only.** This document is not current readiness evidence. Its official-source, testing, metric, logging, security, deployment, and board-readiness statements are recommendations or historical assumptions, not verified current capabilities. See `docs/PROJECT_STATE.md`; presentation remains blocked. + The EMU Regulation Assistant is a demo RAG (Retrieval-Augmented Generation) system answering staff questions about Eastern Mediterranean University rules using indexed official sources. It currently provides cited answers in English and Turkish via a web chat UI. We recommend comprehensive end-to-end verification (functional, security, performance, UX, etc.) and prioritizing fixes and enhancements to make the demo board-ready. Key actions include building test suites (functional and load), tightening security (input validation, encryption, access control), hardening deployment (CI/CD, monitoring), and polishing the UI/UX (visual design, onboarding, responsiveness). This report lists specific test items and tasks, assesses risks, defines success criteria, and outlines a demo storyboard and roadmap. **Key recommendations:** Automate and run extensive QA tests (including cross-browser and accessibility checks【15†L33-L42】【18†L255-L263】); fix high-impact bugs first (e.g. retrieval accuracy, citation formatting); enhance backend robustness (CI/CD, monitoring, security hardening) before the demo; and apply UI/UX improvements (branding, hierarchy, contextual help【22†L132-L140】). Define and track metrics like answer accuracy and latency【13†L209-L217】【13†L246-L254】. Prepare clear demo slides (problem statement, live Q&A example, metrics, roadmap). The following sections detail prioritized tasks, test plans, risk mitigations, and recommended metrics for stakeholders. diff --git a/docs/EVIDENCE_MAP.md b/docs/EVIDENCE_MAP.md new file mode 100644 index 0000000..1f51c29 --- /dev/null +++ b/docs/EVIDENCE_MAP.md @@ -0,0 +1,13 @@ +# Evidence Map + +| Claim | Direct evidence | Current status | +| --- | --- | --- | +| Local bilingual regulation retrieval implementation exists | `emu_advisor/**`, tests, system spec | Verified in code/tests | +| English/Turkish corpora stay separate | routing/retrieval/evaluation tests | Verified in code/tests | +| Historical crawl produced 8,714 chunks / 119 sources / 22 PDFs | dated docs plus prior audit fresh crawl | Historical observation; exact old artifact absent | +| Top-5 retrieval reached 100% on named tracked sets | dated outputs and prior audit rerun | Local retrieval result, not answer correctness | +| 60 cases were human-reviewed by university staff | self-asserted row metadata and automated commit only | Unverified; must be retracted or evidenced | +| 100% answer accuracy/precision/groundedness | proxy metric implementation | Unsupported semantic claim | +| Browser UI works responsively | browser smoke and prior audit | Verified on tested Chromium scopes | +| Production transcript boundary is protected | server route policy | False before EMU-B001 | +| Windows locked install works | unguarded Linux-only dependency | False before EMU-B001 | diff --git a/docs/EXPERIMENT_PROTOCOL.md b/docs/EXPERIMENT_PROTOCOL.md new file mode 100644 index 0000000..89cf436 --- /dev/null +++ b/docs/EXPERIMENT_PROTOCOL.md @@ -0,0 +1,29 @@ +# Experiment Protocol + +Workflow schema: `agentic-workflow/v2` + +## Objective + +Measure retrieval and observable response behavior for separate English/Turkish regulation corpora +without conflating structural proxies with semantic answer quality. + +## Inputs and baselines + +- Name the exact evaluation set, corpus/run manifest, code commit, dependency lock, configuration, + mode, embedding/retrieval backend, and local-model status. +- Hash every immutable input/output. A mutable-site rebuild is a new corpus version. +- Keep assistant-curated regression cases distinct from independently held-out/adjudicated cases. + +## Metrics + +- Valid without human semantic labels: expected-source/chunk retrieval rank, top-k retrieval, + expected-evidence citation match, response-mode behavior, citation presence, and latency. +- Invalid without independent adjudication: answer correctness, citation precision, groundedness, + legal correctness, or human-review claims. + +## Stopping and publication rules + +- Stop on cross-language corpus mixing, out-of-scope source, missing manifest, unsupported label, + or an overconfident answer unsupported by expected evidence. +- Full crawl/model experiments require explicit authorization because they use network/compute. +- Do not publish a benchmark result until an independent rerun reproduces the named manifest. diff --git a/docs/MIGRATION_BACKLOG.md b/docs/MIGRATION_BACKLOG.md index 13c92db..4fb8da0 100644 --- a/docs/MIGRATION_BACKLOG.md +++ b/docs/MIGRATION_BACKLOG.md @@ -1,6 +1,6 @@ # Migration Backlog -Last updated: 2026-05-05 +Last updated: 2026-09-06 This backlog translates the current spec and old-demo evidence into implementation work. It is not validation that the work is complete. @@ -51,3 +51,12 @@ This backlog translates the current spec and old-demo evidence into implementati - Show source language/corpus indicators. - Show bottom citations with title, section/article, page number for PDFs, URL/path, and traceability metadata. - Support streaming and progressive answer display when local generation is slow. + +## EMU-B001 Corrections and Follow-up + +- The tracked 60-case set is now explicitly assistant-curated/pending independent review; do not restore verified-gold metadata without the durable evidence contract. +- New automated runs use the versioned proxy schema and expected-evidence citation matching. Independent semantic adjudication remains future work. +- Capability-protected chat sessions, default-off persistence/logging, explicit corpus mode, exact-host redirect validation, and Windows CI are implemented in EMU-B001 and require final independent verification. +- Implement the immutable run-manifest design before any reproducible benchmark claim. +- Resolve corpus rights, live artifact/service validation, public-exposure authentication, and presentation/release approval in later human-reviewed batches. +- `.old/` is absent in this checkout; do not treat stale references to it as active repository state. diff --git a/docs/MODEL_EVALUATION.md b/docs/MODEL_EVALUATION.md new file mode 100644 index 0000000..f566aa1 --- /dev/null +++ b/docs/MODEL_EVALUATION.md @@ -0,0 +1,33 @@ +# Model Evaluation + +Workflow schema: `agentic-workflow/v2` + +## Evaluation sets + +`v1_gold` is currently assistant-curated despite its present metadata; no durable independent +human-review artifact is committed. `v1_hard` is an assistant-curated regression set and +`emu_gold_seed` is provisional. None is an independently held-out semantic-quality gold standard. + +## Current measurable behaviors + +- Retrieval rank/top-k against expected source/chunk labels. +- Correct response mode for answer/refusal/clarification/conflict cases. +- Citation presence and, after EMU-B001, expected-evidence citation match. +- Extractive/generation timing and explicit model-unavailable fallback. + +Current response/citation/groundedness scores are proxy composites and must not be described as +semantic answer correctness, citation precision, or claim-level grounding. + +## Unverified areas + +- Per-case human adjudication and reviewer provenance. +- Immutable historical corpus/result reproduction. +- Live Qdrant and current Ollama quality/latency on target hardware. +- Held-out evaluation free from implementation iteration. +- Official/institutional approval and legal interpretation. + +## Comparison boundary + +Historical recorded numbers may be retained only as dated local observations with their metric +definitions. They may not be compared as model-quality improvements when corpus, inputs, or proxy +definitions differ. diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index 2f1f371..5bc9065 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -1,118 +1,59 @@ # Project State -Last updated: 2026-05-22 - -## Current Snapshot - -- The current product source of truth is `EMU_RAG_Current_System_Specs.md`. -- The current deliverable is a demo-first, local-only EMU Regulation Assistant, not a full production release. -- The root workspace is now the primary Git repository on branch `main`. -- `.old/` is an ignored local archive of the previous NLPCrawler demo. -- The old demo archive includes a Python RAG pipeline, FastAPI backend, static UI, Ollama integration, and an evaluation harness. -- Live crawl, canonical chunk, and metrics artifacts have been generated under ignored `artifacts/`. -- `eval_sets/v1_gold.jsonl` is now a 60-case assistant-curated candidate set with review status `assistant_curated_pending_human_review`; it is not yet a human-reviewed gold set. -- `eval_sets/emu_gold_seed.jsonl` is now a 50-case provisional seed converted from `docs/gold-set-comprehensive-analysis.md`; it is source-binding and human-review pending. -- Root `README.md` and `docs/DEMO_METRICS_SNAPSHOT.md` now describe the publishable local demo boundary, commands, metrics, and known limits. -- `docs/PROJECT_STATUS_PROGRESS_PLAN.md` now provides a concise current-status, progress, metrics, limitations, and forward-plan summary. -- `docs/SPRINT_PLAN.md` now sequences implementation and testing into 24-48 hour sprints. -- Sprint 1 has added the root canonical schema package, JSONL validator, schema docs, fixtures, and unit tests. -- Sprints 2-17 now have root implementation scaffolds, local tests, and a real-corpus demo validation pass; see `docs/SPRINT_STATUS.md`. -- Board-readiness continuation work from `docs/EMUAdvisor Full Analysis.md` has added strict request validation, optional admin-token protection, security headers, CI, browser-smoke tooling, local analytics, review helpers, benchmark probes, and board-demo documentation. - -## Active Objective - -Harden and productize the V1 EMU Regulation Assistant around the current spec: - -- Narrow V1 to official EMU regulations and official linked PDFs. -- Preserve separate English and Turkish corpora with no EN/TR cross-corpus search in V1. -- Replace the English-focused embedding default. -- Move toward a Qdrant-backed hybrid retrieval stack. -- Add canonical source/chunk metadata and traceability. -- Establish realistic verification with bilingual gold questions. - -## Current Implementation Evidence - -- Old-demo pipeline scripts live under `.old/` as numbered Python files from crawl through reranking. -- `.old/backend/server.py` exposes FastAPI endpoints for `/`, `/ask`, `/fetch`, `/fetch/status`, and `/whoami`. -- `.old/backend/rag_adapter.py` handles retrieval invocation, clarification, confidence gating, extraction answers, and optional Ollama generation. -- `.old/backend/config.json` points both English and Turkish index paths to `mevzuat_crawl/index_v4_dedup` and enables Ollama model `qwen2.5:14b-instruct`. -- `.old/requirements-full.txt` documents the full old-demo Python dependency surface and intentionally leaves `torch` unpinned. -- `emu_advisor/schema.py` defines canonical document/chunk validation and legacy chunk mapping. -- `emu_advisor/validate_jsonl.py` validates canonical document or chunk JSONL files. -- `emu_advisor/html_ingest.py` and `emu_advisor/pdf_ingest.py` emit canonical chunks from official-scope HTML/PDF sources; HTML ingestion now preserves table summaries, row-level evidence, and derived academic salary facts. -- `emu_advisor/routing.py` handles language/corpus/scope routing, including ASCII-written Turkish detection, and keeps retrieval within the detected-language corpus. -- `emu_advisor/evaluation.py` validates expanded bilingual evaluation sets and computes top-k retrieval metrics. -- `emu_advisor/evaluation.py` also accepts provisional gold-seed cases when they are explicitly marked as pending source binding and excluded from hard quality claims. -- `emu_advisor/pipeline.py` performs official-host crawl/build into canonical ignored artifacts. -- `emu_advisor/metrics.py` runs evaluation and writes dashboard-ready JSON, Markdown, per-case CSV, human-review CSV, failure-analysis summaries, and cheap/balanced/expensive mode comparisons. -- `emu_advisor/corpus.py` loads `artifacts/demo_corpus/latest/chunks.jsonl` when present and falls back to fixture chunks otherwise. -- `emu_advisor/generation.py` provides local Ollama `qwen3:8b` generation with extractive fallback on failure. -- `emu_advisor/query_understanding.py` adds local LLM-first query rewriting for standalone retrieval queries, Turkish ASCII normalization, and EN/TR mid-conversation follow-ups with deterministic fallback. -- `emu_advisor/embeddings.py`, `store.py`, `retrieval.py`, `modes.py`, and `index.py` provide local retrieval infrastructure plus a Qdrant adapter/build CLI with local fallback. -- `emu_advisor/answer.py` and `citations.py` implement deterministic answerability, refusal, conflict, table answers, scholarship topic bundles, fallback, and citation behavior. -- `emu_advisor/admin.py` provides snapshot/diff/activation workflow. -- `emu_advisor/server.py`, `static/`, `audit_log.py`, and `load_test.py` provide a landing page at `/`, unified UI at `/admin` (User chat + Diagnostics tabs), sanitized `/chat` and `/chat/stream`, full `/ask`, LLM status diagnostics, privacy logging, and load simulation. -- `static/landing.html`, `static/admin.html`, `static/shared.js`, `static/user-chat.js`, and `static/admin-diagnostics.js` replace the former split `index.html` / `app.js` public chat surface. -- The unified UI now has persistent light/dark theme tokens, a header theme toggle, high-contrast toolbar controls, and a bounded streaming message pane; user-mode chat retrieval is internally pinned to balanced mode, and citations/extractive evidence are revealed only after streaming completes. -- `emu_advisor/eval_review.py` exports human-review CSVs and can bind provisional seed cases against a built corpus artifact. -- `emu_advisor/benchmark.py` runs local embedding and generated-mode probes without promoting generated mode by default. -- `emu_advisor/readiness.py` generates `docs/BOARD_DEMO_READINESS.md` with explicit pass, partial, and blocked gates. -- `tools/browser_smoke.py` provides an optional Playwright smoke for desktop/mobile public chat rendering. -- `tests/fixtures/` contains valid and invalid canonical chunk fixtures. - -## Verification State - -- Sprint 0 repo ownership baseline is complete: root is the primary Git repository and `.old/` is ignored local legacy reference code. -- Root documentation file listing was verified. -- Legacy `.old/` Python syntax scan passed for 15 files. -- Legacy `.old/backend` import smoke failed because `.old/backend/rag_adapter.py` imports `key` from `anyio`, which is not available in the installed AnyIO package. -- Root schema unit tests passed: `python -m unittest discover -s tests`. -- Full root test suite passed: 69 tests after LLM-first query-understanding, ASCII Turkish routing, and EN/TR language-switch regression coverage. -- Canonical valid chunk fixture passed JSONL validation. -- Canonical invalid chunk fixture failed validation as expected with field-level errors. -- Root package/test/tool syntax scan passed for 38 files. -- Candidate evaluation set validation passed for 60 cases: 30 English, 30 Turkish, 48 answerable, 4 refusal, 4 clarification, and 4 conflict/cross-source cases. -- Hard regression set validation passed for 50 cases: 25 English, 25 Turkish, 48 answerable, and 2 refusal cases. -- Targeted real-corpus probes for ASCII Turkish prompts (`burs oranlari nelerdir`, `not itirazi nasil yapilir`, `basvuru belgeleri nelerdir`, `buna nasil basvururum`) route to `regulations_tr`; contextless follow-ups still depend on the query-understanding rewrite for meaningful standalone retrieval. -- Provisional gold seed validation is expected to pass for 50 source-binding-pending cases, but those cases are not human-reviewed or adjudicated. -- FastAPI app import smoke passed: app title `EMU Regulation Assistant`. -- Live crawl completed from `https://mevzuat.emu.edu.tr/content.htm` and `https://mevzuat.emu.edu.tr/Content-en.htm`: 123 pages, 22 PDFs, 8,714 chunks, 4 crawl errors. -- Active corpus status: 119 sources/documents, 3,878 English chunks, 4,836 Turkish chunks, 8,599 HTML chunks, 115 PDF chunks. -- Structured evidence status: 493 table summaries, 7,601 table rows, 8 derived salary facts, 497 normal text chunks, and 115 PDF chunks without table metadata. -- Live metrics over the 60-case assistant-curated candidate set are presentable for extractive mode: top-5 retrieval 100%, response accuracy 100%, rejection accuracy 100%, clarification accuracy 100%, citation coverage 100%, extractive latency p50 674 ms / p95 1,267 ms. -- Hard regression metrics over `eval_sets/v1_hard.jsonl` are presentable: 50 cases, top-5 retrieval 100%, response accuracy 100%, rejection accuracy 100%, citation coverage 100%, extractive latency p50 468 ms / p95 2,867 ms, 0 failed cases. -- Generated mode with local Ollama model `qwen3:8b` is implemented and fallback-safe, but the latest bounded 2-second smoke metrics run timed out; generated metrics are marked unavailable in `artifacts/metrics/latest_generated`. -- Mode comparison now has an executable path through `python -m emu_advisor.metrics run --all-modes --cases eval_sets\emu_gold_seed.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\metrics\mode_comparison`. -- Latest provisional seed mode comparison wrote ignored artifacts under `artifacts/metrics/mode_comparison`: balanced total score 0.863 / top-5 84.0% / p50 315 ms / 8 failed cases; expensive total score 0.863 / top-5 84.0% / p50 342 ms / 8 failed cases; cheap total score 0.831 / top-5 80.0% / p50 276 ms / 10 failed cases. -- Embedded local Qdrant indexing was validated with `artifacts/qdrant/latest`, 8,714 chunks, 256 dimensions, and collection `emu_regulations`. -- Board readiness report currently marks the demo as `partial`: presentable locally, but blocked on human-reviewed gold status and service-backed Qdrant validation. -- No campus/server deployment environment is documented yet. - -## Known Risks - -- The old demo still defaults to `intfloat/e5-base-v2` in multiple places, but active root code now supports hash fallback plus optional local Ollama embeddings. -- Qdrant adapter and index CLI exist in root code; embedded local Qdrant is validated, but a Docker/live Qdrant service build/run has not been validated in this environment. -- Canonical document/chunk schema is implemented and integrated into local ingestion, pipeline build, corpus loading, store, retrieval, metrics, and citation tests. -- PDF handling requirements are stronger than the evidence visible in the old demo code and docs. -- The `.old/` archive contains `__pycache__` files, but `.old/` is ignored by the root repo. -- The `.old/` backend currently fails import in this environment because of an existing AnyIO import issue. -- Generated mode depends on an available local Ollama `qwen3:8b`; the current bounded smoke run timed out at 2 seconds, so extractive fallback remains the validated continuity path. -- Admin/debug endpoints are protected only when `EMU_ADVISOR_ADMIN_TOKEN` is configured; production profile now requires that token at startup. -- Optional Playwright browser smoke depends on local browser availability and is skip-safe when dependencies are unavailable. -- Latest UI smoke passed for `/admin?view=user`: no EN/TR cross-corpus controls were present, the streamed answer completed before the evidence reveal appeared, and the reveal button exposed citations plus extractive evidence while staying hidden by default. -- Broad scholarship prompts use deterministic grouped subqueries; p50 remains under 1 second, but p95 is higher than direct extractive questions. -- The 60-case evaluation set and 50-case hard regression set are assistant-curated pending human review; do not label either as human-reviewed. -- The 50-case provisional gold seed is converted from the comprehensive analysis document, but several cases still need exact source/chunk binding and human adjudication before the scores can be presented as gold-standard results. - -## Next Actions - -1. Human-review `artifacts/metrics/latest/human_review.csv` and promote/repair `eval_sets/v1_gold.jsonl` only after manual labels are complete. -2. Run the all-mode provisional seed benchmark and inspect `artifacts/metrics/mode_comparison/comparison.md`. -3. Manually review the zero-failure candidate and hard-regression metrics for overfitting risk, especially table-derived salary facts and grouped scholarship answers. -4. Bind and adjudicate the provisional gold seed cases before presenting them as gold-standard quality claims. -5. Run the Qdrant index against a Docker/live Qdrant service once Docker or a managed local service is available. -6. Diagnose `qwen3:8b` runtime latency beyond the 2-second smoke limit and benchmark streaming first-token latency on target hardware. -7. Benchmark `qwen3-embedding:4b` indexing on target hardware. -8. Confirm target deployment hardware and local-service permissions with IT before latency or serving commitments. -9. Run the optional Playwright smoke on a machine with browser dependencies installed. -10. Configure `EMU_ADVISOR_ADMIN_TOKEN` before any production-profile demo. +Last updated: 2026-09-06 + +Active branch: `remediation/evidence-security-portability` + +Active batch: none; last closed batch `EMU-B001` + +State: EMU-B001 complete with residual risks; presentation/publication/release gates remain blocked + +## Product boundary + +EMUAdvisor is an independent local-only English/Turkish assistant for public EMU regulations. It is not an official university service or final decision-maker. Runtime retrieval/generation remains local; ingestion alone may contact the exact official source host under the source policy. + +## Current implementation + +- FastAPI application with static user/diagnostic UI. +- Deterministic routing, hybrid local/Qdrant retrieval, cited extractive answers, refusal/clarification paths, and optional local Ollama generation. +- Explicit `fixture` versus `artifact` corpus mode. Fixture mode is limited to dev/test and visibly labeled; unsafe or missing configurations fail closed. +- Official ingestion requires exact-host HTTPS and validates redirects/final URLs. Local fixtures retain non-official provenance. +- Server-issued per-session capability protects public continuation/read/export/clear. Enumeration is admin-only and message-minimized. +- Transcript persistence, audit logging, and raw-query logging are separate positive opt-ins, all off by default. +- Platform-aware pinned dependencies and Ubuntu/Windows Python 3.12 core CI. + +## Evaluation truth + +`eval_sets/v1_gold.jsonl` contains 60 assistant-curated regression cases pending independent human review. The filename is retained for compatibility. The previous blanket `human_reviewed_verified` metadata has no durable per-case review evidence and is not relied upon. + +New automated output uses `emu-advisor-automated-proxy/v2`. It measures observable retrieval/evidence/behavior/format/latency proxies. It does not semantically grade answer correctness, citation precision, claim grounding, or legal reliability. Historical percentages are legacy/unverified proxy output because their required corpus/metric manifests are unavailable here. + +## Active blockers + +- Independent human adjudication and evidence reference. +- Implemented immutable run manifests and a reproducible artifact-backed rerun. +- Corpus rights/distribution decision. +- Live Qdrant/Ollama/target hardware and deployment validation. +- Public-Internet identity/threat model beyond local session capabilities. + +Therefore benchmark, presentation, publication, tag, release, deployment, and production-readiness claims remain blocked. + +## EMU-B001 verification + +Fresh independent testing returned `PASS_WITH_RISKS` on repaired snapshot +`d56a5dcdd63dbe2a1be14d42b3e16156250ab0394a849757c4269bac4ec11b26` after an initial +documentation-ordering `FAIL` was repaired and retested. Syntax, focused/full tests, evaluation +validation, review status, publication guard, browser smoke, Windows clean installation, dependency +audit, protected-input comparison, immutable CI refs, and tracked-tree credential scan passed. + +Remote Ubuntu CI, live crawling, artifact-backed local services, immutable-run reproduction, +history-aware secret review, human adjudication, source-content rights, and Internet-grade identity +remain unverified or deferred. See `QA_REPORT.md` and `RISK_REGISTER.md`. + +## Protected local data + +Ignored `artifacts/**` and `logs/**` may contain private transcripts, queries, corpora, indexes, or generated evidence. EMU-B001 neither reads nor deletes existing files. Owner-approved cleanup or migration is a separate operation. + +## Workflow authority + +Use `DEV_STATE.md` for cycle status, `BLUEPRINT.md` for the accepted batch, `RISK_REGISTER.md` for risk status, and `QA_REPORT.md` for verified evidence. diff --git a/docs/PROJECT_STATUS_PROGRESS_PLAN.md b/docs/PROJECT_STATUS_PROGRESS_PLAN.md index 9c369e8..228edc3 100644 --- a/docs/PROJECT_STATUS_PROGRESS_PLAN.md +++ b/docs/PROJECT_STATUS_PROGRESS_PLAN.md @@ -1,200 +1,29 @@ -# Project Status, Progress, and Plan +# Project Status and Progress Plan -Last updated: 2026-05-22 +## Current status -## Executive Summary +EMUAdvisor has an active local software implementation and a substantial automated test suite. EMU-B001 completed its bounded evidence-label, privacy, corpus-provenance, source-scope, and Windows-portability hardening with an independent `PASS_WITH_RISKS`. This does not make the repository presentation-ready. -EMUAdvisor is now a presentable local-only demo for answering staff-facing questions about official EMU regulations. The root repository is the active implementation surface, while `.old/` is retained only as an ignored legacy reference. +## Completed in EMU-B001 -The current system can crawl the official regulation source scope, build canonical chunks, route English and Turkish questions to separate corpora, answer with citations, refuse out-of-scope questions, show conflicts, handle table-derived salary facts, and return grouped extractive answers for broad scholarship questions. +- Honest assistant-curated/pending-review evaluation metadata. +- Versioned automated proxy schema with expected-evidence citation matching. +- Capability-isolated public chat sessions and admin-only enumeration. +- Default-off transcript persistence, audit logging, and raw-query logging. +- Explicit, visibly labeled fixture mode and fail-closed artifact mode. +- Exact-host HTTPS redirect validation and non-laundered fixture provenance. +- Platform-aware Python lock and Windows/Ubuntu Python 3.12 CI design. -This is not production-ready and must not be described as an official EMU decision system. The strongest current claim is: a local demo has been validated against assistant-curated candidate and hard-regression evaluation sets. +Local verification passed, including a fresh Windows Python 3.12 environment. Remote Ubuntu CI and +the live/artifact/human/security gates below remain future actions. -## Product Boundary +## Next milestones after EMU-B001 -V1 is scoped to official EMU regulations from `mevzuat.emu.edu.tr` plus official linked PDFs that belong to that regulation source set. +1. Obtain independent review and a privacy-safe durable attestation for the evaluation set. +2. Implement the immutable run-manifest design from `docs/DATA_MANIFEST.md`. +3. Resolve source-content rights and the lawful artifact-sharing boundary. +4. Perform a controlled live crawl and artifact-backed proxy run with hashes. +5. Validate local services and target hardware, then perform deployment/security review. +6. Reassess presentation/publication only after those gates pass. -V1 supports English and Turkish, but the corpora remain separate because the sources are not guaranteed to be one-to-one translations. V1 no longer exposes EN/TR cross-corpus answers or comparison. - -V1 does not cover events, advising, course/program information, private records, workflow automation, email, scheduling, or general university chatbot behavior. - -## Achieved So Far - -- Root repo ownership is established for future `emu-advisor` publication. -- Canonical document/chunk schema and JSONL validation are implemented. -- HTML and PDF ingestion are implemented for official-scope sources. -- Table-aware HTML ingestion now emits table summaries, row-level table chunks, and derived salary facts. -- Live crawl/index pipeline exists at `python -m emu_advisor.pipeline build ...`. -- Corpus loading prefers `artifacts/demo_corpus/latest/chunks.jsonl` and falls back to fixture data only when no live corpus exists. -- English/Turkish routing and out-of-scope routing are implemented without EN/TR corpus mixing. -- Hybrid lexical+dense retrieval is implemented with local hash embeddings and optional Ollama embeddings. -- Qdrant backend support is implemented with local fallback and an index build CLI. -- Deterministic answerability gates handle answer, refusal, clarification, and conflict modes before generation. -- Citation objects preserve chunk, document, URL, version hash, crawl timestamp, and supporting chunk metadata. -- Broad scholarship prompts use deterministic grouped extractive retrieval instead of sending large raw contexts to the LLM. -- Local Ollama `qwen3:8b` generated mode is implemented with immediate extractive fallback. -- FastAPI demo endpoints and static UI are implemented. -- The root UI is now split into a simple deployed-style chatbot at `/` and a diagnostics-heavy admin console at `/admin`. -- A sanitized `POST /chat` endpoint now returns only public answer state, language, citations, and user-facing evidence groups, while `POST /ask` remains the full admin/debug endpoint. -- Metrics runner writes JSON, Markdown, per-case CSV, human-review CSV, and failure analysis. -- Metrics can now compare cheap, balanced, and expensive modes in one run with `--all-modes`. -- `eval_sets/emu_gold_seed.jsonl` captures the provisional 50-case seed from `docs/gold-set-comprehensive-analysis.md`. -- README, release-candidate notes, sprint status, run protocol, demo metrics snapshot, and repo map are tracked. - -## Current Demo Evidence - -Latest validated corpus snapshot: - -| Item | Current Value | -|---|---:| -| Official pages crawled | 123 | -| PDFs crawled | 22 | -| Crawl errors | 4 | -| Sources/documents | 119 | -| Total chunks | 8,714 | -| English chunks | 3,878 | -| Turkish chunks | 4,836 | -| HTML chunks | 8,599 | -| PDF chunks | 115 | -| Table summaries | 493 | -| Table rows | 7,601 | -| Derived salary facts | 8 | - -Latest `eval_sets/v1_gold.jsonl` metrics: - -| Metric | Value | -|---|---:| -| Cases | 60 | -| Review status | assistant-curated, pending human review | -| Retrieval top-5 | 100% | -| Response accuracy | 100% | -| Rejection accuracy | 100% | -| Clarification accuracy | 100% | -| Citation coverage | 100% | -| Extractive latency p50 | 674 ms | -| Extractive latency p95 | 1,267 ms | -| Failed cases | 0 | - -Latest `eval_sets/v1_hard.jsonl` metrics: - -| Metric | Value | -|---|---:| -| Cases | 50 | -| Review status | assistant-curated hard regression | -| Salary-table cases | 24 | -| Scholarship-bundle cases | 24 | -| Refusal cases | 2 | -| Retrieval top-5 | 100% | -| Response accuracy | 100% | -| Rejection accuracy | 100% | -| Citation coverage | 100% | -| Extractive latency p50 | 468 ms | -| Extractive latency p95 | 2,867 ms | -| Failed cases | 0 | - -Generated-mode status: - -- Local Ollama service and `qwen3:8b` were detected. -- The bounded 2-second smoke run timed out. -- Generated metrics are marked unavailable in `artifacts/metrics/latest_generated`. -- Extractive and grouped answers remain the validated demo path. - -## Current Architecture - -The active root pipeline is: - -```text -official EMU HTML/PDF sources - -> emu_advisor.pipeline - -> artifacts/demo_corpus/latest/chunks.jsonl - -> emu_advisor.corpus - -> emu_advisor.retrieval - -> emu_advisor.answer - -> optional emu_advisor.generation - -> emu_advisor.server + static UI - -> emu_advisor.metrics -``` - -Core implementation areas: - -- `emu_advisor/html_ingest.py`: HTML/text/table/derived-fact ingestion. -- `emu_advisor/pdf_ingest.py`: PDF text extraction with page metadata. -- `emu_advisor/retrieval.py`: hybrid retrieval, structured-evidence boosts, source quality demotion. -- `emu_advisor/answer.py`: evidence gating, table answers, scholarship topic bundles, conflict/refusal/clarification. -- `emu_advisor/store.py` and `emu_advisor/index.py`: local store plus Qdrant adapter and build CLI. -- `emu_advisor/server.py`: FastAPI app, demo UI endpoints, corpus/metrics/LLM status, ask endpoints. -- `emu_advisor/metrics.py`: evaluation runner and artifact writer. - -## Current Limitations - -- The evaluation sets are assistant-curated. They need manual review before being called human-reviewed gold metrics. -- The provisional gold seed is useful for mode tuning, but it is explicitly pending exact source/chunk binding and human review. -- Zero-failure metrics are useful for regression tracking, but they may overfit current source labels and need independent review. -- Live Docker/service Qdrant has not been validated in this environment; only embedded local Qdrant has been validated. -- Generated mode is fallback-safe but not currently reliable under the bounded smoke timeout. -- PDF table structure is still weaker than HTML table handling. -- Campus/server deployment constraints and hardware assumptions are not documented yet. -- The `.old/` archive remains useful historically but is not the active implementation path. - -## Plan From Here - -### Phase 1: Evaluation Credibility - -- Human-review `artifacts/metrics/latest/human_review.csv`. -- Review `eval_sets/v1_gold.jsonl` and `eval_sets/v1_hard.jsonl` case by case. -- Bind, repair, and review `eval_sets/emu_gold_seed.jsonl` before calling it gold-standard evaluation. -- Mark incorrect or weak labels and repair expected source URLs/chunk IDs. -- Add failure-analysis notes for any repaired cases. -- Keep the label `assistant_curated_pending_human_review` until manual review is complete. - -### Phase 2: Retrieval Quality Hardening - -- Continue adding hard regression cases for real observed failures, not broad synthetic expansion. -- Improve table evidence for non-salary numeric facts where HTML export structure is irregular. -- Improve answer extraction so table answers prefer the most relevant derived or row-level evidence and avoid unrelated nearby rows. -- Add more explicit support for multi-source answers where the correct answer spans separate regulations. - -### Phase 3: Production Retrieval Path - -- Validate Qdrant against a live Docker or service-backed Qdrant instance. -- Decide production profile defaults for `EMU_ADVISOR_VECTOR_BACKEND`, `EMU_ADVISOR_QDRANT_URL`, and collection naming. -- Add operational checks for missing or stale Qdrant indexes. -- Benchmark index build time and query latency on target hardware. - -### Phase 4: Local Model Reliability - -- Diagnose `qwen3:8b` timeout behavior outside the 2-second smoke limit. -- Benchmark first-token latency and total latency for generated mode. -- Compare generated availability across cheap, balanced, and expensive mode runs only after extractive mode metrics are stable. -- Decide whether generated mode should remain opt-in only for demo usage. -- Benchmark `qwen3-embedding:4b` indexing quality and runtime against the current hash baseline. - -### Phase 5: Demo Packaging And Publication - -- Keep README and demo docs aligned with the latest metrics. -- Add screenshots or captured sample outputs for GitHub presentation. -- Add a clear "not production / not official decision" warning near demo entry points. -- Prepare a clean GitHub repository as `emu-advisor` once local docs, ignores, and artifacts are reviewed. - -### Phase 6: Deployment Planning - -- Identify target deployment hardware and local-service permissions. -- Decide whether the demo should run as a simple local Uvicorn service, a Windows service, Docker Compose, or another campus-friendly setup. -- Document backup/rebuild procedures for crawl artifacts, Qdrant index artifacts, and metrics artifacts. -- Define a refresh cadence for official-source crawling and manual evaluation review. - -## Immediate Next Actions - -1. Run the all-mode provisional seed benchmark and inspect mode failures. -2. Manually label the current human-review CSVs and repair any weak cases. -3. Bind and adjudicate the provisional gold-seed cases. -4. Run a live Qdrant service validation instead of embedded-only Qdrant. -5. Diagnose Ollama `qwen3:8b` latency and decide whether generated mode should be hidden, opt-in, or demo-only. -6. Add more hard cases for table/numeric facts discovered during manual demo testing. -7. Prepare GitHub publication polish: screenshots, sample outputs, and final known-limits wording. - -## Current Readiness Judgment - -The project is ready for a local, staff-facing demonstration and for continued GitHub publication preparation. - -The project is not ready for production use, official policy interpretation, or unattended deployment until the evaluation set is human-reviewed, live Qdrant is validated, local model behavior is characterized, and deployment constraints are documented. +Historical project percentages are legacy automated proxy observations and are not used as current semantic evidence. diff --git a/docs/PUBLICATION_CHECKLIST.md b/docs/PUBLICATION_CHECKLIST.md index d0368f2..e94618c 100644 --- a/docs/PUBLICATION_CHECKLIST.md +++ b/docs/PUBLICATION_CHECKLIST.md @@ -1,36 +1,19 @@ # Publication Checklist -Use this before changing EMUAdvisor from private to public. +Current decision: `BLOCKED` -## Required before publication +- [x] Code/runtime boundary documented. +- [x] Tracked evaluation set relabeled assistant-curated/pending independent review. +- [x] Automated metrics renamed as evidence/behavior proxies. +- [x] Fixture mode visibly distinguished from artifact mode. +- [x] Transcript persistence, audit logging, and raw-query logging default off. +- [x] Public transcript operations bound to a per-session capability. +- [ ] Independent evaluation review with privacy-safe durable evidence. +- [ ] Implemented immutable run manifests with input/output hashes. +- [ ] Reproducible artifact-backed corpus/proxy run. +- [ ] Corpus redistribution/licensing determination. +- [ ] Live Qdrant/Ollama/target hardware validation. +- [ ] Deployment threat model and public-exposure decision. +- [ ] Independent release/presentation QA approval. -- [x] Publishing rights confirmed by the repository owner. -- [x] MIT license present. -- [x] `eval_sets/v1_gold.jsonl` recorded as human-reviewed and verified by the project author and university staff. -- [x] Admin tokens excluded from URL/query-parameter flows. -- [x] Browser diagnostics use tab-scoped session storage plus authorization headers. -- [x] Pinned dependency snapshot present in `requirements-lock.txt`. -- [x] Core CI and browser smoke are required jobs. -- [ ] Keep `.old/` archive ignored and out of the clean public tree unless explicitly promoted. -- [ ] Keep generated crawl, index, metrics, screenshots, conversations, and logs under ignored artifact paths. -- [ ] Run the final branch through `python tools/publication_guard.py` and GitHub Actions. -- [ ] Refresh `docs/BOARD_DEMO_READINESS.md` when deployment/runtime evidence changes. -- [ ] Confirm README commands match the supported run protocol. -- [ ] Check the current tree and Git history for secrets or non-public university material. - -## Required wording - -- Describe the project as independent research/software, not an official EMU administrative service. -- State that outputs are informational and are not final university decisions. -- Describe `v1_gold` as the verified human-reviewed 60-case benchmark. -- Keep `v1_hard` identified as a hard regression suite unless separately reviewed/documented as gold. -- Keep `emu_gold_seed` identified as provisional unless it completes the same review process. -- Describe recorded benchmark metrics as fixed local evaluation results, not production guarantees. -- State that production deployment remains unvalidated until service-backed Qdrant and target runtime/hardware are verified. - -## Optional publication assets - -- Public UI screenshot. -- Diagnostics screenshot with tokens and sensitive local logs excluded. -- Sample output snippets from `docs/DEMO_METRICS_SNAPSHOT.md`. -- Architecture diagram based on the README data flow. +Do not create a tag, release, deployment, social preview, screenshot package, or presentation-readiness claim while any unchecked blocker remains. diff --git a/docs/RELEASE_CANDIDATE.md b/docs/RELEASE_CANDIDATE.md index 072eefb..eb0e35d 100644 --- a/docs/RELEASE_CANDIDATE.md +++ b/docs/RELEASE_CANDIDATE.md @@ -1,78 +1,16 @@ -# V1 Release Candidate Notes +# Release Candidate Status -Last updated: 2026-05-05 +Status: `NOT A RELEASE CANDIDATE` -## Status +The repository name and history are retained, but release, deployment, GitHub-presentation, and production-readiness actions are blocked. -The repository now has a root implementation path for every planned sprint plus a presentable real-corpus demo pass. It is still not production-ready, but it is suitable for a local staff-facing demo and GitHub publication as `emu-advisor`. +Reasons: -## Verified Locally +- the 60-case tracked set is assistant-curated and pending independent review; +- historical metrics are legacy automated proxies without the immutable artifacts needed for reproduction; +- the run-manifest design is documented but not implemented; +- corpus distribution rights are unresolved; +- live source/services/target hardware and a deployment threat model are unverified; +- session capabilities provide local isolation, not full public-Internet identity. -- Canonical schema validation. -- HTML ingestion from official-scope EMU regulation URLs. -- PDF ingestion with page-number metadata using local `pypdf`. -- Language/corpus routing for English and Turkish. -- Machine-readable 60-case assistant-curated bilingual candidate evaluation set. -- Local deterministic embedding baseline. -- Qdrant adapter and local fallback vector/payload store. -- Hybrid lexical+dense retrieval and mode presets. -- Answerability gate, refusal, clarification, conflict display, citations, and extractive fallback. -- Snapshot/diff/activation workflow. -- FastAPI demo endpoints and EMU-branded static UI. -- Privacy-preserving audit log format. -- Active-session load simulation. -- Root live crawl/index CLI for `mevzuat.emu.edu.tr`. -- Corpus artifact loader with demo fallback. -- Metrics runner and saved dashboard artifacts. -- API endpoints for `/metrics`, `/corpus/status`, and `/llm/status`. -- Demo README and tracked metrics snapshot. -- Embedded local Qdrant index build at `artifacts/qdrant/latest`. -- Table-aware HTML ingestion with row-level chunks and derived academic salary facts. -- Scholarship topic-bundle answer path for broad scholarship questions. -- Local Ollama `qwen3:8b` generated-answer fallback and status diagnostics. - -## Current Demo Metrics - -Latest ignored artifacts: - -- Corpus: `artifacts/demo_corpus/latest/chunks.jsonl` -- Metrics: `artifacts/metrics/latest/metrics.json`, `metrics.md`, `per_case.csv`, `human_review.csv` - -Measured on 2026-05-05 over the 60-case assistant-curated candidate set: - -- Corpus: 123 crawled pages, 22 PDFs, 8,714 chunks, 119 sources/documents. -- Language split: 3,878 English chunks and 4,836 Turkish chunks. -- Structured evidence: 493 table summaries, 7,601 table rows, 8 derived salary facts. -- Retrieval top-5: 100%. -- Response accuracy: 100%. -- Rejection accuracy: 100%. -- Clarification accuracy: 100%. -- Citation coverage: 100%. -- Extractive latency: p50 674 ms, p95 1,267 ms. -- Hard regression set: 50 cases, top-5 retrieval 100%, response accuracy 100%, rejection accuracy 100%, citation coverage 100%, extractive p50 468 ms. -- Generated mode: `qwen3:8b` was detected but the bounded 2-second smoke metric run timed out; extractive fallback remains the validated path. - -## Not Yet Validated - -- Human-reviewed 50-60 question gold set. -- Docker/live Qdrant service deployment with `qdrant_client`. -- Human-rated local LLM generation quality, streaming first-token latency, or GPU serving. -- Campus server deployment constraints. - -## Demo Command - -```powershell -python -m emu_advisor.pipeline build --seed https://mevzuat.emu.edu.tr/content.htm --seed https://mevzuat.emu.edu.tr/Content-en.htm --out artifacts\demo_corpus\latest --max-pages 1000 --include-pdfs -python -m emu_advisor.index build --chunks artifacts\demo_corpus\latest\chunks.jsonl --backend qdrant --collection emu_regulations --qdrant-path artifacts\qdrant\latest -python -m emu_advisor.metrics run --cases eval_sets\v1_gold.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\metrics\latest -python -m emu_advisor.metrics run --cases eval_sets\v1_hard.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\metrics\hard_latest -$env:EMU_ADVISOR_VECTOR_BACKEND="qdrant" -$env:EMU_ADVISOR_QDRANT_PATH="artifacts\qdrant\latest" -python -m uvicorn emu_advisor.server:app --host 127.0.0.1 --port 8000 -``` - -Then open `http://127.0.0.1:8000`. - -## Release Gate - -Do not publish this as production. It is suitable for a presentable local demo, GitHub publication as `emu-advisor`, and continuing implementation toward reviewed metrics and production retrieval infrastructure. +EMU-B001 may close software controls with residual risks. That outcome is not release approval. diff --git a/docs/REPO_MAP.md b/docs/REPO_MAP.md index 17ff5bc..e10b5ad 100644 --- a/docs/REPO_MAP.md +++ b/docs/REPO_MAP.md @@ -1,254 +1,43 @@ -# Repo Map +# Repository Map -Last updated: 2026-05-07 +## Authority -## Workspace Shape +- Product boundary: `EMU_RAG_Current_System_Specs.md`. +- Active workflow: `DEV_STATE.md` and `BLUEPRINT.md`. +- Current implementation/evidence state: `docs/PROJECT_STATE.md`. +- Risks and verification: `RISK_REGISTER.md` and `QA_REPORT.md`. +- Commands: `docs/RUN_PROTOCOL.md`. -```text -EMUAdvisor/ - EMU_RAG_Current_System_Specs.md - requirements.txt - requirements-dev.txt - AGENTS.md - docs/ - DEMO_METRICS_SNAPSHOT.md - eval_spec.md - PROJECT_STATE.md - PROJECT_STATUS_PROGRESS_PLAN.md - REPO_MAP.md - RUN_PROTOCOL.md - SCHEMA.md - SPRINT_PLAN.md - SPRINT_STATUS.md - VERSION_LOG.md - MIGRATION_BACKLOG.md - RELEASE_CANDIDATE.md - BOARD_DEMO_READINESS.md - DEMO_STORYBOARD.md - PUBLICATION_CHECKLIST.md - emu_advisor/ - __init__.py - admin.py - answer.py - audit_log.py - citations.py - corpus.py - benchmark.py - demo.py - embeddings.py - evaluation.py - eval_review.py - generation.py - html_ingest.py - load_test.py - metrics.py - modes.py - pdf_ingest.py - pipeline.py - retrieval.py - readiness.py - routing.py - server.py - schema.py - store.py - text.py - index.py - validate_jsonl.py - eval_sets/ - emu_gold_seed.jsonl - v1_gold.jsonl - v1_hard.jsonl - artifacts/ # ignored generated crawl, corpus, metrics, and review outputs - static/ - landing.html - admin.html - shared.js - user-chat.js - admin-diagnostics.js - style.css - tests/ - fixtures/ - canonical_chunks.valid.jsonl - canonical_chunks.invalid.jsonl - test_schema.py - tools/ - browser_smoke.py - .github/ - workflows/ - ci.yml - .old/ - README.md - requirements-full.txt - 1.BasicCrawlV2.py - 2.ClassifyContentV2.py - 3.ExtractRowsV7.py - 4.CompileDataV6.py - 5.ChunkerV5.py - 5.2.DedupChunks.py - 5.3.PostprocessChunks.py - 6.BuildIndex.py - 6.TestRetrieve.py - 7.RetrieveHybrid.py - 8.RerankMultilingualV7_3.py - EvaluateRetrieval.py - bm25_utils.py - backend/ - README.md - config.json - server.py - rag_adapter.py - requirements.txt - static/ - index.html - app.js - style.css -``` +`.old/` is absent from this checkout. Any historical README material is non-authoritative even if it exists in Git history. -## Repo Type +## Runtime -- Research/prototype workspace for a RAG assistant. -- Current root is the primary Git repository for future `emu-advisor` work. -- `.old/` is an ignored local archive of the previous runnable Python/FastAPI demo. +- `emu_advisor/server.py`: FastAPI routes, explicit corpus/profile startup, admin boundary, capability-protected chat endpoints. +- `emu_advisor/conversation_store.py`: in-memory/default session store, optional bounded persistence, capability hashes. +- `emu_advisor/audit_log.py`: minimized optional audit records. +- `emu_advisor/corpus.py`: explicit fixture/artifact loading and legacy-metric refusal. +- `emu_advisor/routing.py`, `retrieval.py`, `answer.py`, `generation.py`: local query, retrieval, cited answer, and optional local model flow. +- `static/`: user/diagnostic UI; fixture warning and same-tab session capability handling. -## Source Areas +## Ingestion and evidence -- `EMU_RAG_Current_System_Specs.md`: current product and architecture specification. -- `requirements.txt` and `requirements-dev.txt`: root runtime and test dependencies. -- `docs/`: operating docs for future Codex and human work. -- `docs/SCHEMA.md`: canonical document/chunk schema contract and validation usage. -- `docs/SPRINT_PLAN.md`: 24-48 hour implementation and testing sequence. -- `docs/SPRINT_STATUS.md`: sprint implementation status and validation boundary. -- `docs/PROJECT_STATUS_PROGRESS_PLAN.md`: current status, achieved progress, validated metrics, limitations, and forward plan. -- `docs/RELEASE_CANDIDATE.md`: live demo release notes, measured metrics, and remaining production gaps. -- `docs/DEMO_METRICS_SNAPSHOT.md`: concise tracked snapshot of current demo corpus, metrics, sample outputs, and limits. -- `docs/BOARD_DEMO_READINESS.md`: generated board-demo readiness status with explicit blocked gates. -- `docs/DEMO_STORYBOARD.md`: repeatable stakeholder demo script. -- `docs/PUBLICATION_CHECKLIST.md`: clean GitHub publication checklist and wording guardrails. -- `docs/eval_spec.md`: scoring rubric, case status rules, and mode-comparison instructions. -- `emu_advisor/schema.py`: dependency-free canonical schema validation and legacy chunk mapping. -- `emu_advisor/validate_jsonl.py`: JSONL validator CLI for canonical records. -- `emu_advisor/html_ingest.py` and `pdf_ingest.py`: canonical source ingestion, including table summaries, row-level chunks, and derived salary facts for HTML. -- `emu_advisor/pipeline.py`: polite official-host crawl/build CLI for canonical demo artifacts. -- `emu_advisor/corpus.py`: active corpus artifact loader, fixture fallback, and corpus status reporting. -- `emu_advisor/metrics.py`: evaluation runner that emits JSON/Markdown/CSV metrics, human-review CSV, failure analysis, and cheap/balanced/expensive comparison reports. -- `emu_advisor/eval_review.py`: review-status summary, human-review CSV export, and provisional seed binding helpers. -- `emu_advisor/benchmark.py`: local embedding and generated-mode benchmark probes. -- `emu_advisor/generation.py`: local Ollama generated-answer adapter with extractive fallback. -- `emu_advisor/retrieval.py`, `store.py`, `embeddings.py`, `modes.py`, and `index.py`: local/Qdrant retrieval stack and index build CLI. -- `emu_advisor/answer.py` and `citations.py`: answerability, table answers, scholarship topic bundles, fallback, conflict, and citations. -- `emu_advisor/server.py` and `static/`: FastAPI demo and UI; `/` is a landing page, `/admin` hosts User chat and Diagnostics modes, `/chat` and `/chat/stream` are sanitized public chat APIs, `/ask` remains the full diagnostic endpoint, `/analytics` summarizes local audit logs, and admin/debug routes can be token-protected. -- `emu_advisor/readiness.py`: board-demo readiness report generator. -- `tools/browser_smoke.py`: optional Playwright desktop/mobile browser smoke. -- `eval_sets/emu_gold_seed.jsonl`: 50-case provisional seed converted from `docs/gold-set-comprehensive-analysis.md`, pending exact source/chunk binding and human review. -- `eval_sets/v1_gold.jsonl`: current 60-case assistant-curated bilingual candidate set pending human review. -- `eval_sets/v1_hard.jsonl`: 50-case assistant-curated hard regression set for table-derived salary and broad scholarship failures. -- `artifacts/`: ignored generated crawl metadata, canonical chunks, snapshots, metrics reports, and review CSVs. -- `tests/`: unit tests and schema fixtures. -- `.old/*.py`: old-demo ingestion, processing, indexing, retrieval, reranking, and evaluation scripts. -- `.old/backend/`: FastAPI backend, RAG adapter, configuration, and static frontend. -- `.old/requirements-full.txt`: full old-demo dependency list. -- `.old/backend/requirements.txt`: minimal backend dependency list. +- `emu_advisor/pipeline.py`: exact-host HTTPS crawl, manual redirect validation, explicit file-fixture provenance. +- `emu_advisor/html_ingest.py`, `pdf_ingest.py`, `schema.py`: canonical record production and validation. +- `emu_advisor/evaluation.py`: protected case loading, retrieval matching, expected-evidence citation matching. +- `emu_advisor/metrics.py`: `emu-advisor-automated-proxy/v2` reports. +- `emu_advisor/eval_review.py`: review export/status tooling. +- `tools/publication_guard.py`: pending-review and claim guard. -## Pipeline Data Flow +## Tests and CI -```text -Official EMU regulation HTML/PDF sources - -> emu_advisor.pipeline - -> artifacts/demo_corpus/latest/raw - -> artifacts/demo_corpus/latest/chunks.jsonl - -> emu_advisor.corpus - -> emu_advisor.retrieval - -> emu_advisor.answer and optional emu_advisor.generation - -> emu_advisor.server and static UI - -> /chat for simple demo or /ask for admin diagnostics - -> emu_advisor.metrics - -> artifacts/metrics/latest/{metrics.json,metrics.md,per_case.csv,human_review.csv} -``` +- `tests/`: unit, regression, security/privacy, provenance, and server tests. +- `tools/browser_smoke.py`: synthetic fixture-mode browser smoke. +- `tools/syntax_check.py`: cross-platform syntax scan. +- `.github/workflows/ci.yml`: Windows/Ubuntu Python 3.12 core checks and Ubuntu browser smoke with immutable action pins. -Legacy reference flow: +## Protected/generated paths -```text -Official EMU regulation HTML/PDF sources - -> 1.BasicCrawlV2.py - -> crawl.sqlite and raw HTML files - -> 2.ClassifyContentV2.py - -> classified JSONL - -> 3.ExtractRowsV7.py - -> rows JSONL - -> 4.CompileDataV6.py - -> document JSONL - -> 5.ChunkerV5.py - -> chunk JSONL - -> 5.2.DedupChunks.py - -> deduplicated chunks and report - -> 5.3.PostprocessChunks.py - -> postprocessed chunks - -> 6.BuildIndex.py - -> BM25 and dense FAISS/numpy index directories - -> 7.RetrieveHybrid.py - -> hybrid candidates - -> 8.RerankMultilingualV7_3.py - -> reranked hits - -> backend/rag_adapter.py - -> extraction or local Ollama synthesis - -> backend/server.py and static UI -``` - -## Runtime And Build Signals - -- Language: Python. -- Active root package: `emu_advisor`. -- Root test runner: `python -m unittest discover -s tests`. -- Root corpus build: `python -m emu_advisor.pipeline build --seed https://mevzuat.emu.edu.tr/content.htm --seed https://mevzuat.emu.edu.tr/Content-en.htm --out artifacts\demo_corpus\latest --max-pages 1000 --include-pdfs`. -- Root metrics run: `python -m emu_advisor.metrics run --cases eval_sets\v1_gold.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\metrics\latest`. -- Root hard metrics run: `python -m emu_advisor.metrics run --cases eval_sets\v1_hard.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\metrics\hard_latest`. -- Root Qdrant index build: `python -m emu_advisor.index build --chunks artifacts\demo_corpus\latest\chunks.jsonl --backend qdrant --collection emu_regulations --qdrant-path artifacts\qdrant\latest` for embedded local Qdrant, or omit `--qdrant-path` for a live Qdrant service. -- Backend framework: FastAPI with Uvicorn. -- Frontend: static HTML, CSS, and JavaScript. -- Crawl storage: ignored raw files plus canonical JSONL artifacts. -- Intermediate data: JSONL and JSON reports. -- Existing retrieval stack: BM25 plus dense embeddings with FAISS when available, numpy fallback otherwise. -- Active root retrieval stack: local/Qdrant lexical+dense hybrid retrieval with hash fallback and optional Ollama embeddings. -- Existing old-demo model tooling: `sentence-transformers`, `transformers`, optional local Ollama. -- Root runtime dependencies: FastAPI, Uvicorn, Pydantic, HTTPX, and pypdf. -- Optional production vector backend dependency: `qdrant-client`. -- Full old-demo dependencies: FastAPI, Uvicorn, Pydantic, Requests, HTTPX, BeautifulSoup, charset-normalizer, numpy, faiss-cpu, sentence-transformers, transformers. -- `torch` is intentionally not pinned because CUDA wheels are platform-specific. - -## Active Versus Legacy - -- Active product direction is the root system spec. -- Active root implementation covers the live demo pipeline from crawl/build through metrics, answer/API, and UI. -- Legacy runnable code is archived under `.old/`. -- Old-demo READMEs are useful for commands but do not override the current spec. -- Generated root `artifacts/` outputs are ignored by Git and currently hold the live demo corpus and metrics. - -## Known Drift - -- The root workspace is version-controlled on `main`. -- `.old/` is ignored by the root repo and should not be treated as active implementation code. -- Old-demo config and scripts default to `intfloat/e5-base-v2`, which conflicts with the bilingual V1 requirement. -- Old-demo config points EN and TR indexes to the same path. -- The active root implementation supports local and Qdrant-backed hybrid retrieval; old-demo code still uses BM25 plus FAISS/numpy. -- The spec requires stronger PDF metadata handling than the visible old-demo docs prove. -- The `.old/` archive contains Python cache files that should remain ignored. -- There is no single standard test command. - -## Entry Points - -- Full legacy pipeline reference: run numbered scripts in `.old/` in sequence. -- Root corpus build: `python -m emu_advisor.pipeline build ...`. -- Root metrics: `python -m emu_advisor.metrics run ...`. -- Root mode comparison: `python -m emu_advisor.metrics run --all-modes --cases eval_sets\emu_gold_seed.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\metrics\mode_comparison`. -- Root Qdrant/local index build: `python -m emu_advisor.index build ...`. -- Root Qdrant health check: `python -m emu_advisor.index health ...`. -- Root review status: `python -m emu_advisor.eval_review status eval_sets\v1_gold.jsonl eval_sets\v1_hard.jsonl eval_sets\emu_gold_seed.jsonl`. -- Root benchmark probe: `python -m emu_advisor.benchmark embedding --cases eval_sets\v1_gold.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --embedding hash`. -- Root readiness report: `python -m emu_advisor.readiness --out docs\BOARD_DEMO_READINESS.md`. -- Retrieval smoke: `6.TestRetrieve.py`, `7.RetrieveHybrid.py`, or `8.RerankMultilingualV7_3.py` with a built index. -- Legacy evaluation: `EvaluateRetrieval.py` with a built index and query set. -- Backend: `python -m uvicorn emu_advisor.server:app --host 127.0.0.1 --port 8000` from the root repo. -- Legacy backend: `python -m uvicorn backend.server:app --host 0.0.0.0 --port 8000` from `.old/`. -- Landing: `http://127.0.0.1:8000` after backend startup. -- User chat: `http://127.0.0.1:8000/admin?view=user`. -- Diagnostics: `http://127.0.0.1:8000/admin?view=diagnostics`. +- Protected tracked inputs: system specification and evaluation sets. +- Authorized EMU-B001 exception: provenance/adjudication fields only in `eval_sets/v1_gold.jsonl`. +- Generated/private and ignored: `artifacts/**`, `logs/**`, local environments/caches, Qdrant state, corpus, metrics, review work, transcripts. +- Do not inspect, commit, migrate, or delete existing private artifacts during ordinary verification. diff --git a/docs/REPO_PROFILE.json b/docs/REPO_PROFILE.json new file mode 100644 index 0000000..1e6f940 --- /dev/null +++ b/docs/REPO_PROFILE.json @@ -0,0 +1,95 @@ +{ + "schema_version": "agentic-workflow/v2", + "root": ".", + "files_scanned": 84, + "scan_truncated": false, + "primary_type": "mixed", + "traits": [ + "software", + "data-ml", + "research-evaluation" + ], + "confidence": "high", + "existing_profile": null, + "scores": { + "software": 15, + "data-ml": 10, + "documentation": 4, + "research": 1 + }, + "evidence": { + "software": [ + "dir:tests", + "ext:.py(40)", + "ext:.js(3)", + "keywords:migration" + ], + "data-ml": [ + "file:requirements.txt", + "keywords:evaluation,metrics" + ], + "documentation": [ + "dir:docs" + ], + "research": [ + "keywords:citation" + ] + }, + "extension_counts": { + ".py": 40, + ".md": 19, + ".jsonl": 6, + ".txt": 3, + ".js": 3, + "": 2, + ".json": 2, + ".html": 2, + ".yml": 1, + ".css": 1 + }, + "existing_governance": [ + "AGENTS.md", + "docs/REPO_MAP.md", + "docs/PROJECT_STATE.md", + "README.md" + ], + "top_level": [ + ".claude/", + ".github/", + ".gitignore", + "AGENTS.md", + "CLAUDE.md", + "EMU_RAG_Current_System_Specs.md", + "Implementation Plan 110526.md", + "LICENSE", + "PUBLICATION.md", + "README.md", + "SECURITY.md", + "artifacts/", + "docs/", + "emu_advisor/", + "eval_sets/", + "logs/", + "requirements-dev.txt", + "requirements-lock.txt", + "requirements.txt", + "static/", + "tests/", + "tools/" + ], + "entry_points": [ + "emu_advisor\\index.py", + "emu_advisor\\server.py" + ], + "commands": [], + "sensitive_paths": [], + "generated_output_roots": [], + "protected_input_roots": [], + "selected_profile": "mixed", + "selected_traits": [ + "software", + "data-ml" + ], + "selection_source": "explicit CLI selection", + "project_name": "EMUAdvisor" +} diff --git a/docs/REPO_PROFILE.md b/docs/REPO_PROFILE.md new file mode 100644 index 0000000..2e48364 --- /dev/null +++ b/docs/REPO_PROFILE.md @@ -0,0 +1,82 @@ +# Repository Profile + +Workflow schema: `agentic-workflow/v2` +Project: EMUAdvisor +Repository profile: mixed +Initialized: 2026-09-05 + +## Classification + +- Primary type: `mixed` +- Secondary traits: software, data-ml +- Confidence: high +- Files scanned: 84 +- Scan truncated: False + +## Evidence + +```json +{ + "software": [ + "dir:tests", + "ext:.py(40)", + "ext:.js(3)", + "keywords:migration" + ], + "data-ml": [ + "file:requirements.txt", + "keywords:evaluation,metrics" + ], + "documentation": [ + "dir:docs" + ], + "research": [ + "keywords:citation" + ] +} +``` + +## Existing governance discovered + +- AGENTS.md +- docs/REPO_MAP.md +- docs/PROJECT_STATE.md +- README.md + +## Discovered entry points + +- `emu_advisor\index.py` +- `emu_advisor\server.py` + +## Discovered commands + +- Unit suite: `python -m unittest discover -s tests` +- Evaluation schema validation: `python -m emu_advisor.evaluation ` +- Review-status summary: `python -m emu_advisor.eval_review status ` +- Publication guard: `python tools/publication_guard.py` +- Browser smoke: `python tools/browser_smoke.py --start-server` +- Backend: `python -m uvicorn emu_advisor.server:app --host 127.0.0.1 --port 8000` +- Optional generated artifacts, live crawl, Qdrant, and Ollama commands are documented in `docs/RUN_PROTOCOL.md` and are not implicit verification. + +## Protected/generated boundaries + +### Protected candidates + +- `eval_sets/**`, system specification, license, Git history, remote/release state, credentials, + reviewer identity, private university material, and ignored `.old/**` prototype evidence. + +### Generated-output candidates + +- `artifacts/**`, `logs/**`, crawl/index/Qdrant outputs, chat transcripts, caches, and local environments. + +## Equivalent-file mappings + +- `EMU_RAG_Current_System_Specs.md` is the product/system scope authority. +- Existing `docs/PROJECT_STATE.md`, `docs/REPO_MAP.md`, `docs/RUN_PROTOCOL.md`, and + `docs/VERSION_LOG.md` satisfy their canonical workflow purposes and were preserved. + +## Profile review + +- [x] Primary type and traits confirmed from application, tests, evaluation code, and audit evidence. +- [x] Authority and protected/generated paths reconciled with `AGENTS.md`. +- [x] Required commands confirmed from `docs/RUN_PROTOCOL.md` and CI. diff --git a/docs/REPRODUCIBILITY.md b/docs/REPRODUCIBILITY.md new file mode 100644 index 0000000..07bf894 --- /dev/null +++ b/docs/REPRODUCIBILITY.md @@ -0,0 +1,14 @@ +# Reproducibility + +Core source/tests are reproducible from the lock only after platform portability is repaired. The +historical full corpus and benchmark output are ignored and absent from this checkout, so published +numbers cannot be exactly reproduced from Git alone. + +Every future result manifest must record: code commit; evaluation-set SHA-256; corpus manifest and +chunk-file SHA-256; source list/version hashes; lock SHA-256; Python/OS/hardware; configuration and +local service versions; command; start/end time; output hashes; failures; and reviewer/adjudication +artifact references. Mutable current-site reruns must receive new identities. + +Do not commit full corpus, private review material, transcripts, audit logs, model weights, or Qdrant +state merely to make a claim reproducible. Prefer lawful source/hash manifests and independently +generated local artifacts. diff --git a/docs/RUN_PROTOCOL.md b/docs/RUN_PROTOCOL.md index 8b8bf53..a239386 100644 --- a/docs/RUN_PROTOCOL.md +++ b/docs/RUN_PROTOCOL.md @@ -1,270 +1,55 @@ # Run Protocol -Last updated: 2026-05-05 +All commands run from the repository root. Python 3.12 is the supported verification target. -## Current Rule - -There is no single standard test command. Use the smallest verification level that matches the change, and do not claim higher validation than was actually run. - -## Setup - -Run active project commands from the root repo. Use `.old/` only as a legacy reference until code is promoted or replaced. - -Root setup: - -```powershell -python -m venv .venv -.\.venv\Scripts\Activate.ps1 -pip install -r requirements-dev.txt -``` - -Legacy demo setup, if needed: - -```powershell -cd "C:\Users\Ali\Desktop\EMUAdvisor\.old" -python -m venv .venv -.\.venv\Scripts\Activate.ps1 -pip install -r backend\requirements.txt -``` - -For legacy full pipeline, retrieval, reranking, and evaluation work: +## Install ```powershell -pip install -r .old\requirements-full.txt +py -3.12 -m venv .venv +.\.venv\Scripts\python.exe -m pip install --upgrade pip +.\.venv\Scripts\python.exe -m pip install -r requirements-lock.txt +.\.venv\Scripts\python.exe -m pip check ``` -Install `torch` separately for the target CUDA/CPU environment. Do not assume a CUDA wheel. +`requirements-lock.txt` is platform-aware: `uvloop` is excluded on Windows. CI verifies both Windows and Ubuntu. Audit tooling is separately pinned in `requirements-audit.txt`. -## Verification Ladder - -1. Documentation-only changes - -```powershell -Get-ChildItem -Recurse -File docs,AGENTS.md -``` - -Check that docs do not contradict `EMU_RAG_Current_System_Specs.md`. - -2. Python syntax check without writing bytecode - -For active root code: - -```powershell -@' -import ast -from pathlib import Path -root = Path(r"C:\Users\Ali\Desktop\EMUAdvisor") -failed = [] -checked = 0 -for base in [root / "emu_advisor", root / "tests"]: - for path in base.rglob("*.py"): - checked += 1 - try: - ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except Exception as exc: - failed.append((str(path), repr(exc))) -if failed: - for path, exc in failed: - print(path, exc) - raise SystemExit(1) -print(f"syntax ok: {checked} files") -'@ | python - -``` - -For legacy `.old/` reference code: +## Safe test environment ```powershell -@' -import ast -from pathlib import Path -root = Path(r"C:\Users\Ali\Desktop\EMUAdvisor\.old") -failed = [] -for path in root.rglob("*.py"): - if ".git" in path.parts or "__pycache__" in path.parts: - continue - try: - ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except Exception as exc: - failed.append((str(path), repr(exc))) -if failed: - for path, exc in failed: - print(path, exc) - raise SystemExit(1) -print("syntax ok") -'@ | python - +$env:EMU_ADVISOR_PROFILE = "test" +$env:EMU_ADVISOR_CORPUS_MODE = "fixture" +$env:EMU_ADVISOR_QUERY_REWRITE = "deterministic" +$env:EMU_ADVISOR_ENABLE_CHAT_PERSISTENCE = "0" +$env:EMU_ADVISOR_ENABLE_AUDIT_LOGGING = "0" +$env:EMU_ADVISOR_LOG_RAW_QUERY = "0" ``` -3. Root schema unit tests +## Core verification ```powershell +python tools/syntax_check.py python -m unittest discover -s tests -python -m emu_advisor.validate_jsonl tests\fixtures\canonical_chunks.valid.jsonl --kind chunk -python -m emu_advisor.evaluation eval_sets\v1_gold.jsonl -python -m emu_advisor.evaluation eval_sets\v1_hard.jsonl -python -c "from emu_advisor.server import app; print(app.title)" -python -m emu_advisor.eval_review status eval_sets\v1_gold.jsonl eval_sets\v1_hard.jsonl eval_sets\emu_gold_seed.jsonl -``` - -4. Backend import/startup smoke, when backend dependencies are installed - -```powershell -cd "C:\Users\Ali\Desktop\EMUAdvisor\.old" -python -c "from backend.server import app; print(app.title)" -``` - -5. Root demo corpus build, when network access to official EMU sources is allowed - -```powershell -python -m emu_advisor.pipeline build --seed https://mevzuat.emu.edu.tr/content.htm --seed https://mevzuat.emu.edu.tr/Content-en.htm --out artifacts\demo_corpus\latest --max-pages 1000 --include-pdfs -python -m emu_advisor.validate_jsonl artifacts\demo_corpus\latest\chunks.jsonl --kind chunk -``` - -Generated artifacts are ignored by Git and written under `artifacts/`. - -6. Root metrics gate, when a built corpus artifact exists - -```powershell -python -m emu_advisor.metrics run --cases eval_sets\v1_gold.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\metrics\latest -python -m emu_advisor.metrics run --cases eval_sets\v1_hard.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\metrics\hard_latest -python -m emu_advisor.metrics run --all-modes --cases eval_sets\emu_gold_seed.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\metrics\mode_comparison -``` - -Human-review artifact helpers: - -```powershell -python -m emu_advisor.eval_review export-csv --cases eval_sets\v1_gold.jsonl --out artifacts\review\v1_gold_review.csv -python -m emu_advisor.eval_review bind-seed --cases eval_sets\emu_gold_seed.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\review\emu_gold_seed.bound.jsonl -``` - -To attempt local generated-mode metrics with a bounded Ollama timeout: - -```powershell -python -m emu_advisor.metrics run --cases eval_sets\v1_gold.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\metrics\latest_generated --include-generation --ollama-model qwen3:8b --ollama-timeout-s 2 -``` - -7. Root Qdrant index build, when Qdrant is available - -```powershell -$env:EMU_ADVISOR_VECTOR_BACKEND="qdrant" -$env:EMU_ADVISOR_QDRANT_URL="http://localhost:6333" -$env:EMU_ADVISOR_QDRANT_COLLECTION="emu_regulations" -python -m emu_advisor.index build --chunks artifacts\demo_corpus\latest\chunks.jsonl --backend qdrant --collection emu_regulations -python -m emu_advisor.index health --backend qdrant --collection emu_regulations --qdrant-url http://localhost:6333 +python -m emu_advisor.evaluation eval_sets/v1_gold.jsonl +python -m emu_advisor.evaluation eval_sets/v1_hard.jsonl +python -m emu_advisor.evaluation eval_sets/emu_gold_seed.jsonl +python -m emu_advisor.eval_review status eval_sets/v1_gold.jsonl eval_sets/v1_hard.jsonl eval_sets/emu_gold_seed.jsonl +python tools/publication_guard.py +python tools/browser_smoke.py --start-server ``` -For embedded local Qdrant storage without a running Qdrant server: +## Dependency audit ```powershell -$env:EMU_ADVISOR_VECTOR_BACKEND="qdrant" -$env:EMU_ADVISOR_QDRANT_PATH="artifacts\qdrant\latest" -python -m emu_advisor.index build --chunks artifacts\demo_corpus\latest\chunks.jsonl --backend qdrant --collection emu_regulations --qdrant-path artifacts\qdrant\latest +python -m pip install -r requirements-audit.txt +python -m pip_audit -r requirements-lock.txt ``` -For production-profile server startup, Qdrant is required: +Record Python, pip, platform, audit-tool, commit, dependency-lock hash, environment, and command output. Do not write test output into protected `artifacts/**` or `logs/**`. -```powershell -$env:EMU_ADVISOR_PROFILE="production" -python -m uvicorn emu_advisor.server:app --host 127.0.0.1 --port 8000 -``` - -8. Legacy pipeline reference, when old-demo commands are needed - -```powershell -python 1.BasicCrawlV2.py --out mevzuat_crawl --seed https://mevzuat.emu.edu.tr/content.htm --max-pages 20 -python 2.ClassifyContentV2.py --crawl-dir mevzuat_crawl --out classified_v2.2.jsonl -python 3.ExtractRowsV7.py --crawl_dir mevzuat_crawl --classified classified_v2.2.jsonl --out rows_v7.jsonl -python 4.CompileDataV6.py --crawl_dir mevzuat_crawl --rows rows_v7.jsonl --out docs_v6.jsonl -python 5.ChunkerV5.py --in docs_v6.jsonl --out chunks_v5.jsonl --max-words 450 --overlap-words 80 -python 5.2.DedupChunks.py --in chunks_v5.jsonl --out chunks_dedup.jsonl --report dedup_report.json -python 5.3.PostprocessChunks.py --in chunks_dedup.jsonl --out chunks_post.jsonl --min-short 240 --split-threshold 2200 -python 6.BuildIndex.py --in chunks_post.jsonl --out-dir index_v4 --embed-model intfloat/multilingual-e5-base --device cpu -``` - -9. Legacy retrieval smoke, when an old-demo index exists - -```powershell -python 6.TestRetrieve.py --index-dir index_v4 --lang en --q "attendance requirement" --k 8 -python 7.RetrieveHybrid.py --index-dir index_v4 --q "salary scales" --lang auto --k 8 -python 8.RerankMultilingualV7_3.py --index-dir index_v4 --q "high honour criteria" --lang auto --k 8 -``` - -10. Root Backend/API/UI smoke - -```powershell -python -m uvicorn emu_advisor.server:app --host 127.0.0.1 --port 8000 -``` - -Then check: - -- `GET http://127.0.0.1:8000/metrics` -- `GET http://127.0.0.1:8000/metrics/modes` -- `GET http://127.0.0.1:8000/analytics` -- `GET http://127.0.0.1:8000/corpus/status` -- `GET http://127.0.0.1:8000/llm/status` -- `POST http://127.0.0.1:8000/chat` -- `POST http://127.0.0.1:8000/ask` -- Browser load at `http://127.0.0.1:8000` -- Browser load at `http://127.0.0.1:8000/admin` - -If `EMU_ADVISOR_ADMIN_TOKEN` is set, call admin/debug endpoints with either: - -```powershell -$headers = @{ Authorization = "Bearer $env:EMU_ADVISOR_ADMIN_TOKEN" } -Invoke-RestMethod http://127.0.0.1:8000/metrics -Headers $headers -``` - -For optional browser QA: - -```powershell -python tools\browser_smoke.py --start-server --skip-if-unavailable -``` - -For local load reporting: - -```powershell -python -m emu_advisor.load_test --chunks artifacts\demo_corpus\latest\chunks.jsonl --active-sessions 50 --max-workers 4 -``` - -For local model probes: - -```powershell -python -m emu_advisor.benchmark embedding --cases eval_sets\v1_gold.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --embedding hash -python -m emu_advisor.benchmark generation --cases eval_sets\v1_gold.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --model qwen3:8b --timeout-s 30 --limit 5 -``` - -For board readiness: - -```powershell -python -m emu_advisor.readiness --out docs\BOARD_DEMO_READINESS.md -``` - -11. Legacy evaluation gate, when an old-demo index and query set exist - -```powershell -python EvaluateRetrieval.py --index-dir index_v4 --out eval_run --k 8 -``` - -For V1 readiness, evaluate 50-60 English/Turkish questions and verify that correct supporting evidence appears in the top 5 when the answer exists. - -## Runtime Service Notes - -- Ollama is optional for old-demo answer synthesis but required when `backend/config.json` has `llm.enabled` set to `true`. -- Root generated-answer mode defaults to local Ollama model `qwen3:8b` at `http://localhost:11434` with `EMU_ADVISOR_LLM_TIMEOUT_S=30` and `EMU_ADVISOR_LLM_PROBE_TIMEOUT_S=10` unless overridden. -- Root demo indexing can use local Ollama embeddings with `EMU_ADVISOR_EMBEDDING=ollama`; otherwise it uses the deterministic hash fallback. -- Qdrant server integration now has a root adapter and index build CLI. Development/test profile falls back to the local store; production profile requires Qdrant. - -## Demo UI - -```powershell -python -m uvicorn emu_advisor.server:app --host 127.0.0.1 --port 8000 -``` +## Runtime modes -Open `http://127.0.0.1:8000`. +Development fixture mode requires explicit `EMU_ADVISOR_PROFILE=dev` and `EMU_ADVISOR_CORPUS_MODE=fixture`. It displays a synthetic-data warning. -## Claiming Results +Artifact mode requires explicit `EMU_ADVISOR_CORPUS_MODE=artifact` and a valid nonempty corpus at `artifacts/demo_corpus/latest/chunks.jsonl`. Production additionally requires `EMU_ADVISOR_ADMIN_TOKEN` and service-backed Qdrant. Missing/unknown/unsafe configurations fail closed. -- Syntax checks validate parseability only. -- Backend import checks validate import/startup only. -- Local retrieval smoke checks validate code paths and demo fixtures; they do not prove real corpus answer quality. -- Live `emu_advisor.metrics` outputs are the acceptable evidence for current demo retrieval, refusal, citation, failure-analysis, and latency claims. -- Manual UI/API checks are required before claiming a user-facing flow works. +Transcript persistence, audit logging, and raw-query logging are opt-in settings described in `SECURITY.md`. Never use real private questions or credentials in automated verification. diff --git a/docs/SPRINT_PLAN.md b/docs/SPRINT_PLAN.md index 0bae0d2..fc8f936 100644 --- a/docs/SPRINT_PLAN.md +++ b/docs/SPRINT_PLAN.md @@ -1,5 +1,7 @@ # Sprint Implementation And Testing Plan +> **Historical sprint plan only.** Goals, acceptance statements, and readiness gates in this file are plans, not evidence of completion. Current authority is `DEV_STATE.md` and `docs/PROJECT_STATE.md`; presentation remains blocked. + Last updated: 2026-04-30 Primary source of truth: `EMU_RAG_Current_System_Specs.md` diff --git a/docs/SPRINT_STATUS.md b/docs/SPRINT_STATUS.md index cf34143..92bbe86 100644 --- a/docs/SPRINT_STATUS.md +++ b/docs/SPRINT_STATUS.md @@ -1,71 +1,13 @@ -# Sprint Status +# Historical Sprint Status and Evidence Correction -Last updated: 2026-05-22 +The earlier sprint implemented the local FastAPI application, ingestion/retrieval paths, evaluation tooling, CI, browser smoke, and documentation surfaces. Those implementation facts remain useful. -This records implementation status for the 24-48 hour sprint plan. `Implemented` means local code paths and tests exist. The presentable-demo pass now also has a live official-corpus crawl, assistant-curated 60-case candidate metrics, and a 50-case hard regression set for table/broad-query failures. +The earlier readiness interpretation does not remain current: -| Sprint | Status | Evidence | -|---|---|---| -| 0 Repo ownership | Implemented | Root repo is primary; `.old/` is ignored legacy archive. | -| 1 Canonical schema | Implemented | `emu_advisor/schema.py`, JSONL validator, fixtures, tests. | -| 2 HTML ingestion | Implemented | `emu_advisor/html_ingest.py` emits canonical HTML chunks with traceability, table summaries, row chunks, and derived salary facts. | -| 3 PDF ingestion | Implemented | `emu_advisor/pdf_ingest.py` extracts text/page metadata via local `pypdf`. | -| 4 Language/scope routing | Implemented | `emu_advisor/routing.py` keeps EN/TR corpora separate in V1. | -| 5 Evaluation set v0 | Implemented | `eval_sets/v1_gold.jsonl` now contains 60 assistant-curated EN/TR cases pending human review. | -| 6 Multilingual embedding replacement | Implemented baseline | `emu_advisor/embeddings.py` defaults to local multilingual hash baseline and supports optional Ollama `qwen3-embedding:4b`. | -| 7 Qdrant foundation | Implemented + embedded validation | `emu_advisor/store.py` provides a Qdrant adapter and local fallback; `emu_advisor/index.py` built the 8,714-chunk embedded Qdrant index at `artifacts/qdrant/latest`. | -| 8 Hybrid retrieval | Implemented | `emu_advisor/retrieval.py` provides lexical+dense RRF retrieval with route filters. | -| 9 Operating modes | Implemented | `emu_advisor/modes.py` defines cheap/balanced/expensive full-pipeline presets. | -| 10 Answerability/conflict | Implemented | `emu_advisor/answer.py` gates answer/refuse/clarify/conflict before generation. | -| 11 Citations/traceability | Implemented | `emu_advisor/citations.py` maps answers to chunk/source/version/page metadata. | -| 12 Streaming/fallback | Implemented | `progressive_answer_events()` yields extractive answer before optional generation. | -| 13 Admin snapshots | Implemented | `emu_advisor/admin.py` stages, diffs, activates, and exports snapshots. | -| 14 UI/API | Implemented demo | `emu_advisor/server.py` plus `static/` EMU-branded demo UI with corpus, metrics, and local LLM status cards. | -| 15 Privacy logging | Implemented | `emu_advisor/audit_log.py` hashes session IDs and logs query diagnostics. | -| 16 Load simulation | Implemented | `emu_advisor/load_test.py` simulates 50 active sessions in tests. | -| 17 Release candidate | Implemented demo | `docs/RELEASE_CANDIDATE.md` records live demo status, commands, measured metrics, and gaps. | -| 18 Real-corpus demo metrics | Implemented | `emu_advisor/pipeline.py`, `emu_advisor/metrics.py`, ignored `artifacts/demo_corpus/latest`, and ignored `artifacts/metrics/latest`. | -| 19 Evaluation-first hardening | Implemented candidate pass | 60-case candidate set, failure analysis, demo README/snapshot, LLM status checks, retrieval scoring hardening, Qdrant backend envs. | -| 20 Structured evidence hardening | Implemented regression pass | Table-aware ingestion, derived salary facts, scholarship topic bundles, language-separated routing, `eval_sets/v1_hard.jsonl`, and refreshed metrics. | -| 21 Demo split and mode benchmark foundation | Implemented | `/` simple chatbot, `/admin` diagnostics console, sanitized `/chat`, `eval_sets/emu_gold_seed.jsonl`, `docs/eval_spec.md`, and `--all-modes` metrics comparison. | -| 22 Evaluation review workflow | Implemented foundation | `emu_advisor/eval_review.py` summarizes pending review status, exports CSV review sheets, and binds provisional seed cases against a corpus artifact. | -| 23 Functional QA expansion | Implemented foundation | Added server tests for validation, special-character handling, admin-token behavior, security headers, analytics, and load-report fields. | -| 24 Browser QA harness | Implemented optional | `tools/browser_smoke.py` runs a skip-safe Playwright desktop/mobile smoke; CI includes an optional browser job. | -| 25 Input validation hardening | Implemented | `AskRequest` and `ChatRequest` trim/reject blank and overlong questions, restrict modes/styles, and return sanitized validation errors. | -| 26 Admin and security headers | Implemented foundation | Optional `EMU_ADVISOR_ADMIN_TOKEN` protects admin/debug routes; production profile requires a token; CSP/no-sniff/frame/referrer headers are set. | -| 27 CI baseline | Implemented | `.github/workflows/ci.yml` runs syntax, unit tests, evaluation validation, review status, and optional browser smoke. | -| 28 Live Qdrant validation | Partial | `python -m emu_advisor.index health` exists; service-backed Qdrant still needs a live Docker/service run. | -| 29 Local embedding benchmark | Implemented foundation | `python -m emu_advisor.benchmark embedding ...` records local embedding retrieval metrics and timings. | -| 30 Generated mode characterization | Implemented foundation | `python -m emu_advisor.benchmark generation ...` probes local Ollama generated-mode latency and keeps extractive-first as the safe default. | -| 31 UI polish pass | Implemented foundation | Public UI now shows scope chips, clearer state labels, citation metadata, improved loading/error states, and admin analytics. | -| 32 Accessibility and responsive fixes | Implemented foundation | Added focus-visible styling, responsive scope/layout handling, better touch target sizing, and optional mobile browser smoke. | -| 33 Performance and load pass | Implemented foundation | `load_test.py` now reports p50/p95, errors, fallback count, and CLI output for 50 active sessions. | -| 34 Local analytics dashboard | Implemented | `/analytics` summarizes local audit logs and `/admin` displays query/event/latency counts without external telemetry. | -| 35 Refresh and staleness workflow | Partial | Existing snapshot/diff activation remains; readiness and publication docs now expose refresh/staleness gates, but full stale-index enforcement still needs a live service pass. | -| 36 Demo storyboard package | Implemented | `docs/DEMO_STORYBOARD.md` provides a repeatable board-demo story flow and example prompts. | -| 37 Publication polish | Implemented foundation | `docs/PUBLICATION_CHECKLIST.md`, README updates, CI, and readiness docs support clean GitHub publication guardrails. | -| 38 Board readiness validation | Implemented partial gate | `emu_advisor/readiness.py` generates `docs/BOARD_DEMO_READINESS.md`; current status is partial due to human-review and live-Qdrant blockers. | +- `v1_gold.jsonl` is assistant-curated and pending independent human review. +- Historical perfect percentages were automated retrieval/behavior/citation-presence proxies, not semantic answer-quality evidence. +- Historical ignored corpus and metric outputs lack the immutable manifest evidence needed for reproduction in this checkout. +- Fixture-backed browser checks establish UI/software behavior only. +- Presentation, publication, release, and deployment remain blocked. -## Validation Boundary - -Validated locally: - -- Unit tests for schema, ingestion, pipeline, routing, retrieval, answer behavior, metrics, admin workflow, API, logging, and load simulation. -- Evaluation gold file schema and categories. -- FastAPI import and test-client smoke. -- Live official crawl: 123 pages, 22 PDFs, 8,714 canonical chunks, 4 crawl errors. -- Structured evidence chunks: 493 table summaries, 7,601 table rows, 8 derived salary facts. -- Live assistant-curated candidate metrics: top-5 retrieval 100%, response accuracy 100%, rejection accuracy 100%, clarification accuracy 100%, citation coverage 100%, extractive p50 674 ms / p95 1,267 ms. -- Hard regression metrics: top-5 retrieval 100%, response accuracy 100%, rejection accuracy 100%, citation coverage 100%, extractive p50 468 ms / p95 2,867 ms. -- Bounded generated metrics with Ollama `qwen3:8b`: model detected, but 2-second smoke generation timed out; generated metrics marked unavailable and extractive fallback preserved. -- Embedded local Qdrant index build: 8,714 chunks, 256 dimensions, collection `emu_regulations`. -- Simple/admin UI split and sanitized chat endpoint have unit/API coverage. -- Provisional 50-case gold seed validates as source-binding pending rather than adjudicated gold. - -Not validated: - -- Docker/live Qdrant service build/run. -- Human-reviewed 50-60 question gold set. -- Human-reviewed provisional gold-seed source/chunk bindings. -- Human-rated generated-answer quality; generated mode is implemented but currently timed out under the bounded 2-second smoke metric. -- Campus deployment constraints. +Current execution state belongs in `DEV_STATE.md`; current product/evidence state belongs in `docs/PROJECT_STATE.md`; historical commits remain in Git. diff --git a/docs/TEST_STRATEGY.md b/docs/TEST_STRATEGY.md new file mode 100644 index 0000000..af5cf44 --- /dev/null +++ b/docs/TEST_STRATEGY.md @@ -0,0 +1,33 @@ +# Test Strategy + +Workflow schema: `agentic-workflow/v2` + +## Levels + +- Unit/API: `python -m unittest discover -s tests`. +- Evaluation schema: run `emu_advisor.evaluation` for all three tracked case sets. +- Review/provenance status: `emu_advisor.eval_review status` and publication guard. +- Dependency: clean install, `pip check`, and `pip-audit` from the locked snapshot. +- Browser: `python tools/browser_smoke.py --start-server` at desktop/mobile and light/dark. +- Artifact-backed metrics/live crawl/Qdrant/Ollama: only when explicitly authorized and inputs exist. + +## Critical behaviors + +- Exact-host source scope and redirect/file-fixture boundaries. +- Separate English/Turkish routing and refusal/clarification behavior. +- Honest proxy metric naming and expected-evidence citation matching. +- No unauthenticated cross-session transcript enumerate/read/export/clear. +- Memory/minimized privacy defaults and expiry behavior. +- Explicit fixture mode; real/production startup refuses a missing real corpus. +- Python 3.12 Windows and Ubuntu installation/core tests. + +## Evidence boundary + +Fixture/browser smoke proves UI/API behavior only. Schema validation is not a benchmark. Retrieval +metrics require a named corpus/evaluation manifest. Semantic answer correctness or human review +requires independent adjudication evidence and cannot be inferred from proxy tests. + +## Release gate + +Independent TEST must return PASS or PASS_WITH_RISKS. Presentation remains blocked while any High +evidence/privacy/source/fixture/portability risk lacks direct evidence. diff --git a/docs/VERSION_LOG.md b/docs/VERSION_LOG.md index 2c54d3a..c046a79 100644 --- a/docs/VERSION_LOG.md +++ b/docs/VERSION_LOG.md @@ -2,6 +2,8 @@ Use this log for meaningful project milestones only. +> **Historical evidence notice.** Performance percentages and readiness language recorded before the 2026-09-06 EMU-B001 correction are legacy automated proxy outputs and historical milestone descriptions. They are not semantic answer-quality measurements, independent review, current presentation evidence, or production-readiness claims. Current status is governed by `DEV_STATE.md` and `docs/PROJECT_STATE.md`. + ## 2026-04-30 - Repo Mapping Baseline - Created root operating docs for the EMUAdvisor workspace. @@ -111,3 +113,15 @@ Use this log for meaningful project milestones only. - `cee36fc Add files via upload` - `064058e Initial commit` + +## 2026-09-06 - EMU-B001 Evidence and Privacy Correction + +- Corrected `v1_gold.jsonl` to assistant-curated/pending independent-review provenance while preserving all protected case content and ordering. +- Replaced new semantic-sounding metric output with versioned automated retrieval/evidence/behavior/format/latency proxies and expected-evidence citation matching. +- Added server-issued per-session capabilities, admin-only minimized enumeration, and default-off transcript/audit/raw-query persistence. +- Required explicit fixture/artifact corpus mode, made fixture state visible, hardened exact-host HTTPS redirects, and stopped fixture provenance laundering. +- Added platform-aware Windows dependencies and Windows/Ubuntu Python 3.12 CI with immutable official-action pins. +- Historical percentages in earlier entries are retained only as legacy automated proxy records; they are not semantic answer-quality or current presentation evidence. +- Presentation, publication, release, deployment, and production-readiness remain blocked. +- Initial independent TEST failed because three historical caveats followed the claims they qualified; a bounded documentation repair moved the caveats ahead of those claims. +- Fresh independent TEST passed the repaired snapshot with residual risks. EMU-B001 closed as `COMPLETE_WITH_RISKS`; remote CI, live/artifact-backed reproduction, human review, rights, history-aware secret review, and Internet-grade deployment security remain gated. diff --git a/docs/eval_spec.md b/docs/eval_spec.md index d770213..26381cb 100644 --- a/docs/eval_spec.md +++ b/docs/eval_spec.md @@ -1,63 +1,36 @@ -# EMU Advisor Evaluation Spec +# Evaluation Specification -Last updated: 2026-05-06 +## Evidence classes -## Purpose +EMUAdvisor separates two evidence classes: -The evaluation set is a legal-RAG benchmark for the local-only EMU Regulation Assistant. It must keep retrieval quality, answer correctness, citation precision, groundedness, format compliance, and latency separate so failures can be fixed in the correct layer. +1. Automated regression proxies, produced by software against labeled inputs. +2. Independent human adjudication, recorded through a privacy-safe durable evidence reference. -## Case Status +Automated output must never be described as semantic answer correctness, citation precision, groundedness, legal reliability, or verified gold. -- `assistant_curated_pending_human_review`: runnable candidate cases; useful for regression, not a human-reviewed claim. -- `provisional_gold_seed_pending_source_binding`: cases converted from `docs/gold-set-comprehensive-analysis.md`; useful for mode comparison, but excluded from hard quality claims until source/chunk binding and human review are complete. -- `human_checked`: reviewed by a human reviewer for answer text and citation sufficiency. -- `adjudicated`: reviewed after disagreement or legal/source ambiguity. +## Tracked sets -## Required Case Fields +- `v1_gold.jsonl`: historical filename; 60 assistant-curated regression cases pending independent review. +- `v1_hard.jsonl`: hard regression cases with their own review state. +- `emu_gold_seed.jsonl`: provisional seed cases, not verified benchmark evidence. -Every executable case should include: +Case IDs, questions, ordering, languages, behaviors, categories, expected corpus/document/chunk/source labels, and keywords are protected evaluation inputs. Review-state changes require traceable evidence. -- `case_id`, `question`, `language`, `expected_behavior`, `category`, and `review_status`. -- At least one source binding: `expected_chunk_ids`, `expected_source_urls`, or provisional expected-answer support. -- `expected_answer_text`, `expected_citation_paths`, `expected_quote_spans`, `answer_format`, `prompt_type`, and `difficulty` when available. -- `grading.must_include` and `grading.must_not_include` for answer-quality review. +## Automated proxy schema -## Scoring +New reports use `emu-advisor-automated-proxy/v2` and `verified: false`. They may report: -The mode benchmark uses these weights: +- expected-evidence retrieval top-k rates; +- answer-mode plus expected-evidence proxy rate; +- refusal and clarification behavior-match rates; +- citation presence rate; +- expected-evidence citation-match rate and evidence level; +- nonempty-format proxy; +- routing, retrieval, extractive, generation, and first-token latency. -| Dimension | Weight | -|---|---:| -| Retrieval correctness | 0.30 | -| Answer correctness | 0.30 | -| Citation precision | 0.20 | -| Groundedness / hallucination proxy | 0.15 | -| Format compliance | 0.05 | +Expected chunks are primary. Exact normalized source URLs are a fallback only when a case lacks expected chunks. Keywords may assist retrieval but cannot prove citation or answer quality. Conflict cases require all distinct expected evidence sides. -Automated scoring is a proxy. Human review remains required before presenting any score as legal-quality validation. +## Human-review contract -## Mode Comparison - -Run all modes with: - -```powershell -python -m emu_advisor.metrics run --all-modes --cases eval_sets\emu_gold_seed.jsonl --chunks artifacts\demo_corpus\latest\chunks.jsonl --out artifacts\metrics\mode_comparison -``` - -Outputs: - -- `artifacts/metrics/mode_comparison/comparison.json` -- `artifacts/metrics/mode_comparison/comparison.md` -- per-mode `metrics.json`, `metrics.md`, `per_case.csv`, and `human_review.csv` - -## Review Rules - -- A correct legal answer must cite the smallest sufficient provision when the provision is known. -- A correct answer with the wrong citation is capped at partial credit. -- A correct citation with an incomplete rule is capped at partial credit. -- Unsupported thresholds, deadlines, exceptions, offices, or article paths are hallucinations. -- English and Turkish corpora must not be mixed in V1 evaluations. - -## Current Boundary - -`eval_sets/emu_gold_seed.jsonl` is provisional. It can drive cheap/balanced/expensive tuning, but the current project must continue to label it as pending source binding and human review. +Any future `human_reviewed_verified` row requires explicit correctness judgments plus a durable evidence object containing a reference, dataset SHA-256, review date, and reviewer role. Repository tools must reject self-asserted verified metadata without that contract. Personal reviewer identity need not be public. diff --git a/emu_advisor/audit_log.py b/emu_advisor/audit_log.py index c07b35f..18df637 100644 --- a/emu_advisor/audit_log.py +++ b/emu_advisor/audit_log.py @@ -2,7 +2,6 @@ from __future__ import annotations -import hashlib import json from dataclasses import dataclass from datetime import datetime, timezone @@ -23,32 +22,37 @@ class AuditEvent: class AuditLogger: - def __init__(self, path: Path, *, salt: str = "emu-advisor-local", max_bytes: int = 1_000_000) -> None: + def __init__( + self, + path: Path, + *, + enabled: bool = True, + include_raw_query: bool = False, + max_bytes: int = 1_000_000, + ) -> None: self.path = path - self.salt = salt + self.enabled = enabled + self.include_raw_query = include_raw_query self.max_bytes = max_bytes def log(self, event: AuditEvent) -> None: + if not self.enabled: + return self.path.parent.mkdir(parents=True, exist_ok=True) self._rotate_if_needed() payload = { "created_at": datetime.now(timezone.utc).isoformat(), "event_type": event.event_type, - "query": event.query, - "session_hash": self._hash_session(event.session_id), "route": event.route, "answer_mode": event.answer_mode, "latency_ms": event.latency_ms, "citation_ids": event.citation_ids, } + if self.include_raw_query: + payload["query"] = event.query with self.path.open("a", encoding="utf-8") as handle: handle.write(json.dumps(payload, ensure_ascii=False, sort_keys=True) + "\n") - def _hash_session(self, session_id: Optional[str]) -> Optional[str]: - if not session_id: - return None - return hashlib.sha256((self.salt + session_id).encode("utf-8")).hexdigest() - def _rotate_if_needed(self) -> None: if not self.path.exists() or self.path.stat().st_size < self.max_bytes: return diff --git a/emu_advisor/conversation_store.py b/emu_advisor/conversation_store.py index 23162dc..09193f2 100644 --- a/emu_advisor/conversation_store.py +++ b/emu_advisor/conversation_store.py @@ -1,29 +1,40 @@ -"""Conversation session store with TTL, message cap, and optional disk persistence.""" +"""Capability-protected conversation sessions with opt-in disk persistence.""" from __future__ import annotations +import hashlib +import hmac import html import json import os +import secrets import time -import uuid from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Mapping, Optional MAX_MESSAGES = 20 -SESSION_TTL_S = 1800 # 30 minutes +SESSION_TTL_S = 1800 DEFAULT_SESSION_STORE_PATH = Path("artifacts") / "chat_sessions.json" +SESSION_ID_BYTES = 16 +CAPABILITY_BYTES = 32 -def _persistence_disabled() -> bool: - return os.getenv("EMU_ADVISOR_DISABLE_CHAT_PERSISTENCE", "").casefold() in {"1", "true", "yes", "on"} +def _enabled(name: str) -> bool: + return os.getenv(name, "").casefold() in {"1", "true", "yes", "on"} + + +@dataclass(frozen=True) +class SessionAccess: + session_id: str + capability: str @dataclass class _Session: session_id: str + capability_hash: str messages: List[Mapping[str, str]] = field(default_factory=list) language: Optional[str] = None created_at: float = field(default_factory=time.time) @@ -36,25 +47,36 @@ class SessionInfo: created_at: float last_active: float message_count: int - last_user_message: Optional[str] = None class ConversationStore: - def __init__(self, persist_path: Optional[Path | str] = None) -> None: + """Store sessions whose public operations require a separate bearer capability.""" + + def __init__(self, persist_path: Optional[Path | str] = None, *, persistence_enabled: Optional[bool] = None) -> None: import threading self._sessions: Dict[str, _Session] = {} self._lock = threading.Lock() - if _persistence_disabled(): - self._persist_path: Optional[Path] = None - else: - configured_path = os.getenv("EMU_ADVISOR_CHAT_STORE_PATH") - self._persist_path = Path(persist_path or configured_path or DEFAULT_SESSION_STORE_PATH) + if persistence_enabled is None: + persistence_enabled = _enabled("EMU_ADVISOR_ENABLE_CHAT_PERSISTENCE") + configured_path = os.getenv("EMU_ADVISOR_CHAT_STORE_PATH") + self._persist_path = Path(persist_path or configured_path or DEFAULT_SESSION_STORE_PATH) if persistence_enabled else None self._load_from_disk() + @staticmethod + def _hash_capability(capability: str) -> str: + return hashlib.sha256(capability.encode("utf-8")).hexdigest() + + @classmethod + def _authorized(cls, session: _Session, capability: Optional[str]) -> bool: + if not capability or len(capability) < 32: + return False + return hmac.compare_digest(session.capability_hash, cls._hash_capability(capability)) + def _session_to_dict(self, session: _Session) -> Dict[str, Any]: return { "session_id": session.session_id, + "capability_hash": session.capability_hash, "messages": list(session.messages[-MAX_MESSAGES:]), "language": session.language, "created_at": session.created_at, @@ -63,7 +85,10 @@ def _session_to_dict(self, session: _Session) -> Dict[str, Any]: def _session_from_dict(self, payload: Mapping[str, Any]) -> Optional[_Session]: session_id = str(payload.get("session_id") or "").strip() - if not session_id: + capability_hash = str(payload.get("capability_hash") or "").strip() + if not session_id or len(capability_hash) != 64: + # Legacy sessions intentionally remain on disk but are not loaded into + # the public capability boundary. return None raw_messages = payload.get("messages") or [] messages: List[Mapping[str, str]] = [] @@ -78,6 +103,7 @@ def _session_from_dict(self, payload: Mapping[str, Any]) -> Optional[_Session]: now = time.time() return _Session( session_id=session_id, + capability_hash=capability_hash, messages=messages, language=str(payload.get("language") or "") or None, created_at=float(payload.get("created_at") or now), @@ -91,6 +117,10 @@ def _load_from_disk(self) -> None: payload = json.loads(self._persist_path.read_text(encoding="utf-8")) except Exception: return + if not isinstance(payload, Mapping) or payload.get("version") != 2: + # Never overwrite or migrate a legacy/private transcript file implicitly. + self._persist_path = None + return raw_sessions = payload.get("sessions", []) if isinstance(payload, Mapping) else [] if not isinstance(raw_sessions, list): return @@ -99,10 +129,11 @@ def _load_from_disk(self) -> None: if not isinstance(item, Mapping): continue session = self._session_from_dict(item) - if session is None: - continue - if now - session.last_active <= SESSION_TTL_S: + if session is not None and now - session.last_active <= SESSION_TTL_S: self._sessions[session.session_id] = session + # Opt-in stores are rewritten after load so expired/capability-less legacy + # entries are removed from the active-format file. + self._save_to_disk() def _save_to_disk(self) -> None: if self._persist_path is None: @@ -110,7 +141,7 @@ def _save_to_disk(self) -> None: try: self._persist_path.parent.mkdir(parents=True, exist_ok=True) payload = { - "version": 1, + "version": 2, "saved_at": time.time(), "ttl_seconds": SESSION_TTL_S, "max_messages": MAX_MESSAGES, @@ -120,72 +151,100 @@ def _save_to_disk(self) -> None: tmp_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") tmp_path.replace(self._persist_path) except Exception: - # Chat persistence must never break the main assistant path. return def _prune(self) -> bool: now = time.time() - expired = [sid for sid, s in self._sessions.items() if now - s.last_active > SESSION_TTL_S] - for sid in expired: - del self._sessions[sid] + expired = [sid for sid, session in self._sessions.items() if now - session.last_active > SESSION_TTL_S] + for session_id in expired: + del self._sessions[session_id] return bool(expired) - def get_session(self, session_id: Optional[str]) -> Optional[Dict[str, Any]]: + def create_session(self) -> SessionAccess: + with self._lock: + self._prune() + while True: + session_id = secrets.token_urlsafe(SESSION_ID_BYTES) + if session_id not in self._sessions: + break + capability = secrets.token_urlsafe(CAPABILITY_BYTES) + self._sessions[session_id] = _Session( + session_id=session_id, + capability_hash=self._hash_capability(capability), + ) + self._save_to_disk() + return SessionAccess(session_id=session_id, capability=capability) + + def owns_session(self, session_id: Optional[str], capability: Optional[str]) -> bool: + if not session_id: + return False + with self._lock: + changed = self._prune() + session = self._sessions.get(session_id) + allowed = session is not None and self._authorized(session, capability) + if changed: + self._save_to_disk() + return allowed + + def get_session(self, session_id: Optional[str], capability: Optional[str]) -> Optional[Dict[str, Any]]: if not session_id: return None with self._lock: changed = self._prune() session = self._sessions.get(session_id) - if session is None: + if session is None or not self._authorized(session, capability): if changed: self._save_to_disk() return None session.last_active = time.time() self._save_to_disk() - return { - "session_id": session.session_id, - "messages": list(session.messages), - "language": session.language, - "created_at": session.created_at, - "last_active": session.last_active, - } + return self._public_session(session) - def create_session(self, session_id: Optional[str] = None) -> str: + def get_session_admin(self, session_id: Optional[str]) -> Optional[Dict[str, Any]]: + if not session_id: + return None with self._lock: - self._prune() - if session_id is None: - session_id = uuid.uuid4().hex - if session_id not in self._sessions: - self._sessions[session_id] = _Session(session_id=session_id) - self._sessions[session_id].last_active = time.time() - self._save_to_disk() - return session_id + changed = self._prune() + session = self._sessions.get(session_id) + if session is None: + if changed: + self._save_to_disk() + return None + return self._public_session(session) - def add_message(self, session_id: str, role: str, text: str) -> None: + @staticmethod + def _public_session(session: _Session) -> Dict[str, Any]: + return { + "session_id": session.session_id, + "messages": list(session.messages), + "language": session.language, + "created_at": session.created_at, + "last_active": session.last_active, + } + + def add_message(self, session_id: str, capability: str, role: str, text: str) -> bool: with self._lock: self._prune() session = self._sessions.get(session_id) - if session is None: - session = _Session(session_id=session_id) - self._sessions[session_id] = session + if session is None or not self._authorized(session, capability): + return False session.messages.append({"role": role, "text": text}) - if len(session.messages) > MAX_MESSAGES: - session.messages = session.messages[-MAX_MESSAGES:] + session.messages = session.messages[-MAX_MESSAGES:] session.last_active = time.time() self._save_to_disk() + return True - def get_history(self, session_id: str, max_exchanges: int = 6) -> List[Mapping[str, str]]: - with self._lock: - changed = self._prune() - session = self._sessions.get(session_id) - if session is None: - if changed: - self._save_to_disk() - return [] - max_msgs = max_exchanges * 2 - session.last_active = time.time() - self._save_to_disk() - return list(session.messages[-max_msgs:]) + def get_history(self, session_id: str, capability: str, max_exchanges: int = 6) -> List[Mapping[str, str]]: + session = self.get_session(session_id, capability) + if session is None: + return [] + return list(session["messages"][-max_exchanges * 2 :]) + + def get_history_admin(self, session_id: str, max_exchanges: int = 6) -> List[Mapping[str, str]]: + session = self.get_session_admin(session_id) + if session is None: + return [] + return list(session["messages"][-max_exchanges * 2 :]) def prune_expired(self) -> int: with self._lock: @@ -196,115 +255,54 @@ def prune_expired(self) -> int: return before - len(self._sessions) def list_sessions(self) -> List[SessionInfo]: - """Return list of active sessions with metadata.""" with self._lock: changed = self._prune() - sessions = [] - for sid, session in self._sessions.items(): - last_user_msg = None - for msg in reversed(session.messages): - if msg.get("role") == "user": - last_user_msg = msg.get("text") - break - sessions.append(SessionInfo( + sessions = [ + SessionInfo( session_id=session.session_id, created_at=session.created_at, last_active=session.last_active, message_count=len(session.messages), - last_user_message=last_user_msg, - )) - # Sort by last active descending - sessions.sort(key=lambda s: s.last_active, reverse=True) + ) + for session in self._sessions.values() + ] + sessions.sort(key=lambda session: session.last_active, reverse=True) if changed: self._save_to_disk() return sessions - def clear_session(self, session_id: str) -> bool: - """Clear all messages in a session, keeping the session active.""" + def clear_session(self, session_id: str, capability: str) -> bool: with self._lock: self._prune() session = self._sessions.get(session_id) - if session is None: + if session is None or not self._authorized(session, capability): return False session.messages = [] session.last_active = time.time() self._save_to_disk() return True - def export_session(self, session_id: str, format: str = "markdown") -> Optional[str]: - """Export session history to specified format.""" + def export_session(self, session_id: str, capability: str, format: str = "markdown") -> Optional[str]: with self._lock: + self._prune() session = self._sessions.get(session_id) - if session is None: + if session is None or not self._authorized(session, capability): return None - - format = format.lower().strip() - if format not in ("markdown", "html"): - format = "markdown" - - messages_text = [] - for msg in session.messages: - role = msg.get("role", "unknown") - text = msg.get("text", "") - if role == "user": - messages_text.append(f"**User:**\n{text}") - elif role == "assistant": - messages_text.append(f"**Assistant:**\n{text}") - else: - messages_text.append(f"**{role}:**\n{text}") - - content = "\n\n".join(messages_text) - - if format == "markdown": - return f"""# Chat Session: {session_id} - -**Started:** {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(session.created_at))} -**Last Active:** {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(session.last_active))} -**Message Count:** {len(session.messages)} - ---- - -## Conversation - -{content} -""" - timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(session.created_at)) - active = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(session.last_active)) - message_blocks = [] - for msg in session.messages: - role = html.escape(str(msg.get("role", "unknown"))) - text = html.escape(str(msg.get("text", ""))).replace("\n", "
") - css_role = "user" if role == "user" else "assistant" - message_blocks.append( - f'
{role.title()}
{text}
' - ) - html_messages = "\n".join(message_blocks) - return f""" - - - - Chat Session: {html.escape(session_id)} - - - -
-

Chat Session: {html.escape(session_id)}

-
Started: {html.escape(timestamp)} | Last Active: {html.escape(active)} | {len(session.messages)} messages
-
-
-{html_messages} -
- -""" + export_format = format.lower().strip() + if export_format not in {"markdown", "html"}: + export_format = "markdown" + if export_format == "markdown": + messages = [] + for message in session.messages: + role = str(message.get("role", "unknown")).title() + messages.append(f"**{role}:**\n{message.get('text', '')}") + return "# Chat transcript\n\n" + "\n\n".join(messages) + "\n" + blocks = [] + for message in session.messages: + role = html.escape(str(message.get("role", "unknown"))) + text = html.escape(str(message.get("text", ""))).replace("\n", "
") + blocks.append(f'
{role.title()}
{text}
') + return "Chat transcript" + "\n".join(blocks) + "" _store: Optional[ConversationStore] = None @@ -315,3 +313,9 @@ def get_store() -> ConversationStore: if _store is None: _store = ConversationStore() return _store + + +def reset_store() -> None: + """Reset process-local state; intended for isolated application/test startup.""" + global _store + _store = None diff --git a/emu_advisor/corpus.py b/emu_advisor/corpus.py index 92fffbf..89e466e 100644 --- a/emu_advisor/corpus.py +++ b/emu_advisor/corpus.py @@ -22,16 +22,50 @@ class CorpusBundle: source: str path: Optional[Path] status: Dict[str, Any] - - -def load_corpus(path: Path = DEFAULT_CORPUS_PATH, *, fallback_to_demo: bool = True) -> CorpusBundle: + mode: str = "artifact" + fixture: bool = False + + +def load_corpus( + path: Path = DEFAULT_CORPUS_PATH, + *, + mode: str, + profile: str, +) -> CorpusBundle: + mode = mode.strip().casefold() + profile = profile.strip().casefold() + if profile not in {"dev", "test", "production"}: + raise RuntimeError("EMU_ADVISOR_PROFILE must be explicitly set to dev, test, or production") + if mode not in {"fixture", "artifact"}: + raise RuntimeError("EMU_ADVISOR_CORPUS_MODE must be explicitly set to fixture or artifact") + if mode == "fixture": + if profile not in {"dev", "test"}: + raise RuntimeError("fixture corpus mode is allowed only in dev or test profiles") + chunks = [] + for raw in demo_chunks(): + chunk = dict(raw) + chunk["source_url"] = f"fixture://{str(chunk['chunk_id']).replace(':', '-')}" + chunk["metadata"] = {**dict(chunk.get("metadata") or {}), "fixture": True, "official_source": False} + chunks.append(chunk) + return CorpusBundle( + chunks=chunks, + source="explicit_fixture", + path=None, + status=corpus_status(chunks, path=None), + mode="fixture", + fixture=True, + ) if path.exists(): chunks = load_chunks_jsonl(path) - return CorpusBundle(chunks=chunks, source="artifact", path=path, status=corpus_status(chunks, path=path)) - if fallback_to_demo: - chunks = [dict(chunk) for chunk in demo_chunks()] - return CorpusBundle(chunks=chunks, source="demo_fallback", path=None, status=corpus_status(chunks, path=None)) - raise FileNotFoundError(f"corpus chunks not found: {path}") + return CorpusBundle( + chunks=chunks, + source="artifact", + path=path, + status=corpus_status(chunks, path=path), + mode="artifact", + fixture=False, + ) + raise FileNotFoundError(f"artifact corpus chunks not found: {path}") def load_chunks_jsonl(path: Path) -> List[Dict[str, Any]]: @@ -64,7 +98,7 @@ def corpus_status(chunks: List[Dict[str, Any]], *, path: Optional[Path]) -> Dict sources = {str(chunk.get("source_url")) for chunk in chunks} crawl_times = sorted({str(chunk.get("last_crawled_at")) for chunk in chunks if chunk.get("last_crawled_at")}) return { - "path": str(path) if path else None, + "path": path.as_posix() if path and not path.is_absolute() else (path.name if path else None), "chunk_count": len(chunks), "document_count": len(documents), "source_count": len(sources), @@ -80,6 +114,17 @@ def load_latest_metrics(path: Path = DEFAULT_METRICS_PATH) -> Dict[str, Any]: if not path.exists(): return {"available": False, "path": str(path)} payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("schema_version") != "emu-advisor-automated-proxy/v2": + return { + "available": True, + "path": str(path), + "schema_version": payload.get("schema_version"), + "legacy": True, + "verified": False, + "message": "Legacy automated proxy output; not semantic answer-quality evidence.", + } payload.setdefault("available", True) payload.setdefault("path", str(path)) + payload.setdefault("legacy", False) + payload.setdefault("verified", False) return payload diff --git a/emu_advisor/evaluation.py b/emu_advisor/evaluation.py index abf569e..abd87c8 100644 --- a/emu_advisor/evaluation.py +++ b/emu_advisor/evaluation.py @@ -86,21 +86,25 @@ def evaluate_hits(case: EvaluationCase, hits: List[Dict[str, Any]]) -> Retrieval if case.expected_behavior in {"refuse", "clarify"}: return RetrievalMetric(case.case_id, case.expected_behavior, None, False, False, False) - expected_ids = {value for value in (case.expected_chunk_id, case.expected_document_id) if value} - expected_ids.update(case.expected_chunk_ids) + expected_ids = set(case.expected_chunk_ids) + if case.expected_chunk_id: + expected_ids.add(case.expected_chunk_id) + expected_documents = {case.expected_document_id} if case.expected_document_id else set() expected_sources = {case.expected_source_url} if case.expected_source_url else set() expected_sources.update(case.expected_source_urls) rank: Optional[int] = None for idx, hit in enumerate(hits, start=1): ids = {str(hit.get("chunk_id", "")), str(hit.get("document_id", ""))} source_url = str(hit.get("source_url", "")) - text = str(hit.get("chunk_text", "")).casefold() - expected_keywords = getattr(case, "expected_answer_keywords", []) - keyword_hit = bool(expected_keywords) and all( - keyword.casefold() in text for keyword in expected_keywords - ) - source_hit = bool(expected_sources) and source_url in expected_sources - if (expected_ids and expected_ids & ids) or source_hit or keyword_hit: + if expected_ids: + matched = bool(expected_ids & ids) + elif expected_documents: + matched = bool(expected_documents & ids) + else: + matched = bool(expected_sources) and _normalize_evidence_url(source_url) in { + _normalize_evidence_url(value) for value in expected_sources + } + if matched: rank = idx break @@ -114,6 +118,62 @@ def evaluate_hits(case: EvaluationCase, hits: List[Dict[str, Any]]) -> Retrieval ) +def citation_matches_expected_evidence( + case: EvaluationCase, + citations: List[Dict[str, Any]], +) -> tuple[bool, str]: + """Return whether citations match labeled evidence and the evidence level used. + + Keywords never establish citation correctness. Chunk identifiers are primary; + exact normalized source URLs are used only when a case has no expected chunks. + Conflict cases require all distinct expected evidence items. + """ + + expected_chunks = set(case.expected_chunk_ids) + if case.expected_chunk_id: + expected_chunks.add(case.expected_chunk_id) + cited_chunks = {str(item.get("chunk_id") or "") for item in citations} + + if expected_chunks: + matched = expected_chunks.issubset(cited_chunks) if case.expected_behavior == "conflict" else bool(expected_chunks & cited_chunks) + level = "expected_chunk" + else: + expected_sources = {_normalize_evidence_url(value) for value in case.expected_source_urls} + if case.expected_source_url: + expected_sources.add(_normalize_evidence_url(case.expected_source_url)) + cited_sources = {_normalize_evidence_url(str(item.get("source_url") or "")) for item in citations} + matched = expected_sources.issubset(cited_sources) if case.expected_behavior == "conflict" else bool(expected_sources & cited_sources) + level = "expected_source" + + if matched and case.expected_citation_paths: + cited_paths = {_citation_path_from_result(item) for item in citations} + expected_paths = {_normalize_citation_path(value) for value in case.expected_citation_paths} + matched = expected_paths.issubset(cited_paths) if case.expected_behavior == "conflict" else bool(expected_paths & cited_paths) + level += "_and_path" + return matched, level + + +def _normalize_evidence_url(value: str) -> str: + return value.strip().rstrip("/").casefold() + + +def _normalize_citation_path(value: str) -> str: + return " ".join(value.strip().casefold().split()) + + +def _citation_path_from_result(citation: Dict[str, Any]) -> str: + explicit = citation.get("citation_path") or citation.get("section_path") + if explicit: + return _normalize_citation_path(str(explicit)) + article = citation.get("article_number") or citation.get("article") + page = citation.get("page_number") or citation.get("page") + if article: + return _normalize_citation_path(f"Art. {article}") + if page: + return _normalize_citation_path(f"p. {page}") + return "" + + def summarize_metrics(metrics: Iterable[RetrievalMetric]) -> Dict[str, Any]: metrics_list = list(metrics) answerable = [m for m in metrics_list if m.expected_behavior == "answer"] diff --git a/emu_advisor/html_ingest.py b/emu_advisor/html_ingest.py index 8782ba9..eea0592 100644 --- a/emu_advisor/html_ingest.py +++ b/emu_advisor/html_ingest.py @@ -660,8 +660,10 @@ def _fold(text: str) -> str: def _require_allowed_source(source_url: str) -> None: parsed = urlparse(source_url) + if parsed.scheme == "fixture" and not parsed.hostname and parsed.path: + return if parsed.scheme not in {"http", "https"}: - raise HtmlScopeError("HTML source URL must be http or https") + raise HtmlScopeError("HTML source URL must be official HTTPS or explicit fixture provenance") if parsed.hostname != ALLOWED_REGULATION_HOST: raise HtmlScopeError(f"source host is outside V1 scope: {parsed.hostname}") diff --git a/emu_advisor/metrics.py b/emu_advisor/metrics.py index fdaf24d..ee13083 100644 --- a/emu_advisor/metrics.py +++ b/emu_advisor/metrics.py @@ -19,7 +19,13 @@ scholarship_group_query, ) from .corpus import load_chunks_jsonl -from .evaluation import EvaluationCase, evaluate_hits, load_cases, validate_case_set +from .evaluation import ( + EvaluationCase, + citation_matches_expected_evidence, + evaluate_hits, + load_cases, + validate_case_set, +) from .generation import OllamaGenerator from .modes import MODE_PRESETS from .retrieval import HybridRetriever @@ -27,9 +33,9 @@ TARGETS = { - "retrieval_top5": 0.85, - "rejection_accuracy": 0.90, - "citation_coverage": 1.0, + "expected_evidence_retrieval_top5_rate": 0.85, + "refusal_behavior_match_rate": 0.90, + "expected_evidence_citation_match_rate": 1.0, "extractive_latency_p50_ms": 1000, } @@ -40,11 +46,11 @@ } SCORE_WEIGHTS = { - "retrieval_correctness": 0.30, - "answer_correctness": 0.30, - "citation_precision": 0.20, - "groundedness": 0.15, - "format_compliance": 0.05, + "retrieval_evidence_proxy": 0.30, + "behavior_evidence_proxy": 0.30, + "citation_match_proxy": 0.20, + "combined_support_proxy": 0.15, + "nonempty_format_proxy": 0.05, } @@ -59,10 +65,12 @@ class CaseResult: retrieval_top1: bool retrieval_top3: bool retrieval_top5: bool - response_correct: bool + behavior_evidence_proxy_pass: bool rejection_correct: bool clarification_correct: bool - citation_covered: bool + citation_present: bool + expected_evidence_citation_match: bool + citation_evidence_level: str route_ms: int retrieve_ms: int extractive_ms: int @@ -70,11 +78,11 @@ class CaseResult: first_token_ms: Optional[int] generated_error: Optional[str] retrieval_score: float - answer_correctness_score: float - citation_precision_score: float - groundedness_score: float - format_compliance_score: float - total_score: float + behavior_evidence_proxy_score: float + citation_match_proxy_score: float + combined_support_proxy_score: float + nonempty_format_proxy_score: float + weighted_proxy_score: float failure_reason: str answer: str citations: str @@ -156,13 +164,17 @@ def run_evaluation( first_token_ms = generated.first_token_ms generated_error = generated.error - citation_covered = bool(citations) if case.expected_behavior in {"answer", "conflict"} else True - response_correct = _response_correct( + citation_present = bool(citations) if case.expected_behavior in {"answer", "conflict"} else True + expected_evidence_citation_match, citation_evidence_level = citation_matches_expected_evidence(case, citations) + if case.expected_behavior not in {"answer", "conflict"}: + expected_evidence_citation_match = True + citation_evidence_level = "not_applicable" + behavior_evidence_proxy_pass = _behavior_evidence_proxy_pass( case, answer.text, metric.top5, actual_mode=answer.mode, - citation_covered=citation_covered, + expected_evidence_citation_match=expected_evidence_citation_match, ) rejection_correct = case.expected_behavior == "refuse" and answer.mode == "refuse" clarification_correct = case.expected_behavior == "clarify" and answer.mode == "clarify" @@ -170,20 +182,21 @@ def run_evaluation( case=case, actual_mode=answer.mode, metric=metric, - response_correct=response_correct, + behavior_evidence_proxy_pass=behavior_evidence_proxy_pass, rejection_correct=rejection_correct, clarification_correct=clarification_correct, - citation_covered=citation_covered, + citation_present=citation_present, + expected_evidence_citation_match=expected_evidence_citation_match, ) score_parts = _score_case( case=case, metric=metric, answer_text=answer.text, actual_mode=answer.mode, - response_correct=response_correct, + behavior_evidence_proxy_pass=behavior_evidence_proxy_pass, rejection_correct=rejection_correct, clarification_correct=clarification_correct, - citation_covered=citation_covered, + expected_evidence_citation_match=expected_evidence_citation_match, failure_reason=failure_reason, ) results.append( @@ -197,22 +210,24 @@ def run_evaluation( retrieval_top1=metric.top1, retrieval_top3=metric.top3, retrieval_top5=metric.top5, - response_correct=response_correct, + behavior_evidence_proxy_pass=behavior_evidence_proxy_pass, rejection_correct=rejection_correct, clarification_correct=clarification_correct, - citation_covered=citation_covered, + citation_present=citation_present, + expected_evidence_citation_match=expected_evidence_citation_match, + citation_evidence_level=citation_evidence_level, route_ms=route_ms, retrieve_ms=retrieve_ms, extractive_ms=extractive_ms, generation_ms=generated_ms, first_token_ms=first_token_ms, generated_error=generated_error, - retrieval_score=score_parts["retrieval_correctness"], - answer_correctness_score=score_parts["answer_correctness"], - citation_precision_score=score_parts["citation_precision"], - groundedness_score=score_parts["groundedness"], - format_compliance_score=score_parts["format_compliance"], - total_score=score_parts["total"], + retrieval_score=score_parts["retrieval_evidence_proxy"], + behavior_evidence_proxy_score=score_parts["behavior_evidence_proxy"], + citation_match_proxy_score=score_parts["citation_match_proxy"], + combined_support_proxy_score=score_parts["combined_support_proxy"], + nonempty_format_proxy_score=score_parts["nonempty_format_proxy"], + weighted_proxy_score=score_parts["weighted_proxy"], failure_reason=failure_reason, answer=answer.text, citations=json.dumps(citations, ensure_ascii=False), @@ -253,6 +268,9 @@ def run_all_modes( generator=generator, ) comparison = { + "schema_version": "emu-advisor-automated-proxy/v2", + "evidence_class": "automated_regression_proxy", + "verified": False, "available": True, "cases_path": str(cases_path), "chunks_path": str(chunks_path), @@ -278,18 +296,22 @@ def summarize_case_results(results: List[CaseResult]) -> Dict[str, Any]: first_token_latencies = [result.first_token_ms for result in successful_generations if result.first_token_ms is not None] summary = { + "schema_version": "emu-advisor-automated-proxy/v2", + "evidence_class": "automated_regression_proxy", + "verified": False, "available": True, "cases": len(results), "answerable_cases": len(answerable), "rejection_cases": len(refusals), "clarification_cases": len(clarifications), - "retrieval_top1": _rate(answerable, lambda result: result.retrieval_top1), - "retrieval_top3": _rate(answerable, lambda result: result.retrieval_top3), - "retrieval_top5": _rate(answerable, lambda result: result.retrieval_top5), - "response_accuracy": _rate(answerable, lambda result: result.response_correct), - "rejection_accuracy": _rate(refusals, lambda result: result.rejection_correct), - "clarification_accuracy": _rate(clarifications, lambda result: result.clarification_correct), - "citation_coverage": _rate(answerable, lambda result: result.citation_covered), + "expected_evidence_retrieval_top1_rate": _rate(answerable, lambda result: result.retrieval_top1), + "expected_evidence_retrieval_top3_rate": _rate(answerable, lambda result: result.retrieval_top3), + "expected_evidence_retrieval_top5_rate": _rate(answerable, lambda result: result.retrieval_top5), + "answer_mode_and_evidence_proxy_rate": _rate(answerable, lambda result: result.behavior_evidence_proxy_pass), + "refusal_behavior_match_rate": _rate(refusals, lambda result: result.rejection_correct), + "clarification_behavior_match_rate": _rate(clarifications, lambda result: result.clarification_correct), + "citation_presence_rate": _rate(answerable, lambda result: result.citation_present), + "expected_evidence_citation_match_rate": _rate(answerable, lambda result: result.expected_evidence_citation_match), "extractive_latency_p50_ms": int(median(extractive_latencies)) if extractive_latencies else None, "extractive_latency_p95_ms": _percentile(extractive_latencies, 0.95), "generated_latency_p50_ms": int(median(generation_latencies)) if generation_latencies else None, @@ -300,23 +322,23 @@ def summarize_case_results(results: List[CaseResult]) -> Dict[str, Any]: "generated_cases_completed": len(successful_generations), "generated_error_cases": len([result for result in generation_attempts if result.generated_error]), "generated_available": bool(successful_generations), - "retrieval_correctness_score": _average(results, lambda result: result.retrieval_score), - "answer_correctness_score": _average(results, lambda result: result.answer_correctness_score), - "citation_precision_score": _average(results, lambda result: result.citation_precision_score), - "groundedness_score": _average(results, lambda result: result.groundedness_score), - "format_compliance_score": _average(results, lambda result: result.format_compliance_score), - "total_score": _average(results, lambda result: result.total_score), + "retrieval_evidence_proxy_score": _average(results, lambda result: result.retrieval_score), + "behavior_evidence_proxy_score": _average(results, lambda result: result.behavior_evidence_proxy_score), + "citation_match_proxy_score": _average(results, lambda result: result.citation_match_proxy_score), + "combined_support_proxy_score": _average(results, lambda result: result.combined_support_proxy_score), + "nonempty_format_proxy_score": _average(results, lambda result: result.nonempty_format_proxy_score), + "weighted_proxy_score": _average(results, lambda result: result.weighted_proxy_score), "failure_counts": _failure_counts(results), "category_metrics": _category_metrics(results), "worst_failed_cases": _worst_failed_cases(results), "targets": TARGETS, } - summary["presentable"] = ( - summary["retrieval_top5"] is not None - and summary["retrieval_top5"] >= TARGETS["retrieval_top5"] - and summary["rejection_accuracy"] is not None - and summary["rejection_accuracy"] >= TARGETS["rejection_accuracy"] - and summary["citation_coverage"] == TARGETS["citation_coverage"] + summary["automated_gate_pass"] = ( + summary["expected_evidence_retrieval_top5_rate"] is not None + and summary["expected_evidence_retrieval_top5_rate"] >= TARGETS["expected_evidence_retrieval_top5_rate"] + and summary["refusal_behavior_match_rate"] is not None + and summary["refusal_behavior_match_rate"] >= TARGETS["refusal_behavior_match_rate"] + and summary["expected_evidence_citation_match_rate"] == TARGETS["expected_evidence_citation_match_rate"] and summary["extractive_latency_p50_ms"] is not None and summary["extractive_latency_p50_ms"] < TARGETS["extractive_latency_p50_ms"] ) @@ -350,21 +372,21 @@ def write_reports(results: List[CaseResult], summary: Dict[str, Any], *, out_dir (out_dir / "metrics.md").write_text(_markdown_summary(summary), encoding="utf-8") -def _response_correct( +def _behavior_evidence_proxy_pass( case: EvaluationCase, answer_text: str, top5: bool, *, actual_mode: str, - citation_covered: bool, + expected_evidence_citation_match: bool, ) -> bool: if case.expected_behavior == "conflict": - return top5 and citation_covered and (actual_mode == "show_conflict" or "conflict" in answer_text.casefold()) + return top5 and expected_evidence_citation_match and (actual_mode == "show_conflict" or "conflict" in answer_text.casefold()) if case.expected_behavior != "answer": return False if not top5: return False - return citation_covered and actual_mode in {"answer", "answer_uncertain"} + return expected_evidence_citation_match and actual_mode in {"answer", "answer_uncertain"} def _rate(results: List[CaseResult], predicate) -> Optional[float]: @@ -414,18 +436,21 @@ def _failure_reason( case: EvaluationCase, actual_mode: str, metric, - response_correct: bool, + behavior_evidence_proxy_pass: bool, rejection_correct: bool, clarification_correct: bool, - citation_covered: bool, + citation_present: bool, + expected_evidence_citation_match: bool, ) -> str: if case.expected_behavior in {"answer", "conflict"}: if not metric.top5: - return "expected_source_missing_from_top5" - if not citation_covered: + return "expected_evidence_missing_from_top5" + if not citation_present: return "missing_citation" - if not response_correct: - return "answer_not_supported_by_expected_source" + if not expected_evidence_citation_match: + return "citation_does_not_match_expected_evidence" + if not behavior_evidence_proxy_pass: + return "answer_mode_or_expected_evidence_proxy_failed" if case.expected_behavior == "refuse" and not rejection_correct: return "false_answer_for_refusal_case" if actual_mode != "refuse" else "" if case.expected_behavior == "clarify" and not clarification_correct: @@ -439,10 +464,10 @@ def _score_case( metric, answer_text: str, actual_mode: str, - response_correct: bool, + behavior_evidence_proxy_pass: bool, rejection_correct: bool, clarification_correct: bool, - citation_covered: bool, + expected_evidence_citation_match: bool, failure_reason: str, ) -> Dict[str, float]: if case.expected_behavior in {"answer", "conflict"}: @@ -454,42 +479,43 @@ def _score_case( retrieval = 0.70 else: retrieval = 0.0 - answer = 1.0 if response_correct else 0.0 - citation = 1.0 if citation_covered else 0.0 - groundedness = 1.0 if response_correct and citation_covered and not failure_reason else 0.0 + answer = 1.0 if behavior_evidence_proxy_pass else 0.0 + citation = 1.0 if expected_evidence_citation_match else 0.0 + support = 1.0 if behavior_evidence_proxy_pass and expected_evidence_citation_match and not failure_reason else 0.0 else: retrieval = 1.0 correct_behavior = rejection_correct if case.expected_behavior == "refuse" else clarification_correct answer = 1.0 if correct_behavior else 0.0 citation = 1.0 - groundedness = 1.0 if correct_behavior else 0.0 + support = 1.0 if correct_behavior else 0.0 format_score = 1.0 if str(answer_text).strip() else 0.0 total = ( - SCORE_WEIGHTS["retrieval_correctness"] * retrieval - + SCORE_WEIGHTS["answer_correctness"] * answer - + SCORE_WEIGHTS["citation_precision"] * citation - + SCORE_WEIGHTS["groundedness"] * groundedness - + SCORE_WEIGHTS["format_compliance"] * format_score + SCORE_WEIGHTS["retrieval_evidence_proxy"] * retrieval + + SCORE_WEIGHTS["behavior_evidence_proxy"] * answer + + SCORE_WEIGHTS["citation_match_proxy"] * citation + + SCORE_WEIGHTS["combined_support_proxy"] * support + + SCORE_WEIGHTS["nonempty_format_proxy"] * format_score ) return { - "retrieval_correctness": retrieval, - "answer_correctness": answer, - "citation_precision": citation, - "groundedness": groundedness, - "format_compliance": format_score, - "total": total, + "retrieval_evidence_proxy": retrieval, + "behavior_evidence_proxy": answer, + "citation_match_proxy": citation, + "combined_support_proxy": support, + "nonempty_format_proxy": format_score, + "weighted_proxy": total, } def _failure_counts(results: List[CaseResult]) -> Dict[str, int]: counts = { "failed_cases": 0, - "expected_source_missing_from_top5": 0, + "expected_evidence_missing_from_top5": 0, "missing_citation": 0, "false_refusal": 0, "false_answer": 0, "missing_clarification": 0, - "answer_not_supported_by_expected_source": 0, + "citation_does_not_match_expected_evidence": 0, + "answer_mode_or_expected_evidence_proxy_failed": 0, } for result in results: failed = bool(result.failure_reason) @@ -511,8 +537,8 @@ def _category_metrics(results: List[CaseResult]) -> Dict[str, Dict[str, Any]]: answerable = [result for result in category_results if result.expected_behavior in {"answer", "conflict"}] out[category] = { "cases": len(category_results), - "retrieval_top5": _rate(answerable, lambda result: result.retrieval_top5), - "response_accuracy": _rate(answerable, lambda result: result.response_correct), + "expected_evidence_retrieval_top5_rate": _rate(answerable, lambda result: result.retrieval_top5), + "answer_mode_and_evidence_proxy_rate": _rate(answerable, lambda result: result.behavior_evidence_proxy_pass), "failed_cases": sum(1 for result in category_results if result.failure_reason), } return out @@ -536,20 +562,29 @@ def _worst_failed_cases(results: List[CaseResult], *, limit: int = 12) -> List[D def _markdown_summary(summary: Dict[str, Any]) -> str: mode = summary.get("mode", "balanced") - lines = ["# EMU Advisor Metrics", "", f"Mode: `{mode}`", f"Presentable: `{summary['presentable']}`", ""] + lines = [ + "# EMU Advisor Automated Regression Proxies", + "", + "These measurements are automated behavior/evidence proxies, not semantic answer-quality or human-review evidence.", + "", + f"Mode: `{mode}`", + f"Automated gate pass: `{summary['automated_gate_pass']}`", + "", + ] for key in [ "cases", - "total_score", - "retrieval_correctness_score", - "answer_correctness_score", - "citation_precision_score", - "groundedness_score", - "format_compliance_score", - "retrieval_top5", - "response_accuracy", - "rejection_accuracy", - "clarification_accuracy", - "citation_coverage", + "weighted_proxy_score", + "retrieval_evidence_proxy_score", + "behavior_evidence_proxy_score", + "citation_match_proxy_score", + "combined_support_proxy_score", + "nonempty_format_proxy_score", + "expected_evidence_retrieval_top5_rate", + "answer_mode_and_evidence_proxy_rate", + "refusal_behavior_match_rate", + "clarification_behavior_match_rate", + "citation_presence_rate", + "expected_evidence_citation_match_rate", "extractive_latency_p50_ms", "extractive_latency_p95_ms", "generated_latency_p50_ms", @@ -584,8 +619,8 @@ def _rank_modes(results: Dict[str, Any]) -> List[Dict[str, Any]]: ranked = [ { "mode": mode, - "total_score": summary.get("total_score"), - "retrieval_top5": summary.get("retrieval_top5"), + "weighted_proxy_score": summary.get("weighted_proxy_score"), + "expected_evidence_retrieval_top5_rate": summary.get("expected_evidence_retrieval_top5_rate"), "extractive_latency_p50_ms": summary.get("extractive_latency_p50_ms"), "failed_cases": summary.get("failure_counts", {}).get("failed_cases"), } @@ -593,7 +628,7 @@ def _rank_modes(results: Dict[str, Any]) -> List[Dict[str, Any]]: ] ranked.sort( key=lambda item: ( - item["total_score"] if item["total_score"] is not None else -1, + item["weighted_proxy_score"] if item["weighted_proxy_score"] is not None else -1, -(item["extractive_latency_p50_ms"] or 999999), ), reverse=True, @@ -603,14 +638,14 @@ def _rank_modes(results: Dict[str, Any]) -> List[Dict[str, Any]]: def _mode_comparison_markdown(comparison: Dict[str, Any]) -> str: lines = ["# EMU Advisor Mode Comparison", ""] - lines.append("| Mode | Total score | Top-5 retrieval | p50 extractive | Failed cases |") + lines.append("| Mode | Weighted proxy | Expected-evidence top-5 | p50 extractive | Failed cases |") lines.append("|---|---:|---:|---:|---:|") for item in comparison.get("ranking", []): lines.append( "| {mode} | {total} | {top5} | {latency} | {failed} |".format( mode=item["mode"], - total=_fmt_float(item.get("total_score")), - top5=_fmt_percent(item.get("retrieval_top5")), + total=_fmt_float(item.get("weighted_proxy_score")), + top5=_fmt_percent(item.get("expected_evidence_retrieval_top5_rate")), latency="-" if item.get("extractive_latency_p50_ms") is None else f"{item['extractive_latency_p50_ms']} ms", failed="-" if item.get("failed_cases") is None else item["failed_cases"], ) diff --git a/emu_advisor/pdf_ingest.py b/emu_advisor/pdf_ingest.py index a8f40f3..f9feb1a 100644 --- a/emu_advisor/pdf_ingest.py +++ b/emu_advisor/pdf_ingest.py @@ -95,8 +95,10 @@ def ingest_pdf_document(input_doc: PdfDocumentInput, *, max_words: int = 220) -> def _require_allowed_pdf_source(source_url: str) -> None: parsed = urlparse(source_url) + if parsed.scheme == "fixture" and not parsed.hostname and parsed.path.lower().endswith(".pdf"): + return if parsed.scheme not in {"http", "https"}: - raise PdfScopeError("PDF source URL must be http or https") + raise PdfScopeError("PDF source URL must be official HTTPS or explicit fixture provenance") if parsed.hostname != ALLOWED_REGULATION_HOST: raise PdfScopeError(f"source host is outside V1 scope: {parsed.hostname}") if not parsed.path.lower().endswith(".pdf"): diff --git a/emu_advisor/pipeline.py b/emu_advisor/pipeline.py index f7f662b..0ee74f9 100644 --- a/emu_advisor/pipeline.py +++ b/emu_advisor/pipeline.py @@ -1,4 +1,4 @@ -"""Corpus crawl/build pipeline for presentable demo artifacts.""" +"""Corpus crawl/build pipeline with explicit source provenance.""" from __future__ import annotations @@ -25,6 +25,8 @@ ALLOWED_HOST = "mevzuat.emu.edu.tr" TRACKING_PARAMS = {"utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", "gclid", "fbclid"} +REDIRECT_CODES = {301, 302, 303, 307, 308} +MAX_REDIRECTS = 5 @dataclass(frozen=True) @@ -68,8 +70,15 @@ def normalize_url(url: str) -> str: def is_allowed_crawl_url(url: str, *, allow_file: bool = False) -> bool: parsed = urlparse(url) if allow_file and parsed.scheme == "file": - return True - return parsed.scheme in {"http", "https"} and parsed.hostname == ALLOWED_HOST + return not parsed.netloc and bool(parsed.path) + if parsed.scheme != "https" or parsed.hostname != ALLOWED_HOST: + return False + if parsed.username or parsed.password: + return False + try: + return parsed.port in {None, 443} + except ValueError: + return False def discover_links(html: str, base_url: str, *, include_pdfs: bool, allow_file: bool = False) -> List[str]: @@ -108,18 +117,19 @@ def build_corpus( errors = 0 pdf_count = 0 - with httpx.Client(timeout=timeout_s, follow_redirects=True) as client: + with httpx.Client(timeout=timeout_s, follow_redirects=False) as client: while queue and len(seen) < max_pages: url = queue.pop(0) if url in seen: continue seen.add(url) fetched_at = datetime.now(timezone.utc).isoformat() - record: Dict[str, Any] = {"url": url, "fetched_at": fetched_at} + record: Dict[str, Any] = {"url": _public_source_url(url), "fetched_at": fetched_at} try: - payload, content_type = _fetch(url, client) + payload, content_type, final_url, redirect_chain = _fetch(url, client, allow_file=allow_file) + source_url = _public_source_url(final_url) raw_path = raw_dir / f"{len(seen):05d}_{_safe_name(url)}" - if url.lower().endswith(".pdf") or "pdf" in content_type.lower(): + if final_url.lower().endswith(".pdf") or "pdf" in content_type.lower(): if not include_pdfs: continue pdf_count += 1 @@ -128,12 +138,13 @@ def build_corpus( pdf_chunks = ingest_pdf_document( PdfDocumentInput( pdf_path=pdf_path, - source_url=_public_source_url(url), - source_title=_title_from_url(url), - language=_language_hint_from_url(url) or "en", + source_url=source_url, + source_title=_title_from_url(final_url), + language=_language_hint_from_url(final_url) or "en", last_crawled_at=fetched_at, ) ) + _mark_fixture_chunks(pdf_chunks, final_url) chunks.extend(pdf_chunks) record.update({"status": "ok", "content_type": content_type, "chunks": len(pdf_chunks), "source_type": "pdf"}) else: @@ -143,17 +154,19 @@ def build_corpus( html_chunks = ingest_html_document( HtmlDocumentInput( html=html, - source_url=_public_source_url(url), + source_url=source_url, source_title=None, language=_language_hint_from_url(url), last_crawled_at=fetched_at, ) ) + _mark_fixture_chunks(html_chunks, final_url) chunks.extend(html_chunks) record.update({"status": "ok", "content_type": content_type, "chunks": len(html_chunks), "source_type": "html"}) - for link in discover_links(html, url, include_pdfs=include_pdfs, allow_file=allow_file): + for link in discover_links(html, final_url, include_pdfs=include_pdfs, allow_file=allow_file): if link not in seen and link not in queue and len(seen) + len(queue) < max_pages * 3: queue.append(link) + record.update({"requested_url": _public_source_url(url), "final_url": source_url, "redirect_chain": redirect_chain}) time.sleep(delay_s) except Exception as exc: errors += 1 @@ -166,7 +179,7 @@ def build_corpus( _write_jsonl(pages, out_dir / "crawl_pages.jsonl") manifest = { "created_at": datetime.now(timezone.utc).isoformat(), - "seeds": seeds, + "seeds": [_public_source_url(seed) for seed in seeds], "max_pages": max_pages, "include_pdfs": include_pdfs, "page_count": len(pages), @@ -176,29 +189,71 @@ def build_corpus( } (out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") snapshot = write_snapshot(chunks, out_dir.parent / "snapshots", label="live-corpus") - (out_dir.parent / "active_snapshot.txt").write_text(str(snapshot.resolve()), encoding="utf-8") + try: + snapshot_pointer = snapshot.relative_to(out_dir.parent).as_posix() + except ValueError: + snapshot_pointer = snapshot.name + (out_dir.parent / "active_snapshot.txt").write_text(snapshot_pointer, encoding="utf-8") return BuildResult(out_dir=out_dir, chunk_count=len(chunks), page_count=len(pages), pdf_count=pdf_count, errors=errors) -def _fetch(url: str, client: httpx.Client) -> tuple[bytes, str]: +def _fetch( + url: str, + client: httpx.Client, + *, + allow_file: bool = False, +) -> tuple[bytes, str, str, List[str]]: parsed = urlparse(url) if parsed.scheme == "file": + if not is_allowed_crawl_url(url, allow_file=allow_file): + raise ValueError("file fixtures require explicit allow_file mode") path = Path(url2pathname(parsed.path)) payload = path.read_bytes() content_type = "application/pdf" if path.suffix.lower() == ".pdf" else "text/html" - return payload, content_type + return payload, content_type, url, [] if not is_allowed_crawl_url(url): raise ValueError(f"refusing out-of-scope crawl URL: {url}") - response = client.get(url, headers={"User-Agent": "EMUAdvisorDemo/1.0"}) - response.raise_for_status() - return response.content, response.headers.get("content-type", "") + current = normalize_url(url) + redirects: List[str] = [] + for _ in range(MAX_REDIRECTS + 1): + if not is_allowed_crawl_url(current): + raise ValueError(f"refusing out-of-scope redirect URL: {current}") + response = client.get(current, headers={"User-Agent": "EMUAdvisorDemo/1.0"}, follow_redirects=False) + response_url = normalize_url(str(response.url)) + if not is_allowed_crawl_url(response_url): + raise ValueError(f"refusing out-of-scope final response URL: {response_url}") + if response.status_code not in REDIRECT_CODES: + response.raise_for_status() + return response.content, response.headers.get("content-type", ""), response_url, redirects + location = response.headers.get("location") + if not location: + raise ValueError("redirect response is missing Location") + target = normalize_url(urljoin(response_url, location)) + if not is_allowed_crawl_url(target): + raise ValueError(f"refusing out-of-scope redirect URL: {target}") + if target in redirects or target == current: + raise ValueError("redirect loop detected") + redirects.append(target) + current = target + raise ValueError(f"redirect limit exceeded ({MAX_REDIRECTS})") def _public_source_url(url: str) -> str: parsed = urlparse(url) if parsed.scheme == "file": - return f"https://{ALLOWED_HOST}/content/fixture/{Path(parsed.path).name}" - return url + return f"fixture:///{Path(parsed.path).name}" + return normalize_url(url) + + +def _mark_fixture_chunks(chunks: List[Dict[str, Any]], source_url: str) -> None: + if urlparse(source_url).scheme != "file": + return + for chunk in chunks: + chunk["metadata"] = { + **dict(chunk.get("metadata") or {}), + "fixture": True, + "official_source": False, + } def _decode_html(payload: bytes, content_type: str) -> str: diff --git a/emu_advisor/readiness.py b/emu_advisor/readiness.py index 7f0b244..6a9f83a 100644 --- a/emu_advisor/readiness.py +++ b/emu_advisor/readiness.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Any, Dict, List, Optional -from .audit_log import summarize_audit_log from .corpus import load_latest_metrics from .eval_review import summarize_review_status @@ -23,28 +22,33 @@ def build_readiness_report(root: Path = ROOT) -> Dict[str, Any]: _file_check(root / "docs" / "EMUAdvisor Full Analysis.md", "full analysis", root=root), _file_check(root / "docs" / "DEMO_STORYBOARD.md", "demo storyboard", root=root), _file_check(root / "docs" / "PUBLICATION_CHECKLIST.md", "publication checklist", root=root), - _file_check(root / "eval_sets" / "v1_gold.jsonl", "verified gold evaluation set", root=root), + _file_check(root / "eval_sets" / "v1_gold.jsonl", "assistant-curated regression set", root=root), _file_check(root / "eval_sets" / "v1_hard.jsonl", "hard regression set", root=root), ] metrics_path = root / "artifacts" / "metrics" / "latest" / "metrics.json" metrics = load_latest_metrics(metrics_path) - if metrics.get("available"): + if metrics.get("available") and not metrics.get("legacy"): checks.append( { "name": "latest metrics artifact", "status": "pass", - "detail": f"top5={metrics.get('retrieval_top5')} citation={metrics.get('citation_coverage')}", + "detail": ( + f"expected-evidence top5={metrics.get('expected_evidence_retrieval_top5_rate')} " + f"citation-match={metrics.get('expected_evidence_citation_match_rate')} (automated proxies)" + ), } ) + elif metrics.get("legacy"): + checks.append({"name": "latest metrics artifact", "status": "blocked", "detail": metrics["message"]}) else: checks.append({"name": "latest metrics artifact", "status": "partial", "detail": "not present in artifacts"}) gold_review = summarize_review_status([root / "eval_sets" / "v1_gold.jsonl"]) checks.append( { - "name": "verified gold review status", - "status": "blocked" if gold_review["pending_cases"] else "pass", + "name": "independent evaluation review status", + "status": "blocked" if gold_review["pending_cases"] else "partial", "detail": f"{gold_review['total_cases']} cases; {gold_review['pending_cases']} pending", } ) @@ -74,12 +78,16 @@ def build_readiness_report(root: Path = ROOT) -> Dict[str, Any]: "detail": os.getenv("EMU_ADVISOR_QDRANT_URL") or "service-backed Qdrant not documented in environment", } ) - analytics = summarize_audit_log(root / "logs" / "audit.jsonl") + analytics = { + "available": False, + "events": None, + "detail": "Audit logging is disabled by default; existing private logs are not inspected by readiness checks.", + } checks.append( { "name": "local analytics log", - "status": "pass" if analytics["events"] else "partial", - "detail": f"{analytics['events']} audit events", + "status": "partial", + "detail": analytics["detail"], } ) status = _overall_status(checks) @@ -114,7 +122,8 @@ def write_markdown_report(report: Dict[str, Any], out: Path) -> None: "## Non-Negotiable Limits", "", "- This is a board-demo readiness report, not a production approval.", - "- `v1_gold` is the human-reviewed verified gold set; auxiliary regression/seed sets have separate review status.", + "- `v1_gold` is an assistant-curated regression set pending independent human review.", + "- Automated measurements are behavior/evidence proxies, not semantic answer-quality evidence.", "- Production-style deployment remains blocked until service-backed Qdrant and target hardware are validated.", "- Generated mode remains extractive-first unless local model latency and answer quality are characterized.", ] @@ -135,7 +144,7 @@ def _display_path(path: Path, *, root: Path) -> str: def _overall_status(checks: List[Dict[str, Any]]) -> str: if any(check["status"] == "blocked" for check in checks): - return "partial" + return "blocked" if any(check["status"] == "partial" for check in checks): return "partial" return "demo_ready" @@ -144,7 +153,7 @@ def _overall_status(checks: List[Dict[str, Any]]) -> str: def _status_sentence(status: str) -> str: if status == "demo_ready": return "The local board demo has the required tracked evidence for a controlled demonstration." - return "The local board demo is presentable with named gaps that must not be described as production-ready." + return "Board-demo and publication readiness are blocked until independent review and reproducible live-corpus evidence exist." def build_parser() -> argparse.ArgumentParser: diff --git a/emu_advisor/server.py b/emu_advisor/server.py index a7ff1aa..6047f72 100644 --- a/emu_advisor/server.py +++ b/emu_advisor/server.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import hmac import os import time from contextlib import asynccontextmanager @@ -28,7 +29,7 @@ from .conversation_detection import is_casual_message from .citations import unique_citations from .text import tokenize -from .conversation_store import get_store +from .conversation_store import ConversationStore from .corpus import load_corpus, load_latest_metrics from .embeddings import create_embedding_model from .generation import DEFAULT_OLLAMA_LLM, OllamaGenerator @@ -44,6 +45,10 @@ USER_CHAT_RETRIEVAL_MODE = "balanced" +def _env_enabled(name: str) -> bool: + return os.getenv(name, "").casefold() in {"1", "true", "yes", "on"} + + class AskRequest(BaseModel): model_config = ConfigDict(extra="forbid") @@ -79,12 +84,44 @@ class ChatRequest(BaseModel): def validate_question(cls, value: str) -> str: return _clean_question(value) + @field_validator("session_id") + @classmethod + def validate_session_id(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return None + value = value.strip() + if not value or len(value) > 128: + raise ValueError("session_id must be between 1 and 128 characters") + return value + + +class ExportRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + session_id: str + format: str = "markdown" + + @field_validator("format") + @classmethod + def validate_format(cls, value: str) -> str: + value = value.lower().strip() + if value not in {"markdown", "html"}: + raise ValueError("format must be 'markdown' or 'html'") + return value + + +class SessionRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + session_id: str + def create_app() -> FastAPI: - corpus = load_corpus() + runtime_profile = os.getenv("EMU_ADVISOR_PROFILE", "").strip().casefold() + corpus_mode = os.getenv("EMU_ADVISOR_CORPUS_MODE", "").strip().casefold() + corpus = load_corpus(mode=corpus_mode, profile=runtime_profile) chunks = corpus.chunks embedder = create_embedding_model(os.getenv("EMU_ADVISOR_EMBEDDING", "hash")) - runtime_profile = os.getenv("EMU_ADVISOR_PROFILE", "dev").casefold() admin_token = os.getenv("EMU_ADVISOR_ADMIN_TOKEN", "").strip() if runtime_profile == "production" and not admin_token: raise RuntimeError("EMU_ADVISOR_ADMIN_TOKEN is required when EMU_ADVISOR_PROFILE=production") @@ -111,8 +148,14 @@ def create_app() -> FastAPI: query_rewrite_model = os.getenv("EMU_ADVISOR_QUERY_REWRITE_MODEL") or os.getenv("EMU_ADVISOR_LLM", DEFAULT_OLLAMA_LLM) query_rewrite_timeout_s = float(os.getenv("EMU_ADVISOR_QUERY_REWRITE_TIMEOUT_S", "3")) query_rewrite_base_url = os.getenv("EMU_ADVISOR_QUERY_REWRITE_BASE_URL", "http://localhost:11434") - store = get_store() - logger = AuditLogger(ROOT_DIR / "logs" / "audit.jsonl") + store = ConversationStore() + audit_path = Path(os.getenv("EMU_ADVISOR_AUDIT_LOG_PATH", str(ROOT_DIR / "logs" / "audit.jsonl"))) + audit_enabled = _env_enabled("EMU_ADVISOR_ENABLE_AUDIT_LOGGING") + logger = AuditLogger( + audit_path, + enabled=audit_enabled, + include_raw_query=_env_enabled("EMU_ADVISOR_LOG_RAW_QUERY"), + ) @asynccontextmanager async def lifespan(_app: FastAPI): @@ -132,7 +175,7 @@ async def validation_error_handler(_request: Request, exc: RequestValidationErro async def board_readiness_middleware(request: Request, call_next): if _requires_admin_token(request, configured_token=admin_token): provided = _provided_admin_token(request) - if not provided or provided != admin_token: + if not provided or not hmac.compare_digest(provided, admin_token): return _with_security_headers( JSONResponse(status_code=401, content={"detail": "admin authentication required"}), request=request, @@ -153,7 +196,13 @@ def admin_console() -> str: @app.get("/health") def health() -> Dict[str, Any]: - return {"ok": True, "chunks": len(chunks), "corpus_source": corpus.source} + return { + "ok": True, + "chunks": len(chunks), + "corpus_source": corpus.source, + "corpus_mode": corpus.mode, + "fixture": corpus.fixture, + } @app.get("/whoami") def whoami() -> Dict[str, Any]: @@ -163,12 +212,16 @@ def whoami() -> Dict[str, Any]: "default_embedding": retriever.embedder.metadata.model_name, "default_llm": generator.model, "chunk_count": len(chunks), + "corpus_mode": corpus.mode, + "fixture": corpus.fixture, } @app.get("/corpus/status") def corpus_status() -> Dict[str, Any]: return { "source": corpus.source, + "corpus_mode": corpus.mode, + "fixture": corpus.fixture, "runtime_profile": runtime_profile, "vector_backend": retriever.vector_backend, "vector_backend_warning": retriever.backend_warning, @@ -191,7 +244,13 @@ def mode_metrics() -> Dict[str, Any]: @app.get("/analytics") def analytics() -> Dict[str, Any]: - return summarize_audit_log(ROOT_DIR / "logs" / "audit.jsonl") + if not audit_enabled: + return { + "available": False, + "events": 0, + "message": "Audit logging is disabled; existing private logs were not inspected.", + } + return summarize_audit_log(audit_path) @app.get("/llm/status") def llm_status(smoke: bool = False) -> Dict[str, Any]: @@ -202,12 +261,12 @@ def ask(request: AskRequest) -> Dict[str, Any]: return _answer_request(request) @app.post("/chat") - def chat(request: ChatRequest) -> Dict[str, Any]: - return _handle_chat(request, prefer_generated=False) + def chat(payload: ChatRequest, request: Request) -> Response: + return _handle_chat(payload, http_request=request, prefer_generated=False) @app.post("/chat/stream") - def chat_stream(request: ChatRequest) -> StreamingResponse: - return _handle_chat_stream(request) + def chat_stream(payload: ChatRequest, request: Request) -> Response: + return _handle_chat_stream(payload, http_request=request) def _chat_retrieve_answer( expanded_query: str, @@ -245,24 +304,34 @@ def _understand_request_query( timeout_s=query_rewrite_timeout_s, ) - def _handle_chat(request: ChatRequest, *, prefer_generated: bool = False) -> Dict[str, Any]: + def _handle_chat( + request: ChatRequest, + *, + http_request: Request, + prefer_generated: bool = False, + ) -> Any: """Handle a chat request with session management, casual detection, and full pipeline.""" - # Get or create session - session_id = request.session_id - session = store.get_session(session_id) - if session is None: - session_id = store.create_session(session_id) - session = store.get_session(session_id) + capability = _session_capability(http_request) + issued_capability: Optional[str] = None + if request.session_id: + session_id = request.session_id + if not store.owns_session(session_id, capability): + return _session_unavailable() + else: + access = store.create_session() + session_id = access.session_id + capability = access.capability + issued_capability = access.capability # Get conversation history for LLM - conversation_history = store.get_history(session_id) + conversation_history = store.get_history(session_id, capability) # Check for casual message first (short-circuit retrieval) is_casual, category, casual_response = is_casual_message(request.question) if is_casual: # Log the casual interaction - store.add_message(session_id, "user", request.question) - store.add_message(session_id, "assistant", casual_response) + store.add_message(session_id, capability, "user", request.question) + store.add_message(session_id, capability, "assistant", casual_response) total_ms = _elapsed_ms(time.perf_counter()) logger.log( @@ -276,7 +345,7 @@ def _handle_chat(request: ChatRequest, *, prefer_generated: bool = False) -> Dic citation_ids=[], ) ) - return { + response = { "session_id": session_id, "answer": casual_response, "state": "casual", @@ -285,15 +354,18 @@ def _handle_chat(request: ChatRequest, *, prefer_generated: bool = False) -> Dic "citations": [], "evidence_groups": [], } + if issued_capability: + response["session_capability"] = issued_capability + return response understanding = _understand_request_query(request.question, conversation_history) resolved_query = understanding.standalone_query route = route_query(resolved_query, language_hint=understanding.retrieval_language) if not route.in_scope: - store.add_message(session_id, "user", request.question) - store.add_message(session_id, "assistant", "I cannot answer questions outside the scope of EMU regulations.") - return { + store.add_message(session_id, capability, "user", request.question) + store.add_message(session_id, capability, "assistant", "I cannot answer questions outside the scope of EMU regulations.") + response = { "session_id": session_id, "answer": "I cannot answer questions outside the scope of EMU regulations.", "state": "out_of_scope", @@ -302,6 +374,9 @@ def _handle_chat(request: ChatRequest, *, prefer_generated: bool = False) -> Dic "citations": [], "evidence_groups": [], } + if issued_capability: + response["session_capability"] = issued_capability + return response extractive_answer, hits = _chat_retrieve_answer(resolved_query, route=route) @@ -320,8 +395,8 @@ def _handle_chat(request: ChatRequest, *, prefer_generated: bool = False) -> Dic generated_answer = generated.text or None # Store messages - store.add_message(session_id, "user", request.question) - store.add_message(session_id, "assistant", generated_answer or extractive_answer.text) + store.add_message(session_id, capability, "user", request.question) + store.add_message(session_id, capability, "assistant", generated_answer or extractive_answer.text) final_answer = generated_answer if prefer_generated and generated_answer else extractive_answer.text citations = [citation.as_dict() for citation in extractive_answer.citations] @@ -339,7 +414,7 @@ def _handle_chat(request: ChatRequest, *, prefer_generated: bool = False) -> Dic ) ) - return { + response = { "session_id": session_id, "answer": final_answer, "extractive_answer": extractive_answer.text, @@ -352,6 +427,9 @@ def _handle_chat(request: ChatRequest, *, prefer_generated: bool = False) -> Dic "evidence_groups": extractive_answer.evidence_groups, "latency_ms": total_ms, } + if issued_capability: + response["session_capability"] = issued_capability + return response def _generate_suggestions(query: str, hits: List[Mapping[str, Any]]) -> List[str]: """Generate context-aware follow-up suggestions based on the query and retrieved hits.""" @@ -405,13 +483,29 @@ def _generate_suggestions(query: str, hits: List[Mapping[str, Any]]) -> List[str return suggestions[:3] # Return up to 3 suggestions - def _handle_chat_stream(request: ChatRequest) -> StreamingResponse: + def _handle_chat_stream( + request: ChatRequest, + *, + http_request: Request, + ) -> StreamingResponse | JSONResponse: """Handle streaming chat request with NDJSON output.""" - session_id = request.session_id - if store.get_session(session_id) is None: - session_id = store.create_session(session_id) + capability = _session_capability(http_request) + issued_capability: Optional[str] = None + if request.session_id: + session_id = request.session_id + if not store.owns_session(session_id, capability): + return _session_unavailable() + else: + access = store.create_session() + session_id = access.session_id + capability = access.capability + issued_capability = access.capability + + session_event = {"type": "session", "session_id": session_id} + if issued_capability: + session_event["session_capability"] = issued_capability - conversation_history = store.get_history(session_id) + conversation_history = store.get_history(session_id, capability) normalized_history = [ {"role": msg.get("role", "user"), "content": msg.get("text", msg.get("content", ""))} for msg in conversation_history @@ -419,11 +513,11 @@ def _handle_chat_stream(request: ChatRequest) -> StreamingResponse: is_casual, category, casual_response = is_casual_message(request.question) if is_casual: - store.add_message(session_id, "user", request.question) - store.add_message(session_id, "assistant", casual_response) + store.add_message(session_id, capability, "user", request.question) + store.add_message(session_id, capability, "assistant", casual_response) def casual_events(): - yield json.dumps({"type": "session", "session_id": session_id}, ensure_ascii=False) + "\n" + yield json.dumps(session_event, ensure_ascii=False) + "\n" yield json.dumps( {"type": "casual", "category": category, "text": casual_response}, ensure_ascii=False, @@ -439,11 +533,11 @@ def casual_events(): if not route.in_scope: refusal = "I cannot answer questions outside the scope of EMU regulations." - store.add_message(session_id, "user", request.question) - store.add_message(session_id, "assistant", refusal) + store.add_message(session_id, capability, "user", request.question) + store.add_message(session_id, capability, "assistant", refusal) def out_of_scope_events(): - yield json.dumps({"type": "session", "session_id": session_id}, ensure_ascii=False) + "\n" + yield json.dumps(session_event, ensure_ascii=False) + "\n" yield json.dumps({"type": "retrieving"}, ensure_ascii=False) + "\n" yield json.dumps({"type": "refusal", "text": refusal}, ensure_ascii=False) + "\n" yield json.dumps({"type": "suggestions", "questions": []}, ensure_ascii=False) + "\n" @@ -452,7 +546,7 @@ def out_of_scope_events(): return StreamingResponse(out_of_scope_events(), media_type="application/x-ndjson") def stream_events(): - yield json.dumps({"type": "session", "session_id": session_id}, ensure_ascii=False) + "\n" + yield json.dumps(session_event, ensure_ascii=False) + "\n" yield json.dumps({"type": "retrieving"}, ensure_ascii=False) + "\n" extractive_answer, hits = _chat_retrieve_answer( @@ -509,15 +603,15 @@ def stream_events(): yield json.dumps({"type": "suggestions", "questions": suggestions}, ensure_ascii=False) + "\n" yield json.dumps({"type": "done"}, ensure_ascii=False) + "\n" - store.add_message(session_id, "user", request.question) - store.add_message(session_id, "assistant", final_assistant_text) + store.add_message(session_id, capability, "user", request.question) + store.add_message(session_id, capability, "assistant", final_assistant_text) return StreamingResponse(stream_events(), media_type="application/x-ndjson") def _answer_request(request: AskRequest, *, event_type: str = "ask") -> Dict[str, Any]: total_started = time.perf_counter() rewrite_started = time.perf_counter() - conversation_history = store.get_history(request.session_id) if request.session_id else [] + conversation_history: List[Mapping[str, str]] = [] understanding = _understand_request_query(request.question, conversation_history) resolved_query = understanding.standalone_query rewrite_ms = _elapsed_ms(rewrite_started) @@ -616,7 +710,7 @@ def _answer_request(request: AskRequest, *, event_type: str = "ask") -> Dict[str @app.post("/ask/stream") def ask_stream(request: AskRequest) -> StreamingResponse: - conversation_history = store.get_history(request.session_id) if request.session_id else [] + conversation_history: List[Mapping[str, str]] = [] understanding = _understand_request_query(request.question, conversation_history) resolved_query = understanding.standalone_query route = route_query(resolved_query, language_hint=understanding.retrieval_language) @@ -660,8 +754,11 @@ def events(): return StreamingResponse(events(), media_type="application/x-ndjson") @app.get("/chat/sessions") - def list_chat_sessions() -> List[Dict[str, Any]]: - """List all active chat sessions with metadata.""" + def list_chat_sessions(request: Request) -> Any: + """List metadata for active sessions to an authenticated administrator.""" + provided = _provided_admin_token(request) + if not admin_token or not provided or not hmac.compare_digest(provided, admin_token): + return JSONResponse(status_code=401, content={"detail": "admin authentication required"}) sessions = store.list_sessions() return [ { @@ -669,18 +766,17 @@ def list_chat_sessions() -> List[Dict[str, Any]]: "created_at": s.created_at, "last_active": s.last_active, "message_count": s.message_count, - "last_user_message": s.last_user_message, } for s in sessions ] @app.get("/chat/session/{session_id}") - def get_chat_session(session_id: str) -> Dict[str, Any]: + def get_chat_session(session_id: str, request: Request) -> Any: """Return one active chat session's visible transcript.""" - session = store.get_session(session_id) + session = store.get_session(session_id, _session_capability(request)) if session is None: - return {"session_id": session_id, "messages": [], "message_count": 0} + return _session_unavailable() messages = [ {"role": msg.get("role", "unknown"), "text": msg.get("text", msg.get("content", ""))} for msg in session.get("messages", []) @@ -693,39 +789,23 @@ def get_chat_session(session_id: str) -> Dict[str, Any]: "messages": messages, } - class ExportRequest(BaseModel): - session_id: str - format: str = "markdown" # "markdown" or "html" - - @field_validator("format") - @classmethod - def validate_format(cls, value: str) -> str: - value = value.lower().strip() - if value not in ("markdown", "html"): - raise ValueError("format must be 'markdown' or 'html'") - return value - @app.post("/chat/export") - def export_chat_session(request: ExportRequest) -> JSONResponse: + def export_chat_session(payload: ExportRequest, request: Request) -> Response: """Export a chat session to Markdown or HTML format.""" - export = store.export_session(request.session_id, request.format) + export = store.export_session(payload.session_id, _session_capability(request) or "", payload.format) if export is None: - return JSONResponse( - status_code=404, - content={"error": f"Session '{request.session_id}' not found"}, - ) + return _session_unavailable() - content_type = "text/markdown" if request.format == "markdown" else "text/html" + content_type = "text/markdown" if payload.format == "markdown" else "text/html" return Response(content=export, media_type=content_type) @app.post("/chat/clear") - def clear_chat_session(request: ChatRequest) -> Dict[str, Any]: + def clear_chat_session(payload: SessionRequest, request: Request) -> Any: """Clear messages from a chat session.""" - success = store.clear_session(request.session_id) + success = store.clear_session(payload.session_id, _session_capability(request) or "") if not success: - # Try to create if doesn't exist (for IDempotency) - store.create_session(request.session_id) - return {"session_id": request.session_id, "cleared": True, "message_count": 0} + return _session_unavailable() + return {"session_id": payload.session_id, "cleared": True, "message_count": 0} return app @@ -787,6 +867,20 @@ def _provided_admin_token(request: Request) -> Optional[str]: return None +def _session_capability(request: Request) -> Optional[str]: + capability = request.headers.get("x-emu-session-capability") + if not capability: + return None + capability = capability.strip() + if len(capability) > 256: + return None + return capability + + +def _session_unavailable() -> JSONResponse: + return JSONResponse(status_code=404, content={"detail": "session unavailable"}) + + def _requires_admin_token(request: Request, *, configured_token: str) -> bool: if not configured_token: return False diff --git a/eval_sets/v1_gold.jsonl b/eval_sets/v1_gold.jsonl index d862b96..144afe3 100644 --- a/eval_sets/v1_gold.jsonl +++ b/eval_sets/v1_gold.jsonl @@ -1,60 +1,60 @@ -{"case_id": "EN-001", "category": "course_registration", "citation_ok": true, "expected_answer_keywords": ["first", "second"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0005", "expected_chunk_ids": ["en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0005"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What must first-year students register for before taking other courses?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-002", "category": "course_registration", "citation_ok": true, "expected_answer_keywords": ["two"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0006", "expected_chunk_ids": ["en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0006"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "When can a student's normal course load be reduced by up to two courses?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-003", "category": "course_registration", "citation_ok": true, "expected_answer_keywords": ["High Honour", "two"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0007", "expected_chunk_ids": ["en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0007"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "How many extra courses may a High Honour student add to the semester course load?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-004", "category": "course_withdrawal", "citation_ok": true, "expected_answer_keywords": ["withdraw", "two"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0017", "expected_chunk_ids": ["en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0017"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What is the maximum number of registered courses a student may withdraw from in one semester?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-005", "category": "course_registration", "citation_ok": true, "expected_answer_keywords": ["dismissed"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0005", "expected_chunk_ids": ["en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0005"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What happens if a student fails to renew registration for two consecutive semesters?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-006", "category": "attendance", "citation_ok": true, "expected_answer_keywords": ["attend"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0005", "expected_chunk_ids": ["en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0005"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What attendance requirement applies to registered courses, laboratories and studios?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-007", "category": "grading_exam", "citation_ok": true, "expected_answer_keywords": ["absenteeism"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0008", "expected_chunk_ids": ["en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0008"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What does the NG grade mean in the EMU grade table?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-008", "category": "leave_freeze", "citation_ok": true, "expected_answer_keywords": ["5 weeks"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0028", "expected_chunk_ids": ["en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0028"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Within how many weeks may a student apply for leave of absence with a valid reason?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-009", "category": "grading_exam", "citation_ok": true, "expected_answer_keywords": ["Re-sit"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0:c0007", "expected_chunk_ids": ["en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0:c0007"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-4-Rules-examinations_and_evaluations.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-4-Rules-examinations_and_evaluations.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "When are re-sit examinations administered for Fall and Spring semester courses?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-010", "category": "grading_exam", "citation_ok": true, "expected_answer_keywords": ["make-up"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0:c0010", "expected_chunk_ids": ["en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0:c0010"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-4-Rules-examinations_and_evaluations.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-4-Rules-examinations_and_evaluations.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What happens if the reason for missing an exam continues during the make-up exam period?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-011", "category": "scholarships", "citation_ok": true, "expected_answer_keywords": ["5000"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0006", "expected_chunk_ids": ["en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0006"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Who may receive the EMU scholarship for students ranked in the first 5000?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-012", "category": "scholarships", "citation_ok": true, "expected_answer_keywords": ["1%"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0017", "expected_chunk_ids": ["en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0017"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "How is the high honour scholarship determined within an EMU program?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-013", "category": "scholarships", "citation_ok": true, "expected_answer_keywords": ["100%", "50%"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0023", "expected_chunk_ids": ["en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0023"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What does postgraduate scholarship refer to in the EMU scholarship rules?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-014", "category": "tuition_refunds", "citation_ok": true, "expected_answer_keywords": ["per course"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730:c0004", "expected_chunk_ids": ["en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730:c0004"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-10-Regulations-RegulationsforTutionFees.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-10-Regulations-RegulationsforTutionFees.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "How do postgraduate students who pay per course pay semester tuition fees?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-015", "category": "leave_freeze", "citation_ok": true, "expected_answer_keywords": ["freeze"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730:c0011", "expected_chunk_ids": ["en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730:c0011"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-10-Regulations-RegulationsforTutionFees.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-10-Regulations-RegulationsforTutionFees.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "May new students freeze registration after the add-drop deadline because of admission, visa or transportation problems?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-016", "category": "research_assistant", "citation_ok": true, "expected_answer_keywords": ["Category C"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0005", "expected_chunk_ids": ["en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0005"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What is Category C in the EMU research assistant by-law?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-017", "category": "research_assistant", "citation_ok": true, "expected_answer_keywords": ["postgraduate"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0009", "expected_chunk_ids": ["en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0009"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What registration condition must a candidate meet to be appointed as a research assistant?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-018", "category": "staff_regulations", "citation_ok": true, "expected_answer_keywords": ["scales"], "expected_behavior": "answer", "expected_chunk_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0011", "expected_chunk_ids": ["en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0011"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What does the academic staff by-law say about the number and scales of academic staff positions?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-019", "category": "staff_regulations", "citation_ok": true, "expected_answer_keywords": ["salaries"], "expected_behavior": "answer", "expected_chunk_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0012", "expected_chunk_ids": ["en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0012"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "How are academic staff salaries and allowances paid under the academic staff by-law?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-020", "category": "staff_regulations", "citation_ok": true, "expected_answer_keywords": ["annual"], "expected_behavior": "answer", "expected_chunk_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0037", "expected_chunk_ids": ["en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0037"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "When are regular annual scale increases given to academic personnel?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-021", "category": "staff_regulations", "citation_ok": true, "expected_answer_keywords": ["scales"], "expected_behavior": "answer", "expected_chunk_id": "en:html:by-law-staffingandemploymentforadministrativestaff-htm:d0437576f3:c0024", "expected_chunk_ids": ["en:html:by-law-staffingandemploymentforadministrativestaff-htm:d0437576f3:c0024"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:by-law-staffingandemploymentforadministrativestaff-htm:d0437576f3", "expected_source_url": "https://mevzuat.emu.edu.tr/By-law_StaffingandEmploymentforAdministrativeStaff.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/By-law_StaffingandEmploymentforAdministrativeStaff.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What does the administrative staff by-law say about positions and scales?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-022", "category": "title_promotion", "citation_ok": true, "expected_answer_keywords": ["Associate Professor"], "expected_behavior": "answer", "expected_chunk_id": "en:html:6-2-rules-academic-staff-title-bylaw-m-htm:5e9b289f3e:c0004", "expected_chunk_ids": ["en:html:6-2-rules-academic-staff-title-bylaw-m-htm:5e9b289f3e:c0004"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-2-rules-academic-staff-title-bylaw-m-htm:5e9b289f3e", "expected_source_url": "https://mevzuat.emu.edu.tr/6-2-Rules-Academic staff title bylaw-m.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-2-Rules-Academic staff title bylaw-m.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What prior title and experience are required for a professor title application?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-023", "category": "housing_facilities", "citation_ok": true, "expected_answer_keywords": ["Housing"], "expected_behavior": "answer", "expected_chunk_id": "en:html:regulations-for-benefiting-from-university-housing-and-guest-house-facilities-htm:ac21a5eb34:c0003", "expected_chunk_ids": ["en:html:regulations-for-benefiting-from-university-housing-and-guest-house-facilities-htm:ac21a5eb34:c0003"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:regulations-for-benefiting-from-university-housing-and-guest-house-facilities-htm:ac21a5eb34", "expected_source_url": "https://mevzuat.emu.edu.tr/REGULATIONS FOR BENEFITING FROM UNIVERSITY HOUSING AND GUEST HOUSE FACILITIES .htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/REGULATIONS FOR BENEFITING FROM UNIVERSITY HOUSING AND GUEST HOUSE FACILITIES .htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Which academic staff members may benefit from university housing facilities?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-024", "category": "research_ethics", "citation_ok": true, "expected_answer_keywords": ["source"], "expected_behavior": "answer", "expected_chunk_id": "en:html:6-1-5-regulation-scientificresearchpublicationethics-htm:b95c7c5f51:c0010", "expected_chunk_ids": ["en:html:6-1-5-regulation-scientificresearchpublicationethics-htm:b95c7c5f51:c0010"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-1-5-regulation-scientificresearchpublicationethics-htm:b95c7c5f51", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1-5-Regulation-ScientificResearchPublicationEthics.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1-5-Regulation-ScientificResearchPublicationEthics.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What publication ethics rule applies when using previously published or unpublished research?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-025", "category": "out_of_scope", "citation_ok": true, "expected_answer_keywords": [], "expected_behavior": "refuse", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "What concerts or campus events are happening at EMU today?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-026", "category": "out_of_scope", "citation_ok": true, "expected_answer_keywords": [], "expected_behavior": "refuse", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Send an email to my department chair and schedule a meeting about my registration.", "review_status": "human_reviewed_verified"} -{"case_id": "EN-027", "category": "ambiguous_query", "citation_ok": true, "expected_answer_keywords": [], "expected_behavior": "clarify", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Which office should verify ambiguous scholarship versus tuition evidence?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-028", "category": "ambiguous_query", "citation_ok": true, "expected_answer_keywords": [], "expected_behavior": "clarify", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Who should verify ambiguous registration or add-drop rules for a specific student case?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-029", "category": "conflict", "citation_ok": true, "expected_answer_keywords": ["salaries"], "expected_behavior": "conflict", "expected_chunk_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0012", "expected_chunk_ids": ["en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0012", "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0005"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm", "https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Are the academic staff salary provisions conflicting or different?", "review_status": "human_reviewed_verified"} -{"case_id": "EN-030", "category": "conflict", "citation_ok": true, "expected_answer_keywords": ["Category"], "expected_behavior": "conflict", "expected_chunk_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0005", "expected_chunk_ids": ["en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0005", "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0012"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm", "https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm"], "is_correct": true, "language": "en", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Do the research assistant appointment and scholarship provisions show any conflict?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-001", "category": "course_registration", "citation_ok": true, "expected_answer_keywords": ["birinci"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0004", "expected_chunk_ids": ["tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0004"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Birinci yıl öğrencisi hangi dönem derslerine kayıt olmakla yükümlüdür?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-002", "category": "course_registration", "citation_ok": true, "expected_answer_keywords": ["Yüksek Şeref"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0005", "expected_chunk_ids": ["tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0005"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Yüksek Şeref öğrencisi normal ders yüküne ek olarak kaç ders alabilir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-003", "category": "course_registration", "citation_ok": true, "expected_answer_keywords": ["Ders Ekleme"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0011", "expected_chunk_ids": ["tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0011"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Ders ekleme veya bırakma süreci ne zaman yapılır?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-004", "category": "course_registration", "citation_ok": true, "expected_answer_keywords": ["kayıtlarını yenilemek"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0012", "expected_chunk_ids": ["tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0012"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Öğrenciler her dönem başında kayıtlarını nasıl yenilemek zorundadır?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-005", "category": "tuition_refunds", "citation_ok": true, "expected_answer_keywords": ["iade"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0013", "expected_chunk_ids": ["tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0013"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Kayıt sildiren öğrenci harç iadesi için hangi esaslara tabidir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-006", "category": "grading_exam", "citation_ok": true, "expected_answer_keywords": ["Devamsızlıktan"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0016", "expected_chunk_ids": ["tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0016"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "NG notu ne anlama gelir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-007", "category": "leave_freeze", "citation_ok": true, "expected_answer_keywords": ["izin"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0035", "expected_chunk_ids": ["tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0035"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Öğrenci izinli ayrılmak için nereye yazılı ve gerekçeli başvuru yapar?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-008", "category": "grading_exam", "citation_ok": true, "expected_answer_keywords": ["3 gün"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57:c0006", "expected_chunk_ids": ["tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57:c0006"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-4-Yonetmelik-SinavveDegerlendirme.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-4-Yonetmelik-SinavveDegerlendirme.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Bütünleme sınavına başvuru dönem notlarının ilanından sonra kaç gün içinde yapılır?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-009", "category": "grading_exam", "citation_ok": true, "expected_answer_keywords": ["Telafi"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57:c0007", "expected_chunk_ids": ["tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57:c0007"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-4-Yonetmelik-SinavveDegerlendirme.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-4-Yonetmelik-SinavveDegerlendirme.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Telafi sınavları dönem içinde mi dönem sonunda mı yapılabilir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-010", "category": "scholarships", "citation_ok": true, "expected_answer_keywords": ["5000"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0005", "expected_chunk_ids": ["tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0005"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "İlk 5000 arasına giren öğrencilere hangi burs verilebilir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-011", "category": "scholarships", "citation_ok": true, "expected_answer_keywords": ["%1"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0014", "expected_chunk_ids": ["tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0014"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Yüksek Şeref bursu programdaki yüzde kaçlık başarı dilimine göre verilir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-012", "category": "scholarships", "citation_ok": true, "expected_answer_keywords": ["%100"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0018", "expected_chunk_ids": ["tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0018"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Lisansüstü burslar hangi oranlarda verilir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-013", "category": "scholarships", "citation_ok": true, "expected_answer_keywords": ["birden fazla"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0026", "expected_chunk_ids": ["tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0026"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Bir öğrenci birden fazla burs ve indirimden aynı anda yararlanabilir mi?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-014", "category": "tuition_refunds", "citation_ok": true, "expected_answer_keywords": ["%20"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-10-y-netmelik-ogrenimharc-htm:e6f41ba36f:c0008", "expected_chunk_ids": ["tr:html:5-1-10-y-netmelik-ogrenimharc-htm:e6f41ba36f:c0008"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-10-y-netmelik-ogrenimharc-htm:e6f41ba36f", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-10-Yönetmelik-OgrenimHarc.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-10-Yönetmelik-OgrenimHarc.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Ders ekleme veya bırakma sürecinde harç iadesi için hangi yüzde uygulanır?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-015", "category": "research_assistant", "citation_ok": true, "expected_answer_keywords": ["araştırma görevlisi"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0003", "expected_chunk_ids": ["tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0003"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Araştırma görevlisi yönetmeliği hangi nitelik ve burs konularını kapsar?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-016", "category": "research_assistant", "citation_ok": true, "expected_answer_keywords": ["akademik yardım"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0005", "expected_chunk_ids": ["tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0005"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Araştırma görevlileri öğrenci akademik yardım masalarında görev alabilir mi?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-017", "category": "research_assistant", "citation_ok": true, "expected_answer_keywords": ["asgari ücret"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0012", "expected_chunk_ids": ["tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0012"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "A ve B kategorisi araştırma görevlisi aylık bursu asgari ücrete göre nasıl belirlenir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-018", "category": "staff_regulations", "citation_ok": true, "expected_answer_keywords": ["barem"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0011", "expected_chunk_ids": ["tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0011"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Akademik personel kadro sayısı ve baremleri hangi cetvellere göre düzenlenir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-019", "category": "staff_regulations", "citation_ok": true, "expected_answer_keywords": ["maaş"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0012", "expected_chunk_ids": ["tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0012"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Akademik personele maaş ve makam ödeneği nasıl verilir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-020", "category": "staff_regulations", "citation_ok": true, "expected_answer_keywords": ["ilk atanma"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0039", "expected_chunk_ids": ["tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0039"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Akademik personelin barem içi artışı ilk atanma tarihine göre nasıl verilir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-021", "category": "staff_regulations", "citation_ok": true, "expected_answer_keywords": ["kadro"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-7-1-yontperskadrocal-htm:1cb7f72b71:c0020", "expected_chunk_ids": ["tr:html:tuzukler-7-1-yontperskadrocal-htm:1cb7f72b71:c0020"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-7-1-yontperskadrocal-htm:1cb7f72b71", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/7-1_YontPersKadroCal.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/7-1_YontPersKadroCal.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Yönetsel hizmetler personeli kadro ve baremleri hangi düzenlemede geçer?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-022", "category": "student_discipline", "citation_ok": true, "expected_answer_keywords": ["45 iş günü"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-2-yonetmelik-ogrencidisiplin-htm:4ada820abe:c0020", "expected_chunk_ids": ["tr:html:5-2-yonetmelik-ogrencidisiplin-htm:4ada820abe:c0020"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-2-yonetmelik-ogrencidisiplin-htm:4ada820abe", "expected_source_url": "https://mevzuat.emu.edu.tr/5-2-Yonetmelik-OgrenciDisiplin.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-2-Yonetmelik-OgrenciDisiplin.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Öğrenci Disiplin Kurulu dosyayı en geç kaç iş günü içinde karara bağlar?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-023", "category": "research_ethics", "citation_ok": true, "expected_answer_keywords": ["akademik etik"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:6-1-5-yonetmelik-bilimselarastirmayayinetigi-htm:fd248ba433:c0004", "expected_chunk_ids": ["tr:html:6-1-5-yonetmelik-bilimselarastirmayayinetigi-htm:fd248ba433:c0004"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:6-1-5-yonetmelik-bilimselarastirmayayinetigi-htm:fd248ba433", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1-5-Yonetmelik-BilimselArastirmaYayinEtigi.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1-5-Yonetmelik-BilimselArastirmaYayinEtigi.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Üniversite akademik etik ilkelerini nasıl bir değer olarak görür?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-024", "category": "research_ethics", "citation_ok": true, "expected_answer_keywords": ["hakem"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:6-3-3-bilimselarastirmalardestekilkeleri-htm:d1b8dace78:c0003", "expected_chunk_ids": ["tr:html:6-3-3-bilimselarastirmalardestekilkeleri-htm:d1b8dace78:c0003"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:6-3-3-bilimselarastirmalardestekilkeleri-htm:d1b8dace78", "expected_source_url": "https://mevzuat.emu.edu.tr/6-3-3-BilimselArastirmalarDestekIlkeleri.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-3-3-BilimselArastirmalarDestekIlkeleri.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Bilimsel araştırma proje önerileri hakem değerlendirmesine nasıl sunulur?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-025", "category": "out_of_scope", "citation_ok": true, "expected_answer_keywords": [], "expected_behavior": "refuse", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Bugünkü kampüs etkinlikleri ve konserler nelerdir?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-026", "category": "out_of_scope", "citation_ok": true, "expected_answer_keywords": [], "expected_behavior": "refuse", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Bölüm başkanıma e-posta gönderip kayıt için toplantı planla.", "review_status": "human_reviewed_verified"} -{"case_id": "TR-027", "category": "ambiguous_query", "citation_ok": true, "expected_answer_keywords": [], "expected_behavior": "clarify", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Hangi ofise sorulmalı: burs mu harç mı olduğundan emin değilim.", "review_status": "human_reviewed_verified"} -{"case_id": "TR-028", "category": "ambiguous_query", "citation_ok": true, "expected_answer_keywords": [], "expected_behavior": "clarify", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Hangi ofise sorulmalı: kayıt dondurma mı ders bırakma mı emin değilim.", "review_status": "human_reviewed_verified"} -{"case_id": "TR-029", "category": "conflict", "citation_ok": true, "expected_answer_keywords": ["maas"], "expected_behavior": "conflict", "expected_chunk_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0012", "expected_chunk_ids": ["tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0012", "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0012"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm", "https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Akademik personel maas hukumleri celiskili mi?", "review_status": "human_reviewed_verified"} -{"case_id": "TR-030", "category": "conflict", "citation_ok": true, "expected_answer_keywords": ["asgari"], "expected_behavior": "conflict", "expected_chunk_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0012", "expected_chunk_ids": ["tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0012", "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0012"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm", "https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm"], "is_correct": true, "language": "tr", "notes": "Human-reviewed and verified by the project author and university staff.", "question": "Arastirma gorevlisi burs kurallari farkli veya celiskili mi?", "review_status": "human_reviewed_verified"} +{"case_id": "EN-001", "category": "course_registration", "citation_ok": null, "expected_answer_keywords": ["first", "second"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0005", "expected_chunk_ids": ["en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0005"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What must first-year students register for before taking other courses?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-002", "category": "course_registration", "citation_ok": null, "expected_answer_keywords": ["two"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0006", "expected_chunk_ids": ["en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0006"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "When can a student's normal course load be reduced by up to two courses?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-003", "category": "course_registration", "citation_ok": null, "expected_answer_keywords": ["High Honour", "two"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0007", "expected_chunk_ids": ["en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0007"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "How many extra courses may a High Honour student add to the semester course load?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-004", "category": "course_withdrawal", "citation_ok": null, "expected_answer_keywords": ["withdraw", "two"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0017", "expected_chunk_ids": ["en:html:5-1-5-rules-course-registration-htm:b9be885ddb:c0017"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-5-rules-course-registration-htm:b9be885ddb", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Rules-Course_Registration.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What is the maximum number of registered courses a student may withdraw from in one semester?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-005", "category": "course_registration", "citation_ok": null, "expected_answer_keywords": ["dismissed"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0005", "expected_chunk_ids": ["en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0005"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What happens if a student fails to renew registration for two consecutive semesters?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-006", "category": "attendance", "citation_ok": null, "expected_answer_keywords": ["attend"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0005", "expected_chunk_ids": ["en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0005"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What attendance requirement applies to registered courses, laboratories and studios?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-007", "category": "grading_exam", "citation_ok": null, "expected_answer_keywords": ["absenteeism"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0008", "expected_chunk_ids": ["en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0008"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What does the NG grade mean in the EMU grade table?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-008", "category": "leave_freeze", "citation_ok": null, "expected_answer_keywords": ["5 weeks"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0028", "expected_chunk_ids": ["en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b:c0028"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-0-regulation-education-examination-success-htm:f6fb21317b", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-0-Regulation-Education_Examination_Success.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "Within how many weeks may a student apply for leave of absence with a valid reason?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-009", "category": "grading_exam", "citation_ok": null, "expected_answer_keywords": ["Re-sit"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0:c0007", "expected_chunk_ids": ["en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0:c0007"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-4-Rules-examinations_and_evaluations.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-4-Rules-examinations_and_evaluations.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "When are re-sit examinations administered for Fall and Spring semester courses?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-010", "category": "grading_exam", "citation_ok": null, "expected_answer_keywords": ["make-up"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0:c0010", "expected_chunk_ids": ["en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0:c0010"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-4-rules-examinations-and-evaluations-htm:2a7e1ba6a0", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-4-Rules-examinations_and_evaluations.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-4-Rules-examinations_and_evaluations.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What happens if the reason for missing an exam continues during the make-up exam period?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-011", "category": "scholarships", "citation_ok": null, "expected_answer_keywords": ["5000"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0006", "expected_chunk_ids": ["en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0006"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "Who may receive the EMU scholarship for students ranked in the first 5000?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-012", "category": "scholarships", "citation_ok": null, "expected_answer_keywords": ["1%"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0017", "expected_chunk_ids": ["en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0017"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "How is the high honour scholarship determined within an EMU program?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-013", "category": "scholarships", "citation_ok": null, "expected_answer_keywords": ["100%", "50%"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0023", "expected_chunk_ids": ["en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95:c0023"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-2-rules-scholarship-regulations-htm:f2245f5c95", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Rules-Scholarship_regulations.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What does postgraduate scholarship refer to in the EMU scholarship rules?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-014", "category": "tuition_refunds", "citation_ok": null, "expected_answer_keywords": ["per course"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730:c0004", "expected_chunk_ids": ["en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730:c0004"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-10-Regulations-RegulationsforTutionFees.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-10-Regulations-RegulationsforTutionFees.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "How do postgraduate students who pay per course pay semester tuition fees?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-015", "category": "leave_freeze", "citation_ok": null, "expected_answer_keywords": ["freeze"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730:c0011", "expected_chunk_ids": ["en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730:c0011"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-1-10-regulations-regulationsfortutionfees-htm:ccbb8c3730", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-10-Regulations-RegulationsforTutionFees.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-10-Regulations-RegulationsforTutionFees.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "May new students freeze registration after the add-drop deadline because of admission, visa or transportation problems?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-016", "category": "research_assistant", "citation_ok": null, "expected_answer_keywords": ["Category C"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0005", "expected_chunk_ids": ["en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0005"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What is Category C in the EMU research assistant by-law?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-017", "category": "research_assistant", "citation_ok": null, "expected_answer_keywords": ["postgraduate"], "expected_behavior": "answer", "expected_chunk_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0009", "expected_chunk_ids": ["en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0009"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What registration condition must a candidate meet to be appointed as a research assistant?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-018", "category": "staff_regulations", "citation_ok": null, "expected_answer_keywords": ["scales"], "expected_behavior": "answer", "expected_chunk_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0011", "expected_chunk_ids": ["en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0011"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What does the academic staff by-law say about the number and scales of academic staff positions?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-019", "category": "staff_regulations", "citation_ok": null, "expected_answer_keywords": ["salaries"], "expected_behavior": "answer", "expected_chunk_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0012", "expected_chunk_ids": ["en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0012"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "How are academic staff salaries and allowances paid under the academic staff by-law?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-020", "category": "staff_regulations", "citation_ok": null, "expected_answer_keywords": ["annual"], "expected_behavior": "answer", "expected_chunk_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0037", "expected_chunk_ids": ["en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0037"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "When are regular annual scale increases given to academic personnel?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-021", "category": "staff_regulations", "citation_ok": null, "expected_answer_keywords": ["scales"], "expected_behavior": "answer", "expected_chunk_id": "en:html:by-law-staffingandemploymentforadministrativestaff-htm:d0437576f3:c0024", "expected_chunk_ids": ["en:html:by-law-staffingandemploymentforadministrativestaff-htm:d0437576f3:c0024"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:by-law-staffingandemploymentforadministrativestaff-htm:d0437576f3", "expected_source_url": "https://mevzuat.emu.edu.tr/By-law_StaffingandEmploymentforAdministrativeStaff.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/By-law_StaffingandEmploymentforAdministrativeStaff.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What does the administrative staff by-law say about positions and scales?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-022", "category": "title_promotion", "citation_ok": null, "expected_answer_keywords": ["Associate Professor"], "expected_behavior": "answer", "expected_chunk_id": "en:html:6-2-rules-academic-staff-title-bylaw-m-htm:5e9b289f3e:c0004", "expected_chunk_ids": ["en:html:6-2-rules-academic-staff-title-bylaw-m-htm:5e9b289f3e:c0004"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-2-rules-academic-staff-title-bylaw-m-htm:5e9b289f3e", "expected_source_url": "https://mevzuat.emu.edu.tr/6-2-Rules-Academic staff title bylaw-m.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-2-Rules-Academic staff title bylaw-m.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What prior title and experience are required for a professor title application?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-023", "category": "housing_facilities", "citation_ok": null, "expected_answer_keywords": ["Housing"], "expected_behavior": "answer", "expected_chunk_id": "en:html:regulations-for-benefiting-from-university-housing-and-guest-house-facilities-htm:ac21a5eb34:c0003", "expected_chunk_ids": ["en:html:regulations-for-benefiting-from-university-housing-and-guest-house-facilities-htm:ac21a5eb34:c0003"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:regulations-for-benefiting-from-university-housing-and-guest-house-facilities-htm:ac21a5eb34", "expected_source_url": "https://mevzuat.emu.edu.tr/REGULATIONS FOR BENEFITING FROM UNIVERSITY HOUSING AND GUEST HOUSE FACILITIES .htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/REGULATIONS FOR BENEFITING FROM UNIVERSITY HOUSING AND GUEST HOUSE FACILITIES .htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "Which academic staff members may benefit from university housing facilities?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-024", "category": "research_ethics", "citation_ok": null, "expected_answer_keywords": ["source"], "expected_behavior": "answer", "expected_chunk_id": "en:html:6-1-5-regulation-scientificresearchpublicationethics-htm:b95c7c5f51:c0010", "expected_chunk_ids": ["en:html:6-1-5-regulation-scientificresearchpublicationethics-htm:b95c7c5f51:c0010"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-1-5-regulation-scientificresearchpublicationethics-htm:b95c7c5f51", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1-5-Regulation-ScientificResearchPublicationEthics.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1-5-Regulation-ScientificResearchPublicationEthics.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What publication ethics rule applies when using previously published or unpublished research?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-025", "category": "out_of_scope", "citation_ok": null, "expected_answer_keywords": [], "expected_behavior": "refuse", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "What concerts or campus events are happening at EMU today?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-026", "category": "out_of_scope", "citation_ok": null, "expected_answer_keywords": [], "expected_behavior": "refuse", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "Send an email to my department chair and schedule a meeting about my registration.", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-027", "category": "ambiguous_query", "citation_ok": null, "expected_answer_keywords": [], "expected_behavior": "clarify", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "Which office should verify ambiguous scholarship versus tuition evidence?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-028", "category": "ambiguous_query", "citation_ok": null, "expected_answer_keywords": [], "expected_behavior": "clarify", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "Who should verify ambiguous registration or add-drop rules for a specific student case?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-029", "category": "conflict", "citation_ok": null, "expected_answer_keywords": ["salaries"], "expected_behavior": "conflict", "expected_chunk_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0012", "expected_chunk_ids": ["en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0012", "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0005"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm", "https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "Are the academic staff salary provisions conflicting or different?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "EN-030", "category": "conflict", "citation_ok": null, "expected_answer_keywords": ["Category"], "expected_behavior": "conflict", "expected_chunk_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0005", "expected_chunk_ids": ["en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732:c0005", "en:html:6-1-staffingemploymentacademicstaff-htm:957ed34295:c0012"], "expected_corpora": ["regulations_en"], "expected_corpus": "regulations_en", "expected_document_id": "en:html:5-4-3-rules-research-assistant-by-law-m-htm:2347c4f732", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Rules-Research_assistant_by-law-m.htm", "https://mevzuat.emu.edu.tr/6-1_StaffingEmploymentAcademicStaff.htm"], "is_correct": null, "language": "en", "notes": "Assistant-curated regression case pending independent human review.", "question": "Do the research assistant appointment and scholarship provisions show any conflict?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-001", "category": "course_registration", "citation_ok": null, "expected_answer_keywords": ["birinci"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0004", "expected_chunk_ids": ["tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0004"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Birinci yıl öğrencisi hangi dönem derslerine kayıt olmakla yükümlüdür?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-002", "category": "course_registration", "citation_ok": null, "expected_answer_keywords": ["Yüksek Şeref"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0005", "expected_chunk_ids": ["tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0005"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Yüksek Şeref öğrencisi normal ders yüküne ek olarak kaç ders alabilir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-003", "category": "course_registration", "citation_ok": null, "expected_answer_keywords": ["Ders Ekleme"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0011", "expected_chunk_ids": ["tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570:c0011"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-5-yonetmelik-derskayit-htm:6c57f64570", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-5-Yonetmelik-DersKayit.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Ders ekleme veya bırakma süreci ne zaman yapılır?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-004", "category": "course_registration", "citation_ok": null, "expected_answer_keywords": ["kayıtlarını yenilemek"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0012", "expected_chunk_ids": ["tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0012"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Öğrenciler her dönem başında kayıtlarını nasıl yenilemek zorundadır?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-005", "category": "tuition_refunds", "citation_ok": null, "expected_answer_keywords": ["iade"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0013", "expected_chunk_ids": ["tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0013"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Kayıt sildiren öğrenci harç iadesi için hangi esaslara tabidir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-006", "category": "grading_exam", "citation_ok": null, "expected_answer_keywords": ["Devamsızlıktan"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0016", "expected_chunk_ids": ["tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0016"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "NG notu ne anlama gelir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-007", "category": "leave_freeze", "citation_ok": null, "expected_answer_keywords": ["izin"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0035", "expected_chunk_ids": ["tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d:c0035"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-5-1-ogrtsnvbasari-htm:46e5dda29d", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/5-1_OgrtSnvBasari.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Öğrenci izinli ayrılmak için nereye yazılı ve gerekçeli başvuru yapar?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-008", "category": "grading_exam", "citation_ok": null, "expected_answer_keywords": ["3 gün"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57:c0006", "expected_chunk_ids": ["tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57:c0006"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-4-Yonetmelik-SinavveDegerlendirme.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-4-Yonetmelik-SinavveDegerlendirme.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Bütünleme sınavına başvuru dönem notlarının ilanından sonra kaç gün içinde yapılır?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-009", "category": "grading_exam", "citation_ok": null, "expected_answer_keywords": ["Telafi"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57:c0007", "expected_chunk_ids": ["tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57:c0007"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-4-yonetmelik-sinavvedegerlendirme-htm:2e84bede57", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-4-Yonetmelik-SinavveDegerlendirme.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-4-Yonetmelik-SinavveDegerlendirme.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Telafi sınavları dönem içinde mi dönem sonunda mı yapılabilir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-010", "category": "scholarships", "citation_ok": null, "expected_answer_keywords": ["5000"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0005", "expected_chunk_ids": ["tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0005"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "İlk 5000 arasına giren öğrencilere hangi burs verilebilir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-011", "category": "scholarships", "citation_ok": null, "expected_answer_keywords": ["%1"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0014", "expected_chunk_ids": ["tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0014"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Yüksek Şeref bursu programdaki yüzde kaçlık başarı dilimine göre verilir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-012", "category": "scholarships", "citation_ok": null, "expected_answer_keywords": ["%100"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0018", "expected_chunk_ids": ["tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0018"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Lisansüstü burslar hangi oranlarda verilir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-013", "category": "scholarships", "citation_ok": null, "expected_answer_keywords": ["birden fazla"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0026", "expected_chunk_ids": ["tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22:c0026"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-2-yonetmelik-burs-indirim-uygulama-htm:cedfc45a22", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-2-Yonetmelik-Burs-Indirim-Uygulama.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Bir öğrenci birden fazla burs ve indirimden aynı anda yararlanabilir mi?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-014", "category": "tuition_refunds", "citation_ok": null, "expected_answer_keywords": ["%20"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-1-10-y-netmelik-ogrenimharc-htm:e6f41ba36f:c0008", "expected_chunk_ids": ["tr:html:5-1-10-y-netmelik-ogrenimharc-htm:e6f41ba36f:c0008"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-1-10-y-netmelik-ogrenimharc-htm:e6f41ba36f", "expected_source_url": "https://mevzuat.emu.edu.tr/5-1-10-Yönetmelik-OgrenimHarc.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-1-10-Yönetmelik-OgrenimHarc.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Ders ekleme veya bırakma sürecinde harç iadesi için hangi yüzde uygulanır?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-015", "category": "research_assistant", "citation_ok": null, "expected_answer_keywords": ["araştırma görevlisi"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0003", "expected_chunk_ids": ["tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0003"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Araştırma görevlisi yönetmeliği hangi nitelik ve burs konularını kapsar?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-016", "category": "research_assistant", "citation_ok": null, "expected_answer_keywords": ["akademik yardım"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0005", "expected_chunk_ids": ["tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0005"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Araştırma görevlileri öğrenci akademik yardım masalarında görev alabilir mi?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-017", "category": "research_assistant", "citation_ok": null, "expected_answer_keywords": ["asgari ücret"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0012", "expected_chunk_ids": ["tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0012"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "A ve B kategorisi araştırma görevlisi aylık bursu asgari ücrete göre nasıl belirlenir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-018", "category": "staff_regulations", "citation_ok": null, "expected_answer_keywords": ["barem"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0011", "expected_chunk_ids": ["tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0011"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Akademik personel kadro sayısı ve baremleri hangi cetvellere göre düzenlenir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-019", "category": "staff_regulations", "citation_ok": null, "expected_answer_keywords": ["maaş"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0012", "expected_chunk_ids": ["tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0012"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Akademik personele maaş ve makam ödeneği nasıl verilir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-020", "category": "staff_regulations", "citation_ok": null, "expected_answer_keywords": ["ilk atanma"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0039", "expected_chunk_ids": ["tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0039"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Akademik personelin barem içi artışı ilk atanma tarihine göre nasıl verilir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-021", "category": "staff_regulations", "citation_ok": null, "expected_answer_keywords": ["kadro"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:tuzukler-7-1-yontperskadrocal-htm:1cb7f72b71:c0020", "expected_chunk_ids": ["tr:html:tuzukler-7-1-yontperskadrocal-htm:1cb7f72b71:c0020"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-7-1-yontperskadrocal-htm:1cb7f72b71", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/7-1_YontPersKadroCal.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/7-1_YontPersKadroCal.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Yönetsel hizmetler personeli kadro ve baremleri hangi düzenlemede geçer?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-022", "category": "student_discipline", "citation_ok": null, "expected_answer_keywords": ["45 iş günü"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:5-2-yonetmelik-ogrencidisiplin-htm:4ada820abe:c0020", "expected_chunk_ids": ["tr:html:5-2-yonetmelik-ogrencidisiplin-htm:4ada820abe:c0020"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-2-yonetmelik-ogrencidisiplin-htm:4ada820abe", "expected_source_url": "https://mevzuat.emu.edu.tr/5-2-Yonetmelik-OgrenciDisiplin.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-2-Yonetmelik-OgrenciDisiplin.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Öğrenci Disiplin Kurulu dosyayı en geç kaç iş günü içinde karara bağlar?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-023", "category": "research_ethics", "citation_ok": null, "expected_answer_keywords": ["akademik etik"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:6-1-5-yonetmelik-bilimselarastirmayayinetigi-htm:fd248ba433:c0004", "expected_chunk_ids": ["tr:html:6-1-5-yonetmelik-bilimselarastirmayayinetigi-htm:fd248ba433:c0004"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:6-1-5-yonetmelik-bilimselarastirmayayinetigi-htm:fd248ba433", "expected_source_url": "https://mevzuat.emu.edu.tr/6-1-5-Yonetmelik-BilimselArastirmaYayinEtigi.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-1-5-Yonetmelik-BilimselArastirmaYayinEtigi.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Üniversite akademik etik ilkelerini nasıl bir değer olarak görür?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-024", "category": "research_ethics", "citation_ok": null, "expected_answer_keywords": ["hakem"], "expected_behavior": "answer", "expected_chunk_id": "tr:html:6-3-3-bilimselarastirmalardestekilkeleri-htm:d1b8dace78:c0003", "expected_chunk_ids": ["tr:html:6-3-3-bilimselarastirmalardestekilkeleri-htm:d1b8dace78:c0003"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:6-3-3-bilimselarastirmalardestekilkeleri-htm:d1b8dace78", "expected_source_url": "https://mevzuat.emu.edu.tr/6-3-3-BilimselArastirmalarDestekIlkeleri.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/6-3-3-BilimselArastirmalarDestekIlkeleri.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Bilimsel araştırma proje önerileri hakem değerlendirmesine nasıl sunulur?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-025", "category": "out_of_scope", "citation_ok": null, "expected_answer_keywords": [], "expected_behavior": "refuse", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Bugünkü kampüs etkinlikleri ve konserler nelerdir?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-026", "category": "out_of_scope", "citation_ok": null, "expected_answer_keywords": [], "expected_behavior": "refuse", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Bölüm başkanıma e-posta gönderip kayıt için toplantı planla.", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-027", "category": "ambiguous_query", "citation_ok": null, "expected_answer_keywords": [], "expected_behavior": "clarify", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Hangi ofise sorulmalı: burs mu harç mı olduğundan emin değilim.", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-028", "category": "ambiguous_query", "citation_ok": null, "expected_answer_keywords": [], "expected_behavior": "clarify", "expected_chunk_id": null, "expected_chunk_ids": [], "expected_corpora": [], "expected_corpus": null, "expected_document_id": null, "expected_source_url": null, "expected_source_urls": [], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Hangi ofise sorulmalı: kayıt dondurma mı ders bırakma mı emin değilim.", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-029", "category": "conflict", "citation_ok": null, "expected_answer_keywords": ["maas"], "expected_behavior": "conflict", "expected_chunk_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0012", "expected_chunk_ids": ["tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0012", "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0012"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c", "expected_source_url": "https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm", "https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Akademik personel maas hukumleri celiskili mi?", "review_status": "assistant_curated_pending_independent_review"} +{"case_id": "TR-030", "category": "conflict", "citation_ok": null, "expected_answer_keywords": ["asgari"], "expected_behavior": "conflict", "expected_chunk_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0012", "expected_chunk_ids": ["tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637:c0012", "tr:html:tuzukler-6-1-akdperskadrocal-htm:3390963e9c:c0012"], "expected_corpora": ["regulations_tr"], "expected_corpus": "regulations_tr", "expected_document_id": "tr:html:5-4-3-yonetmelik-arastirmagorevlisigorevburs-htm:bff0120637", "expected_source_url": "https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm", "expected_source_urls": ["https://mevzuat.emu.edu.tr/5-4-3-Yonetmelik-ArastirmaGorevlisiGorevBurs.htm", "https://mevzuat.emu.edu.tr/Tuzukler/6-1_AkdPersKadroCal.htm"], "is_correct": null, "language": "tr", "notes": "Assistant-curated regression case pending independent human review.", "question": "Arastirma gorevlisi burs kurallari farkli veya celiskili mi?", "review_status": "assistant_curated_pending_independent_review"} diff --git a/requirements-audit.txt b/requirements-audit.txt new file mode 100644 index 0000000..a32eab8 --- /dev/null +++ b/requirements-audit.txt @@ -0,0 +1,2 @@ +# Audit runner authority; application dependencies remain in requirements-lock.txt. +pip-audit==2.10.1 diff --git a/requirements-lock.txt b/requirements-lock.txt index c229914..e1a79a4 100644 --- a/requirements-lock.txt +++ b/requirements-lock.txt @@ -4,6 +4,7 @@ anyio==4.14.2 certifi==2026.7.22 charset-normalizer==3.4.9 click==8.4.2 +colorama==0.4.6; sys_platform == "win32" fastapi==0.141.1 greenlet==3.5.4 grpcio==1.83.0 @@ -23,7 +24,8 @@ protobuf==7.35.1 pydantic==2.13.4 pydantic_core==2.46.4 pyee==13.0.1 -pypdf==6.15.0 +pypdf==6.17.0 +pywin32==312; sys_platform == "win32" python-dotenv==1.2.2 PyYAML==6.0.3 qdrant-client==1.19.0 @@ -33,6 +35,6 @@ typing-inspection==0.4.2 typing_extensions==4.16.0 urllib3==2.7.0 uvicorn==0.52.1 -uvloop==0.22.1 +uvloop==0.22.1; sys_platform != "win32" watchfiles==1.2.0 websockets==17.0.1 diff --git a/requirements.txt b/requirements.txt index e9407b9..a8b6e58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ fastapi>=0.104.0 uvicorn[standard]>=0.24.0 pydantic>=2.0.0 -pypdf>=4.0.0 +pypdf>=6.16.1 httpx>=0.25.0 qdrant-client>=1.9.0 diff --git a/shared/audit.md b/shared/audit.md new file mode 100644 index 0000000..9e8bebf --- /dev/null +++ b/shared/audit.md @@ -0,0 +1,9 @@ +# Shared Audit + +- 2026-09-05T00:00:00Z | repo-bootstrap | initialize governance | workflow files | classification report | complete +- 2026-09-05T18:50:00Z | root | accept EMU-B001 and acquire writer lock | governance only | bootstrap audit PASS and baseline hashes | complete +- 2026-09-06T04:21:00Z | tester | independently test initial frozen snapshot | snapshot e195f40ca72ebe3bb1491814dc5121066ae58bb737bfbe5c6b9a824d4854a02f | caveat-order evidence | FAIL +- 2026-09-06T04:22:00Z | tester | independently retest bounded repair | snapshot d56a5dcdd63dbe2a1be14d42b3e16156250ab0394a849757c4269bac4ec11b26 | full/focused/security/evaluation/browser/dependency evidence | PASS_WITH_RISKS +- 2026-09-06T04:23:00Z | docs-qa | reconcile and close EMU-B001 | workflow/state/QA/risk/current-status records | acceptance mapping and residual-risk review | COMPLETE_WITH_RISKS +- 2026-09-06T04:24:00Z | repo-bootstrap audit | validate closure governance | DEV_STATE and BLUEPRINT | active-batch mismatch | FAIL then corrected +- 2026-09-06T04:25:00Z | repo-bootstrap audit | revalidate closure governance | workflow pack | no errors or warnings | PASS diff --git a/shared/context.md b/shared/context.md new file mode 100644 index 0000000..b8367f5 --- /dev/null +++ b/shared/context.md @@ -0,0 +1,8 @@ +# Shared Context + +- Workflow schema: agentic-workflow/v2 +- Phase: CLOSE +- Active batch: EMU-B001 (cycle closed) +- Owner: root +- Latest verdict: PASS_WITH_RISKS on repaired snapshot `d56a5dcdd63dbe2a1be14d42b3e16156250ab0394a849757c4269bac4ec11b26` +- Product boundary: local-only regulation retrieval demo; EMU-B001 is complete with risks; benchmark/presentation/publication/release/deployment remain blocked pending human review, live/reproducible evidence, rights, history-aware secret review, and Internet-grade security diff --git a/shared/errors.md b/shared/errors.md new file mode 100644 index 0000000..8945c5f --- /dev/null +++ b/shared/errors.md @@ -0,0 +1,4 @@ +# Shared Errors + +- 2026-09-06T04:21:00Z | EMU-B001 | Initial TEST `FAIL`: disclaimers in `docs/EMUAdvisor Full Analysis.md`, `docs/SPRINT_PLAN.md`, and `docs/VERSION_LOG.md` followed the historical claims they qualified. Resolved by bounded caveat-order repair and fresh independent retest; the FAIL remains part of the audit trail. +- 2026-09-06T04:24:00Z | EMU-B001 | First docs-QA bootstrap audit `FAIL`: `DEV_STATE.md` batch value did not exactly match `BLUEPRINT.md`. Resolved by retaining `Active batch: EMU-B001` alongside the closed cycle status; rerun evidence is recorded in `docs/BOOTSTRAP_AUDIT.md`. diff --git a/shared/history.md b/shared/history.md new file mode 100644 index 0000000..da69c16 --- /dev/null +++ b/shared/history.md @@ -0,0 +1,7 @@ +# Shared History + +- 2026-09-05: Repository governance bootstrap initialized. +- 2026-09-06: EMU-B001 initial independent TEST failed on late historical caveats; a bounded documentation-only repair followed. +- 2026-09-06: Fresh independent TEST returned PASS_WITH_RISKS on the repaired snapshot. +- 2026-09-06: Docs-QA reconciled acceptance evidence, preserved residual release gates, closed EMU-B001 as COMPLETE_WITH_RISKS, and released the cooperative lock. +- 2026-09-06: Closure governance audit passed after one recorded active-batch field correction. diff --git a/shared/locks.json b/shared/locks.json new file mode 100644 index 0000000..35fa40f --- /dev/null +++ b/shared/locks.json @@ -0,0 +1,4 @@ +{ + "schema_version": "agentic-workflow/v2", + "locks": [] +} diff --git a/shared/messages.md b/shared/messages.md new file mode 100644 index 0000000..409a5fb --- /dev/null +++ b/shared/messages.md @@ -0,0 +1,3 @@ +# Shared Messages + +No pending messages. diff --git a/shared/status.md b/shared/status.md new file mode 100644 index 0000000..a68cf95 --- /dev/null +++ b/shared/status.md @@ -0,0 +1,14 @@ +# Shared Status + +[2026-09-05T18:15:00Z] [repo-bootstrap] [BOOTSTRAP] [READY_FOR_DEV_LOOP] [EMU-B001] [Mixed software/data-ML/research profile; three AGENTS warnings repaired.] +[2026-09-05T18:20:00Z] [root] [PLAN] [IN_PROGRESS] [EMU-B001] [Independent mapper/risk reviews complete; architect plan requested.] +[2026-09-05T18:45:00Z] [root] [PLAN] [READY] [EMU-B001] [Independent plan accepted; implementation remains bounded by evidence/privacy/source/portability controls.] +[2026-09-05T18:50:00Z] [root] [IMPLEMENT] [IN_PROGRESS] [EMU-B001] [Baseline recorded and cooperative root lock acquired.] +[2026-09-06T04:20:00Z] [root] [TEST] [IN_PROGRESS] [EMU-B001] [Executor checks green; freezing diff for independent TEST.] + +[2026-09-06T04:21:00Z] [tester] [TEST] [FAIL] [EMU-B001] [Initial snapshot failed because historical caveats appeared after the claims they qualified.] +[2026-09-06T04:22:00Z] [tester] [TEST] [PASS_WITH_RISKS] [EMU-B001] [Fresh repaired snapshot passed; external, human, live-service, reproducibility, rights, identity, and maintenance risks remain.] +[2026-09-06T04:23:00Z] [docs-qa] [CLOSE] [COMPLETE_WITH_RISKS] [EMU-B001] [Evidence reconciled, residual gates retained, and cooperative writer lock released.] +[2026-09-06T04:25:00Z] [docs-qa] [CLOSE] [COMPLETE_WITH_RISKS] [EMU-B001] [Post-correction governance audit PASS with no errors or warnings.] + +[2026-09-05T00:00:00Z] [repo-bootstrap] [BOOTSTRAP] [COMPLETE] [NONE] [Governance pack initialized; validation pending.] diff --git a/static/admin-diagnostics.js b/static/admin-diagnostics.js index 1ce8acb..e0ab8ff 100644 --- a/static/admin-diagnostics.js +++ b/static/admin-diagnostics.js @@ -165,12 +165,14 @@ const modes = await modesResponse.json(); const analytics = await analyticsResponse.json(); corpusCount.textContent = corpus.chunk_count ?? "-"; - top5.textContent = formatPercent(metrics.retrieval_top5); - reject.textContent = formatPercent(metrics.rejection_accuracy); + top5.textContent = metrics.legacy ? "legacy" : formatPercent(metrics.expected_evidence_retrieval_top5_rate); + reject.textContent = metrics.legacy ? "legacy" : formatPercent(metrics.refusal_behavior_match_rate); latency.textContent = metrics.extractive_latency_p50_ms == null ? "-" : `${metrics.extractive_latency_p50_ms}ms`; auditCount.textContent = analytics.events ?? "-"; corpusDetails.innerHTML = renderDetails({ source: corpus.source, + corpus_mode: corpus.corpus_mode, + fixture: corpus.fixture, runtime_profile: corpus.runtime_profile, vector_backend: corpus.vector_backend, embedding_model: corpus.embedding_model, @@ -229,7 +231,7 @@ return Object.entries(modes) .map(([name, preset]) => { const latest = payload.results?.[name] || {}; - const score = latest.total_score == null ? "-" : Number(latest.total_score).toFixed(2); + const score = latest.weighted_proxy_score == null ? "-" : Number(latest.weighted_proxy_score).toFixed(2); return `\n
\n

${escapeHtml(name)}

\n

${escapeHtml(preset.local_llm_label || "")}

\n
\n
fanout
${escapeHtml(String(preset.retrieval_fanout))}
\n
rerank
${preset.rerank_enabled ? "on" : "off"}
\n
context
${escapeHtml(String(preset.max_context_chunks))}
\n
score
${score}
\n
\n
\n `; }) .join(""); diff --git a/static/admin.html b/static/admin.html index a7b1ffb..e5d5196 100644 --- a/static/admin.html +++ b/static/admin.html @@ -28,6 +28,9 @@

EMU Regulation Assistant

+
English and Turkish corpora are routed separately Official mevzuat sources only @@ -55,17 +58,19 @@

EMU Regulation Assistant

+ + + + +
- -

This demo is informational only and does not provide a final official university decision.

@@ -102,8 +107,8 @@

Admin authentication

-chunks
-
-top-5 retrieval
-
-rejection accuracy
+
-expected-evidence top-5 proxy
+
-refusal-mode match
-p50 extractive latency
-local LLM
-audit events
diff --git a/static/landing.html b/static/landing.html index cd62a9d..901e337 100644 --- a/static/landing.html +++ b/static/landing.html @@ -8,6 +8,9 @@
+

Eastern Mediterranean University

EMU Regulation Assistant

diff --git a/static/shared.js b/static/shared.js index 9f4777b..829943a 100644 --- a/static/shared.js +++ b/static/shared.js @@ -168,3 +168,12 @@ toggleTheme, }; })(window); +fetch("/health") + .then((response) => response.json()) + .then((status) => { + const banner = document.querySelector("#fixture-banner"); + if (banner && status.fixture === true) { + banner.hidden = false; + } + }) + .catch(() => {}); diff --git a/static/style.css b/static/style.css index 3b7f54a..317ef35 100644 --- a/static/style.css +++ b/static/style.css @@ -1129,3 +1129,16 @@ summary { min-height: 46px; } } +.fixture-banner { + margin: 0 0 1rem; + padding: 0.85rem 1rem; + border: 2px solid #9a3412; + border-radius: 0.6rem; + background: #ffedd5; + color: #7c2d12; + font-weight: 700; +} + +.fixture-banner[hidden] { + display: none; +} diff --git a/static/user-chat.js b/static/user-chat.js index 2f715ef..1614445 100644 --- a/static/user-chat.js +++ b/static/user-chat.js @@ -8,8 +8,8 @@ const messages = document.querySelector("#messages"); const sessionIdInput = document.querySelector("#session-id"); const newSessionBtn = document.querySelector("#new-session-btn"); - const historyBtn = document.querySelector("#history-btn"); const exportBtn = document.querySelector("#export-btn"); + const clearBtn = document.querySelector("#clear-session-btn"); const exportFormat = document.querySelector("#export-format"); const advancedToggle = document.querySelector("#advanced-toggle"); const advancedPanel = document.querySelector("#advanced-panel"); @@ -18,19 +18,28 @@ return; } - let currentSessionId = sessionStorage.getItem("emuSessionId") || "user-ui"; + let currentSessionId = sessionStorage.getItem("emuSessionId"); + let currentSessionCapability = sessionStorage.getItem("emuSessionCapability"); let messageCounter = 0; const stickyScrollThreshold = 96; - if (!sessionStorage.getItem("emuSessionId")) { - sessionStorage.setItem("emuSessionId", currentSessionId); - } if (sessionIdInput) { - sessionIdInput.value = currentSessionId; + sessionIdInput.value = currentSessionId || "Created after first message"; + } + + function sessionHeaders() { + const headers = { "Content-Type": "application/json" }; + if (currentSessionCapability) { + headers["X-EMU-Session-Capability"] = currentSessionCapability; + } + return headers; } function restoreCurrentSessionIfAvailable() { - fetch(`/chat/session/${encodeURIComponent(currentSessionId)}`) + if (!currentSessionId || !currentSessionCapability) { + return; + } + fetch(`/chat/session/${encodeURIComponent(currentSessionId)}`, { headers: sessionHeaders() }) .then((r) => r.json()) .then((payload) => { if (payload.messages && payload.messages.length > 0) { @@ -40,27 +49,16 @@ .catch(() => {}); } - function generateSessionId() { - return `session-${Math.random().toString(36).slice(2, 11)}`; - } - function createNewSession() { - const newSessionId = generateSessionId(); + currentSessionId = null; + currentSessionCapability = null; + sessionStorage.removeItem("emuSessionId"); + sessionStorage.removeItem("emuSessionCapability"); if (sessionIdInput) { - sessionIdInput.value = newSessionId; + sessionIdInput.value = "Created after first message"; } - currentSessionId = newSessionId; - sessionStorage.setItem("emuSessionId", newSessionId); messages.innerHTML = ""; - addMessage("assistant", "New session created. Ask a question about EMU regulations."); - showHistoryPanel(false); - } - - function showHistoryPanel(show) { - const panel = document.querySelector("#history-panel"); - if (panel) { - panel.hidden = !show; - } + addMessage("assistant", "A private session will be created when you send your next question."); } function renderSessionMessages(payload) { @@ -75,22 +73,6 @@ }); } - function loadSessionFromHistory(sessionId) { - currentSessionId = sessionId; - if (sessionIdInput) { - sessionIdInput.value = sessionId; - } - sessionStorage.setItem("emuSessionId", sessionId); - showHistoryPanel(false); - fetch(`/chat/session/${encodeURIComponent(sessionId)}`) - .then((r) => r.json()) - .then(renderSessionMessages) - .catch(() => { - messages.innerHTML = ""; - addMessage("assistant", `Session ${sessionId} loaded. Continue the conversation below.`); - }); - } - function addMessage(role, text) { const stickToBottom = shouldStickToBottom(); const article = document.createElement("article"); @@ -265,9 +247,12 @@ } function exportSession(format = "markdown") { + if (!currentSessionId || !currentSessionCapability) { + return; + } fetch("/chat/export", { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: sessionHeaders(), body: JSON.stringify({ session_id: currentSessionId, format }), }) .then((r) => r.blob()) @@ -318,10 +303,10 @@ try { const response = await fetch("/chat/stream", { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: sessionHeaders(), body: JSON.stringify({ question: value, - session_id: currentSessionId, + session_id: currentSessionId || null, }), }); @@ -358,6 +343,10 @@ if (payload.type === "session") { currentSessionId = payload.session_id; sessionStorage.setItem("emuSessionId", currentSessionId); + if (payload.session_capability) { + currentSessionCapability = payload.session_capability; + sessionStorage.setItem("emuSessionCapability", currentSessionCapability); + } if (sessionIdInput) { sessionIdInput.value = currentSessionId; } @@ -412,33 +401,26 @@ newSessionBtn?.addEventListener("click", createNewSession); - historyBtn?.addEventListener("click", () => { - fetch("/chat/sessions") - .then((r) => r.json()) - .then((sessions) => { - const list = document.querySelector("#history-list"); - if (list) { - list.innerHTML = sessions - .map( - (s) => ` -

  • - -
    • - ` - ) - .join(""); - } - showHistoryPanel(true); - }); - }); - exportBtn?.addEventListener("click", () => { exportSession(exportFormat?.value || "markdown"); }); + clearBtn?.addEventListener("click", async () => { + if (!currentSessionId || !currentSessionCapability) { + createNewSession(); + return; + } + const response = await fetch("/chat/clear", { + method: "POST", + headers: sessionHeaders(), + body: JSON.stringify({ session_id: currentSessionId }), + }); + if (response.ok) { + messages.innerHTML = ""; + addMessage("assistant", "This session transcript was cleared."); + } + }); + advancedToggle?.addEventListener("click", () => { const open = advancedPanel?.hidden !== false; if (advancedPanel) { @@ -458,13 +440,6 @@ if (event.target.classList.contains("toggle-supporting-results")) { toggleSupportingResults(event.target); } - if (event.target.id === "close-history-btn" || event.target.closest("#close-history-btn")) { - showHistoryPanel(false); - } - const historyItem = event.target.closest(".history-item"); - if (historyItem?.dataset.session) { - loadSessionFromHistory(historyItem.dataset.session); - } }); document.querySelectorAll("[data-view-tab]").forEach((tab) => { diff --git a/tests/test_board_readiness_tools.py b/tests/test_board_readiness_tools.py index 530879d..68e6b9e 100644 --- a/tests/test_board_readiness_tools.py +++ b/tests/test_board_readiness_tools.py @@ -78,7 +78,7 @@ def test_readiness_report_can_write_markdown(self) -> None: exists = out.exists() self.assertTrue(exists) - self.assertEqual(report["status"], "partial") + self.assertEqual(report["status"], "blocked") def _case_payload() -> dict: diff --git a/tests/test_ingestion_routing_evaluation.py b/tests/test_ingestion_routing_evaluation.py index 2adeba1..55b206a 100644 --- a/tests/test_ingestion_routing_evaluation.py +++ b/tests/test_ingestion_routing_evaluation.py @@ -175,9 +175,8 @@ def test_seed_evaluation_set_is_machine_readable(self) -> None: self.assertEqual(sum(1 for case in cases if case.expected_behavior == "refuse"), 4) self.assertEqual(sum(1 for case in cases if case.expected_behavior == "clarify"), 4) self.assertEqual(sum(1 for case in cases if case.expected_behavior == "conflict"), 4) - self.assertEqual({case.review_status for case in cases}, {"human_reviewed_verified"}) - self.assertTrue(all(case.is_correct is True for case in cases)) - self.assertTrue(all(case.citation_ok is True for case in cases)) + self.assertEqual({case.review_status for case in cases}, {"assistant_curated_pending_independent_review"}) + self.assertTrue(all(case.is_correct is None and case.citation_ok is None for case in cases)) for case in cases: if case.expected_behavior in {"answer", "conflict"}: self.assertTrue(case.expected_chunk_ids or case.expected_source_urls) diff --git a/tests/test_pipeline_metrics_generation.py b/tests/test_pipeline_metrics_generation.py index 26c736e..b6e3b54 100644 --- a/tests/test_pipeline_metrics_generation.py +++ b/tests/test_pipeline_metrics_generation.py @@ -67,10 +67,16 @@ def test_fixture_pipeline_builds_canonical_chunks_without_network(self) -> None: self.assertEqual(result.errors, 0) self.assertGreaterEqual(result.chunk_count, 1) - self.assertEqual(chunks[0]["source_url"], "https://mevzuat.emu.edu.tr/content/fixture/fixture.htm") + self.assertEqual(chunks[0]["source_url"], "fixture:///fixture.htm") + self.assertTrue(chunks[0]["metadata"]["fixture"]) + self.assertFalse(chunks[0]["metadata"]["official_source"]) + self.assertNotIn(str(root), json.dumps(chunks[0])) self.assertEqual(chunks[0]["language"], "en") self.assertTrue((out / "manifest.json").exists()) self.assertTrue((out.parent / "active_snapshot.txt").exists()) + self.assertNotIn(str(root), (out / "manifest.json").read_text(encoding="utf-8")) + self.assertNotIn(str(root), (out / "crawl_pages.jsonl").read_text(encoding="utf-8")) + self.assertNotIn(str(root), (out.parent / "active_snapshot.txt").read_text(encoding="utf-8")) def test_fixture_pipeline_uses_content_language_when_url_is_ambiguous(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -110,10 +116,13 @@ def test_metrics_runner_writes_dashboard_review_and_failure_analysis_artifacts(s summary = run_evaluation(cases_path=cases, chunks_path=chunks, out_dir=out) self.assertEqual(summary["cases"], 60) - self.assertEqual(summary["retrieval_top5"], 1.0) + self.assertEqual(summary["expected_evidence_retrieval_top5_rate"], 1.0) self.assertEqual(summary["mode"], "balanced") self.assertIn("mode_preset", summary) - self.assertIn("total_score", summary) + self.assertEqual(summary["schema_version"], "emu-advisor-automated-proxy/v2") + self.assertIn("weighted_proxy_score", summary) + self.assertNotIn("response_accuracy", summary) + self.assertNotIn("groundedness", summary) self.assertIn("failure_counts", summary) self.assertIn("category_metrics", summary) self.assertIn("worst_failed_cases", summary) diff --git a/tests/test_security_provenance.py b/tests/test_security_provenance.py new file mode 100644 index 0000000..b3e1ebb --- /dev/null +++ b/tests/test_security_provenance.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import json +import os +import tempfile +import unittest +import re +from pathlib import Path +from unittest.mock import patch + +import httpx +from fastapi.testclient import TestClient + +os.environ.setdefault("EMU_ADVISOR_PROFILE", "test") +os.environ.setdefault("EMU_ADVISOR_CORPUS_MODE", "fixture") +os.environ.setdefault("EMU_ADVISOR_QUERY_REWRITE", "deterministic") + +from emu_advisor.conversation_store import ConversationStore +from emu_advisor.corpus import load_corpus +from emu_advisor.evaluation import EvaluationCase, citation_matches_expected_evidence +from emu_advisor.pipeline import _fetch, is_allowed_crawl_url +from emu_advisor.server import create_app +from tools.publication_guard import verify_human_review_contract + + +def _case(*, behavior: str = "answer", chunks: list[str] | None = None, sources: list[str] | None = None) -> EvaluationCase: + chunks = chunks or [] + sources = sources or [] + return EvaluationCase( + case_id="synthetic", + question="synthetic question", + language="en", + expected_behavior=behavior, + expected_corpus="regulations_en", + expected_document_id=None, + expected_chunk_id=chunks[0] if chunks else None, + expected_source_url=sources[0] if sources else None, + expected_corpora=["regulations_en"], + expected_chunk_ids=chunks, + expected_source_urls=sources, + expected_answer_keywords=["keyword"], + category="synthetic", + is_correct=None, + citation_ok=None, + review_status="assistant_curated_pending_independent_review", + notes="synthetic", + ) + + +class SessionCapabilityTests(unittest.TestCase): + def test_cross_session_read_export_and_clear_are_uniformly_denied(self) -> None: + with patch.dict( + os.environ, + { + "EMU_ADVISOR_PROFILE": "test", + "EMU_ADVISOR_CORPUS_MODE": "fixture", + "EMU_ADVISOR_QUERY_REWRITE": "deterministic", + "EMU_ADVISOR_ENABLE_CHAT_PERSISTENCE": "0", + "EMU_ADVISOR_ENABLE_AUDIT_LOGGING": "0", + }, + clear=False, + ): + client = TestClient(create_app()) + first = client.post("/chat", json={"question": "Hi"}).json() + second = client.post("/chat", json={"question": "Hello"}).json() + own = {"X-EMU-Session-Capability": first["session_capability"]} + other = {"X-EMU-Session-Capability": second["session_capability"]} + + self.assertEqual(client.get(f"/chat/session/{first['session_id']}", headers=own).status_code, 200) + denied = [ + client.get(f"/chat/session/{first['session_id']}", headers=other), + client.post("/chat/export", headers=other, json={"session_id": first["session_id"], "format": "markdown"}), + client.post("/chat/clear", headers=other, json={"session_id": first["session_id"]}), + client.post("/chat", headers=other, json={"session_id": first["session_id"], "question": "continue"}), + ] + self.assertEqual({response.status_code for response in denied}, {404}) + self.assertEqual({response.json()["detail"] for response in denied}, {"session unavailable"}) + + def test_persistence_is_opt_in_and_stores_only_capability_hash(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "sessions.json" + disabled = ConversationStore(path, persistence_enabled=False) + disabled.create_session() + self.assertFalse(path.exists()) + + enabled = ConversationStore(path, persistence_enabled=True) + access = enabled.create_session() + payload = path.read_text(encoding="utf-8") + self.assertNotIn(access.capability, payload) + self.assertIn("capability_hash", payload) + + def test_legacy_persistence_file_is_not_loaded_or_rewritten(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "legacy.json" + original = '{"version":1,"sessions":[{"session_id":"legacy","messages":[]}]}' + path.write_text(original, encoding="utf-8") + store = ConversationStore(path, persistence_enabled=True) + store.create_session() + self.assertEqual(path.read_text(encoding="utf-8"), original) + self.assertEqual(store.list_sessions()[0].message_count, 0) + + +class CorpusModeAndSourceTests(unittest.TestCase): + def test_corpus_mode_fails_closed_and_fixture_is_explicit(self) -> None: + with self.assertRaises(RuntimeError): + load_corpus(mode="", profile="test") + with self.assertRaises(RuntimeError): + load_corpus(mode="fixture", profile="production") + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(FileNotFoundError): + load_corpus(Path(tmp) / "missing.jsonl", mode="artifact", profile="test") + fixture = load_corpus(mode="fixture", profile="test") + self.assertTrue(fixture.fixture) + self.assertTrue(all(str(chunk["source_url"]).startswith("fixture://") for chunk in fixture.chunks)) + + def test_real_source_scope_is_https_exact_host_without_credentials_or_bad_port(self) -> None: + self.assertTrue(is_allowed_crawl_url("https://mevzuat.emu.edu.tr/rules.htm")) + self.assertFalse(is_allowed_crawl_url("http://mevzuat.emu.edu.tr/rules.htm")) + self.assertFalse(is_allowed_crawl_url("https://user@mevzuat.emu.edu.tr/rules.htm")) + self.assertFalse(is_allowed_crawl_url("https://mevzuat.emu.edu.tr:8443/rules.htm")) + self.assertFalse(is_allowed_crawl_url("https://evil.example/rules.htm")) + + def test_cross_host_redirect_is_rejected_before_target_request(self) -> None: + requested: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requested.append(str(request.url)) + return httpx.Response(302, headers={"Location": "https://evil.example/stolen"}, request=request) + + with httpx.Client(transport=httpx.MockTransport(handler), follow_redirects=False) as client: + with self.assertRaisesRegex(ValueError, "out-of-scope redirect"): + _fetch("https://mevzuat.emu.edu.tr/start", client) + self.assertEqual(requested, ["https://mevzuat.emu.edu.tr/start"]) + + +class EvidenceProxyTests(unittest.TestCase): + def test_citation_requires_expected_chunk_and_conflict_requires_both(self) -> None: + single = _case(chunks=["expected:c1"]) + self.assertEqual(citation_matches_expected_evidence(single, [{"chunk_id": "wrong:c1"}]), (False, "expected_chunk")) + self.assertEqual(citation_matches_expected_evidence(single, [{"chunk_id": "expected:c1"}]), (True, "expected_chunk")) + + conflict = _case(behavior="conflict", chunks=["left:c1", "right:c1"]) + self.assertFalse(citation_matches_expected_evidence(conflict, [{"chunk_id": "left:c1"}])[0]) + self.assertTrue(citation_matches_expected_evidence(conflict, [{"chunk_id": "left:c1"}, {"chunk_id": "right:c1"}])[0]) + + def test_source_fallback_is_used_only_without_expected_chunks(self) -> None: + source = "https://mevzuat.emu.edu.tr/rule.htm" + source_only = _case(sources=[source]) + self.assertEqual(citation_matches_expected_evidence(source_only, [{"source_url": source + "/"}]), (True, "expected_source")) + chunk_primary = _case(chunks=["expected:c1"], sources=[source]) + self.assertFalse(citation_matches_expected_evidence(chunk_primary, [{"source_url": source}])[0]) + + def test_human_review_status_cannot_self_certify(self) -> None: + with self.assertRaises(ValueError): + verify_human_review_contract( + {"review_status": "human_reviewed_verified", "is_correct": True, "citation_ok": True} + ) + + +class PortabilityContractTests(unittest.TestCase): + def test_ci_has_cross_platform_core_and_immutable_official_action_refs(self) -> None: + workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") + self.assertIn("ubuntu-latest", workflow) + self.assertIn("windows-latest", workflow) + self.assertIn("EMU_ADVISOR_CORPUS_MODE: fixture", workflow) + refs = re.findall(r"uses:\s+(actions/(?:checkout|setup-python))@([0-9a-f]+)", workflow) + self.assertGreaterEqual(len(refs), 4) + self.assertTrue(all(len(sha) == 40 for _action, sha in refs)) + + def test_uvloop_lock_is_excluded_on_windows(self) -> None: + lock = Path("requirements-lock.txt").read_text(encoding="utf-8") + self.assertIn('uvloop==0.22.1; sys_platform != "win32"', lock) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_server_logging_load.py b/tests/test_server_logging_load.py index 88970da..fb38593 100644 --- a/tests/test_server_logging_load.py +++ b/tests/test_server_logging_load.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os import tempfile import unittest from pathlib import Path @@ -8,6 +9,9 @@ from fastapi.testclient import TestClient +os.environ.setdefault("EMU_ADVISOR_PROFILE", "test") +os.environ.setdefault("EMU_ADVISOR_CORPUS_MODE", "fixture") + from emu_advisor.audit_log import AuditEvent, AuditLogger from emu_advisor.corpus import CorpusBundle, corpus_status from emu_advisor.demo import demo_chunks @@ -24,7 +28,10 @@ def setUp(self) -> None: "os.environ", { "EMU_ADVISOR_QUERY_REWRITE": "deterministic", - "EMU_ADVISOR_DISABLE_CHAT_PERSISTENCE": "1", + "EMU_ADVISOR_PROFILE": "test", + "EMU_ADVISOR_CORPUS_MODE": "fixture", + "EMU_ADVISOR_ENABLE_CHAT_PERSISTENCE": "0", + "EMU_ADVISOR_ENABLE_AUDIT_LOGGING": "0", }, clear=False, ) @@ -181,7 +188,7 @@ def test_chat_stream_returns_ndjson_session_and_answer(self) -> None: ): response = client.post( "/chat/stream", - json={"question": "What is the attendance requirement?", "session_id": "test-stream"}, + json={"question": "What is the attendance requirement?"}, ) self.assertEqual(response.status_code, 200) @@ -189,6 +196,8 @@ def test_chat_stream_returns_ndjson_session_and_answer(self) -> None: lines = [json.loads(line) for line in response.text.splitlines() if line.strip()] types = [line["type"] for line in lines] self.assertEqual(types[0], "session") + self.assertIn("session_capability", lines[0]) + self.assertNotEqual(lines[0]["session_id"], lines[0]["session_capability"]) self.assertIn("extractive_answer", types) self.assertLess(types.index("generated_delta"), types.index("extractive_answer")) self.assertEqual(types[-1], "done") @@ -209,7 +218,7 @@ def capture_retrieve(self, query, *args, **kwargs): chat = client.post("/chat", json={"question": "What is the attendance requirement?"}) stream = client.post( "/chat/stream", - json={"question": "What is the attendance requirement?", "session_id": "balanced-stream"}, + json={"question": "What is the attendance requirement?"}, ) self.assertEqual(chat.status_code, 200) @@ -217,11 +226,17 @@ def capture_retrieve(self, query, *args, **kwargs): self.assertGreaterEqual(len(observed_modes), 2) self.assertEqual(set(observed_modes), {"balanced"}) - def test_chat_sessions_list_does_not_error(self) -> None: - client = TestClient(create_app()) - response = client.get("/chat/sessions") - self.assertEqual(response.status_code, 200) - self.assertIsInstance(response.json(), list) + def test_chat_sessions_are_admin_only_and_omit_message_text(self) -> None: + with patch.dict("os.environ", {"EMU_ADVISOR_ADMIN_TOKEN": "secret"}, clear=False): + client = TestClient(create_app()) + client.post("/chat", json={"question": "private attendance question"}) + denied = client.get("/chat/sessions") + allowed = client.get("/chat/sessions", headers={"X-EMU-Admin-Token": "secret"}) + self.assertEqual(denied.status_code, 401) + self.assertEqual(allowed.status_code, 200) + self.assertIsInstance(allowed.json(), list) + self.assertNotIn("private attendance question", json.dumps(allowed.json())) + self.assertTrue(all("last_user_message" not in item for item in allowed.json())) def test_casual_chat_returns_friendly_response(self) -> None: client = TestClient(create_app()) @@ -322,15 +337,19 @@ def capture_retrieve(self, query, *args, **kwargs): client = TestClient(create_app()) first = client.post( "/chat", - json={"question": "What is the attendance requirement?", "session_id": "language-switch-chat"}, + json={"question": "What is the attendance requirement?"}, ) + session_id = first.json()["session_id"] + session_headers = {"X-EMU-Session-Capability": first.json()["session_capability"]} second = client.post( "/chat", - json={"question": "buna nasil basvururum", "session_id": "language-switch-chat"}, + headers=session_headers, + json={"question": "buna nasil basvururum", "session_id": session_id}, ) stream = client.post( "/chat/stream", - json={"question": "buna nasil basvururum", "session_id": "language-switch-chat"}, + headers=session_headers, + json={"question": "buna nasil basvururum", "session_id": session_id}, ) self.assertEqual(first.status_code, 200) @@ -345,10 +364,10 @@ def capture_retrieve(self, query, *args, **kwargs): class AuditLoggingTests(unittest.TestCase): - def test_audit_log_hashes_session_id_and_keeps_debug_fields(self) -> None: + def test_audit_log_omits_query_and_session_identifier_by_default(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "audit.jsonl" - logger = AuditLogger(path, salt="test") + logger = AuditLogger(path) logger.log( AuditEvent( event_type="ask", @@ -363,9 +382,19 @@ def test_audit_log_hashes_session_id_and_keeps_debug_fields(self) -> None: payload = json.loads(path.read_text(encoding="utf-8")) self.assertNotIn("user@example.com", json.dumps(payload)) + self.assertNotIn("attendance requirement", json.dumps(payload)) self.assertEqual(payload["answer_mode"], "answer") self.assertEqual(payload["citation_ids"], ["doc:c1"]) - self.assertIsNotNone(payload["session_hash"]) + self.assertNotIn("session_hash", payload) + + def test_raw_query_logging_requires_explicit_opt_in(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "audit.jsonl" + logger = AuditLogger(path, include_raw_query=True) + logger.log(AuditEvent("ask", "synthetic question", "synthetic-session", {}, "refuse", 1, [])) + payload = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(payload["query"], "synthetic question") + self.assertNotIn("synthetic-session", json.dumps(payload)) class LoadSimulationTests(unittest.TestCase): diff --git a/tools/browser_smoke.py b/tools/browser_smoke.py index 033d159..3369fbb 100644 --- a/tools/browser_smoke.py +++ b/tools/browser_smoke.py @@ -54,6 +54,7 @@ def run_playwright_smoke(args) -> int: page.goto(admin_url, wait_until="networkidle") assert "EMU Regulation Assistant" in page.title() assert page.locator("#user-panel").is_visible() + assert page.locator("#fixture-banner").is_visible() set_theme(page, "light") exercise_user_chat(page) assert_readable_controls(page) @@ -112,6 +113,17 @@ def exercise_user_chat(page) -> None: page.locator(".answer-state").first.wait_for(timeout=15000) assert page.locator(".answer-text, .public-citations").first.is_visible() wait_for_send_ready(page, timeout_s=15) + capability_state = page.evaluate( + """() => ({ + sessionId: sessionStorage.getItem("emuSessionId"), + capability: sessionStorage.getItem("emuSessionCapability"), + url: window.location.href, + })""" + ) + assert capability_state["sessionId"] + assert capability_state["capability"] + assert capability_state["sessionId"] != capability_state["capability"] + assert capability_state["capability"] not in capability_state["url"] state = page.evaluate( """before => { const messages = document.querySelector("#messages"); @@ -144,11 +156,9 @@ def wait_for_send_ready(page, *, timeout_s: float) -> None: def assert_readable_controls(page) -> None: selectors = [ - "#history-btn", "#export-btn", "#export-format", "#new-session-btn", - "#advanced-toggle", "#theme-toggle", ".view-tab.is-active", ".view-tab:not(.is-active)", diff --git a/tools/publication_guard.py b/tools/publication_guard.py index fa54c73..1e96c15 100644 --- a/tools/publication_guard.py +++ b/tools/publication_guard.py @@ -10,21 +10,34 @@ GOLD = ROOT / "eval_sets" / "v1_gold.jsonl" -def verify_gold() -> None: +def verify_evaluation_provenance() -> None: rows = [json.loads(line) for line in GOLD.read_text(encoding="utf-8").splitlines() if line.strip()] if not rows: - raise SystemExit("verified gold set is empty") + raise SystemExit("assistant-curated regression set is empty") failures = [] for row in rows: - if row.get("review_status") != "human_reviewed_verified": + if row.get("review_status") != "assistant_curated_pending_independent_review": failures.append(f"{row.get('case_id')}: review_status={row.get('review_status')!r}") - if row.get("is_correct") is not True: - failures.append(f"{row.get('case_id')}: is_correct must be true") - if row.get("citation_ok") is not True: - failures.append(f"{row.get('case_id')}: citation_ok must be true") + if row.get("is_correct") is not None: + failures.append(f"{row.get('case_id')}: is_correct must remain null pending independent review") + if row.get("citation_ok") is not None: + failures.append(f"{row.get('case_id')}: citation_ok must remain null pending independent review") + note = str(row.get("notes") or "").casefold() + if "human-reviewed" in note or "university staff" in note: + failures.append(f"{row.get('case_id')}: unsupported human-review note") if failures: - raise SystemExit("verified gold metadata regression:\n" + "\n".join(failures)) - print(f"verified gold metadata ok: {len(rows)} cases") + raise SystemExit("evaluation provenance regression:\n" + "\n".join(failures)) + print(f"assistant-curated pending-review metadata ok: {len(rows)} cases") + + +def verify_human_review_contract(row: dict) -> None: + """Reject future self-certified human-review labels without durable evidence.""" + if row.get("review_status") != "human_reviewed_verified": + return + evidence = row.get("review_evidence") + required = {"reference", "dataset_sha256", "reviewed_at", "reviewer_role"} + if not isinstance(evidence, dict) or not required.issubset(evidence) or row.get("is_correct") is not True or row.get("citation_ok") is not True: + raise ValueError("human_reviewed_verified requires correctness judgments and a durable review_evidence reference") def verify_public_tree() -> None: @@ -36,13 +49,25 @@ def verify_public_tree() -> None: forbidden = { "?admin_token=": "admin tokens must not be passed in URLs", "&admin_token=": "admin tokens must not be passed in URLs", + "reviewed by university staff": "unsupported university-staff review claim", + "verified human-reviewed gold": "unsupported verified-gold claim", } failures = [] + claim_audit_records = { + Path("docs/CLAIM_REGISTER.md"), + Path("docs/EVIDENCE_MAP.md"), + } extensions = {".md", ".py", ".js", ".html", ".yml", ".yaml", ".txt"} for path in ROOT.rglob("*"): if not path.is_file() or path.suffix.lower() not in extensions: continue - if ".git" in path.parts or path == Path(__file__): + relative = path.relative_to(ROOT) + if ( + ".git" in path.parts + or path == Path(__file__) + or relative in claim_audit_records + or any(part in {"artifacts", "logs", ".old"} for part in path.parts) + ): continue try: text = path.read_text(encoding="utf-8") @@ -57,7 +82,7 @@ def verify_public_tree() -> None: def main() -> int: - verify_gold() + verify_evaluation_provenance() verify_public_tree() return 0 diff --git a/tools/syntax_check.py b/tools/syntax_check.py new file mode 100644 index 0000000..cd5756f --- /dev/null +++ b/tools/syntax_check.py @@ -0,0 +1,28 @@ +"""Cross-platform Python syntax check used by CI.""" + +from __future__ import annotations + +import ast +from pathlib import Path + + +def main() -> int: + failed: list[tuple[str, str]] = [] + checked = 0 + for base in (Path("emu_advisor"), Path("tests"), Path("tools")): + for path in base.rglob("*.py"): + checked += 1 + try: + ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + except Exception as exc: + failed.append((str(path), repr(exc))) + if failed: + for path, error in failed: + print(path, error) + return 1 + print(f"syntax ok: {checked} files") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 211fdf63293be062f1dfa0d52c6875f0c00b9b6c Mon Sep 17 00:00:00 2001 From: Ali Farrokhnejad Date: Sun, 6 Sep 2026 07:33:56 +0300 Subject: [PATCH 2/2] Record green cross-platform CI evidence --- DEV_LOG.md | 7 +++++++ DEV_STATE.md | 3 ++- QA_REPORT.md | 4 ++-- RISK_REGISTER.md | 7 +++---- docs/PROJECT_STATE.md | 7 +++++-- docs/PROJECT_STATUS_PROGRESS_PLAN.md | 5 +++-- docs/VERSION_LOG.md | 2 +- shared/audit.md | 1 + shared/history.md | 1 + shared/status.md | 1 + 10 files changed, 26 insertions(+), 12 deletions(-) diff --git a/DEV_LOG.md b/DEV_LOG.md index 069243c..e40341c 100644 --- a/DEV_LOG.md +++ b/DEV_LOG.md @@ -51,3 +51,10 @@ Initialized: 2026-09-05 - Preserved the transient browser-smoke timeout and Starlette/httpx deprecation warning as maintenance risks; two subsequent browser-smoke passes provide bounded local evidence, not a reliability guarantee. - Closed the cycle as `COMPLETE_WITH_RISKS`; benchmark, presentation, publication, release, deployment, merge, and production-readiness actions remain blocked. - The first closure audit returned `FAIL` because `DEV_STATE.md` replaced the canonical batch ID with a descriptive closed value while `BLUEPRINT.md` retained `EMU-B001`; docs-QA restored the exact batch ID without changing the closed cycle status, and the rerun returned `PASS` with no errors or warnings. + +## 2026-09-06 - Commit, PR, and remote CI evidence + +- Committed the closed batch as `9a717d0` and pushed `remediation/evidence-security-portability` without rewriting history or changing the repository name. +- Opened review PR #2 against `main`; no merge, release, deployment, or publication action was taken. +- GitHub Actions runs `34011634890` and `34011648717` both passed Ubuntu core, Windows core, and browser smoke. +- Remote CI closes the platform-run gate for this commit; the human, live-source, artifact, rights, history-review, identity, and deployment gates remain open. diff --git a/DEV_STATE.md b/DEV_STATE.md index 2e01c3c..355bc20 100644 --- a/DEV_STATE.md +++ b/DEV_STATE.md @@ -13,5 +13,6 @@ Initialized: 2026-09-05 - Blockers: Benchmark/presentation/publication/release/deployment remain blocked by missing independent human adjudication, reproducible artifact-backed evidence, corpus-rights decision, live-service validation, history-aware secret review, and Internet-grade identity/security review - Current risks: No Critical risk is open; residual High and Medium gates remain in `RISK_REGISTER.md` - Tester verdict: PASS_WITH_RISKS on repaired snapshot `d56a5dcdd63dbe2a1be14d42b3e16156250ab0394a849757c4269bac4ec11b26` -- Next action: Root performs the final integrity review, commits the closed batch, pushes the remediation branch, and opens a review PR without merging or releasing +- Remote evidence: commit `9a717d0` is pushed in PR #2; both push and pull-request CI runs passed Ubuntu core, Windows core, and browser smoke +- Next action: Human review and merge decision for PR #2; do not release, deploy, publish, or begin presentation work - Last updated: 2026-09-06 diff --git a/QA_REPORT.md b/QA_REPORT.md index dd43074..05cc1f2 100644 --- a/QA_REPORT.md +++ b/QA_REPORT.md @@ -36,6 +36,7 @@ Initialized: 2026-09-05 - Dependency audit: no known vulnerabilities after `pypdf==6.17.0` update. - Protected-input comparison: all 60 `v1_gold` records preserved content/order and changed only the four authorized provenance/adjudication fields; other protected paths have no diff. - CI/source checks: Windows/Ubuntu Python 3.12 matrix and immutable official Action SHAs verified; tracked-tree credential scan found no matches. +- Remote CI: GitHub Actions runs `34011634890` and `34011648717` passed Ubuntu core, Windows core, and browser smoke for commit `9a717d0` / PR #2. ## Bootstrap validation @@ -51,13 +52,12 @@ Initialized: 2026-09-05 | Expected-evidence citations | Focused wrong/matching/fallback/conflict tests | PASS | | Transcript isolation/privacy defaults | Focused capability, cross-session, admin, persistence, legacy-file, and audit-minimization tests | PASS | | Corpus/source provenance | Explicit-mode startup matrix, fixture provenance, and redirect-prevalidation tests | PASS_WITH_RISKS — live official-host crawl not run | -| Windows/Ubuntu portability | Fresh Windows 3.12 install/full suite; CI matrix and immutable refs inspected | PASS_WITH_RISKS — remote Ubuntu CI pending | +| Windows/Ubuntu portability | Fresh Windows 3.12 install/full suite; both remote CI matrices passed Ubuntu core, Windows core, and browser smoke | PASS | | Protected boundaries | Diff comparison, tracked-tree credential scan, and no live/generated/private artifact action | PASS_WITH_RISKS — history-aware scan and owner cleanup remain separate | | Full verification | Syntax 42 files, full suite 81/81, three evaluation validations, review status, guard, browser smoke, dependency audit | PASS_WITH_RISKS — one transient browser timeout and deprecation warning retained | ## Unavailable or deferred checks -- Remote Ubuntu CI: run and retain the GitHub Actions result after push. - Live exact-host crawl and redirect chain: perform only in a separately authorized, controlled live-ingestion batch. - Artifact-backed corpus, Qdrant, Ollama, target hardware, and deployment: reproduce with an implemented immutable run manifest before claims. - Human semantic review/institutional attestation: obtain privacy-safe durable reviewer evidence; automated checks cannot substitute. diff --git a/RISK_REGISTER.md b/RISK_REGISTER.md index 05f0feb..39919fe 100644 --- a/RISK_REGISTER.md +++ b/RISK_REGISTER.md @@ -15,21 +15,20 @@ Initialized: 2026-09-05 | EMU-R04 | High | Mitigated by default; owner-data action open | Persistence, audit logging, and raw-query logging are separate positive opt-ins and privacy tests pass. Existing ignored transcript/log files were intentionally neither read nor deleted. | Keep ignored files protected; owner decides cleanup/migration separately and validates opt-in retention in any deployment. | | EMU-R05 | High | Mitigated synthetically; live verification open | Exact-host HTTPS, redirect-hop/final validation, credential/port rejection, and non-laundered fixture provenance are covered by tests. No live official-host crawl was run. | Run a separately authorized controlled live crawl and retain redirect/provenance evidence before source claims. | | EMU-R06 | High | Mitigated in configuration; artifact gate open | Corpus mode is explicit, fixture is dev/test-only and visible, and invalid/unset/production-fixture configurations fail closed. Artifact-backed corpus startup was not exercised. | Validate a hashed nonempty artifact and target-service path before corpus-backed demonstration claims. | -| EMU-R07 | High | Mitigated locally; remote CI gate open | Platform markers, Windows dependencies, Windows/Ubuntu Python 3.12 CI, and immutable Action refs are present; a fresh Windows 3.12 install/full suite passed. Remote Ubuntu CI has not run on this branch. | Require green remote matrix checks after push before merge or release consideration. | +| EMU-R07 | High | Mitigated — EMU-B001 | Platform markers, Windows dependencies, immutable Action refs, and a fresh Windows 3.12 install/full suite passed. GitHub Actions runs `34011634890` and `34011648717` also passed Ubuntu core, Windows core, and browser smoke. | Preserve the platform matrix and require it to remain green for later changes. | | EMU-R08 | Medium | Partially mitigated — later | Audit tooling and official Actions are pinned, but lock generation inputs/hashes remain incompletely documented. | Record reproducible lock-generation inputs and integrity hashes in a later dependency-maintenance batch. | | EMU-R09 | High | Open — later | Published benchmark inputs/results lack immutable corpus/eval/commit/environment hashes and the public set has been iterated against implementation. | Add immutable run manifests and a held-out/adjudicated evaluation protocol before publishing benchmark claims. | | EMU-R10 | Medium | Open — later | MIT covers software, not necessarily redistribution of regulation-derived corpus/evaluation content. | Keep corpus out of Git, publish minimal/synthetic fixtures and source/hash manifests, and obtain rights confirmation before substantial redistribution. | | EMU-R11 | Medium | Open — later | Corpus/metric writers can overwrite canonical targets without a common staged atomic promotion policy. | Write unique run directories, validate, atomically promote pointers, and retain rollback metadata. | | EMU-R12 | High | Mitigated — EMU-B001 | Bootstrap audit passed; scope, protected inputs, baseline, one writer, rollback, independent TEST, and docs-QA evidence are recorded. | Preserve the workflow evidence and reacquire a scoped lock for any future batch. | | EMU-R13 | Medium | Open — later | Current-tree common-secret scan is clean, but a dedicated redacted history-aware scan is not recorded in the publication gate. | Run a history-aware secret/non-public-material review before presentation or release. | -| EMU-R14 | Medium | Open — maintenance | Browser smoke passed twice after one transient timeout; tests also emit a Starlette/httpx deprecation warning. Current evidence supports the batch but not long-term reliability. | Observe remote CI, investigate recurrence, and update the compatible dependency/test-client path before deprecation becomes failure. | +| EMU-R14 | Medium | Open — maintenance | Browser smoke passed twice locally after one transient timeout and passed in both remote CI runs; tests also emit a Starlette/httpx deprecation warning. Current evidence supports the batch but not long-term reliability. | Investigate any timeout recurrence and update the compatible dependency/test-client path before deprecation becomes failure. | ## Release rule No benchmark, presentation, publication, network-exposed demo, tag, release, deployment, or production-readiness claim may proceed while the residual gates in EMU-R01 through EMU-R07, -EMU-R09, EMU-R10, and EMU-R13 remain unresolved. A green remote Windows/Ubuntu CI result is also -required before merge consideration. Local implementation/testing may use synthetic fixtures without +EMU-R09, EMU-R10, and EMU-R13 remain unresolved. Local implementation/testing may use synthetic fixtures without reading or publishing ignored user artifacts. Full live crawling, Qdrant/Ollama validation, corpus redistribution, and deletion of existing local transcripts/logs require separate explicit scope or owner action. diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index 5bc9065..3d644c2 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -46,8 +46,11 @@ documentation-ordering `FAIL` was repaired and retested. Syntax, focused/full te validation, review status, publication guard, browser smoke, Windows clean installation, dependency audit, protected-input comparison, immutable CI refs, and tracked-tree credential scan passed. -Remote Ubuntu CI, live crawling, artifact-backed local services, immutable-run reproduction, -history-aware secret review, human adjudication, source-content rights, and Internet-grade identity +Commit `9a717d0` is available in review PR #2. GitHub Actions runs `34011634890` and `34011648717` +both passed Ubuntu core, Windows core, and browser smoke. + +Live crawling, artifact-backed local services, immutable-run reproduction, history-aware secret +review, human adjudication, source-content rights, and Internet-grade identity remain unverified or deferred. See `QA_REPORT.md` and `RISK_REGISTER.md`. ## Protected local data diff --git a/docs/PROJECT_STATUS_PROGRESS_PLAN.md b/docs/PROJECT_STATUS_PROGRESS_PLAN.md index 228edc3..b0294b9 100644 --- a/docs/PROJECT_STATUS_PROGRESS_PLAN.md +++ b/docs/PROJECT_STATUS_PROGRESS_PLAN.md @@ -14,8 +14,9 @@ EMUAdvisor has an active local software implementation and a substantial automat - Exact-host HTTPS redirect validation and non-laundered fixture provenance. - Platform-aware Python lock and Windows/Ubuntu Python 3.12 CI design. -Local verification passed, including a fresh Windows Python 3.12 environment. Remote Ubuntu CI and -the live/artifact/human/security gates below remain future actions. +Local verification passed, including a fresh Windows Python 3.12 environment. Both GitHub Actions +run sets passed Ubuntu core, Windows core, and browser smoke. The live/artifact/human/security gates +below remain future actions. ## Next milestones after EMU-B001 diff --git a/docs/VERSION_LOG.md b/docs/VERSION_LOG.md index c046a79..46a9677 100644 --- a/docs/VERSION_LOG.md +++ b/docs/VERSION_LOG.md @@ -124,4 +124,4 @@ Use this log for meaningful project milestones only. - Historical percentages in earlier entries are retained only as legacy automated proxy records; they are not semantic answer-quality or current presentation evidence. - Presentation, publication, release, deployment, and production-readiness remain blocked. - Initial independent TEST failed because three historical caveats followed the claims they qualified; a bounded documentation repair moved the caveats ahead of those claims. -- Fresh independent TEST passed the repaired snapshot with residual risks. EMU-B001 closed as `COMPLETE_WITH_RISKS`; remote CI, live/artifact-backed reproduction, human review, rights, history-aware secret review, and Internet-grade deployment security remain gated. +- Fresh independent TEST passed the repaired snapshot with residual risks. EMU-B001 closed as `COMPLETE_WITH_RISKS`; both remote CI matrices passed for commit `9a717d0`, while live/artifact-backed reproduction, human review, rights, history-aware secret review, and Internet-grade deployment security remain gated. diff --git a/shared/audit.md b/shared/audit.md index 9e8bebf..63c4d71 100644 --- a/shared/audit.md +++ b/shared/audit.md @@ -7,3 +7,4 @@ - 2026-09-06T04:23:00Z | docs-qa | reconcile and close EMU-B001 | workflow/state/QA/risk/current-status records | acceptance mapping and residual-risk review | COMPLETE_WITH_RISKS - 2026-09-06T04:24:00Z | repo-bootstrap audit | validate closure governance | DEV_STATE and BLUEPRINT | active-batch mismatch | FAIL then corrected - 2026-09-06T04:25:00Z | repo-bootstrap audit | revalidate closure governance | workflow pack | no errors or warnings | PASS +- 2026-09-06T04:32:22Z | root | push closed batch and verify remote CI | commit 9a717d0 / PR #2 | Actions runs 34011634890 and 34011648717 | PASS diff --git a/shared/history.md b/shared/history.md index da69c16..a9829ca 100644 --- a/shared/history.md +++ b/shared/history.md @@ -5,3 +5,4 @@ - 2026-09-06: Fresh independent TEST returned PASS_WITH_RISKS on the repaired snapshot. - 2026-09-06: Docs-QA reconciled acceptance evidence, preserved residual release gates, closed EMU-B001 as COMPLETE_WITH_RISKS, and released the cooperative lock. - 2026-09-06: Closure governance audit passed after one recorded active-batch field correction. +- 2026-09-06: Commit `9a717d0` was pushed in PR #2; both GitHub Actions run sets passed Ubuntu core, Windows core, and browser smoke. No merge or release followed. diff --git a/shared/status.md b/shared/status.md index a68cf95..da530d7 100644 --- a/shared/status.md +++ b/shared/status.md @@ -10,5 +10,6 @@ [2026-09-06T04:22:00Z] [tester] [TEST] [PASS_WITH_RISKS] [EMU-B001] [Fresh repaired snapshot passed; external, human, live-service, reproducibility, rights, identity, and maintenance risks remain.] [2026-09-06T04:23:00Z] [docs-qa] [CLOSE] [COMPLETE_WITH_RISKS] [EMU-B001] [Evidence reconciled, residual gates retained, and cooperative writer lock released.] [2026-09-06T04:25:00Z] [docs-qa] [CLOSE] [COMPLETE_WITH_RISKS] [EMU-B001] [Post-correction governance audit PASS with no errors or warnings.] +[2026-09-06T04:32:22Z] [root] [REVIEW] [CI_PASS] [EMU-B001] [Commit 9a717d0 pushed in PR #2; both push and PR runs passed Ubuntu core, Windows core, and browser smoke.] [2026-09-05T00:00:00Z] [repo-bootstrap] [BOOTSTRAP] [COMPLETE] [NONE] [Governance pack initialized; validation pending.]