diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1b0a00e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,47 @@ +version: 2 + +updates: + # Actions are the highest-value thing to keep current: a stale third-party + # action is a supply-chain risk in every workflow that uses it. + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + day: monday + commit-message: + prefix: "ci" + groups: + actions: + patterns: ["*"] + + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + day: monday + commit-message: + prefix: "deps" + # One PR for the dev toolchain, separate PRs for anything in the request + # path: a ruff bump needs a glance, a FastAPI or SQLAlchemy bump needs + # reading. + groups: + dev-tooling: + patterns: ["ruff", "mypy", "pytest*", "pre-commit", "build", "twine", "mkdocs*"] + dependency-type: development + ignore: + # Retrieval quality is pinned to specific model and runtime versions and + # is measured on the eval set, not on whatever pip resolves this week. + # These move when someone re-runs the golden set, not on a schedule. + - dependency-name: torch + - dependency-name: transformers + - dependency-name: sentence-transformers + - dependency-name: FlagEmbedding + open-pull-requests-limit: 5 + + - package-ecosystem: docker + directory: "/" + schedule: + interval: weekly + day: monday + commit-message: + prefix: "docker" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..81119ee --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,60 @@ +name: Publish documentation + +on: + push: + branches: [master] + paths: + - "docs/**" + - "mkdocs.yml" + - ".github/workflows/docs.yml" + pull_request: + paths: + - "docs/**" + - "mkdocs.yml" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + name: Build the site + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - run: pip install mkdocs-material mkdocs-static-i18n + + # --strict is the point of running this on pull requests too: it fails on + # a broken internal link or a page missing from the navigation, which is + # exactly the breakage nobody notices until a reader hits it. + - run: mkdocs build --strict + + - uses: actions/configure-pages@v5 + if: github.event_name != 'pull_request' + - uses: actions/upload-pages-artifact@v3 + if: github.event_name != 'pull_request' + with: + path: site + + deploy: + name: Deploy to Pages + needs: build + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/README.en.md b/README.en.md new file mode 100644 index 0000000..78192bf --- /dev/null +++ b/README.en.md @@ -0,0 +1,156 @@ +# Praxis + +[Русский](README.md) · **English** · [Documentation](https://drobyshevdev.github.io/praxis/) + +A legal assistant for Russian law. It answers a question and cites the specific +provisions the answer rests on — each citation checked separately by a model. When the +law does not support a claim, it says so. + +Praxis answering a question with verified citations + +## The problem + +A lawyer has three options today, and each is awkward in its own way. + +ChatGPT and other general-purpose models invent: they cite articles that do not exist, +confuse revisions, and are confidently wrong. That cannot be trusted in practice, where +the price of a mistake is a lost case or a financial loss. + +KonsultantPlyus and Garant give you a document search, not an answer. Matching the +provisions and drawing the conclusion is still the lawyer's job, and the subscription is +expensive. + +Search engines return forums, superseded revisions and SEO noise. + +Nothing sits between these options. No tool answers a question at the speed of a model +but with references you can open and check. That is the space Praxis occupies. + +## Who it is for + +Lawyers and in-house counsel at small and medium businesses without access to expensive +reference systems. Accountants and HR staff with questions about the Tax, Labour and +Administrative Offences Codes. Sole traders and founders. Individuals with everyday +disputes. + +The Russian market comes first because the data is available, the audience is legible and +the problems are concrete. The same architecture transfers to EU and US law, where open +data is even more plentiful (EUR-Lex, CourtListener, the Caselaw Access Project). + +## How it works + +The question goes through an agent that plans the search. Retrieval is hybrid over the +corpus of provisions (BM25 plus dense embeddings), a reranker selects the best, and +cross-referenced provisions are pulled in through an article-to-article graph. The +generator assembles an answer in which every claim is bound to a provision, and the +Citation Verifier checks every binding. + +What separates Praxis from a wrapper around chat-with-PDF: + +**Citation Verifier.** A separate NLI model checks each reference for entailment: does +the text of the provision support this specific claim? What is not confirmed is not +presented as fact. + +**Agentic self-RAG.** The agent breaks a complex question into sub-queries, searches again +and reformulates until it has enough grounding. The chain of reasoning is visible in the +answer. + +**GraphRAG.** Provisions reference each other ("in accordance with article 15"). That is a +ready-made graph: a question about damages surfaces article 393, and the graph pulls in +article 15 that it names. + +**Extractive by default.** Without an LLM key Praxis does not compose text — it quotes the +applicable provisions verbatim with references, and such an answer cannot hallucinate. +Synthesis through Claude is enabled by a key and passes the same per-sentence citation +check. + +**Measured quality.** recall@k, MRR and citation precision are computed on a golden set, +not judged by eye. + +## Data + +The statutory data is available. Codes and federal laws are published in machine-readable +form at pravo.gov.ru, and that is what the main path is built on: the repository contains +a parser for the official text (`statute_parser`) and a sample corpus of the Civil Code; +the full corpus is loaded the same way. + +The full text of the Civil Code is already extracted into the repository +(`corpus/gk-rf.json` — 1,712 articles, 4,717 provisions, all four parts, sourced from +Wikisource) and is loaded through `PRAXIS_CORPUS_DIR`. The revision in force should be +verified against pravo.gov.ru. + +Judicial practice is harder. There is no open structured corpus for Russia comparable to +the Caselaw Access Project, and kad.arbitr and the GAS "Pravosudie" system give up their +data reluctantly. That is the next stage, as a separate pipeline. + +## Quality + +A run of the eval harness over the golden set (12 questions, `praxis-eval`): + +| Metric | Real models (RTX 4060) | Offline fallback | +|---|---|---| +| recall@5 | 1.00 | 1.00 | +| MRR | 1.00 | 0.90 | +| hit-rate | 1.00 | 1.00 | +| mean confidence | 0.88 | 0.63 | +| citation precision | 0.29 | 0.40 | + +Real models: BGE-M3 for embeddings, bge-reranker-v2-m3 for reranking, rubert-NLI for +citation checking, all on GPU. The needed provision always reaches the top of the +results. Citation precision is understated because the golden set has one reference +article per question while the system also returns adjacent relevant provisions. That is +fixed by labelling several correct articles per question. + +On the full Civil Code corpus (4,717 provisions, an 18-question golden set) the real +models hold recall@5 0.92, MRR 0.94, hit-rate 1.0 and confidence 0.80. The offline +fallback drops to recall 0.64 at that size — on real data, real models are not optional. + +## Roadmap + +- v0. Skeleton, domain models, legal-aware chunking, BM25 baseline. Done. +- v1. Hybrid retrieval and reranking, Citation Verifier, self-RAG, eval, FastAPI and a web + UI. Done. Every ML component has a real implementation on GPU or through Claude, and a + deterministic offline fallback. +- v2. Real ingestion of the official text, GraphRAG over cross-references, a GPU run. Done. +- v3. The full corpus of codes and federal laws from pravo.gov.ru, judicial practice and a + provision-to-case graph, span-level citation highlighting in the UI, an expanded golden + set. +- v4. The full Civil Code extracted into the repository, a public API (`/v1`) with CORS and + a Python client, competitive analysis. Desktop and mobile on the same API are next. + +How to run it — [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md). + +## Stack and ecosystem + +Python 3.12, FastAPI, Postgres with pgvector, Docker. Retrieval: BM25 and BGE-M3 dense +embeddings, bge-reranker-v2-m3 as the reranker. Citation checking: NLI on GPU. The LLM is +plugged in through a provider: Claude for synthesis, Russian providers (GigaChat, +YandexGPT) for scenarios with data-residency requirements. + +The project uses two libraries from the same organisation: +[glia](https://github.com/DrobyshevDev/glia) for the agent loop in LLM mode (search is +exposed as a glia tool, and the trace comes from its trajectory), and +[mlango](https://github.com/DrobyshevDev/mlango) for a tracked golden eval through its +evals subsystem (`integrations/mlango_eval`, `manage.py evaluate`). + +Architecture — [ARCHITECTURE.md](ARCHITECTURE.md). + +## Openness and clients + +Everything is open (Apache-2.0) and runs locally without third-party tokens. The core +functionality — search, citation checking, answers — needs neither keys nor paid services: +local models download from HuggingFace for free, and the default answer is extractive, the +verbatim text of the provisions. Claude is an optional path to synthesis, not a condition +of the system working. Paid features and a subscription are for later. + +A single API core (`/v1`) serves every client: the web UI now, desktop and mobile as thin +clients on the same API later. For ML practitioners there is a public API with CORS, an +OpenAPI schema and a Python client — [docs/API.md](docs/API.md). + +Competitive analysis and where the project wins — +[docs/COMPETITIVE.md](docs/COMPETITIVE.md). + +--- + +**Not legal advice.** Praxis surfaces provisions and checks that a citation supports a +claim. It does not assess your situation, does not account for procedural context, and +does not replace a lawyer. diff --git a/README.md b/README.md index 514e3cb..26d0848 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Praxis +**Русский** · [English](README.en.md) · [Документация](https://drobyshevdev.github.io/praxis/) + Юридический ассистент по российскому праву. Отвечает на вопрос и приводит ссылки на конкретные нормы, каждую из которых отдельно проверяет модель. Если подтверждения в законе нет, то сообщает об этом. @@ -114,9 +116,11 @@ BGE-M3, реранкер bge-reranker-v2-m3. Проверка цитат: NLI н провайдера: Claude для синтеза, RU-провайдеры (GigaChat, YandexGPT) для сценариев с требованием резидентности. -Проект использует библиотеки автора (DenisDrobyshev): glia — агентный цикл в LLM-режиме -(поиск оформлен как glia-инструмент, трейс из trajectory); mlango — трекаемый golden-eval -через его подсистему evals (`integrations/mlango_eval`, `manage.py evaluate`). +Проект использует две библиотеки той же организации: +[glia](https://github.com/DrobyshevDev/glia) — агентный цикл в LLM-режиме (поиск оформлен +как glia-инструмент, трейс из trajectory); [mlango](https://github.com/DrobyshevDev/mlango) — +трекаемый golden-eval через его подсистему evals (`integrations/mlango_eval`, +`manage.py evaluate`). Архитектура — [ARCHITECTURE.md](ARCHITECTURE.md). diff --git a/docs/API.md b/docs/API.md index 9ab07f9..47efe15 100644 --- a/docs/API.md +++ b/docs/API.md @@ -37,7 +37,8 @@ curl -s http://localhost:8077/v1/search \ -d '{"query":"толкование договора","top_k":5}' ``` -Python-клиент (только стандартная библиотека, [clients/python/praxis_client.py](../clients/python/praxis_client.py)): +Python-клиент (только стандартная библиотека, +[clients/python/praxis_client.py](https://github.com/DrobyshevDev/praxis/blob/master/clients/python/praxis_client.py)): ```python from praxis_client import PraxisClient diff --git a/docs/index.en.md b/docs/index.en.md new file mode 100644 index 0000000..2d2afcf --- /dev/null +++ b/docs/index.en.md @@ -0,0 +1,90 @@ +# Praxis + +**A legal assistant for Russian law.** It answers a question and cites the specific +provisions the answer rests on — each citation checked separately by a model. When the +law does not support a claim, it says so instead of producing plausible text. + +```bash +docker compose up app # → http://localhost:8077, no API keys required +``` + +## Why + +A lawyer working on Russian law has three options today, each awkward in its own way. +General-purpose models invent: they cite articles that do not exist, confuse revisions, +and are confidently wrong. Commercial reference systems search documents rather than +answer, and are expensive. Search engines return forums, superseded revisions and SEO +noise. + +Nothing sits between them: no tool answers at the speed of a model with references you +can open and check. That is the gap Praxis fills. + +## How it works + +The question goes through an agent that plans the search. Retrieval is hybrid over the +corpus of provisions — BM25 plus dense embeddings — a reranker selects the best, and +provisions connected by cross-reference are pulled in through an article-to-article +graph. The generator assembles an answer in which every claim is bound to a provision, +and the Citation Verifier checks every binding. + +**Citation Verifier.** A separate NLI model checks each reference for entailment: does +the text of the provision actually support this specific claim? What is not confirmed is +not presented as fact. + +**Agentic self-RAG.** The agent decomposes a complex question into sub-queries, searches +again and reformulates until it has enough grounding. The chain of reasoning is visible +in the answer. + +**GraphRAG.** Provisions reference each other ("in accordance with article 15"). That is +a ready-made graph: a question about damages surfaces article 393, and the graph pulls in +article 15 that it names. + +**Extractive by default.** Without an LLM key Praxis does not compose text — it quotes +the applicable provisions verbatim with references, and an answer like that cannot +hallucinate. Synthesis through Claude is enabled by a key and passes the same +per-sentence citation check. + +## Quality + +The eval harness over the golden set (12 questions, `praxis-eval`): + +| Metric | Real models (RTX 4060) | Offline fallback | +|---|---|---| +| recall@5 | 1.00 | 1.00 | +| MRR | 1.00 | 0.90 | +| hit-rate | 1.00 | 1.00 | +| mean confidence | 0.88 | 0.63 | +| citation precision | 0.29 | 0.40 | + +Real models: BGE-M3 for embeddings, bge-reranker-v2-m3 for reranking, rubert-NLI for +citation checking, all on GPU. The needed provision always reaches the top of the +results. Citation precision is understated because the golden set has one reference +article per question while the system also returns adjacent relevant provisions. + +On the full Civil Code corpus (4,717 provisions, an 18-question golden set) the real +models hold recall@5 0.92, MRR 0.94, hit-rate 1.0 and confidence 0.80. The offline +fallback drops to recall 0.64 at that size — on real data the real models are not +optional. + +## Data + +Codes and federal laws are published in machine-readable form at pravo.gov.ru, and the +repository contains a parser for the official text (`statute_parser`). The full text of +the Civil Code is already extracted (`corpus/gk-rf.json` — 1,712 articles, 4,717 +provisions, all four parts) and is loaded through `PRAXIS_CORPUS_DIR`. The revision in +force should always be checked against the official source. + +There is no open structured corpus of Russian judicial practice comparable to the +Caselaw Access Project; that is the next pipeline rather than something already shipped. + +## Next + +- [Public API](API.md) — the `/v1` endpoints, the response format, the Python client. +- [Development](DEVELOPMENT.md) — running it locally and what CI checks. +- [Comparison](COMPETITIVE.md) — where Praxis wins and where it does not. +- [Architecture](https://github.com/DrobyshevDev/praxis/blob/master/ARCHITECTURE.md) — layers and data flow. + +!!! warning "Not legal advice" + Praxis surfaces provisions and checks that a citation supports a claim. It does not + assess your situation, does not account for procedural context, and does not replace + a lawyer. Always verify the revision in force against the official source. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..56c4b49 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,83 @@ +# Praxis + +**Юридический ассистент по российскому праву.** Отвечает на вопрос и приводит ссылки на +конкретные нормы, каждую из которых отдельно проверяет модель. Если подтверждения в законе +нет — сообщает об этом, а не выдаёт правдоподобный текст. + +```bash +docker compose up app # → http://localhost:8077, ключи не нужны +``` + +## Зачем + +У юриста сейчас три варианта, и каждый неудобен по-своему. Модели общего назначения +выдумывают: ссылаются на несуществующие статьи, путают редакции, ошибаются уверенно. +Справочные системы дают поиск по документам, а не ответ, и стоят дорого. Поисковики +отдают форумы, устаревшие редакции и SEO-мусор. + +Между этими вариантами пусто: нет инструмента, который отвечает со скоростью модели, но +со ссылками, которые можно открыть и проверить. Praxis занимает это место. + +## Как работает + +Вопрос идёт через агента, который планирует поиск. Дальше гибридный поиск по корпусу норм +(BM25 + плотные эмбеддинги), реранкер отбирает лучшее, связанные по ссылкам нормы +дотягиваются через граф «статья → статья». Генератор собирает ответ, где каждое +утверждение привязано к норме, и Citation Verifier проверяет каждую привязку. + +**Citation Verifier.** Каждую ссылку отдельная NLI-модель проверяет на entailment: +подтверждает ли текст нормы конкретный тезис. Неподтверждённое не подаётся как факт. + +**Agentic self-RAG.** Сложный вопрос агент разбивает на под-запросы, до-ищет и +переформулирует, пока не наберёт оснований. Ход рассуждения виден в ответе. + +**GraphRAG.** Нормы ссылаются друг на друга («в соответствии со статьёй 15»). Это готовый +граф: вопрос про убытки поднимает ст. 393, а граф дотягивает упомянутую в ней ст. 15. + +**Экстрактивный режим по умолчанию.** Без ключа к LLM Praxis не сочиняет текст, а +дословно приводит применимые нормы со ссылками — такой ответ не может галлюцинировать. +Синтез через Claude включается ключом и проходит ту же проверку цитат по каждому +предложению. + +## Качество + +Прогон eval-харнесса по golden set (12 вопросов, `praxis-eval`): + +| Метрика | Реальные модели (RTX 4060) | Офлайн-fallback | +|---|---|---| +| recall@5 | 1.00 | 1.00 | +| MRR | 1.00 | 0.90 | +| hit-rate | 1.00 | 1.00 | +| mean confidence | 0.88 | 0.63 | +| citation precision | 0.29 | 0.40 | + +Реальные модели: BGE-M3 (эмбеддинги), bge-reranker-v2-m3 (реранк), rubert-NLI (проверка +цитат), всё на GPU. Нужная норма всегда попадает в топ выдачи. Citation precision занижен +потому, что в golden по одной эталонной статье на вопрос, а система приводит и смежные +релевантные нормы. + +На полном корпусе ГК (4717 норм, golden из 18 вопросов) реальные модели держат recall@5 +0.92, MRR 0.94, hit-rate 1.0, уверенность 0.80. Офлайн-fallback на таком объёме падает до +recall 0.64 — на реальных данных без настоящих моделей не обойтись. + +## Данные + +Кодексы и федеральные законы доступны в машиночитаемом виде на pravo.gov.ru; в +репозитории лежит парсер официального текста (`statute_parser`). Полный текст ГК РФ уже +выгружен (`corpus/gk-rf.json` — 1712 статей, 4717 норм, все четыре части) и подключается +через `PRAXIS_CORPUS_DIR`. Актуальность редакции нужно сверять с pravo.gov.ru. + +Открытого структурированного корпуса судебной практики уровня Caselaw Access для России +нет — это следующий этап, отдельным пайплайном. + +## Дальше + +- [Открытый API](API.md) — эндпоинты `/v1`, формат ответа, Python-клиент. +- [Разработка](DEVELOPMENT.md) — как поднять локально и что гоняет CI. +- [Сравнение с альтернативами](COMPETITIVE.md) — где Praxis выигрывает и где нет. +- [Архитектура](https://github.com/DrobyshevDev/praxis/blob/master/ARCHITECTURE.md) — слои и поток данных. + +!!! warning "Не юридическая консультация" + Praxis показывает нормы и проверяет, что цитата подтверждает тезис. Он не оценивает + вашу ситуацию, не учитывает процессуальный контекст и не заменяет юриста. Редакцию + нормы всегда сверяйте с официальным источником. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..d5e5870 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,101 @@ +site_name: Praxis +site_description: >- + Юридический ассистент по российскому праву: ответ со ссылками на нормы, + каждая из которых проверена отдельно. +site_url: https://drobyshevdev.github.io/praxis/ +repo_url: https://github.com/DrobyshevDev/praxis +repo_name: DrobyshevDev/praxis +edit_uri: edit/master/docs/ +copyright: >- + Apache-2.0. Praxis не оказывает юридических услуг и не заменяет консультацию юриста. + +theme: + name: material + language: ru + features: + # navigation.instant is deliberately absent: it breaks the i18n language + # switcher, and the switcher matters more here than the transition. + - navigation.tracking + - navigation.sections + - navigation.top + - navigation.footer + - content.code.copy + - content.action.edit + - search.suggest + - search.highlight + - toc.follow + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: indigo + accent: indigo + toggle: + icon: material/weather-night + name: Тёмная тема + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: indigo + accent: indigo + toggle: + icon: material/weather-sunny + name: Светлая тема + icon: + repo: fontawesome/brands/github + +plugins: + - search: + lang: ru + # One file per translation: docs/index.md is Russian, docs/index.en.md is + # English. Russian is the default because the subject is Russian law and the + # reference pages quote it directly; a page with no translation falls back + # rather than disappearing from the navigation. + - i18n: + docs_structure: suffix + fallback_to_default: true + reconfigure_material: true + reconfigure_search: true + languages: + - locale: ru + name: Русский + default: true + build: true + - locale: en + name: English + build: true + site_name: Praxis + site_description: >- + A legal assistant for Russian law that answers with the provisions + it rests on, each citation verified separately. + nav_translations: + Обзор: Overview + Открытый API: Public API + Разработка: Development + Сравнение с альтернативами: Comparison + +markdown_extensions: + - admonition + - attr_list + - def_list + - footnotes + - md_in_html + - tables + - toc: + permalink: true + - pymdownx.details + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + +nav: + - Обзор: index.md + - Открытый API: API.md + - Разработка: DEVELOPMENT.md + - Сравнение с альтернативами: COMPETITIVE.md + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/DrobyshevDev/praxis