diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 01e3d79..0eceb8f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,10 +1,7 @@ * @shigabeev -/.github/ @shigabeev -/LICENSING.md @shigabeev -/SECURITY.md @shigabeev -/MANIFEST.in @shigabeev -/evaluation/ @shigabeev +/.github/workflows/ @shigabeev +/benchmarks/ @shigabeev /src/address_normalizer/data/ @shigabeev -/training/ @shigabeev +/tools/ @shigabeev /pyproject.toml @shigabeev diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 4d75763..90aadda 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,13 +1,2 @@ -# These are supported funding model platforms - github: shigabeev -patreon: # Replace with a single Patreon username -open_collective: # Replace with a single Open Collective username ko_fi: frappuccino_o -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -liberapay: # Replace with a single Liberapay username -issuehunt: # Replace with a single IssueHunt username -otechie: # Replace with a single Otechie username -lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry -custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 2c7a5f9..25d9c2e 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -1,69 +1,39 @@ name: Bug report -description: Report a reproducible API, CLI, installation, or packaging defect -title: "[Bug] " -labels: - - bug - - needs-triage +description: Report incorrect parsing, spans, detection, CLI, or packaging +title: "bug: " +labels: ["needs-triage"] body: - type: markdown attributes: - value: | - Thanks for a report maintainers can reproduce. For incorrect extracted - fields, use the Parsing failure form instead. Never include secrets, - private addresses, or unredacted production data. + value: Use a synthetic or redacted address. Never post private customer data. - type: input id: version attributes: - label: Package version or commit - description: Output of `python -c "import address_normalizer; print(address_normalizer.__version__)"` and, for a checkout, the commit SHA. - placeholder: 2.0.0a2 / abc1234 - validations: - required: true - - type: input - id: python - attributes: - label: Python and operating system - placeholder: Python 3.12.4 on Ubuntu 24.04 + label: Version + placeholder: 2.0.0a2 validations: required: true - type: textarea - id: install + id: input attributes: - label: Installation and exact command - description: Include the install source and the smallest command that triggers the defect. - render: shell + label: Minimal input + render: text validations: required: true - type: textarea id: actual attributes: - label: Actual behavior - description: Include the complete exception and traceback where applicable. + label: Actual result + render: json validations: required: true - type: textarea id: expected attributes: - label: Expected behavior - description: Explain the observable result you expected and why. + label: Expected result and why validations: required: true - type: textarea - id: reproduction - attributes: - label: Minimal reproduction - description: Prefer a self-contained snippet with synthetic data. - render: python - validations: - required: true - - type: checkboxes - id: checks + id: context attributes: - label: Submission checks - options: - - label: I reproduced this on the version or commit named above. - required: true - - label: I removed secrets and personal or production address data. - required: true - - label: This is not a security vulnerability requiring private disclosure. - required: true + label: Environment or additional context diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 2466385..54f2b1f 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,8 +1,5 @@ blank_issues_enabled: false contact_links: - name: Security vulnerability - url: https://github.com/shigabeev/address-normalizer/security/policy - about: Read the private-reporting policy; do not disclose vulnerabilities publicly. - - name: Support and usage questions - url: https://github.com/shigabeev/address-normalizer/blob/master/SUPPORT.md - about: Check support scope and troubleshooting before opening an issue. + url: https://github.com/shigabeev/address-normalizer/security/advisories/new + about: Report vulnerabilities privately. diff --git a/.github/ISSUE_TEMPLATE/data-provenance.yml b/.github/ISSUE_TEMPLATE/data-provenance.yml deleted file mode 100644 index 9aadb8d..0000000 --- a/.github/ISSUE_TEMPLATE/data-provenance.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Data provenance -description: Record source, permission, redistribution, or benchmark-integrity information -title: "[Provenance] " -labels: - - data - - provenance - - needs-triage -body: - - type: markdown - attributes: - value: | - Do not upload the dataset or sample rows unless redistribution is - clearly permitted. This form records evidence for maintainer review; it - does not itself authorize use. - - type: input - id: source - attributes: - label: Source and publisher - placeholder: Dataset title — publishing organization - validations: - required: true - - type: input - id: url - attributes: - label: Canonical source URL - placeholder: https://... - validations: - required: true - - type: textarea - id: version - attributes: - label: Version, revision, and retrieval date - description: Include immutable revision IDs and checksums where available. - validations: - required: true - - type: textarea - id: terms - attributes: - label: License or terms evidence - description: Link the exact terms and quote only the clause needed to explain use or redistribution. - validations: - required: true - - type: dropdown - id: intended_use - attributes: - label: Proposed role - options: - - Provenance correction only - - Independent sealed evaluation - - Train or validation data - - Redistributed derived artifact - - Runtime package data - - Other or undecided - validations: - required: true - - type: textarea - id: processing - attributes: - label: Filtering, transformations, and split policy - description: Include deduplication, grouping, leakage prevention, and personal-data handling. - validations: - required: true - - type: textarea - id: risks - attributes: - label: Open questions and risks - description: Note conflicting terms, stale snapshots, bias, private data, or attribution obligations. - validations: - required: true - - type: checkboxes - id: checks - attributes: - label: Submission checks - options: - - label: I have not attached unlicensed, private, or redistribution-restricted data. - required: true - - label: I am not treating this form as legal approval to use the source. - required: true diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml index 5e745cf..5b091be 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -1,51 +1,23 @@ name: Feature request -description: Propose a user problem that fits the small offline parser boundary -title: "[Feature] " -labels: - - enhancement - - needs-triage +description: Propose a focused addition to the parser +title: "feature: " +labels: ["needs-triage"] body: - - type: markdown - attributes: - value: | - Start with the user problem. The core package will remain offline, - dependency-free at runtime, and separate from FIAS/GAR resolution. - type: textarea id: problem attributes: - label: User problem - description: Who needs this, in what workflow, and what fails today? - validations: - required: true - - type: textarea - id: outcome - attributes: - label: Smallest useful outcome - description: Describe observable behavior without prescribing an implementation. + label: Problem + description: Describe the user need, not an implementation. validations: required: true - type: textarea - id: alternatives + id: example attributes: - label: Alternatives considered - description: Could application code, a resolver, or an integration example solve this outside the core package? + label: Minimal example and desired result validations: required: true - type: textarea - id: compatibility - attributes: - label: API, data, and maintenance implications - description: Note serialized-shape changes, runtime cost, dependencies, data provenance, and who will maintain it. - validations: - required: true - - type: checkboxes - id: boundary + id: tradeoffs attributes: - label: Product boundary - options: - - label: The request does not require bundled FIAS/GAR data or address verification in the parser. - required: true - - label: The request does not require a hidden download, mandatory service, or new scientific runtime. - required: true - - label: I searched existing issues and documentation for this need. - required: true + label: Scope and trade-offs + description: Explain why this belongs in the offline parser rather than a FIAS/GAR resolver. diff --git a/.github/ISSUE_TEMPLATE/parsing-failure.yml b/.github/ISSUE_TEMPLATE/parsing-failure.yml deleted file mode 100644 index 0f0b3ec..0000000 --- a/.github/ISSUE_TEMPLATE/parsing-failure.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Parsing failure -description: Share one redacted or synthetic address with expected fields and spans -title: "[Parse] " -labels: - - parsing - - needs-triage -body: - - type: markdown - attributes: - value: | - One minimal case is more useful than a private corpus dump. Replace real - personal data with a synthetic address that still reproduces the - behavior. Confidence is not a correctness probability. - - type: input - id: version - attributes: - label: Package version or commit - placeholder: 2.0.0a2 / abc1234 - validations: - required: true - - type: textarea - id: input - attributes: - label: Minimal address input - description: Use synthetic or safely redacted text; preserve punctuation and Unicode relevant to the failure. - render: text - validations: - required: true - - type: textarea - id: actual - attributes: - label: Current `parse(...).as_dict()` output - render: json - validations: - required: true - - type: textarea - id: expected - attributes: - label: Expected fields and offsets - description: List each expected value and `[start, end)` span. Explain ambiguous interpretations. - placeholder: | - street: value="...", span=[0, 10] - house_num: value="...", span=[12, 13] - validations: - required: true - - type: dropdown - id: domain - attributes: - label: Input domain - description: This helps compare evidence without mixing benchmark domains. - options: - - Synthetic minimal reproduction - - User-entered or noisy address - - Clean registry-style address - - Historical v1 compatibility case - - Other or unknown - validations: - required: true - - type: textarea - id: impact - attributes: - label: Application impact - description: Which downstream decision is wrong, and can resolver/review policy catch it? - validations: - required: true - - type: checkboxes - id: checks - attributes: - label: Submission checks - options: - - label: The input is synthetic or safely redacted and contains no private address data. - required: true - - label: I checked warnings, alternatives, and unparsed output rather than only normalized text. - required: true - - label: I understand that this report may become a public regression example. - required: true diff --git a/.github/ISSUE_TEMPLATE/security-contact.yml b/.github/ISSUE_TEMPLATE/security-contact.yml deleted file mode 100644 index 563cf53..0000000 --- a/.github/ISSUE_TEMPLATE/security-contact.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Private security contact request -description: Request a private channel only when GitHub private vulnerability reporting is unavailable -title: "[Security contact] Private channel requested" -labels: - - security - - needs-triage -body: - - type: markdown - attributes: - value: | - Do not describe the vulnerability here. This public issue only asks a - maintainer to arrange a private reporting channel. Use GitHub's private - vulnerability report instead whenever it is available. - - type: checkboxes - id: disclosure - attributes: - label: Public disclosure check - options: - - label: I have included no vulnerability details, exploit, secret, private address, or other sensitive information in this issue. - required: true diff --git a/.github/REPOSITORY_SETTINGS.md b/.github/REPOSITORY_SETTINGS.md deleted file mode 100644 index 4af7775..0000000 --- a/.github/REPOSITORY_SETTINGS.md +++ /dev/null @@ -1,66 +0,0 @@ -# Suggested repository settings - -These are maintainer-facing settings for the GPL-3.0-only v2 package. - -## About section - -**Description** - -> Small offline Python parser for unstructured Russian addresses, with typed -> fields, source offsets, warnings, and no bundled FIAS/GAR database. - -**Website** - -Leave empty until a maintained documentation or package page exists. - -**Topics** - -```text -address-parsing -python -russian -nlp -offline -fias -gar -data-quality -``` - -`fias` and `gar` describe the downstream integration boundary, not bundled -registry data or identifier lookup. Do not add `geocoding`, `address-validation`, -`production-ready`, or an accuracy claim. - -## Community settings - -- Enable Issues and the issue-form chooser. -- Enable private vulnerability reporting before announcing a release. -- Keep blank issues disabled while the structured forms are new. -- Use Discussions only if a maintainer is prepared to moderate and answer them. -- Do not enable automatic deletion of branches or merge methods without first - checking the release and backport process. - -## Branch and review settings - -For the release branch, require: - -- pull requests and at least one maintainer review; -- approval from code owners for package data, training, workflows, and release - metadata; -- passing required checks; -- resolution of review conversations; -- no force pushes or branch deletion. - -Keep pull-request workflows unprivileged. Do not use `pull_request_target` to -check out or execute contributor code. - -## Labels - -Create the labels in [`docs/triage.md`](../docs/triage.md) manually or with an -explicitly reviewed maintainer script. No GitHub API calls have been made for -these suggestions. - -## Release state - -The license scope and data/model provenance decisions are recorded in -`LICENSING.md`. Publishing still requires the tagged build, artifact, and -protected-environment checks in `docs/releasing.md`. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 125ed48..3ec61f7 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,62 +1,12 @@ -> Contributions are accepted under GPL-3.0-only. Confirm that every submitted -> code, data, model, and generated artifact may be contributed under that -> license. See `LICENSING.md`. +## What problem does this solve? -## Linked issue or reproduced bug + -Closes # +## What changed? - +## Verification -## Behavior and maintenance impact - - - -## Evidence - - - -- Test that fails before and passes after: -- Focused test result: -- Full `pytest` result: -- Other checks: - -## Benchmark and artifact impact - - - -- Before/after metrics by relevant domain and field: -- Known regressions: -- Model-size delta: -- Wheel-size delta: -- Data source, version, license/terms, and checksum: - -## Author verification - -- [ ] This change is linked to an issue or includes a complete reproduced bug. -- [ ] I added a test that fails before the change and passes afterward, or - explained why no test applies. -- [ ] I ran the focused tests and the complete local suite and reported exact - results above. -- [ ] Original source offsets, ambiguity, warnings, alternatives, and unparsed - evidence are preserved where relevant. -- [ ] I did not add a runtime dependency, network call, hidden download, - FIAS/GAR data, or service requirement. -- [ ] I checked relevant benchmark domains rather than optimizing only one - aggregate score. -- [ ] I understand every submitted change, including automated or AI-assisted - portions, and can explain and maintain it. -- [ ] I have the right to submit every code, data, model, and documentation - artifact in this pull request. -- [ ] No secrets, private addresses, production logs, caches, or large external - corpora are included. +- [ ] Added or updated a focused test +- [ ] `pytest` passes +- [ ] Relevant benchmark still passes +- [ ] No private address data, runtime dependency, or hidden network behavior diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bce3176..7543cae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,17 +2,12 @@ name: CI on: push: - branches: [master, main] + branches: [master] pull_request: - workflow_dispatch: permissions: contents: read -concurrency: - group: ci-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - jobs: test: name: Python ${{ matrix.python-version }} @@ -22,106 +17,47 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - name: Check out source - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - cache: pip - cache-dependency-path: pyproject.toml - - name: Install test tools and package - run: | - python -m pip install "pytest==8.4.2" - python -m pip install --no-deps . - - name: Run tests - run: python -m pytest + - run: python -m pip install pytest==8.4.2 + - run: python -m pip install --no-deps . + - run: python -m pytest static-analysis: runs-on: ubuntu-latest steps: - - name: Check out source - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - cache: pip - cache-dependency-path: pyproject.toml - - name: Install mypy - run: python -m pip install "mypy==1.17.1" - - name: Check package typing - run: python -m mypy + - run: python -m pip install mypy==1.17.1 + - run: python -m mypy model-and-regression: runs-on: ubuntu-latest steps: - - name: Check out source - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - - name: Install package - run: python -m pip install --no-deps . - - name: Verify compact model - run: | - python training/evaluate_compact_tagger.py - python training/train_compact_tagger.py \ - --output "${RUNNER_TEMP}/regenerated-model.json" \ - --report "${RUNNER_TEMP}/regenerated-model-report.json" - cmp src/address_normalizer/data/model.json "${RUNNER_TEMP}/regenerated-model.json" - - name: Enforce legacy-reference release gates - run: | - python evaluation/evaluate.py \ - --data evaluation/legacy_reference_500.jsonl \ - --gates evaluation/release_gates.json + - run: python -m pip install --no-deps . + - run: python tools/benchmark.py --check package: runs-on: ubuntu-latest - env: - SOURCE_DATE_EPOCH: "1704067200" steps: - - name: Check out source - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - cache: pip - cache-dependency-path: pyproject.toml - - name: Install build and metadata tools - run: python -m pip install "build==1.3.0" "twine==6.2.0" - - name: Verify publication license and provenance - run: python scripts/check_license.py --require-publishable - - name: Build byte-identical wheel and source distribution twice - run: python scripts/build_reproducibly.py --output dist - - name: Inspect contents, dependencies, checksums, and size - run: | - python scripts/check_artifacts.py dist \ - --write-manifest artifact-manifest.json - python scripts/test_artifact_policy.py dist - python -m twine check dist/*.whl dist/*.tar.gz - - name: Install wheel without an index - run: | - python -m venv "${RUNNER_TEMP}/wheel-venv" - "${RUNNER_TEMP}/wheel-venv/bin/python" -m pip install \ - --no-index --no-deps dist/*.whl - cp scripts/smoke_installed.py "${RUNNER_TEMP}/smoke_installed.py" - cd "${RUNNER_TEMP}" - "${RUNNER_TEMP}/wheel-venv/bin/python" -I smoke_installed.py - "${RUNNER_TEMP}/wheel-venv/bin/address-normalizer" \ - "Москва, Тверская 1" > single.json - echo "Ополченская 5-30" | - "${RUNNER_TEMP}/wheel-venv/bin/address-normalizer" --jsonl > batch.jsonl - "${RUNNER_TEMP}/wheel-venv/bin/python" -I -c \ - "import json; json.load(open('single.json')); [json.loads(line) for line in open('batch.jsonl')]" - - name: Upload inspected distributions - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: distributions - path: | - dist/ - artifact-manifest.json - if-no-files-found: error - retention-days: 14 + - run: python -m pip install build==1.3.0 twine==6.2.0 + - run: python -m build + - run: python -m twine check dist/* + - run: | + python -m venv /tmp/address-normalizer-smoke + /tmp/address-normalizer-smoke/bin/pip install --no-index --no-deps dist/*.whl + cd /tmp + /tmp/address-normalizer-smoke/bin/python -c \ + "from address_normalizer import parse; assert parse('Тверская 1').house_num.value == '1'" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 390488a..bd0c83f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,10 +1,10 @@ -name: Build and publish release +name: Release on: workflow_dispatch: inputs: version: - description: "Expected PEP 440 package version (for example, 2.0.0a2)" + description: "Package version, for example 2.0.0a2" required: true type: string repository: @@ -12,73 +12,40 @@ on: required: true default: testpypi type: choice - options: - - testpypi - - pypi + options: [testpypi, pypi] run-name: Release ${{ inputs.version }} to ${{ inputs.repository }} permissions: contents: read -concurrency: - group: release-${{ github.ref }} - cancel-in-progress: false - jobs: build: runs-on: ubuntu-latest - env: - SOURCE_DATE_EPOCH: "1704067200" steps: - - name: Check out selected release commit - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - with: - fetch-depth: 0 - - name: Require the immutable matching release tag - env: - EXPECTED_VERSION: ${{ inputs.version }} - run: python scripts/check_release_ref.py --version "${EXPECTED_VERSION}" - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - cache: pip - cache-dependency-path: pyproject.toml - - name: Install release tools - run: | - python -m pip install \ - "build==1.3.0" \ - "mypy==1.17.1" \ - "pytest==8.4.2" \ - "twine==6.2.0" - - name: Verify tests and typing - run: | - python -m pip install --no-deps . - python -m pytest - python -m mypy - - name: Verify license and model provenance - run: python scripts/check_license.py --require-publishable - - name: Build and inspect candidate + - run: python -m pip install build==1.3.0 pytest==8.4.2 twine==6.2.0 + - run: python -m pip install --no-deps . + - name: Verify tag and version env: EXPECTED_VERSION: ${{ inputs.version }} run: | - python scripts/build_reproducibly.py \ - --output release-artifact/packages - python scripts/check_artifacts.py release-artifact/packages \ - --expected-version "${EXPECTED_VERSION}" \ - --write-manifest release-artifact/artifact-manifest.json - python scripts/test_artifact_policy.py release-artifact/packages - python -m twine check \ - release-artifact/packages/*.whl \ - release-artifact/packages/*.tar.gz - - name: Upload candidate for maintainer review - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + PACKAGE_VERSION="$(python -c 'import address_normalizer; print(address_normalizer.__version__)')" + test "${GITHUB_REF_TYPE}" = "tag" + test "${GITHUB_REF_NAME}" = "v${EXPECTED_VERSION}" + test "${PACKAGE_VERSION}" = "${EXPECTED_VERSION}" + - run: python -m pytest + - run: python tools/benchmark.py --check + - run: python -m build + - run: python -m twine check dist/* + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: release-candidate-${{ inputs.version }} - path: release-artifact/ + name: distributions-${{ inputs.version }} + path: dist/ if-no-files-found: error - retention-days: 30 publish: needs: build @@ -90,40 +57,20 @@ jobs: contents: write id-token: write steps: - - name: Check out the exact candidate source - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - python-version: "3.12" - - name: Download the inspected candidate - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - name: release-candidate-${{ inputs.version }} - path: release-artifact/ - - name: Recheck license, version, contents, and checksums - env: - EXPECTED_VERSION: ${{ inputs.version }} - run: | - python scripts/check_license.py --require-publishable - python scripts/check_artifacts.py release-artifact/packages \ - --expected-version "${EXPECTED_VERSION}" \ - --verify-manifest release-artifact/artifact-manifest.json - - name: Publish with PyPI Trusted Publishing - uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # release/v1 + name: distributions-${{ inputs.version }} + path: dist/ + - uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # release/v1 with: - packages-dir: release-artifact/packages/ + packages-dir: dist/ repository-url: ${{ inputs.repository == 'testpypi' && 'https://test.pypi.org/legacy/' || 'https://upload.pypi.org/legacy/' }} - skip-existing: false - verbose: true - name: Create GitHub release if: ${{ inputs.repository == 'pypi' }} env: GH_TOKEN: ${{ github.token }} run: | - gh release create "${GITHUB_REF_NAME}" \ - release-artifact/packages/* \ - release-artifact/artifact-manifest.json \ + gh release create "${GITHUB_REF_NAME}" dist/* \ --verify-tag \ --title "address-normalizer ${GITHUB_REF_NAME}" \ --generate-notes diff --git a/.gitignore b/.gitignore index 9ca1970..cb43515 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,8 @@ __pycache__/ *.py[cod] *.egg-info/ .pytest_cache/ +.mypy_cache/ .cache/ .venv/ build/ dist/ -/artifact-manifest.json -/release-artifact/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f34a25f..20a7cfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,40 +1,13 @@ # Changelog -All notable user-visible changes will be recorded here. This project follows -[Semantic Versioning](https://semver.org/) and uses -[PEP 440](https://peps.python.org/pep-0440/) version syntax. +## 2.0.0a2 — 2026-07-29 -## Unreleased +- Added a dependency-free typed parser for Russian address strings. +- Added conservative address-span detection for free-form messages. +- Preserved source offsets, unparsed content, warnings, and alternatives. +- Added JSON/JSONL CLI support and a compact bundled sequence model. +- Published separate benchmark domains and a detailed 500-row failure table. +- Added Russian and English documentation. +- Licensed the project under GPL-3.0-only. -No user-visible changes yet. - -## 2.0.0a2 - 2026-07-29 - -### Added - -- Typed, dependency-free v2 parsing API and JSON/JSONL command-line interface. -- Lazy `parse_iter()` batches, predictable batch input errors, and public - typed-dictionary serialization schemas. -- Conservative `detect_addresses()` message-span detection with typed, - offset-preserving results and a positive/negative behavior fixture. -- Compact bundled sequence model for residual unmarked text. -- Independent evaluation reports with explicit metric-family names and release - regression gates; legacy report keys remain compatibility aliases. -- A committed results index, full historical benchmark report, and explicit - documentation of the hybrid rules/structured-perceptron runtime stack. -- A complete 500-row diagnostic CSV with per-field outcomes, scenario columns, - triage hypotheses, and a representative failure summary. -- Distribution inspection, isolated-wheel smoke tests, size limits, and - artifact checksum manifests. -- GNU GPL v3 licensing, recorded model provenance, a Russian README, and - protected TestPyPI/PyPI Trusted Publishing. - -### Changed - -- Source distributions now exclude evaluation corpora, training inputs, tests, - notebooks, caches, and historical v1 assets. -- CI covers every declared Python minor version from 3.10 through 3.14. -- Message-span detection is explicitly presented as a conservative alpha - feature rather than a complete arbitrary-prose recognizer. - -`2.0.0a1` was an internal development version and was not published. +`2.0.0a1` was an unpublished development version. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7da58d..04338e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,144 +1,27 @@ # Contributing -Thank you for helping make Russian address parsing easier to inspect and trust. -Useful contributions here are usually small: one clearly reproduced behavior, -one bounded change, and evidence a maintainer can rerun. +Keep changes small and explain the user-visible problem they solve. -## License - -The project is licensed under GNU GPL v3 (`GPL-3.0-only`). By submitting a -contribution, you confirm that you have the right to provide it under that -license. Code, examples, generated models, and data-derived artifacts need -clear provenance; see [`LICENSING.md`](LICENSING.md). - -## Choose the right report - -- [Bug report](https://github.com/shigabeev/address-normalizer/issues/new?template=bug-report.yml): - installation, API, CLI, packaging, or deterministic runtime failures. -- [Parsing failure](https://github.com/shigabeev/address-normalizer/issues/new?template=parsing-failure.yml): - one address whose extracted fields, spans, warning, or alternative are - unexpected. -- [Feature request](https://github.com/shigabeev/address-normalizer/issues/new?template=feature-request.yml): - a user problem, not a preselected implementation. -- [Data provenance](https://github.com/shigabeev/address-normalizer/issues/new?template=data-provenance.yml): - source, permission, redistribution, or benchmark-integrity information. -- Security-sensitive reports follow [`SECURITY.md`](SECURITY.md), never a public - issue with exploit or private-address details. - -Remove or replace personal data before posting. A synthetic address that -reproduces the behavior is preferable. - -## Before a pull request - -1. link an existing issue or provide a complete, locally reproducible bug; -2. agree on scope before public API, model, data, dependency, or workflow work; -3. keep the change narrowly focused and preserve unrelated behavior; -4. add a test that fails before the fix and passes afterward; -5. explain the behavior in your own words, including maintenance implications; -6. report before/after output and relevant benchmark domains; -7. run the checks below and include exact results in the pull request. - -A benchmark delta alone is not a product improvement. Parser or model changes -must not trade away another domain, field, original offsets, ambiguity, or -unparsed evidence to improve an aggregate score. - -## Local setup and checks - -Runtime development needs no third-party package dependency: +## Setup ```bash python -m pip install -e . +python -m pip install pytest mypy pytest +python -m mypy ``` -For a parsing change, show the focused failing test first, then run the complete -suite. For example: - -```bash -pytest tests_v2/test_api.py -q -pytest -``` - -For documentation examples: - -```bash -python examples/basic.py -printf '%s\n' 'Ополченская 5-30' | python examples/jsonl_etl.py -python -m compileall -q examples -``` - -Model, evaluation, build, and release work has additional checks and provenance -requirements. Read -[`training/README.md`](https://github.com/shigabeev/address-normalizer/blob/master/training/README.md) -and -[`evaluation/README.md`](https://github.com/shigabeev/address-normalizer/blob/master/evaluation/README.md) -before starting it. Large -external data and its preparation dependencies must stay outside the runtime -package. - -## Parsing-change evidence - -Include: - -- the smallest synthetic or redacted input that reproduces the problem; -- expected fields and `[start, end)` offsets; -- output before and after the change; -- a regression test; -- an explanation of warnings, alternatives, and unparsed content affected; -- results for every relevant committed regression gate. - -Do not copy examples from a sealed test set into tests or tune against that set -while continuing to call it untouched. Never commit private addresses or -unreviewed production logs. - -## Model or benchmark changes - -Discuss these in an issue before implementation. A proposal must identify: - -- source, publisher, version/revision, URL, and retrieval date; -- license or terms and whether redistribution is allowed; -- transformations and filters; -- split and deduplication policy, including leakage prevention; -- exact reproduction command and checksums; -- before/after per-domain and per-field metrics; -- model and wheel size deltas; -- known regressions and rejected alternatives. - -Keep historical, noisy-window, nationwide clean-address, and official-registry -scores separate. Do not select only the friendliest metric or average -incompatible domains. - -## Changes requiring maintainer agreement - -Ask before changing: - -- public API or serialized result shape; -- supported Python versions; -- confidence semantics or review policy; -- runtime dependencies; -- model, training data, or data preparation; -- FIAS/GAR integration inside the core package; -- CI, permissions, release, or publishing workflows; -- large generated or binary artifacts. - -The core package must remain small, dependency-free at runtime, offline, and -independent of any bundled FIAS/GAR database or service. - -## Human accountability and automated assistance - -AI-assisted contributions can be reviewed. The human author must be able to: - -- explain every behavior change and why the approach is maintainable; -- identify the test that proves the bug and fix; -- reproduce claimed benchmark results; -- answer review questions and support follow-up repairs; -- confirm that submitted code and data may be contributed. +Parser changes should include: -Unexplained generated changes, benchmark-only optimizations, bulk formatting, -and changes whose author cannot maintain them will be closed. +1. a minimal synthetic or redistributable failing input; +2. expected component values and source spans; +3. a regression test that fails before the fix; +4. the full test result and, when relevant, `python tools/benchmark.py --check`. -## Review and triage +Do not hide ambiguity by discarding warnings, alternatives, or unparsed text. +Do not add runtime dependencies, network calls, or a registry database without +first discussing the product boundary in an issue. -Maintainers use [`docs/triage.md`](docs/triage.md) for labels, duplicate handling, -security routing, benchmark evidence, and review boundaries. Bounded starter -proposals are in [`docs/good-first-issues.md`](docs/good-first-issues.md). +Never publish private customer addresses in an issue or test. Contributors must +have the right to submit all code, data, and generated artifacts. Contributions +are licensed under GPL-3.0-only. diff --git a/LICENSING.md b/LICENSING.md deleted file mode 100644 index 85e1fc8..0000000 --- a/LICENSING.md +++ /dev/null @@ -1,44 +0,0 @@ -# Licensing and model provenance - -## Maintainer decision - -Effective 2026-07-29, the repository maintainer selected the GNU General -Public License version 3 for this project. The repository is distributed under -the `GPL-3.0-only` SPDX expression. [`LICENSE`](LICENSE) is the unmodified -`gpl-3.0` template returned by GitHub's license API so GitHub and package tools -can identify it consistently. - -The maintainer also authorizes redistribution of the historical -`ref/references.xlsx` workbook, the deterministic -`evaluation/legacy_reference_500.jsonl` derivative, and the compact model -derived from those rows as repository and package artifacts under -`GPL-3.0-only`. - -## Provenance record - -- The workbook first appears in repository commit - `4a72605b0204e2ba4c21f09d74c249b066c41021`, authored by Ilya Shigabeev - (`beat@live.ru`). -- Repository history for the workbook and v2 model contains only the - maintainer identities `Ilya Shigabeev` and `frappuccino`, using the same - `beat@live.ru` email address. -- The source-verifiable 500-row derivative is committed as - `evaluation/legacy_reference_500.jsonl`. -- `training/train_compact_tagger.py` deterministically regenerates the bundled - `src/address_normalizer/data/model.json`. -- `training/model_evaluation.json` records the source dataset digest, split, - training algorithm, tuning, test results, and artifact size. -- CI regenerates the model and requires a byte-for-byte match with the bundled - artifact. - -This record covers artifacts distributed by this repository and package. Large -external Deepparse, RedMadRobot, and Moscow source corpora are not included in -the wheel or source distribution; their separate source and license records are -documented in `evaluation/DATA_SOURCES.md`. - -## Contributions - -Contributions are accepted under the repository's `GPL-3.0-only` license. -Contributors must have the right to submit their code, data, and generated -artifacts and must record data/model provenance as described in -[`CONTRIBUTING.md`](CONTRIBUTING.md). diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index abcee35..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,26 +0,0 @@ -include LICENSE -include README.md -include README.ru.md -include CHANGELOG.md -include CONTRIBUTING.md -include LICENSING.md -include SECURITY.md -include SUPPORT.md -include pyproject.toml -recursive-include src/address_normalizer *.py -include src/address_normalizer/py.typed -include src/address_normalizer/data/model.json -recursive-include docs *.md -recursive-include examples *.md *.py - -exclude requirements-legacy.txt -prune .cache -prune evaluation -prune ref -prune tests_v2 -prune training -global-exclude *.py[cod] -global-exclude __pycache__ -global-exclude .DS_Store -global-exclude *.ipynb -global-exclude *.xlsx diff --git a/PRODUCTION_PLAN.md b/PRODUCTION_PLAN.md deleted file mode 100644 index 5cb4716..0000000 --- a/PRODUCTION_PLAN.md +++ /dev/null @@ -1,348 +0,0 @@ -# Production plan for address-normalizer v2 - -Status: P0/P1 hardening implemented for `2.0.0a2` on -`codex/v2-parser-sprint`; GPL-3.0-only licensing and model provenance are -recorded, and the alpha release is in progress. - -This document records the current evidence and the gates for a releasable v2. -It does not declare the package production-ready. Publication still requires a -tagged, reproducible build and protected TestPyPI/PyPI release checks. - -## Current-state audit - -### Product boundary - -Version 2 is a small, deterministic, offline Russian address **parser**. It -extracts an unverified interpretation with original character offsets. It is -not a FIAS/GAR resolver, geocoder, spelling authority, address validator, -service, or database. - -- Version: `2.0.0a2`. -- Supported Python declared in package metadata: 3.10 through 3.14. -- Runtime dependencies: none. -- Public entry points: `parse()`, `parse_many()`, lazy `parse_iter()`, - conservative `detect_addresses()`, `ParsedAddress`, `DetectedAddress`, - `AddressPart`, `Alternative`, and the matching serialized `TypedDict` - schemas. -- CLI: one-address JSON and stdin JSONL modes. -- Runtime model: 37,130-byte JSON linear-chain tagger. -- `2.0.0a2` candidate wheel: 45,843 bytes. -- Large corpora are ignored under `.cache/external/`; none is package data. -- Historical v1 root files are retained but are outside the `src/` package. - -### API and behavior - -The parser combines offset-preserving tokenization, explicit marker and numeric -rules, a compact sequence tagger for residual text, and deterministic -post-processing. Results retain raw substrings, `[start, end)` offsets, -unparsed spans, warnings, alternatives, and bounded confidence values. -Confidence is decision strength, not a calibrated probability. - -Closed API audit items: - -- `parse()`, `parse_many()`, and `parse_iter()` now have a consistent, - explicitly tested input-error contract. -- Empty input, Unicode whitespace, `ё/е`, long and malformed input, one-shot - iterables, and concurrent calls have explicit tests and documentation. -- `as_dict()` has a documented JSON-compatible schema with public `TypedDict` - types. -- Batch parsing remains eagerly list-based. The additive `parse_iter()` helper - provides one-pass lazy iteration and documents consumption and error timing. -- Supported source values, warning codes, alternative reasons, and the review - policy are part of the public alpha contract. - -### Reliability evidence - -The four evidence domains remain separate because their sources, schemas, and -metrics answer different questions: - -| Domain | Size | Primary metric | `2.0.0a2` baseline | -| --- | ---: | --- | ---: | -| Historical bank-shaped reference | 500 rows | exact component micro F1 | 95.9% | -| RedMadRobot noisy address windows | 578 windows | same-label span-overlap F1 | 58.7% | -| Deepparse nationwide clean strings | 100,000 rows | character-overlap F1 | 66.2% | -| Moscow official clean buildings | 15,196 rows | exact component-value F1 | 85.4% | - -Additional named baselines: - -- Historical exact-address match: 80.4%; no residual word/number tokens: 76.8%. -- RedMadRobot street-and-house slice: 78.8% span-overlap F1. -- Deepparse binary span-overlap F1: 84.4%; exact token-boundary sequence: - 8.0%. -- Moscow exact full-address match: 64.9%; street F1: 66.2%; house F1: - 98.2%; корпус F1: 99.6%; строение F1: 97.0%. -- Compact tagger grouped test: 96.5% token accuracy, 94.3% complete-sequence - accuracy, and 96.9% micro entity F1. The split has no meaningful district or - settlement coverage. - -The historical 500-row gate is small enough for normal CI. RedMadRobot and the -large corpora stay opt-in and must not be downloaded by routine test or build -jobs. No implementation work in this sprint may be tuned against those sealed -external test results. - -### Packaging, release, and repository - -- `setuptools` builds from `src/`; model JSON and `py.typed` are declared as - package data. -- Project metadata names the repository, Python range, console script, and has - no runtime dependencies. -- CI now tests every declared Python minor from 3.10 through 3.14, checks - deterministic model regeneration, runs the legacy gate, and builds - distributions. -- Release gates cover strict typing, metadata validation, exact wheel - allowlisting/forbidden-content inspection, clean wheel installation, CLI - smoke tests, explicit wheel/model budgets, sdist review, and a Trusted - Publishing workflow with named GitHub environments. -- Contributor, security, CODEOWNERS, PR, issue, support, triage, starter-work, - release, examples, and launch-draft assets are present. -- The worktree initially contained unrelated untracked notebook artifacts: - `Untitled.ipynb` and `.ipynb_checkpoints/`. They are not part of this sprint - and must remain untouched and uncommitted. - -## Users and primary use cases - -1. **Application developer** — install a tiny wheel, parse one user-supplied - address, inspect ambiguity, then query the application's own FIAS/GAR - resolver. -2. **Data/ETL engineer** — stream JSONL or an iterable of strings through an - offline, dependency-free parser while retaining raw values and offsets for - audit and correction. -3. **API developer** — expose the typed result from FastAPI or another service - without hidden network, filesystem, or process requirements. -4. **Evaluator/data steward** — reproduce named gates, understand the exact - scoring boundary and provenance, and keep independent domains separate. -5. **Contributor/maintainer** — reproduce a bug, add a fail-before/pass-after - test, measure cross-domain and artifact impact, and release only through a - reviewable workflow. - -Non-users include anyone needing existence verification, FIAS/GAR identifiers, -geocoding, authoritative correction, fuzzy registry search, or an embedded -current registry. Those needs require a downstream resolver. - -## Public API contract - -For the v2 pre-release series: - -- `parse(text: str) -> ParsedAddress` parses exactly one string and raises - `TypeError` for non-strings. -- `parse_many(addresses: Iterable[str]) -> list[ParsedAddress]` consumes the - iterable once, preserves order, returns an eager list, and uses the same - per-item validation as `parse()`. -- `parse_iter(addresses: Iterable[str]) -> Iterator[ParsedAddress]` preserves - order, consumes once, avoids preloading, and raises element errors when - iteration reaches them. -- `detect_addresses(text: str) -> tuple[DetectedAddress, ...]` returns ordered, - non-overlapping half-open spans in a free-form message. Detection is - conservative and requires a street marker plus a building, or an explicit - address cue plus a parseable street and building. -- Empty and whitespace-only strings return a valid empty `ParsedAddress`; they - do not invent components. -- `AddressPart.raw == ParsedAddress.raw[start:end]`; offsets are half-open - indices into the original Python string. -- `as_dict()` returns a JSON-compatible, stable v2 schema. Component keys remain - present with `null` when absent; tuple fields serialize as arrays. -- `normalized` is a convenient rendering of the parser's unverified - interpretation, not a canonical registry address. -- Confidence is a bounded ranking/review signal. No threshold may be described - as a probability or guarantee. -- Warnings, alternatives, and unparsed content are public information, not - debug output. Additive warning codes or alternative reasons may appear in - pre-releases; removing result fields or changing their meaning requires - migration notes. -- Parsing performs no network calls, downloads, service startup, or filesystem - writes. A process-local immutable model may be cached and shared across - threads. - -## Release decisions - -The maintainer selected GPL-3.0-only and authorized redistribution of the -historical workbook, its 500-row derivative, and the compact model. The complete -decision and reproducible provenance chain are recorded in `LICENSING.md`. -External Moscow, Deepparse, and RedMadRobot source corpora remain outside the -distribution under their separately recorded terms. - -### Engineering blockers for a stable `2.0.0` - -- Close all P0 gates below on every supported Python version. -- Independently review a documented sample of the legacy evaluation rows or - create a replacement gold benchmark with a sealed final split. -- Improve or explicitly accept the weak administrative and exact-street - behavior; do not hide it behind aggregate metrics. -- Freeze and document the v2 serialization, warning, and compatibility policy. -- Perform a TestPyPI rehearsal through the protected release environment. - -## Prioritized work - -### P0 — required before the next published pre-release - -- Make API input/serialization/empty-input behavior typed, tested, and - documented. -- Add adversarial regression tests for punctuation, casing, `ё/е`, Unicode - whitespace, compound houses, корпус/строение, apartments, missing markers, - reordered components, ambiguous numeric tails, one-shot iterables, and - concurrency. -- Add a strict CI packaging job that builds sdist/wheel, validates metadata and - contents, enforces wheel/model size budgets, installs the wheel into a clean - environment, and smoke-tests import plus both CLI modes without network. -- Add an explicit type-checker configuration and gate the public package. -- Add a GitHub Actions release workflow using OIDC Trusted Publishing, - immutable artifacts, protected environments, and an explicit version-tag - check. -- Add a changelog and release checklist that put licensing/provenance before - publication. -- Rewrite the README around the two-minute path, honest boundaries, named - benchmarks, result review, FIAS/GAR handoff, API/CLI reference, migration, - troubleshooting, and “Should I use this?” guidance. -- Add executable examples and issue/PR/security/support/triage assets. - -Acceptance criteria: - -- Unit tests pass on locally available interpreters and CI covers every - declared Python minor. -- Static type checking passes with the selected, committed configuration. -- The historical 500-row gate meets every value in - `evaluation/release_gates.json`; no named external baseline regresses as a - side effect of code changes. -- Two consecutive deterministic model builds are byte-identical. -- Built metadata validates; a clean environment installs only the built wheel, - imports from outside the checkout, parses an address, and runs single/JSONL - CLI smoke tests. -- Wheel runtime content is allowlisted and excludes `.cache`, corpora, - notebooks, training/evaluation data, legacy code, secrets, and forbidden - dependencies. -- Model is at most 65,536 bytes; wheel is at most 262,144 bytes. Actual - values are recorded in release evidence. -- README examples execute, README/package metadata render checks pass, and every - numeric reliability claim names its domain and metric. - -### P1 — high value for the beta - -- Exercise and document the lazy batch iterator in streaming integrations. -- Add review-policy recipes for strict/manual/lenient queues based on warnings, - alternatives, unparsed spans, and decision-strength confidence. -- Add reproducible FastAPI, ETL, JSONL, and customer-managed FIAS/GAR resolver - examples without adding runtime dependencies. -- Make benchmark report schemas and metric names self-explanatory; add cheap - fixture-level evaluator tests and provenance validation to routine CI. -- Publish a bounded set of useful starter-issue proposals and a release demo - script/recording plan. -- Document alternative-comparison methodology without unverified competitor - claims, plus maintainer-approved draft launch copy. - -Acceptance criteria: - -- Lazy parsing consumes a generator once and does not materialize it. -- Example source files are syntax-checked; dependency-bearing examples clearly - separate optional dependencies from the core package. -- Fixture-level evaluation runs offline in normal CI, while large downloads - require explicit commands. -- Contributor templates require a linked/reproduced problem, fail-before and - pass-after evidence, cross-domain consideration, and human understanding. - -### P2 — after beta evidence - -- Curate training/validation data for street and administrative fields, with - explicit rights and a new untouched final test. -- Add confidence calibration only if a representative labeled validation set - supports it; otherwise retain decision-strength semantics. -- Report confidence intervals and regional/source-system slices. -- Rehearse TestPyPI, verify attestations and installation, then promote an - unchanged artifact to PyPI with maintainer approval. -- Consider separately versioned integration adapters; keep the core runtime - registry-independent. - -## Decisions and unresolved maintainer questions - -Recorded engineering decisions: - -- Keep `parse_many()` eager and list-returning for compatibility. -- Prefer an additive lazy helper over changing `parse_many()` semantics. -- Keep runtime dependency-free and all data preparation dependencies separate. -- Keep the four benchmark domains and their metric families separate. -- Use PEP 440 progression `2.0.0aN` → `2.0.0bN` → `2.0.0rcN` → `2.0.0`; - do not skip directly from this alpha to stable. -- Build once per release tag and publish that reviewed artifact through Trusted - Publishing; never rebuild between test and publish. - -Maintainer decisions still required: - -1. Should historical v1 remain in the default branch for v2 stable, move to a - named archival directory/branch, or be removed only in a future major - cleanup? -2. Which GitHub environment will protect TestPyPI/PyPI publication, and who may - approve it? -3. What administrative/street quality threshold is acceptable for beta, and - who will perform the independent row review? -4. Should the eventual stable support policy include every Python minor - 3.10–3.14, or follow a rolling set once 3.10 reaches end of upstream support? - -## Proposed release sequence - -1. Complete and review P0. -2. Release `2.0.0a2` to TestPyPI from an approved tag; verify hashes, - attestations, wheel contents, offline install, CLI, and rollback procedure. -3. Release `2.0.0a2` to PyPI only with explicit maintainer approval. -4. Expand human-reviewed validation coverage and close chosen quality targets; - publish `2.0.0b1`. -5. Freeze API/serialization and documentation, run the sealed final evaluation - once, and publish `2.0.0rc1`. -6. Promote the reviewed release-candidate code and evidence to `2.0.0`; rebuild - only for a new version if any input changes. - -## Evidence record for this sprint - -Recorded on 2026-07-28 and rerun for the 2026-07-29 release candidate: - -- Unit suite: 74 tests passed; fresh installed-wheel smoke tests passed on - Python 3.10 and 3.14. CI covers 3.10 through 3.14. -- Strict `mypy==1.17.1`: success on all nine public package modules. -- Compact tagger: deterministic regeneration was byte-identical; the model is - 37,130 bytes with SHA-256 - `c23c4f3cf308b1f36f56d0679df278d66b3aeae41c70a2144d88606199f3eb36`. -- Historical 500-row gate: 80.4% exact-address match, 76.8% no-unparsed rate, - and 95.8538% exact component micro F1; every committed gate passed. -- External baselines remained exactly unchanged: RedMadRobot span-overlap F1 - 58.7462% (street-and-house slice 78.8392%); Deepparse span-overlap F1 - 84.3948%, character-overlap F1 66.2289%, and token-label F1 66.4946%; - Moscow exact component-value micro F1 85.3620% and exact-address match - 64.9316%. -- Two consecutive final builds were byte-identical. The wheel is 45,843 bytes - (SHA-256 - `e98a49d0d7e8514485230bdccab043e6dedf0991c177a1c183fd561f734b1650`); - the sdist is 73,601 bytes (SHA-256 - `e439ab16e0af50424d36c519c6fc309bafb3167af35ad3fb3b21f4f5b892bb1c`). -- The model remained 37,130 bytes, the wheel stayed below its 256 KiB budget, - and the larger source archive includes both English and Russian - documentation. -- Wheel metadata, archive safety, exact runtime allowlist, hashes, artifact - policy rejection tests, Twine rendering, no-index/no-dependency installation, - import, one-address CLI, and JSONL CLI checks all passed. -- GPL-3.0-only licensing and model provenance were subsequently approved on - 2026-07-29; `release-policy.toml` now enforces the publishable state. - -Committed benchmark baselines must not be rewritten merely to make a change -look successful. Timing-only fields may vary when the reports are reproduced. - -Detection and diagnostics addendum, recorded on 2026-07-29: - -- `detect_addresses()` adds conservative, offset-preserving message-span - detection without changing parser output on the historical regression. -- The detection behavior fixture contains 30 narrow messages: 18 positive and - 12 negative, with 20 exact address spans. All currently pass, but this - authored fixture is not an independent accuracy benchmark. -- The historical diagnostic table contains all 500 rows and more than 75 - columns covering per-field outcomes, scenario dimensions, warnings, - unparsed evidence, and heuristic triage causes. It records 402 exact and 98 - non-exact rows. -- Unit tests pass on locally available Python 3.10, 3.13, and 3.14: 74 passed - per interpreter. Strict `mypy==1.17.1` passes all nine package modules. -- The historical parsing gate remains unchanged at 80.4% exact-address match, - 76.8% no-unparsed rate, and 95.8538% exact component micro F1. -- On complete RedMadRobot messages, the development-only detector diagnostic - reports 98.0% any-overlap precision, 68.1% recall, 80.3% F1, and 100% - negative-message specificity. Its failures were inspected, so it is not a - sealed final-test result. -- The reproducible wheel is now 33,104 bytes with SHA-256 - `15f38aa4dde86da620d05d4e8e620624769b26430c7b365b9f62d2575c87c8ed`; - the sdist is 52,762 bytes with SHA-256 - `e45303316773c560473e16e12fe2f574967d4593cf3d0b7c778d7fa9a3d7dda3`. diff --git a/README.en.md b/README.en.md new file mode 100644 index 0000000..3acd1cb --- /dev/null +++ b/README.en.md @@ -0,0 +1,129 @@ +# address-normalizer + +A small offline parser for Russian addresses. It extracts typed fields while +preserving source substrings and offsets, supports Python 3.10+, and has no +runtime dependencies. + +[Русская версия](https://github.com/shigabeev/address-normalizer#readme) + +## Install + +```bash +python -m pip install --pre address-normalizer +``` + +## Use + +```python +from address_normalizer import parse + +result = parse("г. Москва, ул. Тверская, д. 4, кв. 12") + +print(result.city.value) # Москва +print(result.street.value) # Тверская +print(result.house_num.value) # 4 +print(result.apartment.value) # 12 +print(result.normalized) # Москва, ул Тверская, д 4, кв 12 +``` + +Every component includes its normalized `value`, exact source substring `raw`, +half-open `start:end` offsets, decision source, and `confidence`. +`result.as_dict()` returns a JSON-compatible dictionary. + +Ambiguity remains visible: + +```python +result = parse("Ополченская 5-30") + +print(result.normalized) # Ополченская, д 5, кв 30 +print(result.warnings) # ("ambiguous_numeric_tail",) +print(result.alternatives) # includes compound house 5-30 +``` + +`confidence` is internal decision strength, not the probability that an address +exists. Review results containing `warnings`, `alternatives`, or `unparsed`. + +### Detect an address in a message + +```python +from address_normalizer import detect_addresses + +message = "Доставить по адресу: Москва, ул. Тверская, д. 13. Позвоните." + +for item in detect_addresses(message): + print(item.text) # Москва, ул. Тверская, д. 13 + print(item.span) # offsets in the original message + print(item.parsed) # ParsedAddress +``` + +Detection is conservative: weak candidates are intentionally rejected to avoid +treating dates or order numbers as addresses. + +### Batches and CLI + +```python +from address_normalizer import parse_many + +results = parse_many(["Тверская 1", "Невский проспект 10"]) +``` + +```bash +address-normalizer "СПб, Невский проспект 10, корп. 2" +printf '%s\n' "Тверская 1" "Ополченская 5-30" | + address-normalizer --jsonl +``` + +## Scope + +The package extracts postal and administrative fields, street and street type, +building/unit fields, source offsets, warnings, alternatives, and unparsed +content. + +It does not validate addresses against FIAS/GAR, return registry IDs, correct +official spelling, or geocode. Pass extracted candidates to a current registry +resolver owned by your application. + +## Implementation + +The runtime is a straightforward hybrid: offset-preserving tokenization, +explicit address and numeric rules, a compact linear sequence tagger for +unmarked words, and deterministic post-processing. It is not an LLM or neural +network. The model is 37 KB; there are no network calls or hidden downloads. + +## Quality + +One headline “accuracy” would mix incompatible tasks, so benchmarks stay +separate: + +| Dataset | Size | Metric | Result | +| --- | ---: | --- | ---: | +| Historical address sample | 500 | exact component micro F1 | 95.9% | +| Noisy address snippets | 578 | span-overlap F1 | 58.7% | +| Clean nationwide addresses | 100,000 | character-overlap F1 | 66.2% | +| Moscow buildings | 15,196 | exact component micro F1 | 85.4% | + +See +[`benchmarks/README.md`](https://github.com/shigabeev/address-normalizer/blob/master/benchmarks/README.md) +for definitions, per-field results, and +the complete 500-row diagnostic table. + +Building numbers, корпус, and строение are strongest. Administrative levels, +exact street boundaries, rare abbreviations, and unmarked numeric tails are +weaker. + +## Development + +```bash +python -m pip install -e . +python -m pip install pytest +pytest +python tools/benchmark.py --check +``` + +Runtime code lives in `src/address_normalizer`, core tests in `tests`, and the +reproducible failure set in `benchmarks`. + +## License + +GNU GPL v3.0 only. See +[`LICENSE`](https://github.com/shigabeev/address-normalizer/blob/master/LICENSE). diff --git a/README.md b/README.md index ca2c79a..82e513d 100644 --- a/README.md +++ b/README.md @@ -1,467 +1,132 @@ # address-normalizer -**Turn an unstructured Russian address into typed, offset-preserving fields—locally, with no runtime dependencies or registry download.** +Небольшой офлайн-парсер российских адресов для Python 3.10+. Извлекает поля, +сохраняет исходные подстроки и смещения, не требует runtime-зависимостей. -[Русская версия](https://github.com/shigabeev/address-normalizer/blob/master/README.ru.md) +[English README](https://github.com/shigabeev/address-normalizer/blob/master/README.en.md) -`address-normalizer` v2 is a small parser for applications that already have, -or plan to choose, their own FIAS/GAR lookup. It extracts components; it does -not verify an address, return a FIAS ID, geocode, or silently download data. - -## 30-second quick start +## Установка ```bash -python -m pip install address-normalizer -address-normalizer "Ополченская 5-30" +python -m pip install --pre address-normalizer ``` -Or use the typed Python API: +## Использование ```python from address_normalizer import parse -result = parse("Ополченская 5-30") +result = parse("г. Москва, ул. Тверская, д. 4, кв. 12") -print(result.normalized) -# Ополченская, д 5, кв 30 +print(result.city.value) # Москва +print(result.street.value) # Тверская +print(result.house_num.value) # 4 +print(result.apartment.value) # 12 +print(result.normalized) # Москва, ул Тверская, д 4, кв 12 ``` -That compact input is ambiguous. The parser keeps the source offsets, its -chosen interpretation, a warning, and a plausible compound-house alternative: - -```json -{ - "normalized": "Ополченская, д 5, кв 30", - "street": { - "value": "Ополченская", - "raw": "Ополченская", - "span": [0, 11], - "confidence": 0.78, - "source": "model" - }, - "house_num": {"value": "5", "span": [12, 13]}, - "apartment": {"value": "30", "span": [14, 16]}, - "warnings": ["ambiguous_numeric_tail"], - "alternatives": [ - { - "components": {"house_num": "5-30"}, - "reason": "A hyphenated numeric tail can also be a compound house number." - } - ] -} -``` - -The complete stable dictionary shape is available through `result.as_dict()`; -the shortened JSON above highlights the decision. Confidence is a bounded -**decision-strength score**, not a calibrated probability. - -## Why this boundary - -The historical v1 application materialized FIAS paths in Elasticsearch. That -made the parser, registry, storage engine, and deployment topology one system. -Version 2 separates those responsibilities: - -- this package tokenizes and extracts candidate address components; -- your application decides which results require review; -- your own current FIAS/GAR index or service verifies candidates and supplies - identifiers. - -No Elasticsearch, FIAS/GAR database, network call, service process, hidden -download, pandas, or scientific runtime is required by the package. The bundled -structured sequence model is about 37 KB. Large evaluation corpora stay outside -the wheel under the ignored `.cache/external/` directory. - -## Should I use this? - -| Your need | Fit | -| --- | --- | -| Extract Russian address fields in an offline Python process | **Yes** | -| Conservatively locate marked street/building spans in messages | **Yes** | -| Preserve raw substrings, offsets, warnings, and alternatives | **Yes** | -| Feed structured candidates into your own resolver | **Yes** | -| Verify current address existence or get a FIAS/GAR ID | **No—add a resolver** | -| Geocode, transliterate, or correct official spelling | **No** | -| Detect every implicit or markerless address in arbitrary prose | **No** | -| Require uniformly strong administrative-field extraction | **Not yet** | - -The current evidence supports conventional street/building extraction better -than isolated administrative names. Read the [benchmark boundaries](#reliability) -before setting automation policy. - -## Architecture - -```mermaid -flowchart LR - A["Address string"] --> C["Offset-preserving tokenizer"] - M["Free-form message"] --> B["Conservative span detector"] - B --> A - C --> D["Postcode and numeric grammar"] - D --> E["Explicit marker rules"] - E --> F["Tiny sequence tagger for residual text"] - F --> G["Deterministic post-processing"] - G --> H["ParsedAddress"] - H --> I["Application review policy"] - I --> J["Customer-managed FIAS/GAR resolver"] -``` - -Direct address strings can skip detection. The parser itself ends at -`ParsedAddress`; review and resolution remain application responsibilities. - -### What the ML stack actually is - -This is not a neural network, transformer, LLM, spaCy, scikit-learn, PyTorch, -or TensorFlow stack. It is a dependency-free hybrid: - -1. regular-expression tokenization and explicit address grammar; -2. marker and numeric extraction rules; -3. a 37 KB sparse linear-chain tagger for only the residual unmarked words; -4. Viterbi decoding and deterministic post-processing. - -The learned tagger is an epoch-averaged structured perceptron with six labels: -`O`, `REGION`, `DISTRICT`, `CITY`, `SETTLEMENT`, and `STREET`. Its JSON -artifact contains sparse lexical/context feature weights and transition -weights. Message detection is currently rule-based; FIAS/GAR lookup is an -optional downstream application stage. - -See [ML stack and runtime boundaries](docs/ml-stack.md) for the features, -training split, artifact details, and the recommended production separation -between detection, extraction, and registry resolution. - -## API reference - -The public package exports: +Каждый компонент содержит `value`, точную исходную подстроку `raw`, полуоткрытый +интервал `start:end`, источник решения и `confidence`. Весь результат можно +сериализовать: ```python -from address_normalizer import ( - AddressPart, - AddressPartDict, - Alternative, - AlternativeDict, - DetectedAddress, - DetectedAddressDict, - ParsedAddress, - ParsedAddressDict, - detect_addresses, - parse, - parse_iter, - parse_many, -) +payload = result.as_dict() # обычный JSON-совместимый dict ``` -### `parse(text: str) -> ParsedAddress` - -Parses one string without verifying it against a registry. Non-string input -raises `TypeError`. Empty or punctuation-only input returns an empty -`ParsedAddress`; decide at your application boundary whether that should be an -error. - -### `parse_many(addresses: Iterable[str]) -> list[ParsedAddress]` - -Consumes any iterable, preserves order, and returns a list. It is convenient -for bounded batches. - -### `parse_iter(addresses: Iterable[str]) -> Iterator[ParsedAddress]` - -Lazily consumes an iterable in order and keeps only one parsed result at a time. -Use it for unbounded files or streams, as shown in -[`examples/jsonl_etl.py`](https://github.com/shigabeev/address-normalizer/blob/master/examples/jsonl_etl.py). -`parse_many()` and `parse_iter()` reject a bare string so it cannot be mistaken -for a batch; a non-string element raises `TypeError` when iteration reaches it. - -### `detect_addresses(text: str) -> tuple[DetectedAddress, ...]` - -Locates conservative address candidates inside a free-form message. A candidate -normally needs a street marker plus a building number, or an explicit -`адрес:` cue plus a parseable street and building. This avoids treating every -place name or number as an address. +Неоднозначность не скрывается: ```python -from address_normalizer import detect_addresses - -message = ( - "Курьер приедет по адресу: Москва, ул. Тверская, " - "д. 13, кв. 4. Позвоните." -) - -for detected in detect_addresses(message): - assert message[detected.start:detected.end] == detected.text - print(detected.span, detected.text) - print(detected.parsed.as_dict()) -``` - -`DetectedAddress.span` indexes the original message. Component offsets inside -`DetectedAddress.parsed` index the extracted `DetectedAddress.text`. Detection -confidence is decision strength, not a probability or registry verification. -Multiple non-overlapping addresses are returned in message order. - -### Result types - -`ParsedAddress` can contain: +result = parse("Ополченская 5-30") -```text -postal_code, region, district, city, settlement, -street, street_type, house_num, corpus, structure, apartment, -unparsed, warnings, alternatives, confidence +print(result.normalized) # Ополченская, д 5, кв 30 +print(result.warnings) # ("ambiguous_numeric_tail",) +print(result.alternatives) # среди вариантов есть дом 5-30 ``` -Each `AddressPart` contains: - -- `value`: lightly normalized extracted value; -- `raw`: exact substring from the input; -- `start`, `end`, and `span`: original `[start, end)` character offsets; -- `confidence`: bounded decision-strength score; -- `source`: `rule`, `model`, `postprocessor`, or `unparsed`. - -`ParsedAddress.as_dict()` returns a `ParsedAddressDict` made of JSON-compatible -built-in values; `AddressPartDict` and `AlternativeDict` describe nested -objects. Its top level also contains `raw`, `normalized`, and overall -`confidence`. The dataclasses are frozen; treat this serialized shape and the -exported names as the v2 alpha contract. - -## Review policy - -Do not turn the overall confidence into a universal accept/reject threshold. -It has not been calibrated as a probability, and a score learned on one input -domain does not establish the error rate on another. - -A safe default policy is: +`confidence` — сила решения внутри парсера, а не вероятность существования +адреса. Результаты с `warnings`, `alternatives` или `unparsed` стоит проверять. -1. require expected business fields, such as `street` and `house_num`; -2. send any result with `warnings`, `alternatives`, or non-empty `unparsed` to - review or resolver-assisted disambiguation; -3. validate every selected part by slicing the original text with its `span`; -4. choose any confidence threshold only on a representative, labeled - validation set from your own traffic; -5. use a current customer-managed FIAS/GAR source to verify existence and - choose among candidates. +### Поиск адреса в сообщении ```python -from address_normalizer import ParsedAddress +from address_normalizer import detect_addresses +message = "Доставить по адресу: Москва, ул. Тверская, д. 13. Позвоните." -def needs_review(result: ParsedAddress) -> bool: - required_fields_missing = result.street is None or result.house_num is None - unresolved_evidence = bool( - result.warnings or result.alternatives or result.unparsed - ) - return required_fields_missing or unresolved_evidence +for item in detect_addresses(message): + print(item.text) # Москва, ул. Тверская, д. 13 + print(item.span) # смещение в исходном сообщении + print(item.parsed) # ParsedAddress ``` -[`examples/fias_gar_http.py`](https://github.com/shigabeev/address-normalizer/blob/master/examples/fias_gar_http.py) -shows a deliberately generic HTTP boundary for a customer-managed resolver. -Adapt its request contract to your index; there is no canonical resolver API in -this package. - -## CLI reference +Детектор консервативный: лучше пропустить слабый кандидат, чем принять номер +заказа или дату за адрес. -Parse one positional address and emit an indented JSON object: - -```bash -address-normalizer "СПб Невский проспект 10 корп 2 кв 15" -``` +### Несколько адресов и CLI -With no positional address, the command reads one address from standard input: +```python +from address_normalizer import parse_many -```bash -printf '%s' 'Москва Тверская д 5/1 кв 9' | address-normalizer +results = parse_many(["Тверская 1", "Невский проспект 10"]) ``` -`--jsonl` reads one address per line and writes one compact JSON object per -line. Blank lines are parsed as empty addresses rather than skipped. - ```bash -printf '%s\n' \ - 'Ополченская 5-30' \ - 'Самара Авроры 7 12' | +address-normalizer "СПб, Невский проспект 10, корп. 2" +printf '%s\n' "Тверская 1" "Ополченская 5-30" | address-normalizer --jsonl ``` -Exit status is zero after successful processing. Invalid invocation is handled -by `argparse`; malformed content is represented in the parse result rather than -treated as a CLI syntax error. - -## Common recipes - -- [`examples/basic.py`](https://github.com/shigabeev/address-normalizer/blob/master/examples/basic.py): - typed single-address parsing and review routing; -- [`examples/jsonl_etl.py`](https://github.com/shigabeev/address-normalizer/blob/master/examples/jsonl_etl.py): - constant-memory JSONL ETL from standard input; -- [`examples/fastapi_app.py`](https://github.com/shigabeev/address-normalizer/blob/master/examples/fastapi_app.py): - an optional FastAPI wrapper without changing the package's runtime - dependencies; -- [`examples/fias_gar_http.py`](https://github.com/shigabeev/address-normalizer/blob/master/examples/fias_gar_http.py): - inspect or send a resolver request to an endpoint you control. - -These examples are integration starting points, not extra behavior hidden in -the core package. - -## Reliability - -There is no single “accuracy” number. The committed reports cover different -domains and matching rules, and the results must not be averaged: - -| Domain | Test size | Primary metric | Result | Important boundary | -| --- | ---: | --- | ---: | --- | -| Historical bank-shaped reference | 500 rows | exact component micro F1 | 95.9% | Silver regression data; not independently re-reviewed for v2 | -| RedMadRobot noisy address windows | 578 windows | same-label binary span-overlap F1 | 58.7% | Address windows are selected using gold annotations | -| Deepparse nationwide clean strings | 100,000 rows | same-label character-overlap F1 | 66.2% | Registry-derived clean strings with a different token schema | -| Moscow official clean buildings | 15,196 rows | exact component-value micro F1 | 85.4% | Moscow-only October 2021 snapshot | - -Message-span detection is measured separately. On complete RedMadRobot messages, -the current development diagnostic reports 98.0% any-overlap precision, 68.1% -recall, and 80.3% F1 for gold windows containing both `STREET` and `HOUSE`. -Those detector failures were inspected during development, so this is not a -sealed final-test result. Exact-boundary F1 is only 23.8% because gold and -detector boundary policies frequently disagree about surrounding city, -postcode, country, and marker text. - -Metric names matter: - -- **exact component micro F1** pools true/false positive and negative component - values across all scored fields, requiring normalized values to match; -- **binary span-overlap F1** counts a one-to-one same-label span as matched if - the character ranges overlap at all, so it is intentionally lenient; -- **character-overlap F1** scores the amount of correctly overlapping - same-label text and exposes partial or merged spans; -- **token-label F1** pools precision, recall, and F1 over labels aligned to the - source tokens; -- **exact full-address match** requires every scored component value in one row - to match and no extra scored component to be produced. - -Additional context prevents misleading comparisons: - -- RedMadRobot street F1 is 49.1%, while house F1 is 90.2%; -- Deepparse also reports 84.4% binary span-overlap F1 and 66.5% token-label F1; -- the Moscow report has 64.9% exact full-address match, 66.2% street F1, and - 98.2% house F1. - -See -[`evaluation/README.md`](https://github.com/shigabeev/address-normalizer/blob/master/evaluation/README.md) -for pinned sources, exact filters, per-field results, commands, and limitations. -[`evaluation/RESULTS.md`](https://github.com/shigabeev/address-normalizer/blob/master/evaluation/RESULTS.md) -indexes every committed benchmark report. -The historical regression gate is reproducible without downloading large -corpora: - -```bash -python evaluation/evaluate.py \ - --data evaluation/legacy_reference_500.jsonl \ - --gates evaluation/release_gates.json \ - --output evaluation/legacy_reference_500_report.json -``` - -The external benchmark commands require separately installed data-preparation -dependencies and downloads. They never become runtime dependencies. - -## Limitations - -- Parsing does not prove that an address exists or is current. -- No FIAS/GAR identifiers or coordinates are returned. -- Administrative-field recall and exact street boundaries are materially - weaker than numeric building fields on current external benchmarks. -- Hyphenated and unmarked numeric tails can remain genuinely ambiguous. -- Normalization is deliberately light; it is not official-spelling correction. -- Confidence is not a probability and has not been calibrated across domains. -- Detection deliberately misses unmarked address-like text without an - `адрес:` cue and marked streets without a building. Its committed 30-message - fixture is a behavior regression set, not a production accuracy benchmark. -- The current package version is an alpha; public API and model behavior may - still change before the beta. - -## Migrating from v1 - -Version 2 is a new product boundary, not a drop-in replacement for the root -v1 Elasticsearch application. +## Граница ответственности -| v1 responsibility | v2 approach | -| --- | --- | -| Imports from root `api.py` / `parsing.py` | Import `parse` from `address_normalizer` | -| FIAS data loaded into Elasticsearch | Operate your registry/index separately | -| Resolver-shaped final result | `ParsedAddress` extraction candidates | -| Corrected or registry-backed values | Lightly normalized values from source text | -| Implicit resolution choice | Explicit `warnings`, `alternatives`, and `unparsed` | -| Service/application deployment | Library API or JSONL CLI | +Библиотека извлекает: -During migration, keep v1 resolution and v2 extraction side by side on recorded -traffic, compare by field, and define domain-specific review gates before -switching writes. Do not compare v1 registry correction with v2 extraction as -if they were the same task. +- индекс, регион, район, город и населённый пункт; +- улицу и тип улицы; +- дом, корпус, строение и квартиру; +- исходные смещения, неразобранный остаток, предупреждения и альтернативы. -The historical root files (`api.py`, `parsing.py`, `upload_fias.py`, -`docker-compose.yaml`, and `requirements-legacy.txt`) are retained as an -architectural record and are not included in the v2 wheel. +Она не проверяет адрес по ФИАС/ГАР, не возвращает ID реестра, не исправляет +официальное написание и не геокодирует. После парсинга передайте поля в свой +актуальный resolver ФИАС/ГАР. -## Troubleshooting +## Как это работает -**`pip install address-normalizer` does not provide this v2 API.** +Runtime — простой гибрид: токенизатор со смещениями, правила адресных маркеров и +чисел, компактный линейный sequence tagger для слов без маркеров, затем +детерминированная постобработка. Это не LLM и не нейросеть. Модель занимает +37 КБ; сетевых запросов и скрытых загрузок нет. -The v2 alpha has not been published. Install from this checkout until a release -is explicitly announced. +## Качество -**A FIAS ID is missing.** +Одна цифра «accuracy» здесь вводит в заблуждение, поэтому разные наборы +публикуются отдельно: -That is expected. Send selected fields to a current resolver you operate; see -the FIAS/GAR HTTP example. +| Набор | Размер | Метрика | Результат | +| --- | ---: | --- | ---: | +| Историческая адресная выборка | 500 | exact component micro F1 | 95,9% | +| Шумные адресные фрагменты | 578 | span-overlap F1 | 58,7% | +| Чистые адреса по России | 100 000 | character-overlap F1 | 66,2% | +| Здания Москвы | 15 196 | exact component micro F1 | 85,4% | -**A high-confidence parse is wrong.** +Подробные определения, результаты по полям и все 500 строк с причинами ошибок: +[`benchmarks/README.md`](https://github.com/shigabeev/address-normalizer/blob/master/benchmarks/README.md). -Confidence expresses parser decision strength, not correctness probability. -Capture the exact input, output, expected fields, and business context in a -[parsing-failure report](https://github.com/shigabeev/address-normalizer/issues/new?template=parsing-failure.yml). +Сильные поля — дом, корпус и строение. Слабее — административные уровни, точная +граница улицы, редкие сокращения и числовые хвосты без маркеров. -**`parse_many()` uses too much memory.** - -It returns a list by contract. Stream records through `parse_iter()` or use the -JSONL CLI/ETL example. - -**Offsets appear wrong after normalization.** - -Offsets index the original `result.raw`, not `result.normalized`. Verify with -`result.raw[part.start:part.end] == part.raw`. - -**A line disappears or appears empty in JSONL output.** - -The CLI emits one result per input line and intentionally keeps blank lines. -Filter blank records in the calling pipeline if that is the desired policy. - -**The parser imports but the bundled model cannot be found.** - -Install the built wheel rather than copying the package directory manually, and -include the exact install command and wheel contents in a bug report. - -## Development +## Разработка ```bash python -m pip install -e . +python -m pip install pytest pytest -python training/train_compact_tagger.py -python training/evaluate_compact_tagger.py -``` - -Large external data preparation has separate, pinned tooling: - -```bash -python -m pip install -r requirements-evaluation.txt -python evaluation/prepare_deepparse.py --download -python evaluation/prepare_datamos.py --download -``` - -Build artifacts: - -```bash -python -m pip install build -python -m build +python tools/benchmark.py --check ``` -Read -[`CONTRIBUTING.md`](https://github.com/shigabeev/address-normalizer/blob/master/CONTRIBUTING.md) -before proposing a change. Bug reports, minimal parsing failures, provenance -information, documentation corrections, and focused pull requests are welcome. +Публичный API находится в `src/address_normalizer`, основные проверки — в +`tests`, воспроизводимый набор ошибок — в `benchmarks`. -## License +## Лицензия -GNU General Public License v3.0 only (`GPL-3.0-only`). See -[`LICENSE`](https://github.com/shigabeev/address-normalizer/blob/master/LICENSE) -and the -[`LICENSING.md`](https://github.com/shigabeev/address-normalizer/blob/master/LICENSING.md) -provenance record. +GNU GPL v3.0 only. Полный текст — в +[`LICENSE`](https://github.com/shigabeev/address-normalizer/blob/master/LICENSE). diff --git a/README.ru.md b/README.ru.md deleted file mode 100644 index 56d7855..0000000 --- a/README.ru.md +++ /dev/null @@ -1,396 +0,0 @@ -# address-normalizer - -**Извлекает типизированные поля из неструктурированного российского адреса, -сохраняет исходные смещения и работает локально — без runtime-зависимостей и -скачивания справочника.** - -[English version](https://github.com/shigabeev/address-normalizer/blob/master/README.md) - -`address-normalizer` v2 — небольшая Python-библиотека для приложений, которые -используют или планируют использовать собственный поиск по ФИАС/ГАР. Она -извлекает компоненты адреса, но не подтверждает существование адреса, не -возвращает идентификатор ФИАС, не геокодирует и ничего не скачивает скрытно. - -## Быстрый старт за 30 секунд - -```bash -python -m pip install address-normalizer -address-normalizer "Ополченская 5-30" -``` - -Типизированный Python API: - -```python -from address_normalizer import parse - -result = parse("Ополченская 5-30") - -print(result.normalized) -# Ополченская, д 5, кв 30 -``` - -Эта короткая запись неоднозначна. Парсер сохраняет исходные смещения, выбранную -интерпретацию, предупреждение и правдоподобную альтернативу с составным номером -дома: - -```json -{ - "normalized": "Ополченская, д 5, кв 30", - "street": { - "value": "Ополченская", - "raw": "Ополченская", - "span": [0, 11], - "confidence": 0.78, - "source": "model" - }, - "house_num": {"value": "5", "span": [12, 13]}, - "apartment": {"value": "30", "span": [14, 16]}, - "warnings": ["ambiguous_numeric_tail"], - "alternatives": [ - { - "components": {"house_num": "5-30"}, - "reason": "A hyphenated numeric tail can also be a compound house number." - } - ] -} -``` - -Полная стабильная форма словаря доступна через `result.as_dict()`. Пример выше -сокращён, чтобы показать принятое решение. `confidence` — ограниченный показатель -силы решения, а не калиброванная вероятность. - -## Почему граница продукта проходит здесь - -Историческое приложение v1 материализовывало пути ФИАС в Elasticsearch. В -результате парсер, реестр, хранилище и схема развёртывания становились одной -системой. Версия 2 разделяет обязанности: - -- библиотека токенизирует строку и извлекает компоненты-кандидаты; -- приложение решает, какие результаты требуют проверки; -- актуальный индекс или сервис ФИАС/ГАР подтверждает кандидатов и возвращает - идентификаторы. - -Библиотеке не нужны Elasticsearch, база ФИАС/ГАР, сетевые запросы, сервисный -процесс, скрытая загрузка, pandas или научный Python-стек. Встроенная -структурная модель занимает около 37 КБ. Большие оценочные корпуса остаются вне -wheel в игнорируемом каталоге `.cache/external/`. - -## Подходит ли библиотека для моей задачи? - -| Задача | Подходит | -| --- | --- | -| Извлечь поля российского адреса в локальном Python-процессе | **Да** | -| Консервативно найти размеченную улицу и дом внутри сообщения | **Да** | -| Сохранить исходные подстроки, смещения, предупреждения и альтернативы | **Да** | -| Передать структурированных кандидатов собственному resolver | **Да** | -| Проверить существование адреса или получить ID ФИАС/ГАР | **Нет — нужен resolver** | -| Геокодировать, транслитерировать или исправить официальное написание | **Нет** | -| Найти каждый неявный адрес без маркеров в произвольном тексте | **Нет** | -| Одинаково хорошо извлекать все административные уровни | **Пока нет** | - -Текущие результаты лучше подтверждают обычные адреса с улицей и домом, чем -изолированные административные названия. Перед автоматизацией прочитайте -[границы измерений](#надёжность). - -## Архитектура - -```mermaid -flowchart LR - A["Строка адреса"] --> C["Токенизатор со смещениями"] - M["Свободное сообщение"] --> B["Консервативный детектор"] - B --> A - C --> D["Индекс и числовая грамматика"] - D --> E["Правила явных маркеров"] - E --> F["Компактный теггер остаточного текста"] - F --> G["Детерминированная постобработка"] - G --> H["ParsedAddress"] - H --> I["Политика проверки приложения"] - I --> J["Resolver ФИАС/ГАР пользователя"] -``` - -Готовая строка адреса может миновать детектор. Сам парсер заканчивает работу на -`ParsedAddress`; проверка и разрешение остаются обязанностью приложения. - -### Что здесь действительно относится к ML - -Это не нейросеть, transformer, LLM, spaCy, scikit-learn, PyTorch или -TensorFlow. Это dependency-free гибрид: - -1. токенизация регулярными выражениями и явная адресная грамматика; -2. правила для маркеров и числовых компонентов; -3. разреженный линейно-цепочечный теггер размером 37 КБ только для оставшихся - слов без маркеров; -4. декодирование Витерби и детерминированная постобработка. - -Обучаемый теггер — усреднённый по эпохам структурный перцептрон с метками `O`, -`REGION`, `DISTRICT`, `CITY`, `SETTLEMENT` и `STREET`. JSON-артефакт содержит -разреженные веса лексических/контекстных признаков и переходов. Детектор -сообщений сейчас основан на правилах; ФИАС/ГАР — необязательный следующий этап -в приложении. - -Подробности: [ML-стек и границы runtime](docs/ml-stack.md). - -## API - -Публичные экспорты: - -```python -from address_normalizer import ( - AddressPart, - AddressPartDict, - Alternative, - AlternativeDict, - DetectedAddress, - DetectedAddressDict, - ParsedAddress, - ParsedAddressDict, - detect_addresses, - parse, - parse_iter, - parse_many, -) -``` - -### `parse(text: str) -> ParsedAddress` - -Разбирает одну строку без проверки по реестру. Для значения не типа `str` -выбрасывает `TypeError`. Пустая строка или строка только из пунктуации возвращает -пустой `ParsedAddress`; приложение само решает, считать ли это ошибкой. - -### `parse_many(addresses: Iterable[str]) -> list[ParsedAddress]` - -Обрабатывает любой конечный iterable, сохраняет порядок и возвращает список. - -### `parse_iter(addresses: Iterable[str]) -> Iterator[ParsedAddress]` - -Лениво обрабатывает iterable по порядку и держит в памяти один результат. -Используйте для файлов и потоков без фиксированного размера. `parse_many()` и -`parse_iter()` отвергают одиночную строку, чтобы не принять её за набор -адресов. - -### `detect_addresses(text: str) -> tuple[DetectedAddress, ...]` - -Консервативно находит адреса в свободном сообщении. Обычно кандидату нужен -маркер улицы вместе с номером дома либо явный префикс `адрес:` и разбираемые -улица/дом. - -```python -from address_normalizer import detect_addresses - -message = ( - "Курьер приедет по адресу: Москва, ул. Тверская, " - "д. 13, кв. 4. Позвоните." -) - -for detected in detect_addresses(message): - assert message[detected.start:detected.end] == detected.text - print(detected.span, detected.text) - print(detected.parsed.as_dict()) -``` - -`DetectedAddress.span` индексирует исходное сообщение. Смещения компонентов в -`DetectedAddress.parsed` относятся к извлечённому `DetectedAddress.text`. -Несколько непересекающихся адресов возвращаются в порядке появления. - -### Результаты - -`ParsedAddress` может содержать: - -```text -postal_code, region, district, city, settlement, -street, street_type, house_num, corpus, structure, apartment, -unparsed, warnings, alternatives, confidence -``` - -Каждый `AddressPart` содержит: - -- `value` — слегка нормализованное значение; -- `raw` — точную подстроку входа; -- `start`, `end`, `span` — полуоткрытые смещения `[start, end)` в исходной - строке; -- `confidence` — силу решения; -- `source` — `rule`, `model`, `postprocessor` или `unparsed`. - -`ParsedAddress.as_dict()` возвращает JSON-совместимый `ParsedAddressDict`. -Датаклассы immutable; экспортированные имена и сериализованная структура — -контракт альфа-версии v2. - -## Политика проверки результата - -Не используйте общую `confidence` как универсальный порог принятия. Это не -вероятность, и она не устанавливает частоту ошибок на другом домене. - -Безопасная начальная политика: - -1. требовать бизнес-поля, например `street` и `house_num`; -2. отправлять результат с `warnings`, `alternatives` или `unparsed` на ручную - либо реестровую проверку; -3. проверять каждую часть срезом исходной строки по `span`; -4. выбирать порог только на размеченной выборке своего трафика; -5. использовать актуальный ФИАС/ГАР для подтверждения существования и выбора - кандидата. - -```python -from address_normalizer import ParsedAddress - - -def needs_review(result: ParsedAddress) -> bool: - required_fields_missing = result.street is None or result.house_num is None - unresolved_evidence = bool( - result.warnings or result.alternatives or result.unparsed - ) - return required_fields_missing or unresolved_evidence -``` - -Пример [`examples/fias_gar_http.py`](examples/fias_gar_http.py) показывает -обобщённую HTTP-границу для resolver, которым управляет пользователь. - -## CLI - -Один адрес: - -```bash -address-normalizer "СПб Невский проспект 10 корп 2 кв 15" -``` - -Без позиционного аргумента команда читает адрес из stdin: - -```bash -printf '%s' 'Москва Тверская д 5/1 кв 9' | address-normalizer -``` - -Режим `--jsonl` читает одну строку адреса и пишет один компактный JSON на строку: - -```bash -printf '%s\n' \ - 'Ополченская 5-30' \ - 'Самара Авроры 7 12' | - address-normalizer --jsonl -``` - -Пустые строки намеренно сохраняются как пустые результаты. - -## Готовые примеры - -- [`examples/basic.py`](examples/basic.py) — один адрес и маршрутизация на - проверку; -- [`examples/detect_in_message.py`](examples/detect_in_message.py) — поиск - адресов внутри сообщения; -- [`examples/jsonl_etl.py`](examples/jsonl_etl.py) — потоковый JSONL ETL с - постоянным объёмом памяти; -- [`examples/fastapi_app.py`](examples/fastapi_app.py) — необязательная FastAPI - обёртка; -- [`examples/fias_gar_http.py`](examples/fias_gar_http.py) — формирование - запроса к собственному resolver ФИАС/ГАР. - -## Надёжность - -Единого числа «accuracy» нет. Отчёты используют разные домены и правила -сопоставления; усреднять их нельзя: - -| Домен | Размер теста | Основная метрика | Результат | Важная граница | -| --- | ---: | --- | ---: | --- | -| Исторический банковский reference | 500 | micro F1 точных компонентов | 95,9% | Silver regression, не новый gold benchmark | -| Шумные окна RedMadRobot | 578 | F1 пересечения span одинакового типа | 58,7% | Окна выбраны с помощью gold-разметки | -| Чистые адреса Deepparse по России | 100 000 | character-overlap F1 | 66,2% | Реестровые строки с другой схемой токенов | -| Официальные здания Москвы | 15 196 | micro F1 точных компонентов | 85,4% | Москва, снимок октября 2021 года | - -Детекция сообщений измеряется отдельно. На полных сообщениях RedMadRobot -диагностика показывает 98,0% precision, 68,1% recall и 80,3% F1 по любому -пересечению для окон с `STREET` и `HOUSE`. Это development diagnostic, а не -независимый финальный тест. Exact-boundary F1 равен 23,8%, потому что gold и -детектор по-разному включают город, индекс, страну и маркеры. - -Дополнительные результаты: - -- исторический тест: 80,4% полностью точных адресов и 76,8% без остатка; -- RedMadRobot: улица 49,1% F1, дом 90,2% F1; -- Deepparse: 84,4% binary span-overlap F1 и 66,5% token-label F1; -- Москва: 64,9% полностью точных адресов, улица 66,2% F1, дом 98,2% F1. - -Полные определения, источники, фильтры, поля и ограничения находятся в -[`evaluation/README.md`](evaluation/README.md), а все сохранённые результаты -проиндексированы в [`evaluation/RESULTS.md`](evaluation/RESULTS.md). - -Портативный regression gate: - -```bash -python evaluation/evaluate.py \ - --data evaluation/legacy_reference_500.jsonl \ - --gates evaluation/release_gates.json \ - --output evaluation/legacy_reference_500_report.json -``` - -## Ограничения - -- Парсинг не доказывает, что адрес существует или актуален. -- Библиотека не возвращает идентификаторы ФИАС/ГАР и координаты. -- Административные поля и точные границы улиц слабее числовых полей здания. -- Дефисные и немаркированные числовые хвосты могут быть неоднозначны. -- Нормализация намеренно лёгкая и не исправляет официальное написание. -- `confidence` не является вероятностью и не калибрована между доменами. -- Детектор намеренно пропускает текст без маркеров/`адрес:` и улицу без дома. -- Версия остаётся alpha; API и поведение модели могут измениться до beta. - -## Миграция с v1 - -Версия 2 задаёт новую границу продукта и не является drop-in replacement для -старого Elasticsearch-приложения: - -| Обязанность v1 | Подход v2 | -| --- | --- | -| Импорты из корневых `api.py` / `parsing.py` | `from address_normalizer import parse` | -| Загрузка ФИАС в Elasticsearch | Отдельный реестр/индекс приложения | -| Готовый результат resolver | Кандидаты в `ParsedAddress` | -| Исправленные значения из реестра | Лёгкая нормализация исходного текста | -| Неявный выбор разрешения | Явные `warnings`, `alternatives`, `unparsed` | -| Развёртывание приложения | Библиотека или JSONL CLI | - -Старые корневые файлы сохранены как архитектурная история и не попадают в wheel. - -## Частые вопросы - -**Почему нет ID ФИАС?** -Это ожидаемо: передайте выбранные поля актуальному resolver, которым вы -управляете. - -**Почему результат с высокой confidence оказался неправильным?** -Confidence означает силу решения парсера, а не вероятность правильности. -Приложите синтетический пример через форму parsing failure. - -**Почему `parse_many()` использует много памяти?** -По контракту он возвращает список. Для потоков используйте `parse_iter()` или -JSONL CLI. - -**Почему смещения выглядят неправильно после нормализации?** -Они индексируют `result.raw`, а не `result.normalized`: -`result.raw[part.start:part.end] == part.raw`. - -**Почему строка исчезает или остаётся пустой в JSONL?** -CLI выдаёт результат на каждую входную строку, включая пустую. Фильтрацию -выполняет вызывающий процесс. - -## Разработка - -```bash -python -m pip install -e . -pytest -python training/train_compact_tagger.py -python training/evaluate_compact_tagger.py -``` - -Подготовка больших внешних данных имеет отдельные зависимости: - -```bash -python -m pip install -r requirements-evaluation.txt -python evaluation/prepare_deepparse.py --download -python evaluation/prepare_datamos.py --download -``` - -Перед pull request прочитайте [`CONTRIBUTING.md`](CONTRIBUTING.md). - -## Лицензия - -GNU General Public License v3.0 only (`GPL-3.0-only`). См. -[`LICENSE`](LICENSE) и запись о происхождении данных/модели в -[`LICENSING.md`](LICENSING.md). diff --git a/SECURITY.md b/SECURITY.md index 5a0b1fd..04da5a0 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,57 +1,16 @@ -# Security policy +# Security -## Supported versions +Version `2.0.0a2` is the currently supported alpha. -| Version | Supported | -| --- | --- | -| Latest 2.0 pre-release | Yes, best effort | -| Historical v1 Elasticsearch application | No | +Please report vulnerabilities through +[GitHub private vulnerability reporting](https://github.com/shigabeev/address-normalizer/security/advisories/new). +Do not include private addresses, credentials, or customer data in a public +issue. -The alpha does not yet have a guaranteed security response or maintenance SLA. +Useful reports include the affected version, a minimal synthetic reproduction, +impact, and any suggested mitigation. -## Report privately - -Do not open a public issue for a suspected vulnerability or include private -addresses, credentials, tokens, or exploit details in public artifacts. - -Use GitHub private vulnerability reporting for this repository if the -**Report a vulnerability** option is available under the Security tab. Include: - -- affected version, commit, and installation method; -- impact and realistic attack conditions; -- minimal reproduction or proof of concept; -- whether secrets, filesystem access, network access, or untrusted package/model - data are involved; -- a safe way to contact you. - -If private vulnerability reporting is unavailable, use the -[security-contact request](https://github.com/shigabeev/address-normalizer/issues/new?template=security-contact.yml). -It asks only for a private channel and must contain no vulnerability details. -A maintainer can then arrange a private channel. This is a routing fallback, -not a place to disclose the vulnerability. - -The project cannot promise a response SLA before a maintainer security contact -and release process are formally established. The reporter should expect an -acknowledgment, impact assessment, coordinated fix, and disclosure timing to be -agreed before publication. - -## Security boundaries - -The v2 runtime is intended to: - -- parse caller-provided text without network access; -- make no filesystem writes during parsing; -- load only its bundled small model; -- require no runtime dependency; -- preserve rather than execute unparsed input. - -Changes to packaging, resource loading, training/evaluation data, generated -models, GitHub Actions, build provenance, and release credentials are -security-sensitive. Pull-request workflows must remain unprivileged and must -not execute contributor code in a privileged `pull_request_target` context. - -Incorrect address extraction is normally a correctness issue, not a -vulnerability. Treat it as security-sensitive when it crosses a trust boundary -or can lead to authorization bypass, unsafe file/network access, secret -exposure, code execution, or a practical denial of service. Otherwise use the -parsing-failure template and redact personal data. +The package is designed to run offline with no runtime dependencies. A network +request, hidden download, filesystem write, or process launch during parsing is +a security bug. Incorrect parsing is usually a correctness issue unless it +crosses a trust boundary or causes unsafe authorization, routing, or disclosure. diff --git a/SUPPORT.md b/SUPPORT.md deleted file mode 100644 index d3ac38d..0000000 --- a/SUPPORT.md +++ /dev/null @@ -1,36 +0,0 @@ -# Support - -`address-normalizer` v2 is an alpha maintained on a best-effort basis. There is -no paid support channel or guaranteed response time. - -Before opening an issue: - -1. read the README troubleshooting and product-boundary sections; -2. reproduce on the current v2 commit; -3. reduce address data to a synthetic or safely redacted example; -4. include package version, Python version, installation source, command, full - output, and expected behavior. - -Use the matching GitHub issue form for bugs, parsing failures, feature requests, -or data provenance. Public issues are the support record; do not send private -addresses, production logs, credentials, or large corpora. - -The project can help explain: - -- documented Python and CLI behavior; -- reproducible installation or packaging failures; -- unexpected parsing fields, offsets, warnings, and alternatives; -- evaluation commands and committed metric definitions; -- whether a proposed feature fits the small offline parser boundary. - -The project cannot provide: - -- a current FIAS/GAR database, identifier, or verification result; -- support for a customer-managed resolver, search cluster, or geocoder; -- legal advice about address data or licensing; -- private application debugging or integration consulting; -- guarantees that an alpha result is suitable for an automated business - decision. - -Security-sensitive reports follow [`SECURITY.md`](SECURITY.md) and must not be -disclosed in a public issue. diff --git a/api.py b/api.py deleted file mode 100644 index aa8d420..0000000 --- a/api.py +++ /dev/null @@ -1,159 +0,0 @@ -import pandas as pd -from elasticsearch import Elasticsearch - -from parsing import optimize_for_search, optimize_housenum, preprocess, extract_index, extract_house - -''' -Не работает без Elastic с загруженным туда ФИАС и проиндексированным на поиск родителей каждой строки -''' -es = Elasticsearch() - - -def verify_address(full_address): - ''' - Ищет адрес в ФИАС - Вход: строка - Выход: словарь с полным адресом по ФИАС и его составляющими - ''' - if full_address == '': - return [] - string = optimize_for_search(full_address) - query = { - 'size': 1, - "query": { - "query_string": { - "fields": ["fullname"], - "query": string, - "fuzziness": "auto", - #"use_dis_max": "true" - # "tie_breaker": 0.3 - } - } - } - response = es.search(index='fias_full_text', doc_type='address', body=query) - - if False: # True чтобы добавить в ответ текст запроса - dic = response["hits"]["hits"][0]["_source"] - dic['query'] = string - return dic - - try: - return response["hits"]["hits"][0]["_source"] - except IndexError: # Если не найдено - return [] - - -def verify_home(dic, aoguid, index): - ''' - Ищет конкретный дом на указанной улице - ''' - query = { - "size": 1, - "query": { - "bool": { - "must": [], - "should": [], ## Here goes your stuff - "must_not": [] - } - } - } - cases = query['query']['bool']['should'] - must = query['query']['bool']['must'] - must_not = query['query']['bool']['must_not'] - must.append({"match": { - "AOGUID": aoguid - }}) - if dic.get('дом', False): - must.append({"match": { - "HOUSENUM": optimize_housenum(dic["дом"]) - }}) - if dic.get('корпус', False): - cases.append({"match": { - "BUILDNUM": optimize_housenum(dic["корпус"]) - }}) - else: - must_not.append({"match": - {"BUILDNUM": "*"} - }) - if dic.get('строение', False): - cases.append({"match": { - "STRUCNUM": optimize_housenum(dic['строение']) - }}) - else: - must_not.append({"match": - {"STRUCNUM": '*'} - }) - if index: - cases.append({"match": { - "POSTALCODE": '"' + index + '"' - }}) - - must.append({"bool": { - - }}) - - response = es.search(index='fias_houses', body=query) - if len(response["hits"]["hits"]) == 0: - dic.update({"комментарий": "дом не найден в ФИАС"}) - return dic - else: - response = response["hits"]["hits"][0]["_source"] - - if response["BUILDNUM"] == response["HOUSENUM"]: - response['Корпус/строение'] = response["BUILDNUM"] - elif len(response["BUILDNUM"]) > len(response["HOUSENUM"]): - response['Корпус/строение'] = response["BUILDNUM"] - elif len(response["BUILDNUM"]) > len(response["HOUSENUM"]): - response['Корпус/строение'] = response["BUILDNUM"] - else: - response['Корпус/строение'] = response["BUILDNUM"] - new_dic = {key.lower(): response[key] for key in ["HOUSENUM", "BUILDNUM", "STRUCNUM", "POSTALCODE", "HOUSEID"]} - new_dic.update({'house query': str(query)}) - - return new_dic - - -def standardize(string, origin=True, debug=False): - ''' - Обёртка для всех методов выше. Разделяет адрес на его составляющие и ищет совпадение в ФИАС. В 90+% случаев находит. - Вход: строка с адресом - Выход: составляющие адреса - ''' - dic = {} - if origin: - dic['origin'] = string - - string = preprocess(string) - address, index = extract_index(string) - try: - index = index.strip() - except AttributeError: - pass - address, house = extract_house(address) - dic['index'] = index - dic['address'] = address - dic.update(verify_address(address)) - if dic.get('street', False): - dic.update(verify_home(house, dic['guid'], index)) - dic.update(house) - return dic - - -def get_addr(strings, progress=True): - ''' - Обрабатывает несколько адресов подряд. - Вход: массив строк - Выход: pandas Dataframe - ''' - dics = [] - n = len(strings) - 1 - for i, line in enumerate(strings): - if progress: - print("Working on {0} of {1}. Progress {2:03.1f}%".format(i, n, (i / n) * 100), end='\r') - dics.append(standardize(line)) - return pd.DataFrame(dics) - - -if __name__ == "__main__": - address = standardize("142703, Московская область, Ленинский район, г.Видное, ул. Школьная, д.78") - print(address['address'] + 'д ' + address['дом']) diff --git a/app.py b/app.py deleted file mode 100644 index 4b5cbe9..0000000 --- a/app.py +++ /dev/null @@ -1,45 +0,0 @@ -import api - -addr = "188640, ЛЕНИНГРАДСКАЯ, ВСЕВОЛОЖСКИЙ, СНТ. ТАВРЫ, Д. 422" -norm_addr = api.standardize(addr, True) -print(norm_addr) - -addr = "РОССИЯ,197373,г. Санкт-Петербург,РАЙОН ПРИМОРСКИЙ,Город САНКТ-ПЕТЕРБУРГ,,Проспект ШУВАЛОВСКИЙ,д. 59,кор. 1,,кв. 9" -norm_addr = api.standardize(addr, True) -print(norm_addr) - -addr = "182108, ВЕЛИКИЕ ЛУКИ, НОВЫЙ, Д.15 20" -norm_addr = api.standardize(addr, True) -print(norm_addr) - -addr = "660042, КРАСНОЯРСКИЙ КРАЙ, Г. КРАСНОЯРСК, УЛ. СВЕРДЛОВСКАЯ, Д. 61, КВ. 25" -norm_addr = api.standardize(addr, True) -print(norm_addr) - -addr = "157980, Костромская область, р-н. КАДЫЙСКИЙ Р-Н Поселок городского ти, , Улица БОЛЬНИЧНАЯ, д. 18, , кв. 1" -norm_addr = api.standardize(addr, True) -print(norm_addr) - -addr = "630000, Новосибирская область, г. НОВОСИБИРСК, , Улица АЛЕКСАНДРА-НЕВСКОГО, д. 6, , кв. 10" -norm_addr = api.standardize(addr, True) -print(norm_addr) - -addr = "190000, г. Санкт-Петербург, г. САНКТ-ПЕТЕРБУРГ, , Улица КУБИНСКАЯ, д. 28, , кв. 100" -norm_addr = api.standardize(addr, True) -print(norm_addr) - -addr = "115114, РОССИЯ, г Москва, Павелецкий 3-й проезд, д.6, корп.А, кв.58" -norm_addr = api.standardize(addr, True) -print(norm_addr) - -addr = "643,РОССИЯ,452920,02,БАШКОРТОСТАН РЕСП,,АГИДЕЛЬ Г,,СТУДЕНЧЕСКАЯ УЛ,14,,,20" -norm_addr = api.standardize(addr, True) -print(norm_addr) - -addr = "423330, 423330,РЕСПУБЛИКА ТАТАРСТАН,Г. АЗНАКАЕВО,,УЛИЦА ШАЙХУТДИНОВА,КВ. 2, Д. 9, ,, г. АЗНАКАЕВО, , , , ," -norm_addr = api.standardize(addr, True) -print(norm_addr) - -# addr = "москва, ленина ул. , д.6" -# norm_addr = api.standardize(addr, True) -# print(norm_addr) \ No newline at end of file diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..c5ab30b --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,88 @@ +# Benchmarks and known failures + +The parser extracts text; it does not resolve an address against FIAS/GAR. +Results from different datasets use different annotation and matching rules and +must not be averaged into one “accuracy” number. + +## Release results + +| Dataset | Rows/windows | Metric | Precision | Recall | F1 / rate | +| --- | ---: | --- | ---: | ---: | ---: | +| Historical address sample | 500 | exact component values, micro | 97.0% | 94.8% | **95.9%** | +| Historical address sample | 500 | every field exact | — | — | **80.4%** | +| Noisy address snippets | 578 | same-label span overlap | 60.0% | 57.5% | **58.7%** | +| Noisy street+house slice | 144 | same-label span overlap | 81.7% | 76.2% | **78.8%** | +| Clean nationwide addresses | 100,000 | character overlap | 72.8% | 60.8% | **66.2%** | +| Moscow buildings | 15,196 | exact component values, micro | 85.4% | 85.3% | **85.4%** | +| Free-message detector | 144 positives | span overlap | 98.0% | 68.1% | **80.3%** | +| Free-message detector | negative messages | specificity | — | — | **100%** | + +The detector’s specificity result belongs to its evaluated negative slice, not +to arbitrary production traffic. + +## Historical 500-row sample + +This is the only benchmark run in normal CI: + +```bash +python tools/benchmark.py --check +``` + +The command reads [`legacy_500.jsonl`](legacy_500.jsonl). +It passes when exact-address rate is at least 80%, exact-component micro F1 is +at least 95%, and at least 75% of rows have no residual word/number spans. + +Per-field exact-value results: + +| Field | Support | Precision | Recall | F1 | +| --- | ---: | ---: | ---: | ---: | +| postal code | 498 | 100.0% | 100.0% | 100.0% | +| region | 58 | 89.8% | 91.4% | 90.6% | +| district | 11 | 90.9% | 90.9% | 90.9% | +| city | 496 | 99.0% | 99.0% | 99.0% | +| settlement | 9 | 80.0% | 44.4% | 57.1% | +| street | 500 | 96.0% | 96.0% | 96.0% | +| street type | 500 | 94.8% | 87.8% | 91.2% | +| house | 500 | 95.8% | 95.0% | 95.4% | +| корпус | 75 | 100.0% | 92.0% | 95.8% | +| строение | 137 | 97.1% | 97.1% | 97.1% | +| apartment/unit | 95 | 96.2% | 80.0% | 87.4% | + +The sample contains 402 exact rows and 98 failures. The most common primary +failure hypotheses are: + +| Cause | Rows | +| --- | ---: | +| conflicting street markers | 22 | +| reference infers an absent street type | 20 | +| ambiguous or unsupported abbreviation | 15 | +| unmarked numeric roles | 14 | +| compound or letter-suffixed number boundary | 7 | +| administrative label/boundary | 6 | + +[`legacy_500_results.csv`](legacy_500_results.csv) +contains every row, +expected and actual values for every field, scenario flags, mismatch category, +unparsed spans, warnings, and a narrow failure hypothesis. These diagnoses are +triage aids, not independently adjudicated ground truth. + +## Interpreting the other datasets + +- The noisy snippet benchmark uses oracle-cropped address windows and + same-label span overlap. It is useful for messy input, not end-to-end message + detection. +- The nationwide set is large and clean but uses a schema adapter and lenient + character overlap. +- The Moscow set is official clean building data. House F1 is 98.2%, while + exact street F1 is 66.2%; this gap is more useful than its aggregate. +- The free-message benchmark measures conservative span detection. Exact-span + F1 is much lower than overlap F1, so callers must inspect returned offsets. + +The bundled 37 KB sequence model was trained with a deterministic, +group-disjoint 70/15/15 split of the historical rows. Its held-out token +accuracy is 96.5%, complete-sequence accuracy 94.3%, and micro entity F1 96.9%. +The split has little district and settlement coverage. + +The maintainer authored and authorized redistribution of the historical source +rows and derived model under GPL-3.0-only. External benchmark source corpora are +not included in this repository. diff --git a/evaluation/legacy_reference_500.jsonl b/benchmarks/legacy_500.jsonl similarity index 100% rename from evaluation/legacy_reference_500.jsonl rename to benchmarks/legacy_500.jsonl diff --git a/evaluation/legacy_reference_500_diagnostics.csv b/benchmarks/legacy_500_results.csv similarity index 100% rename from evaluation/legacy_reference_500_diagnostics.csv rename to benchmarks/legacy_500_results.csv diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 06041b1..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,91 +0,0 @@ -version: '2.2' -services: - es01: - image: docker.elastic.co/elasticsearch/elasticsearch:7.7.1 - container_name: es01 - restart: always - environment: - - node.name=es01 - - cluster.name=es-docker-cluster - - discovery.seed_hosts=es02,es03 - - cluster.initial_master_nodes=es01,es02,es03 - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - data01:/usr/share/elasticsearch/data - ports: - - 9200:9200 - networks: - - elastic - - es02: - image: docker.elastic.co/elasticsearch/elasticsearch:7.7.1 - container_name: es02 - restart: always - environment: - - node.name=es02 - - cluster.name=es-docker-cluster - - discovery.seed_hosts=es01,es03 - - cluster.initial_master_nodes=es01,es02,es03 - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - data02:/usr/share/elasticsearch/data - ports: - - 9201:9201 - networks: - - elastic - - es03: - image: docker.elastic.co/elasticsearch/elasticsearch:7.7.1 - container_name: es03 - restart: always - environment: - - node.name=es03 - - cluster.name=es-docker-cluster - - discovery.seed_hosts=es01,es02 - - cluster.initial_master_nodes=es01,es02,es03 - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - data03:/usr/share/elasticsearch/data - ports: - - 9202:9202 - networks: - - elastic - - kib01: - image: docker.elastic.co/kibana/kibana:7.7.1 - restart: always - container_name: kib01 - ports: - - 5601:5601 - environment: - ELASTICSEARCH_URL: http://es01:9200 - ELASTICSEARCH_HOSTS: http://es01:9200 - networks: - - elastic - -volumes: - data01: - driver: local - data02: - driver: local - data03: - driver: local - -networks: - elastic: - driver: bridge diff --git a/docs/good-first-issues.md b/docs/good-first-issues.md deleted file mode 100644 index 496b5be..0000000 --- a/docs/good-first-issues.md +++ /dev/null @@ -1,116 +0,0 @@ -# Good first issue proposals - -These are issue drafts, not work already authorized. A maintainer should assign -an owner, confirm the acceptance criteria still match `main`, and create the -issue before adding `good first issue`. - -All proposals are ready for discussion under the repository's GPL-3.0-only -contribution terms. - -## Available now: minimal missing-marker failure set - -**Why it matters:** Current external reports show that unmarked administrative -and street fields are weaker than numeric fields. Small public reproductions -help describe that boundary without sharing production data. - -**Scope** - -- Create three synthetic Russian address inputs covering distinct missing-marker - patterns. -- For each, record `parse(...).as_dict()`, expected component values, exact - `[start, end)` offsets, and why the expected interpretation is unambiguous. -- Submit them in one parsing-failure issue; do not change code or tests. - -**Acceptance criteria** - -- No real person or private address is used. -- Examples are not copied from committed sealed-test failure samples. -- Every expected `raw` value equals the source slice at its proposed span. -- The three examples represent different patterns, not spelling variants. - -## Available now: resolver contract field review - -**Why it matters:** The generic resolver example must be understandable to teams -that operate different FIAS/GAR indexes. - -**Scope** - -- Run `examples/fias_gar_http.py` in dry-run mode on two synthetic addresses, - including one with an alternative. -- Review whether `raw`, `components`, `warnings`, and `alternatives` are enough - to adapt at an application boundary. -- Open one documentation issue with confusing names or missing explanation; do - not propose a universal FIAS/GAR API. - -**Acceptance criteria** - -- No network request or private endpoint is used. -- The issue distinguishes parser output from resolver verification. -- Suggestions do not add FIAS/GAR data, authentication policy, or a network - dependency to the package. - -## CLI stdin and JSONL contract tests - -**Why it matters:** The CLI is the smallest integration surface for shell and -batch users, but its line-preservation behavior should be executable -documentation. - -**Scope** - -- Add subprocess tests for one positional address, single-address stdin, JSONL - order, CRLF input, Unicode, and a blank JSONL line. -- Assert JSON structure and process exit status, not whitespace formatting - except where the CLI contract requires it. - -**Acceptance criteria** - -- Each test fails for a demonstrated contract break, not only a fabricated - internal change. -- Tests invoke the installed entry point or the documented module boundary. -- No network, temp data outside the test directory, or timing assertion. - -## Public typed-dictionary example check - -**Why it matters:** `as_dict()` is the JSON boundary and exports typed-dictionary -shapes. A small checked example can catch documentation drift. - -**Scope** - -- Add a type-check fixture assigning `parse(...).as_dict()` to - `ParsedAddressDict`. -- Exercise one optional component, one `AddressPartDict`, warnings, and - alternatives. -- Document the selected type checker and exact command. - -**Acceptance criteria** - -- No new runtime dependency. -- The type-check dependency remains development-only. -- The fixture checks public imports rather than private implementation types. -- The command is run in CI only after the maintainer agrees on the type-check - policy. - -## Span-verification recipe - -**Why it matters:** Consumers need a safe way to prove every extracted `raw` -substring maps back to the original input. - -**Scope** - -- Add a compact example that iterates every populated part plus `unparsed` and - asserts `result.raw[start:end] == part.raw`. -- Cover Unicode whitespace and `ё` without normalizing the source first. -- Link it from the README review policy. - -**Acceptance criteria** - -- Uses only the public API and standard library. -- Demonstrates original-text offsets, not offsets into `normalized`. -- Includes a runnable command and expected output. - -## Tasks that are not good first issues - -Do not label compact-model retraining, benchmark threshold changes, new runtime -dependencies, public API redesign, release workflows, license selection, or -FIAS/GAR resolution as starter tasks. They require maintainer decisions and -cross-domain review. diff --git a/docs/launch.md b/docs/launch.md deleted file mode 100644 index f7cec08..0000000 --- a/docs/launch.md +++ /dev/null @@ -1,264 +0,0 @@ -# Launch drafts and evidence plan - -**Status: release facts verified; destination-specific posts remain drafts.** - -The maintainer approved GPL-3.0-only licensing, the recorded reference-data and -model provenance, and publication of `2.0.0a2` on 2026-07-29. The drafts below -are ready for final editorial review after the public package and release URLs -resolve. Nothing in this file authorizes automated posting to third-party -channels. - -## One-line problem and solution - -> Extract typed fields and original offsets from messy Russian addresses in a -> small offline Python package—then verify them against the FIAS/GAR resolver -> you already control. - -This line deliberately says “extract,” not “validate,” “resolve,” or “geocode.” - -## Facts to verify immediately before launch - -- release tag and package version; -- public license and exact repository/path scope; -- PyPI and GitHub release URLs; -- Python support matrix; -- clean wheel install and CLI/API output; -- wheel and bundled-model byte sizes; -- runtime dependency list; -- all committed report checksums and baseline values; -- current limitations and unresolved data terms; -- release notes and migration boundary. - -Never copy an approximate size or score from an older draft into a release. - -## Draft: GitHub release - -### address-normalizer 2.0.0a2: a small offline Russian address parser - -`address-normalizer` v2 extracts typed address components while preserving raw -substrings, character offsets, warnings, unparsed text, and alternative -interpretations. - -```python -from address_normalizer import parse - -result = parse("Ополченская 5-30") -print(result.normalized) -# Ополченская, д 5, кв 30 -print(result.warnings) -# ("ambiguous_numeric_tail",) -``` - -What is intentionally outside the package: - -- no bundled FIAS/GAR database; -- no address verification, identifier lookup, or geocoding; -- no Elasticsearch or service process; -- no hidden downloads or network calls; -- no runtime dependencies. - -The wheel is `45,843` bytes and its bundled model is `37,130` bytes in this -release. Independent benchmark results are published by domain rather than -averaged in the -[README reliability table](https://github.com/shigabeev/address-normalizer#reliability). -Numeric building fields are currently stronger than administrative fields and -exact street extraction. - -Install: `python -m pip install --pre address-normalizer==2.0.0a2` - -Documentation: [English README](https://github.com/shigabeev/address-normalizer#readme) -and [Russian README](https://github.com/shigabeev/address-normalizer/blob/master/README.ru.md) - -Migration notes: [Changelog](https://github.com/shigabeev/address-normalizer/blob/master/CHANGELOG.md) - -License: GPL-3.0-only for the repository and distributed package. - -## Draft: Show HN - -**Title** - -> Show HN: address-normalizer – small offline parsing for messy Russian addresses - -**Text** - -I rebuilt an old Russian address project around a narrower boundary. The new -Python package extracts components and preserves source offsets, warnings, -unparsed text, and alternatives. It does not bundle FIAS/GAR, run -Elasticsearch, verify existence, or make network calls. - -The motivating example is `Ополченская 5-30`: the parser selects house `5` and -apartment `30`, but retains compound house `5-30` as an alternative for a -downstream resolver. - -The runtime has no dependencies; verified artifact sizes for this release are -`45,843` and `37,130` bytes. The README publishes four non-comparable -benchmark domains and their limitations instead of one headline “accuracy” -number. Current weaknesses are administrative recall and exact street -boundaries. - -I would value feedback on the typed result contract, ambiguity handling, and -the boundary between extraction and a customer-managed FIAS/GAR resolver: -https://github.com/shigabeev/address-normalizer - -## Draft: Habr - -**Заголовок** - -> Маленький офлайн-парсер российских адресов без встроенного ФИАС и Elasticsearch - -**Лид** - -Старый `address-normalizer` решал сразу две задачи: разбирал строку и искал -адрес в заранее загруженном ФИАС. В версии 2 граница уже: библиотека только -извлекает компоненты, сохраняет исходные смещения, неоднозначности и -неразобранный остаток. Проверка существования и выбор идентификатора остаются -за актуальным справочником пользователя. - -**План текста** - -1. Почему парсинг и разрешение адреса — разные задачи. -2. Разбор `Ополченская 5-30`: выбранный вариант и сохранённая альтернатива. -3. Конвейер: токенизация → правила → компактная модель → постобработка. -4. Типизированный API и JSONL без runtime-зависимостей. -5. Как передать поля в собственный ФИАС/ГАР-сервис. -6. Четыре отдельных бенчмарка и почему их нельзя усреднять. -7. Слабые места: административные поля, точные границы улиц, неоднозначные - числовые хвосты. -8. Размеры артефактов, воспроизводимая команда и планы после alpha. - -**Финал** - -Исходники и методика: https://github.com/shigabeev/address-normalizer. -Особенно полезны синтетические -примеры ошибок с ожидаемыми полями и смещениями; реальные частные адреса -публиковать не нужно. - -## Draft: Reddit / r/Python - -**Title** - -> address-normalizer v2: dependency-free, offline Russian address field extraction - -**Body** - -I released `2.0.0a2` of a small Python library that extracts Russian -address components and preserves original character spans. It is intentionally -not a FIAS/GAR database, validator, geocoder, or service. - -The runtime has no dependencies and makes no network calls. `parse`, -`parse_iter`, and `parse_many` return frozen typed results with warnings, -alternatives, and JSON-compatible serialization. The README includes a FastAPI -wrapper, streaming JSONL ETL, and a generic boundary for a resolver you operate. - -I have kept the evidence separated across historical regression, noisy address -windows, nationwide clean strings, and an official Moscow snapshot. The weakest -current areas are administrative fields and exact street extraction. - -Repository and reproducible numbers: https://github.com/shigabeev/address-normalizer - -Feedback on API ergonomics and failure reporting is welcome. Please use -synthetic or redacted addresses. - -## Draft: Telegram / LinkedIn - -> `Ополченская 5-30` — это дом 5, квартира 30 или дом 5-30? -> -> `address-normalizer` v2 разбирает российские адресные строки офлайн, сохраняет -> исходные смещения и не скрывает неоднозначность. Внутри нет ФИАС/ГАР, -> Elasticsearch, сетевых запросов и runtime-зависимостей: библиотека извлекает -> кандидатов, а актуальный справочник пользователя их проверяет. -> -> В README есть типизированный API, JSONL, FastAPI-пример, интеграционная граница -> с собственным resolver и четыре раздельных бенчмарка с ограничениями. -> -> https://github.com/shigabeev/address-normalizer/releases/tag/v2.0.0a2 - -Before using this short post, add the verified license and release status in the -linked page; do not let brevity conceal them. - -## Reproducible benchmark command - -The small committed regression requires no external dataset download: - -```bash -python evaluation/evaluate.py \ - --data evaluation/legacy_reference_500.jsonl \ - --gates evaluation/release_gates.json -``` - -Record the commit SHA, Python version, full JSON output, wall-clock environment, -and whether the gate passed. Describe it as a historical silver regression, not -an independent nationwide production score. - -The large and external benchmark commands live in `evaluation/README.md`. Run -them only in a separate environment with pinned data artifacts. Report all -defined metrics and per-field results; do not rerun a sealed set repeatedly -while tuning. - -## Alternative-comparison methodology - -A fair comparison starts by aligning product boundaries. - -1. **Classify the alternative.** Is it an extractor, address validator, - FIAS/GAR resolver, geocoder, tokenizer/model, or complete service? Do not - rank different tasks on one “accuracy” axis. -2. **Pin public versions.** Record package/service version, source revision, - configuration, model/data revision, date, Python/platform, and exact command. -3. **Use permitted, task-matched data.** Separate noisy user input, clean - registry strings, administrative-only text, and historical compatibility. - Prevent entity/building groups from leaking across splits. -4. **Map schemas in writing.** Publish every field mapping and unsupported - field. Do not score corrected registry values as extractor truth when those - values do not occur in the input. -5. **Report multiple exact metrics.** At minimum publish exact component values, - exact full-address rows, character/span behavior, per-field precision/recall/ - F1, abstentions/unparsed evidence, and failure slices when supported. -6. **Measure operations separately.** Report artifact/download size, runtime - dependencies, cold/warm latency or throughput, peak memory, required service - or registry, network behavior, and hardware. Never infer an unmeasured value. -7. **Preserve failure evidence.** Count extra fields, missing fields, - alternatives, and discarded text. Do not award a cleaner score for hiding - ambiguity. -8. **Invite correction.** Publish the harness, raw aggregate report, license/ - provenance notes, and a contact path. Label unavailable results - “not measured,” not zero. - -Before naming a competitor in public, verify its current official documentation -and reproduce the claim. Do not repeat marketing copy, old architecture sizes, -or license assumptions as fact. - -## Release demo / terminal recording plan - -Target length: 75–100 seconds, one continuous recording, no edits that hide -installation or network activity. - -1. Start in a new virtual environment with network disabled after the wheel is - already available locally. -2. Show the wheel filename and exact byte size. -3. Install the wheel from its local path and show that no runtime dependency is - installed. -4. Run `Ополченская 5-30`; point to spans, warning, and compound-house - alternative rather than only normalized text. -5. Pipe two lines through `address-normalizer --jsonl`. -6. Run the FIAS/GAR HTTP example without `--send` and explain that it prints an - application-owned resolver request but performs no network call. -7. Show the four-domain benchmark table and the administrative/street - limitations. -8. End on the install command, repository URL, license, and alpha status. - -Save the command transcript beside the recording. Do not show tokens, internal -resolver URLs, shell history, private addresses, local usernames, or unpublished -benchmark data. - -## Launch checklist - -- [x] The licensing and provenance blockers are resolved in writing. -- [ ] A release actually exists at every linked URL. -- [x] Placeholder markers are gone. -- [ ] Install, API, CLI, wheel, and benchmark commands were rerun from the tag. -- [x] Sizes and metrics match the candidate artifacts. -- [x] Limitations remain adjacent to the claims they qualify. -- [x] No post implies validation, FIAS ID lookup, geocoding, or calibrated - confidence. -- [ ] Maintainer approved each destination-specific draft. -- [x] Nothing has been posted by automation. diff --git a/docs/ml-stack.md b/docs/ml-stack.md deleted file mode 100644 index 60ba04e..0000000 --- a/docs/ml-stack.md +++ /dev/null @@ -1,118 +0,0 @@ -# ML stack and runtime boundaries - -## Short answer - -`address-normalizer` is a hybrid rules-and-ML extractor. It is not a -transformer, neural network, LLM, or registry-backed parser. - -The only learned runtime component is a 37,130-byte sparse linear-chain -sequence tagger. It labels residual word tokens after deterministic syntax has -already extracted explicit address components. Message-level address detection -is currently rule-based. - -The package has no runtime dependencies outside the Python standard library and -makes no network calls. - -## Runtime pipeline - -| Stage | Implementation | Learned? | Responsibility | -| --- | --- | --- | --- | -| Message detection | Regular expressions, clause boundaries, evidence scoring, parser validation | No | Find conservative street-and-building candidate spans in prose | -| Tokenization | Offset-preserving regular-expression tokenizer | No | Split words, numbers, and punctuation without losing source offsets | -| Explicit extraction | Marker dictionaries and numeric grammars | No | Extract postcode, region/city/street markers, house, корпус, строение, and apartment | -| Residual labeling | Sparse linear-chain sequence tagger with Viterbi decoding | Yes | Label remaining unmarked words as region, district, city, settlement, street, or other | -| Post-processing | Deterministic boundary and numeric-tail heuristics | No | Fill plausible implicit street/house fields and retain warnings or alternatives | -| Registry resolution | Not part of this package | N/A | Verify existence, resolve abbreviations in context, choose a FIAS/GAR object, and return canonical values | - -Direct address strings enter at tokenization. Free-form messages first pass -through detection; every accepted detection is then parsed by the same address -parser. - -## The learned model - -The runtime model is an epoch-averaged structured perceptron: - -- labels: `O`, `REGION`, `DISTRICT`, `CITY`, `SETTLEMENT`, `STREET`; -- representation: sparse emission and transition weights serialized as JSON; -- decoding: first-order Viterbi sequence decoding; -- token features: lowercased word, word/number/punctuation kind, one- and - two-character prefixes, one- to three-character suffixes, sequence position, - neighboring token text and kind, and digit length; -- artifact: `src/address_normalizer/data/model.json`; -- artifact size: 37,130 bytes; -- stored parameters: 1,240 emission weights and 39 transition weights; -- training algorithm and inference: Python standard library only. - -This is closest in spirit to a small CRF-style linear sequence model, but it is -trained with a structured perceptron objective. It does not calculate neural -embeddings, use pretrained language representations, or produce calibrated -probabilities. - -The model only sees residual word tokens that rules have not consumed. It -therefore does not learn house-number grammar, address detection in prose, or -registry identity. - -## Training data and split - -The current model corpus is derived from the 500-row historical reference: - -- 429 sequence examples from 108 canonical address groups; -- deterministic 70/15/15 group split using SHA-256; -- 301 training, 58 validation, and 70 test sequence examples; -- epoch count selected on validation only; -- final artifact retrained on the 359 train-plus-validation examples; -- 10 selected epochs and seed `2017`. - -The group split prevents variants of the same canonical administrative/street -identity from appearing across train and test. The final 70-example sequence -test has 115 tokens, so its 96.5% token accuracy and 96.9% entity F1 are useful -regression evidence but not a production-scale claim. It contains no -independently measured district or settlement tokens. - -The much larger Deepparse and Moscow datasets are evaluation sources and -potential future training material. They are not used by the current bundled -model. - -## What FIAS/GAR changes - -Text extraction and registry resolution are different tasks. - -The text `пр. Ленина` is locally ambiguous because `пр.` can abbreviate more -than one street type. A full location and building can nevertheless identify -one registry object. Likewise, whether `с. 1` or `4А` is a building/unit role -may be resolved by the set of valid objects at the rest of the address. - -The intended production boundary is therefore: - -1. detect an inclusive address span in a message; -2. extract source-faithful component candidates and offsets; -3. query a current FIAS/GAR index with all available context; -4. rank registry candidates and return the canonical object; -5. retain the original text and parser warnings for auditability. - -The parser should not be trained to fabricate a registry-backed value that is -absent from the text merely to match a historical reference. The resolver can -enrich or correct the extracted candidate because it has the missing registry -knowledge. - -## Current production limitation - -The message detector is deliberately high-precision and rule-based. Its -complete-message RedMadRobot diagnostic has 98.0% overlap precision but only -68.1% recall. That makes it suitable when false positives are expensive, but -not yet sufficient when production requires finding most implicit, malformed, -or markerless addresses. - -A higher-recall production system should add a separately trained -message-level address-span model on representative messages, while keeping -parsing and FIAS/GAR resolution as distinct measured stages: - -```text -message -> address-span detector -> component parser -> FIAS/GAR resolver -``` - -Each stage should have its own metric: span recall/precision, conditional -component accuracy, resolver top-k recall, and end-to-end business success. - -All committed benchmark outputs and their precise scopes are indexed in -[`evaluation/RESULTS.md`](../evaluation/RESULTS.md). diff --git a/docs/releasing.md b/docs/releasing.md deleted file mode 100644 index d766751..0000000 --- a/docs/releasing.md +++ /dev/null @@ -1,98 +0,0 @@ -# Release process - -The GPL-3.0-only license and compact-model provenance are recorded in -`LICENSING.md` and enforced by `release-policy.toml`. - -## Version strategy - -`src/address_normalizer/__init__.py` is the single version source. Setuptools -reads `__version__` into package metadata; do not add a second literal version -to `pyproject.toml`. - -The build gate uses the fixed `SOURCE_DATE_EPOCH` shown below. Setuptools already -produces a byte-identical wheel with that epoch; the gate also canonicalizes -sdist tar ownership, modes, ordering, timestamps, and gzip metadata before -requiring two byte-identical builds. Do not replace this with a hash comparison -that tolerates differences. - -Use PEP 440 versions and matching `v`-prefixed Git tags: - -| Stage | Package version | Tag | Intended use | -| --- | --- | --- | --- | -| Alpha | `2.0.0a2` | `v2.0.0a2` | API and model may still change | -| Beta | `2.0.0b1` | `v2.0.0b1` | Feature-complete external testing | -| Release candidate | `2.0.0rc1` | `v2.0.0rc1` | Stable API, release fixes only | -| Stable | `2.0.0` | `v2.0.0` | Supported public release | - -Development snapshots, if needed, use `.devN` and are not published. Increment -the pre-release number rather than replacing an existing artifact. PyPI files -are immutable release records. - -## Release checklist - -### Maintainer decisions - -- [x] Record the chosen license and its exact scope. -- [x] Review and record historical contribution provenance. -- [x] Confirm redistribution rights for the compact model and its source data. -- [x] Add the license file and PEP 639 `license`/`license-files` metadata. -- [x] Set both statuses in `release-policy.toml` to `approved` and record - repository-relative evidence files for each decision. -- [x] Change the CI license gate from `--expect-blocked` to - `--require-publishable`. - -### Version and evidence - -- [ ] Move relevant entries from `Unreleased` into a dated changelog section. -- [ ] Set `address_normalizer.__version__` to the intended PEP 440 version. -- [ ] Run all unit, type, deterministic-model, and fixed regression gates. -- [ ] Review each external benchmark separately; do not average domains or tune - against a sealed final split. -- [ ] Confirm documentation describes confidence as decision strength, not - calibrated probability. - -### Distribution - -Release tooling requires Python 3.12, matching GitHub Actions. This does not -change the package's Python 3.10+ runtime support. Run from a clean checkout: - -```bash -SOURCE_DATE_EPOCH=1704067200 \ - python scripts/build_reproducibly.py --output dist -python scripts/check_artifacts.py dist --write-manifest artifact-manifest.json -python scripts/test_artifact_policy.py dist -python -m twine check dist/*.whl dist/*.tar.gz -``` - -- [ ] Confirm the wheel has no runtime dependencies. -- [ ] Confirm the model is at most 64 KiB and the wheel at most 256 KiB. -- [ ] Inspect `artifact-manifest.json` checksums and source revision. -- [ ] Install the wheel into a clean virtual environment with - `--no-index --no-deps`. -- [ ] Run `scripts/smoke_installed.py` with isolated Python from outside the - checkout. -- [ ] Confirm the wheel and sdist contain no corpora, caches, notebooks, - training data, legacy files, credentials, or generated bytecode. - -### GitHub and PyPI - -- [ ] Configure the PyPI project to trust this repository's `release.yml` - workflow and its protected `pypi` environment. -- [ ] Require maintainer approval on the `pypi` environment. -- [ ] With explicit maintainer approval, create and push the signed tag matching - the package version (for example, `v2.0.0rc1`). -- [x] Enable publication only after all licensing gates pass. -- [ ] Run the workflow manually from the matching tag, first for `testpypi` and - then for `pypi`; the selected ref, entered version, checkout commit, and - wheel metadata must all match. -- [ ] Review the build artifact and provenance manifest before approving the - environment deployment. - -Never use a long-lived PyPI API token. The workflow grants `id-token: write` -only to the protected publish job and uses PyPI Trusted Publishing. - -## Rollback - -Published files cannot be replaced. If a release is wrong, yank it on PyPI, -document why in the changelog, fix forward with a new version, and preserve the -original checksums and provenance record. diff --git a/docs/triage.md b/docs/triage.md deleted file mode 100644 index 3881af6..0000000 --- a/docs/triage.md +++ /dev/null @@ -1,106 +0,0 @@ -# Issue and pull-request triage - -This guide makes review predictable without pretending maintainers have an SLA. -It is guidance for humans; no labels or repository settings are applied by -automation. - -## First response - -For each new issue: - -1. remove public secrets or personal address data from view and follow the - security/privacy route; -2. confirm it concerns v2 rather than the unsupported historical v1 service; -3. request the exact version, minimal command/input, current output, and expected - result if missing; -4. reproduce before labeling a parser behavior as a confirmed bug; -5. identify the domain: synthetic, noisy/user-entered, clean registry, - historical compatibility, or unknown; -6. check for a duplicate and link the canonical issue; -7. keep proposed implementation separate from the user problem. - -Parsing failures should preserve punctuation, Unicode, output spans, warnings, -alternatives, and unparsed content. Prefer synthetic minimal cases; never ask -for a private corpus dump in a public issue. - -## Suggested labels - -| Label | Suggested color | Use | -| --- | --- | --- | -| `needs-triage` | `D4C5F9` | Reproduction or ownership has not been established | -| `bug` | `D73A4A` | Confirmed behavior contradicts the documented contract | -| `parsing` | `B60205` | Component, span, warning, or ambiguity behavior | -| `cli` | `1D76DB` | Command-line and JSONL behavior | -| `api` | `0052CC` | Public Python API or serialized result | -| `documentation` | `0075CA` | User or contributor documentation | -| `evaluation` | `5319E7` | Metrics, gates, adapters, or benchmark reports | -| `data` | `7057FF` | Dataset or generated-data concern | -| `provenance` | `8A2BE2` | Source, terms, attribution, or redistribution evidence | -| `packaging` | `0E8A16` | Build, wheel, install, metadata, or compatibility | -| `security` | `B60205` | Public tracking only after private disclosure is safe | -| `good first issue` | `7057FF` | Bounded task with exact acceptance criteria and mentor | -| `help wanted` | `008672` | Maintainer has defined scope and will review work | -| `blocked: decision` | `FBCA04` | Explicit maintainer/product choice is required | -| `needs-reproduction` | `FEF2C0` | Report lacks a locally repeatable case | -| `needs-provenance` | `F9D0C4` | Data/model source or permission evidence is incomplete | -| `duplicate` | `CFD3D7` | Canonical issue is linked before closing | -| `wontfix` | `FFFFFF` | Intentionally outside scope, with reason documented | - -Do not use `good first issue` on vague refactors, broad parser improvement, -model retraining, release workflows, or tasks blocked by an unstated design -decision. - -## Parsing failure disposition - -- **Confirmed regression:** add `bug` and `parsing`; record the last known good - version if known. -- **Known limitation:** link the README limitation, retain the example if it - adds a meaningful input class, and avoid promising a fix. -- **Resolver responsibility:** explain that extraction does not verify existence - or choose a FIAS/GAR ID. -- **Ambiguous ground truth:** preserve alternatives; do not force a single label - merely to close the issue. -- **Private or licensed data:** remove it from public view and ask for a - synthetic reproduction. - -A public failure example should become a regression test only after provenance, -privacy, expected spans, and the licensing/contribution policy allow it. - -## Pull-request review - -For eligible changes, require: - -- a linked issue or complete reproduced bug; -- a test that demonstrates failure before and success afterward; -- the author's own explanation of behavior and maintenance implications; -- exact focused and full-suite results; -- before/after metrics for every relevant benchmark domain; -- model and wheel size deltas where relevant; -- preservation of offsets, ambiguity, warnings, alternatives, and unparsed - evidence; -- no hidden network, registry, service, data, or runtime-dependency expansion. - -An unexplained generated patch is not reviewable evidence. Ask the author to -reduce it and explain it; do not reverse-engineer a bulk submission on their -behalf. - -## Benchmark-resistant review - -Reject or redesign a change when it: - -- tunes against a sealed test while continuing to describe it as untouched; -- improves only a lenient overlap metric while exact value or another field - regresses; -- averages historical, noisy, nationwide-clean, and Moscow-clean domains; -- discards unparsed text or ambiguity to manufacture a cleaner score; -- moves data-preparation dependencies into the runtime; -- increases model/package size without a measured, justified tradeoff. - -Require a validation-driven decision and report every measured domain -separately, including rejected regressions. - -## Closing language - -Be direct about scope. A useful close explains which documented boundary applies, -links the canonical issue or recipe, and states what new evidence would justify -reopening. “Not planned” without an explanation is not sufficient. diff --git a/evaluation/DATA_SOURCES.md b/evaluation/DATA_SOURCES.md deleted file mode 100644 index 9256cac..0000000 --- a/evaluation/DATA_SOURCES.md +++ /dev/null @@ -1,83 +0,0 @@ -# External data sources - -These sources complement the historical workbook. None should be added to model -training until its role, license, and split policy are recorded. - -## 1. RedMadRobot Russian PII NER benchmark - -- Source: -- License: MIT -- Size: 2,841 sentences, including 1,252 location/address entity spans -- Useful subset: 493 rows, 578 address windows, 1,010 fields supported by this - parser -- Strength: manually BIO-annotated, production-log-shaped inputs, hard negatives -- Limitation: personal values are replaced; some examples are synthetic - document templates; the adapter uses gold annotations to crop address windows -- Status: integrated as an independent external regression benchmark - -The exact revision and file SHA-256 are pinned in -`evaluate_redmadrobot.py`. The source CSV is downloaded into `.cache/` and is -not committed or used for training. - -## 2. Deepparse worldwide addresses, Russia configuration - -- Source: -- License: CC BY 4.0 -- Pinned candidate revision: `cb61e5e49db87f8c3586b5494149f612460f8992` -- Russian shard: 13,152,918 annotated addresses; 371,595,309-byte Parquet file -- Fields: street number/name, unit, municipality, district, county, province, - postal code, and country -- Strength: nationwide scale and an independently defined token schema -- Limitation: curated from libpostal/open geographic data rather than raw user - input; punctuation is removed; there is no predefined sealed split -- Integrated filter: retain street/house/unit-bearing rows, validate all token - labels, deduplicate normalized text, map labels to package fields, and group - building identities before deterministic 90/5/5 splitting -- Result: 6,314,158 unique usable rows and a 100,000-row sealed test sample -- Status: integrated as a reproducible external corpus and clean-address - benchmark; the current compact model has not been trained on it - -The committed manifest pins the source and generated artifact checksums. Report -binary span, character-overlap, and token metrics separately from the -RedMadRobot real-input-shape benchmark. - -## 3. Moscow official address registry - -- Source package: - -- Published origin: Moscow open-data portal, dataset ID 60562, Department of - City Property -- Pinned snapshot: version 3.630, released 15 October 2021 -- Size: 440,399 records; 121,581,813-byte download archive -- Fields include full and simplified address strings plus region, city, - settlement, street/road element, house, корпус, строение, room, and FIAS ID -- Strength: official structured truth and building-level identifiers -- Integrated filter: retain active official Moscow/GKN records with valid FIAS - UUID, street, and house; deduplicate normalized simplified addresses; group - building identities before deterministic 90/5/5 splitting -- Result: 307,274 unique usable rows and 15,196 exact-value test rows -- Limitation: Moscow-only, clean legal formatting, and a stale 2021 snapshot -- Terms: the archive embeds Russian government open-data terms; the mirror - describes the package as CC-BY-SA. Confirm redistribution before publishing - derived rows. -- Status: integrated as a reproducible historical official-address benchmark - -## Evaluation policy - -Keep the three domains separate: - -1. historical bank-shaped regression data for compatibility; -2. RedMadRobot address windows for external noisy-input generalization; -3. Deepparse or official registries for broad clean-address coverage. - -Never average them into one headline accuracy number. Publish per-field metrics -and domain-specific slices, and reserve a new untouched dataset before changing -rules or features in response to observed external failures. - -## Local artifacts - -All raw and derived data stays under `.cache/external/` and is ignored by Git. -The runtime wheel contains none of it. The committed `deepparse_manifest.json` -and `datamos_manifest.json` record exact filters, counts, revisions, checksums, -split policies, and limitations; the matching report JSON files record the -first untuned scores. diff --git a/evaluation/FAILURE_ANALYSIS.md b/evaluation/FAILURE_ANALYSIS.md deleted file mode 100644 index d1ac0ec..0000000 --- a/evaluation/FAILURE_ANALYSIS.md +++ /dev/null @@ -1,102 +0,0 @@ -# Failure analysis - -This note reviews the current 500-row historical parsing regression and the -complete-message detection diagnostic. It separates parser behavior, ambiguous -input, and questionable reference expectations instead of treating every -mismatch as the same kind of model error. - -## Historical parsing regression - -Current result: - -- 500 rows; -- 402 exact rows and 98 non-exact rows; -- 80.4% exact-address rate; -- 95.8538% exact component-value micro F1; -- 76.8% of rows with no unparsed word or number span. - -`legacy_reference_500_diagnostics.csv` contains more than 75 columns for every -row. `legacy_reference_500_failure_summary.json` contains the complete primary -and secondary cause counts. - -The primary heuristic cause partitions the 98 non-exact rows: - -| Primary likely cause | Rows | -| --- | ---: | -| Conflicting street markers | 22 | -| Reference infers a street type absent from raw text | 20 | -| Ambiguous or unsupported abbreviation | 15 | -| Unmarked numeric-role ambiguity | 14 | -| Compound/letter-number boundary | 7 | -| Administrative label or boundary | 6 | -| Street-type recognition | 4 | -| Reference conflicts with explicit numeric marker | 3 | -| Numeric component not recognized | 3 | -| Four single-row causes | 4 | - -The primary assignment is a deterministic triage aid. Some rows have secondary -causes, and only a reviewer with authority over the reference schema can -adjudicate whether the parser or expected value should change. - -## Ten representative failures - -These ten were checked against the raw string, expected fields, current output, -and unparsed evidence. This is a behavior review, not verification against an -authoritative address registry. - -| ID | Narrow scenario | Observed mismatch | Reason | -| --- | --- | --- | --- | -| `legacy-good-0005` | `ул.Рязанский проспект` | Expected `пр-кт`, actual `ул` | Two explicit street markers compete. The parser selects the first and leaves `проспект` unparsed. This needs a marker-precedence or ambiguity policy. | -| `legacy-good-0359` | `Батайский проезд, 17, 294` | House becomes `294`; apartment missing | Both numeric roles are unmarked. Rightmost-tail logic selects the last number as house, leaving `17` unparsed. The input cannot be resolved safely without a numeric-role convention or alternative. | -| `legacy-good-0165` | `пр.Ленинского Комсомола` | Expected `пр-кт`, actual type missing | `пр.` is ambiguous between `проспект` and `проезд`; the current grammar does not silently choose. The expected value makes a choice not recoverable from the abbreviation alone. | -| `legacy-good-0751` | `ул.Ореховый бул.` | Street includes `Бул`; type is `ул` instead of `б-р` | Conflicting markers cause first-marker selection and a street boundary error. Both the type decision and value boundary need review. | -| `legacy-good-0089` | `Михайлово-Ярцевское п, Исаково д` | District missing; settlement receives district value | Suffix one-letter administrative markers collide with the parser's prefix-oriented marker grammar, shifting the component label and leaving `Исаково д` unparsed. | -| `legacy-good-0795` | `Ордынка Б., ... с.1` | Punctuation differs; type and structure missing | The reference supplies an implicit `ул`, while `с.` is an ambiguous unsupported short form. The remaining street difference is punctuation-only. This row mixes reference inference, abbreviation policy, and normalization. | -| `legacy-good-0006` | `г. Москва Денисовский переулок` | City absorbs street text; street becomes `Переулок` | A missing separator between city and street causes boundary merging before the suffix street marker is interpreted. | -| `legacy-good-0935` | `Котляковская, 4` | Expected `ул`, actual type missing | No street-type marker exists in the raw string. Scoring an inferred `ул` as an extraction error is a reference-policy issue and should be reviewed before changing the parser. | -| `legacy-good-0029` | `ул. Одоевского` | Expected `пр-д`, actual `ул` | The raw text explicitly says `ул`, while the reference expects `проезд`. The parser follows the source; this is a direct source/reference conflict. | -| `legacy-good-0036` | `стр. 4А` | Expected house `4А`; actual structure `4А` | The raw marker explicitly says `строение`, while the reference places the value in `house_num`. The parser follows the marker; the expected schema needs adjudication. | - -At least 24 rows should receive reference review before parser optimization: -20 inferred-but-absent street types, one explicit street-type conflict, and -three explicit numeric-marker conflicts. Fixing the parser to match these rows -without adjudication would increase the score while making source-faithful -extraction worse. - -## Complete-message address detection - -`evaluate_redmadrobot_detection.py` reconstructs all 2,841 source messages and -defines a gold address window only when a location cluster contains both -`STREET` and `HOUSE`. There are 144 such gold spans in 135 messages; the other -2,706 messages are annotated negatives for this narrow detection definition. - -Current development result: - -| Metric | Result | -| --- | ---: | -| Any-overlap precision | 98.0% | -| Any-overlap recall | 68.1% | -| Any-overlap F1 | 80.3% | -| Exact-boundary F1 | 23.8% | -| Negative-message specificity | 100.0% | - -The 107 non-exact messages carry explicit overlapping reason tags: - -| Detection failure tag | Messages | -| --- | ---: | -| Missed address | 44 | -| Span includes context outside gold | 33 | -| Span drops gold text | 33 | -| Spurious address | 1 | - -The main missed-address scenarios are markerless or transliterated addresses, -reversed component order, unusual abbreviations, and gold spans without the -strong street/building evidence required by the conservative detector. Boundary -differences commonly involve optional city, postcode, country, company/person -text, or trailing unit fields. - -The RedMadRobot detector failures were inspected while developing the current -algorithm. This report is therefore a development diagnostic and must not be -described as untouched final-test performance. A replacement final set needs -complete representative messages, hard negatives, grouped entities, and an -explicit boundary policy. diff --git a/evaluation/README.md b/evaluation/README.md deleted file mode 100644 index 7acf53d..0000000 --- a/evaluation/README.md +++ /dev/null @@ -1,368 +0,0 @@ -# Evaluation - -This directory measures two separate tasks: - -1. **address parsing** after an address string or oracle-cropped window is - already available; -2. **address detection** of half-open address spans inside a free-form message. - -Do not use parsing scores as evidence that the package can find addresses in -arbitrary prose. Do not use the small detection fixture as a production -accuracy claim. - -[`RESULTS.md`](RESULTS.md) indexes every committed JSON/CSV result artifact and -the command that reproduces it. - -## What parsing accuracy currently means - -The historical regression requires case-insensitive exact component values -after whitespace and `ё/е` folding. It reports: - -- exact-address rate: every public component value matches on one row; -- no-unparsed rate: no residual word or number spans remain; -- exact component-value micro precision, recall, and F1; -- the same exact-value metrics per public field. - -RedMadRobot instead uses one-to-one same-label span overlap on gold-cropped -address windows. Deepparse reports binary span overlap, character overlap, -token labels, and exact complete sequences. Moscow uses exact component values. -These metrics and domains are intentionally not interchangeable. - -## Historical 500-row regression - -`legacy_reference_500.jsonl` is a deterministic SHA-256 selection of 500 unique -rows from the `Good` worksheet in `ref/references.xlsx`. A row is eligible only -when: - -- the street and house are present; -- supported administrative fields appear as exact token sequences in the source - address; -- house, корпус/строение, and apartment values fit the parser's numeric schema; -- the legacy hierarchy can be represented by the v2 API. - -This filtering prevents corrected FIAS values from being scored as if the -parse-only package were expected to correct or resolve them. - -The workbook was already published in the historical repository as a reference -sample. The selected rows have **not** been independently re-reviewed during the -v2 work and are not bundled in the wheel. A canonical-grouped subset now trains -the compact tagger, while disjoint groups are reserved for validation and test. -Describe the data as a legacy reference or silver corpus—not a new gold dataset. - -Run the alpha regression gate: - -```bash -python evaluation/evaluate.py \ - --data evaluation/legacy_reference_500.jsonl \ - --gates evaluation/release_gates.json \ - --output evaluation/legacy_reference_500_report.json -``` - -This is the portable release gate: it uses only the committed 500-row fixture, -requires no download, and runs as part of the normal test suite. The report -names its exact-value aggregate `exact_component_value_micro`; the legacy -`micro` key remains as a compatibility alias for existing gate files. -The complete generated result is committed as -[`legacy_reference_500_report.json`](legacy_reference_500_report.json). - -The `2.0.0a1` baseline is: - -- 500 rows; -- 80.4% exact-address match; -- 95.9% micro field F1; -- 76.8% with no residual word or number tokens. - -The evaluator reports precision, recall, and F1 for every public field and keeps -a bounded failure sample. Gate thresholds are intentionally just below the -measured deterministic baseline: they prevent regressions but do not establish -production accuracy. - -### Row-level failure diagnostics - -Generate the complete 500-row diagnostic table and summary: - -```bash -python evaluation/analyze_failures.py -``` - -[`legacy_reference_500_diagnostics.csv`](legacy_reference_500_diagnostics.csv) -contains one row per test example and explicit columns for: - -- expected, actual, and match/missing/extra/wrong status for every field; -- mismatch, missing, extra, and wrong-value field lists; -- parser confidence, warnings, alternatives, and unparsed spans; -- punctuation, Unicode whitespace, marker position, administrative, unit, - compound-number, numeric-sequence, ordinal, and repeated-city scenarios; -- triage priority, failure types, likely causes, and a readable summary. - -The likely-cause fields are deterministic hypotheses for triage. They have not -been independently human-verified and must not be presented as causal ground -truth. The current summary contains 402 exact rows and 98 non-exact rows. -Among those failures, 61 involve `street_type`, 25 `house_num`, 21 `apartment`, -and 20 `street`; one row can contribute to several counts. - -The primary heuristic cause partitions all 98 rows: - -| Primary likely cause | Rows | Interpretation | -| --- | ---: | --- | -| Conflicting street markers | 22 | More than one type marker competes | -| Reference infers absent street type | 20 | Expected type is not explicit in raw text | -| Ambiguous/unsupported abbreviation | 15 | `пр.`, `с.`, `ком.`, or a typo needs review | -| Unmarked numeric-role ambiguity | 14 | Bare numbers can be house/corpus/unit | -| Compound or letter-number boundary | 7 | Slash, hyphen, or letter suffix is split | -| Administrative label/boundary | 6 | Adjacent administrative values merge or shift | -| Street-type recognition | 4 | A visible supported-looking marker is missed | -| Reference conflicts with numeric marker | 3 | Raw `стр.` conflicts with expected `house_num` | -| Numeric component not recognized | 3 | House/unit evidence is missed | -| Four single-row causes | 4 | Label confusion, reference conflict, or span/extra field | - -Additional likely-cause tags intentionally overlap—for example, an unmarked -numeric row can also contain label confusion and a missing apartment. Both the -primary partition and all secondary tags are retained in the CSV. - -[`FAILURE_ANALYSIS.md`](FAILURE_ANALYSIS.md) walks through ten representative -rows and explains why at least 24 failures require reference adjudication before -parser optimization. - -## Free-form message detection - -`detection_reference.jsonl` contains 30 deliberately narrow positive and -negative messages. Every row records the message, exact expected substrings, -scenario family, context style, address style, boundary style, polarity, -ambiguity, and notes. - -Run: - -```bash -python evaluation/evaluate_detection.py -``` - -The current conservative detector exactly matches all 20 annotated address -spans and returns no span for all 12 negative messages. That is 100% on this -small authored regression fixture only. It is not independent or large enough -for an accuracy claim. The next meaningful detector benchmark should annotate -complete, representative messages—including hard negatives—without -oracle-cropping. - -An additional diagnostic runs the detector on all 2,841 complete reconstructed -RedMadRobot messages rather than gold-cropped snippets: - -```bash -python evaluation/evaluate_redmadrobot_detection.py \ - --data .cache/external/redmadrobot-pii-benchmark-f77ea831.csv -``` - -Only gold clusters containing both `STREET` and `HOUSE` are address positives. -The current development snapshot has 144 such gold spans in 135 messages: - -| Detection metric | Result | -| --- | ---: | -| Any-overlap precision | 98.0% | -| Any-overlap recall | 68.1% | -| Any-overlap F1 | 80.3% | -| Exact-boundary F1 | 23.8% | -| Negative-message specificity | 100.0% | - -Exact-boundary scoring is much lower because the BIO gold span and detector -have different boundary policies—for example, one may include a city or -country while the other returns the parseable street/building/unit substring. -The complete diagnostic contains 107 non-exact messages: 44 missed-address, -33 context-inclusion, 33 dropped-gold-text, and one spurious-address tag. -Tags overlap. - -These failures were inspected while developing the detector, so this -RedMadRobot detection report is now a development diagnostic, not a sealed -final test. A production claim needs a new untouched message-level benchmark -whose annotation policy explicitly defines optional city, postcode, country, -person-name, and trailing-unit boundaries. - -## Independent external benchmark - -The repository also includes an adapter for -[RedMadRobot's MIT-licensed Russian PII NER benchmark](https://huggingface.co/datasets/redmadrobot-rnd/pii_benchmark). -Its 2,841 manually BIO-annotated sentences combine -production-log-shaped inputs (with real personal values replaced), synthetic -document-style examples, and manually filtered hard negatives. The external -data is pinned by Git revision and SHA-256 but is not committed or used for -training. - -Run the external evaluation once with: - -```bash -python evaluation/evaluate_redmadrobot.py --download \ - --output evaluation/redmadrobot_report.json -``` - -Subsequent runs can omit `--download`. The adapter extracts minimal address -windows from the gold BIO annotations and scores one-to-one, same-label span -overlap for `REGION`, `DISTRICT`, `CITY`, `STREET`, and `HOUSE`. This measures -address parsing after an address window has already been identified; it is not -an address-in-arbitrary-text detection score. `COUNTRY` is retained as context -but is not scored because it is not currently a public parser field. - -Treat this set as sealed evaluation data: do not train on it, tune thresholds -against it, or turn its failures into model features without replacing it with a -new untouched final test. - -The first untuned baseline covers 1,010 address spans in 578 snippets from 493 -source rows: - -| Slice | Snippets | Micro span F1 | -| --- | ---: | ---: | -| All address windows | 578 | 58.7% | -| Two or more distinct fields | 217 | 73.6% | -| Contains both street and house | 144 | 78.8% | -| Administrative fields only | 403 | 41.7% | - -Per-field F1 on all windows is 52.4% region, 44.5% district, 59.8% city, 49.1% -street, and 90.2% house. The large difference from the legacy regression is -important evidence: the parser is useful for conventional street-and-house -inputs, but it currently defaults too readily to `STREET` on isolated -administrative names and has weak administrative recall. - -## Large external corpora - -Install the data-only tools in a separate environment: - -```bash -python -m venv .venv-evaluation -.venv-evaluation/bin/python -m pip install -r requirements-evaluation.txt -``` - -The runtime wheel remains dependency-free. Raw and derived files are written to -the ignored `.cache/external/` directory. - -### Nationwide clean addresses: Deepparse - -Prepare the complete pinned Russian shard: - -```bash -.venv-evaluation/bin/python evaluation/prepare_deepparse.py \ - --download --overwrite -``` - -The streaming pipeline verifies the 371,595,309-byte source by SHA-256, checks -all 13,152,918 token/tag sequences, filters administrative-only strings, -deduplicates normalized text, maps the external tags to package fields, and -assigns canonical building groups to deterministic 90/5/5 splits. The result is: - -- 6,314,158 unique usable rows; -- 5,681,842 train, 316,586 validation, and 315,730 test rows; -- 5,293,689 street-and-house rows, including 679,076 with a unit; -- a 557,321,339-byte labeled Parquet corpus; -- a deterministic 100,000-row compressed JSONL sample from test only. - -Run the large test: - -```bash -python evaluation/evaluate_deepparse.py \ - --output evaluation/deepparse_report.json -``` - -The report uses explicit metric-family keys: - -- `span_overlap_micro`: binary same-label span matching with any overlap; -- `character_overlap_micro`: overlapping-character precision, recall, and F1; -- `token_label_micro`: aligned source-token label precision, recall, and F1; -- `exact_address_rate`: exact labels and exact span boundaries for a full row; -- `exact_token_sequence_rate`: exact complete token-label sequence. - -The original `micro`, `character_micro`, and `token_micro` names remain -compatibility aliases. Each generated report includes `metric_definitions`; -do not compare or average values from different metric families. - -The initial untuned 100,000-row result is: - -| Measure | Result | -| --- | ---: | -| Binary same-label span-overlap F1 | 84.4% | -| Character-overlap F1 | 66.2% | -| Token-label F1 | 66.5% | -| Exact token-boundary sequence | 8.0% | -| Throughput | 7,245 rows/s | - -Binary span overlap is intentionally lenient: any overlapping same-label span is -a match. Character and token metrics expose partial values and merged spans, but -also penalize schema-boundary differences such as the source labeling `дом 12` -as one entity while the package returns the value `12`. Publish all three, not -only the largest number. - -Character-overlap F1 by field is 99.9% postcode, 22.3% region, 22.6% district, -52.2% city, 76.7% street, 59.0% house, and 47.1% apartment. The set is national -in scale but consists of curated open-geographic addresses rather than noisy -user input. - -### Official clean addresses: Moscow registry - -Prepare the pinned October 2021 city snapshot: - -```bash -.venv-evaluation/bin/python evaluation/prepare_datamos.py \ - --download --overwrite -``` - -The filter retains only addresses that are on Moscow territory, official, -registered in the address registry, present in GKN, have a valid FIAS UUID, and -contain structured street and house values. It removes normalized duplicates -and groups street/house/корпус/строение identities before splitting. - -The result contains 307,274 unique active official addresses: 276,368 train, -15,710 validation, and 15,196 test. The portable filtered artifact is -26,050,795 bytes compressed. - -Run exact-value evaluation: - -```bash -python evaluation/evaluate_datamos.py \ - --output evaluation/datamos_report.json -``` - -The exact-value aggregate is named `exact_component_value_micro`; `micro` -remains a compatibility alias for the first published report schema. - -| Measure | Result | -| --- | ---: | -| Exact component-value micro F1 | 85.4% | -| Exact full-address match | 64.9% | -| Street F1 | 66.2% | -| House F1 | 98.2% | -| Корпус F1 | 99.6% | -| Строение F1 | 97.0% | - -This is the strongest current clean-building benchmark because it scores exact -structured values rather than mere span overlap. It is still not a production -claim: the snapshot is old, Moscow-only, and legally formatted. - -The archive embeds the original portal dataset ID, publisher, version, source -URL, and Russian government open-data terms. The mirror describes the package -as CC-BY-SA. Keep both records and confirm redistribution terms before -publishing derived rows. - -## Current evidence, kept separate - -| Domain | Test size | Primary measure | Baseline | -| --- | ---: | --- | ---: | -| Historical bank-shaped reference | 500 | exact field micro F1 | 95.9% | -| RedMadRobot noisy address windows | 578 | binary span-overlap F1 | 58.7% | -| Deepparse nationwide clean strings | 100,000 | character-overlap F1 | 66.2% | -| Moscow official clean buildings | 15,196 | exact component-value F1 | 85.4% | - -These numbers answer different questions and must not be averaged into one -“accuracy” claim. - -## Promoting this to a gold benchmark - -Before making a production-quality claim: - -1. confirm that the legacy workbook may be retained and used for evaluation; -2. have a person review at least 300 rows against the raw text and v2 schema; -3. record reviewer, decision, notes, and review date; -4. exclude corrected registry values that do not occur in the raw input; -5. group variations of one canonical address into the same data split; -6. keep a final test split that is never used to tune rules or the model; -7. publish field metrics, confidence intervals, slice failures, and limitations; -8. do not distribute address rows unless their provenance permits it. - -The existing reference is valuable enough to drive engineering now, but the -remaining human review is a release-management task, not something automation -should silently pretend to have completed. diff --git a/evaluation/RESULTS.md b/evaluation/RESULTS.md deleted file mode 100644 index 5f87209..0000000 --- a/evaluation/RESULTS.md +++ /dev/null @@ -1,91 +0,0 @@ -# Committed evaluation results - -This file is the index of durable benchmark results stored in the repository. -It separates message detection, address parsing, learned-model evaluation, and -software tests because they answer different questions. - -## Result snapshot - -| Task and domain | Rows or examples | Primary result | Committed report | -| --- | ---: | ---: | --- | -| Historical component parsing | 500 addresses | 95.85% exact component-value micro F1; 80.4% exact full address | [`legacy_reference_500_report.json`](legacy_reference_500_report.json) | -| Historical failure analysis | 500 addresses, 98 non-exact | Complete row-level diagnoses and scenario flags | [`legacy_reference_500_diagnostics.csv`](legacy_reference_500_diagnostics.csv), [`legacy_reference_500_failure_summary.json`](legacy_reference_500_failure_summary.json) | -| Compact learned tagger | 70 group-disjoint sequence examples | 96.52% token accuracy; 96.88% micro entity F1 | [`../training/model_evaluation.json`](../training/model_evaluation.json) | -| Authored message detection fixture | 30 messages, 20 spans | 100% exact span F1; regression fixture only | [`detection_report.json`](detection_report.json) | -| Complete-message detection diagnostic | 2,841 messages, 144 gold spans | 98.0% overlap precision; 68.1% recall; 80.3% F1 | [`redmadrobot_detection_report.json`](redmadrobot_detection_report.json) | -| RedMadRobot oracle-cropped parsing | 578 address windows | 58.75% same-label span-overlap F1 | [`redmadrobot_report.json`](redmadrobot_report.json) | -| Deepparse nationwide clean parsing | 100,000 addresses | 84.39% binary span-overlap F1; 66.23% character-overlap F1 | [`deepparse_report.json`](deepparse_report.json) | -| Official Moscow clean-building parsing | 15,196 addresses | 85.36% exact component-value F1; 64.93% exact full address | [`datamos_report.json`](datamos_report.json) | - -These numbers are not interchangeable and must not be averaged. Open each -report's `scope`, `limitations`, and `metric_definitions` before using a result. - -## Source fixtures and manifests - -The reports are accompanied by the exact portable inputs or source metadata -needed to interpret or reproduce them: - -| Artifact | Purpose | -| --- | --- | -| [`legacy_reference_500.jsonl`](legacy_reference_500.jsonl) | Portable historical parser fixture | -| [`release_gates.json`](release_gates.json) | Minimum non-regression thresholds for that fixture | -| [`detection_reference.jsonl`](detection_reference.jsonl) | Narrow positive and negative detector scenarios | -| [`deepparse_manifest.json`](deepparse_manifest.json) | Pinned source, preparation, and split metadata | -| [`datamos_manifest.json`](datamos_manifest.json) | Pinned Moscow source, filtering, and split metadata | -| [`DATA_SOURCES.md`](DATA_SOURCES.md) | Provenance and licensing notes | - -Large external raw/test corpora are intentionally not committed. Their reports -contain pinned source revisions and checksums; preparation writes external data -under the ignored `.cache/external/` directory. - -## Reproduction commands - -Historical parsing and gates: - -```bash -python evaluation/evaluate.py \ - --data evaluation/legacy_reference_500.jsonl \ - --gates evaluation/release_gates.json \ - --output evaluation/legacy_reference_500_report.json -``` - -Historical failure diagnostics: - -```bash -python evaluation/analyze_failures.py -``` - -Learned tagger: - -```bash -python training/train_compact_tagger.py -python training/evaluate_compact_tagger.py -``` - -Detection: - -```bash -python evaluation/evaluate_detection.py -python evaluation/evaluate_redmadrobot_detection.py \ - --data .cache/external/redmadrobot-pii-benchmark-f77ea831.csv -``` - -External parsing: - -```bash -python evaluation/evaluate_redmadrobot.py \ - --data .cache/external/redmadrobot-pii-benchmark-f77ea831.csv -python evaluation/evaluate_deepparse.py -python evaluation/evaluate_datamos.py -``` - -See [`README.md`](README.md) for the exact data preparation commands and -benchmark boundaries. - -## Software tests - -`pytest` verifies API, offsets, behavior, evaluation adapters, failure -diagnostics, artifact loading, and regression gates. A passing test count is an -execution/CI result rather than a model-quality metric, so transient console -logs are not committed as benchmark evidence. The benchmark JSON/CSV artifacts -above are the durable results. diff --git a/evaluation/analyze_failures.py b/evaluation/analyze_failures.py deleted file mode 100644 index 1030b7d..0000000 --- a/evaluation/analyze_failures.py +++ /dev/null @@ -1,603 +0,0 @@ -#!/usr/bin/env python3 -"""Create a row-level diagnostic table for the historical reference set. - -The likely-cause labels are deterministic triage hints, not human-verified -causal ground truth. They make recurring failure shapes visible before a -maintainer decides whether the parser, the reference label, or both need work. -""" - -from __future__ import annotations - -import argparse -from collections import Counter -import csv -import json -from pathlib import Path -import re -import sys -from typing import Any, Iterable - - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from address_normalizer import parse -from address_normalizer.types import ParsedAddress - - -FIELDS = ( - "postal_code", - "region", - "district", - "city", - "settlement", - "street", - "street_type", - "house_num", - "corpus", - "structure", - "apartment", -) -NUMERIC_FIELDS = {"house_num", "corpus", "structure", "apartment"} -ADMIN_FIELDS = {"region", "district", "city", "settlement"} -STREET_TYPE_PATTERNS = { - "ул": re.compile(r"(? str | None: - if value is None: - return None - return " ".join(value.casefold().replace("ё", "е").split()) - - -def _boundary_fold(value: str | None) -> str | None: - folded = _fold(value) - if folded is None: - return None - return re.sub(r"[\W_]+", "", folded) - - -def _load_rows(path: Path) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): - if not line.strip(): - continue - row = json.loads(line) - if not isinstance(row, dict) or not isinstance(row.get("raw"), str): - raise ValueError(f"{path}:{line_number}: invalid evaluation row") - if not isinstance(row.get("expected"), dict): - raise ValueError(f"{path}:{line_number}: expected must be an object") - rows.append(row) - return rows - - -def _values(result: ParsedAddress) -> dict[str, str | None]: - return { - field: ( - getattr(result, field).value - if getattr(result, field) is not None - else None - ) - for field in FIELDS - } - - -def _field_status(expected: str | None, actual: str | None) -> str: - wanted = _fold(expected) - observed = _fold(actual) - if wanted == observed: - return "match" - if wanted is not None and observed is None: - return "missing" - if wanted is None and observed is not None: - return "extra" - return "wrong_value" - - -def _street_marker_style(raw: str, expected_street: str | None) -> str: - markers = list(ANY_STREET_MARKER_RE.finditer(raw)) - if not markers: - return "absent" - if len(markers) > 1: - return "multiple" - if not expected_street: - return "present_unknown_position" - street_index = _fold(raw).find(_fold(expected_street) or "") - if street_index < 0: - return "present_street_not_literal" - return "prefix" if markers[0].start() <= street_index else "suffix" - - -def _scenario_columns( - raw: str, - expected: dict[str, Any], -) -> dict[str, str]: - expected_street = ( - str(expected["street"]) if expected.get("street") is not None else None - ) - expected_city = ( - str(expected["city"]) if expected.get("city") is not None else None - ) - marker_style = _street_marker_style(raw, expected_street) - booleans = { - "has_postal_code": bool(POSTAL_RE.search(raw)), - "has_country_phrase": bool(COUNTRY_RE.search(raw)), - "has_administrative_expected": any( - expected.get(field) is not None for field in ADMIN_FIELDS - ), - "has_unit_expected": any( - expected.get(field) is not None - for field in ("corpus", "structure", "apartment") - ), - "has_street_marker": bool(ANY_STREET_MARKER_RE.search(raw)), - "has_house_marker": bool(HOUSE_MARKER_RE.search(raw)), - "has_unit_marker": bool(UNIT_MARKER_RE.search(raw)), - "has_compact_punctuation": bool(COMPACT_PUNCTUATION_RE.search(raw)), - "has_unicode_whitespace": any( - character.isspace() and character != " " for character in raw - ), - "has_compound_number": bool(COMPOUND_NUMBER_RE.search(raw)), - "has_slash_number": bool(re.search(r"\d\s*/\s*\d", raw)), - "has_hyphenated_number": bool(re.search(r"\d\s*-\s*\d", raw)), - "has_letter_suffix_number": bool(LETTER_SUFFIX_RE.search(raw)), - "has_unmarked_numeric_sequence": bool( - UNMARKED_NUMERIC_SEQUENCE_RE.search(raw) - ), - "has_ordinal_street": bool(ORDINAL_STREET_RE.search(raw)), - "has_ambiguous_abbreviation": bool( - AMBIGUOUS_ABBREVIATION_RE.search(raw) - ), - "has_multiword_street": bool( - expected_street and len(expected_street.split()) > 1 - ), - "has_repeated_city": bool( - expected_city - and _fold(raw).count(_fold(expected_city) or "") > 1 - ), - } - tags = [ - name.removeprefix("has_") - for name, present in booleans.items() - if present - ] - tags.append(f"street_marker_{marker_style}") - return { - **{name: str(value).lower() for name, value in booleans.items()}, - "street_marker_style": marker_style, - "scenario_tags": "|".join(tags), - } - - -def _expected_street_marker_is_visible( - raw: str, - expected_type: str | None, -) -> bool: - if expected_type is None: - return False - pattern = STREET_TYPE_PATTERNS.get(expected_type) - return bool(pattern and pattern.search(raw)) - - -def _diagnose( - raw: str, - expected: dict[str, str | None], - actual: dict[str, str | None], - statuses: dict[str, str], - result: ParsedAddress, -) -> tuple[list[str], list[str], str]: - mismatch_fields = [ - field for field, status in statuses.items() if status != "match" - ] - failure_types = sorted({statuses[field] for field in mismatch_fields}) - causes: set[str] = set() - - expected_by_value = { - _fold(value): field - for field, value in expected.items() - if value is not None - } - for field in mismatch_fields: - wanted = expected.get(field) - observed = actual.get(field) - wanted_folded = _fold(wanted) - observed_folded = _fold(observed) - - if observed_folded is not None and observed_folded in expected_by_value: - if expected_by_value[observed_folded] != field: - causes.add("component_label_confusion") - - if field == "street_type": - if AMBIGUOUS_ABBREVIATION_RE.search(raw): - causes.add("ambiguous_or_unsupported_abbreviation") - elif ( - wanted is not None - and not _expected_street_marker_is_visible(raw, wanted) - ): - if ANY_STREET_MARKER_RE.search(raw): - causes.add("reference_conflicts_with_explicit_street_type") - else: - causes.add("reference_infers_missing_street_type") - elif len(list(ANY_STREET_MARKER_RE.finditer(raw))) > 1: - causes.add("conflicting_street_markers") - else: - causes.add("street_type_recognition") - continue - - if field in NUMERIC_FIELDS: - if AMBIGUOUS_ABBREVIATION_RE.search(raw): - causes.add("ambiguous_or_unsupported_abbreviation") - if UNMARKED_NUMERIC_SEQUENCE_RE.search(raw): - causes.add("unmarked_numeric_role_ambiguity") - if wanted and observed and ( - wanted_folded in (observed_folded or "") - or (observed_folded or "") in (wanted_folded or "") - ): - causes.add("compound_or_letter_number_boundary") - elif wanted and ( - "/" in wanted or "-" in wanted or LETTER_SUFFIX_RE.search(wanted) - ): - causes.add("compound_or_letter_number_boundary") - elif statuses[field] == "missing": - causes.add("numeric_component_not_recognized") - elif statuses[field] == "extra": - causes.add("spurious_numeric_component") - else: - causes.add("numeric_value_or_role") - continue - - if field in ADMIN_FIELDS: - if statuses[field] == "missing": - causes.add("administrative_component_missed") - elif statuses[field] == "extra": - causes.add("spurious_administrative_component") - else: - causes.add("administrative_label_or_boundary") - continue - - if field == "street": - if _boundary_fold(wanted) == _boundary_fold(observed): - causes.add("normalization_only_difference") - elif wanted_folded and observed_folded and ( - wanted_folded in observed_folded - or observed_folded in wanted_folded - ): - causes.add("street_span_boundary") - elif statuses[field] == "missing": - causes.add("street_not_recognized") - else: - causes.add("street_label_or_value") - continue - - causes.add(f"{field}_{statuses[field]}") - - for expected_field in NUMERIC_FIELDS: - wanted = _fold(expected.get(expected_field)) - if wanted is None: - continue - for actual_field in NUMERIC_FIELDS - {expected_field}: - if ( - _fold(actual.get(actual_field)) == wanted - and expected.get(actual_field) is None - and NUMERIC_MARKER_PATTERNS[actual_field].search(raw) - ): - causes.add("reference_conflicts_with_explicit_numeric_marker") - summary = "; ".join( - f"{field}: expected={expected.get(field)!r}, actual={actual.get(field)!r}" - for field in mismatch_fields - ) - return failure_types, sorted(causes), summary - - -def _primary_cause(causes: Iterable[str]) -> str: - available = set(causes) - priority = ( - "reference_conflicts_with_explicit_numeric_marker", - "reference_conflicts_with_explicit_street_type", - "reference_infers_missing_street_type", - "ambiguous_or_unsupported_abbreviation", - "conflicting_street_markers", - "unmarked_numeric_role_ambiguity", - "compound_or_letter_number_boundary", - "component_label_confusion", - "numeric_component_not_recognized", - "numeric_value_or_role", - "administrative_component_missed", - "administrative_label_or_boundary", - "spurious_administrative_component", - "street_span_boundary", - "normalization_only_difference", - "street_type_recognition", - "street_label_or_value", - "street_not_recognized", - "spurious_numeric_component", - ) - return next( - (cause for cause in priority if cause in available), - sorted(available)[0] if available else "", - ) - - -def diagnose_row(row: dict[str, Any]) -> dict[str, str]: - """Return one flat, CSV-ready diagnostic record.""" - - raw = str(row["raw"]) - expected = { - field: ( - str(row["expected"][field]) - if row["expected"].get(field) is not None - else None - ) - for field in FIELDS - } - result = parse(raw) - actual = _values(result) - statuses = { - field: _field_status(expected[field], actual[field]) - for field in FIELDS - } - mismatch_fields = [ - field for field, status in statuses.items() if status != "match" - ] - missing_fields = [ - field for field, status in statuses.items() if status == "missing" - ] - extra_fields = [ - field for field, status in statuses.items() if status == "extra" - ] - wrong_value_fields = [ - field for field, status in statuses.items() if status == "wrong_value" - ] - failure_types, causes, failure_summary = _diagnose( - raw, - expected, - actual, - statuses, - result, - ) - if not mismatch_fields: - triage_priority = "none" - diagnosis_status = "not_applicable" - elif causes == ["reference_infers_missing_street_type"]: - triage_priority = "reference_review" - diagnosis_status = "heuristic_needs_human_review" - elif missing_fields or "component_label_confusion" in causes: - triage_priority = "high" - diagnosis_status = "heuristic_needs_human_review" - elif causes == ["normalization_only_difference"]: - triage_priority = "low" - diagnosis_status = "heuristic_needs_human_review" - else: - triage_priority = "medium" - diagnosis_status = "heuristic_needs_human_review" - - source = row.get("source") - source_row = source.get("row") if isinstance(source, dict) else None - output = { - "id": str(row.get("id", "")), - "dataset": "legacy_reference_500", - "source_row": "" if source_row is None else str(source_row), - "raw": raw, - "review_status": str(row.get("review_status", "unspecified")), - "exact_address": str(not mismatch_fields).lower(), - "triage_priority": triage_priority, - "diagnosis_status": diagnosis_status, - "parser_confidence": f"{result.confidence:.6f}", - "expected_component_count": str( - sum(value is not None for value in expected.values()) - ), - "actual_component_count": str( - sum(value is not None for value in actual.values()) - ), - "mismatch_count": str(len(mismatch_fields)), - "mismatch_fields": "|".join(mismatch_fields), - "missing_fields": "|".join(missing_fields), - "extra_fields": "|".join(extra_fields), - "wrong_value_fields": "|".join(wrong_value_fields), - "failure_types": "|".join(failure_types), - "primary_likely_cause": _primary_cause(causes), - "likely_causes": "|".join(causes), - "failure_summary": failure_summary, - "warnings": "|".join(result.warnings), - "unparsed_spans": json.dumps( - [part.raw for part in result.unparsed], - ensure_ascii=False, - separators=(",", ":"), - ), - "alternatives": json.dumps( - [alternative.as_dict() for alternative in result.alternatives], - ensure_ascii=False, - separators=(",", ":"), - ), - **_scenario_columns(raw, expected), - } - for field in FIELDS: - output[f"expected_{field}"] = expected[field] or "" - output[f"actual_{field}"] = actual[field] or "" - output[f"status_{field}"] = statuses[field] - return output - - -def _representative_failures( - diagnostics: Iterable[dict[str, str]], - limit: int = 10, -) -> list[dict[str, str]]: - selected: list[dict[str, str]] = [] - used_causes: set[str] = set() - failures = [ - row for row in diagnostics if row["exact_address"] == "false" - ] - for row in failures: - causes = row["likely_causes"].split("|") - if any(cause not in used_causes for cause in causes): - selected.append(row) - used_causes.update(causes) - if len(selected) == limit: - return selected - for row in failures: - if row not in selected: - selected.append(row) - if len(selected) == limit: - break - return selected - - -def summarize(diagnostics: list[dict[str, str]]) -> dict[str, Any]: - """Summarize failure counts without collapsing metric domains.""" - - failures = [ - row for row in diagnostics if row["exact_address"] == "false" - ] - cause_counts: Counter[str] = Counter() - primary_cause_counts: Counter[str] = Counter() - field_counts: Counter[str] = Counter() - scenario_counts: Counter[str] = Counter() - for row in failures: - cause_counts.update(filter(None, row["likely_causes"].split("|"))) - primary_cause_counts.update([row["primary_likely_cause"]]) - field_counts.update(filter(None, row["mismatch_fields"].split("|"))) - scenario_counts.update(filter(None, row["scenario_tags"].split("|"))) - sample_columns = ( - "id", - "raw", - "mismatch_fields", - "primary_likely_cause", - "likely_causes", - "failure_summary", - "unparsed_spans", - ) - return { - "dataset": "legacy_reference_500", - "rows": len(diagnostics), - "exact_rows": len(diagnostics) - len(failures), - "failed_rows": len(failures), - "diagnostic_semantics": ( - "likely_causes are deterministic triage hypotheses and require " - "human review; they are not causal ground truth" - ), - "failure_rows_by_likely_cause": dict(cause_counts.most_common()), - "failure_rows_by_primary_likely_cause": dict( - primary_cause_counts.most_common() - ), - "failure_rows_by_mismatch_field": dict(field_counts.most_common()), - "scenario_tags_on_failure_rows": dict(scenario_counts.most_common()), - "representative_failure_sample": [ - {column: row[column] for column in sample_columns} - for row in _representative_failures(diagnostics) - ], - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument( - "--data", - type=Path, - default=ROOT / "evaluation/legacy_reference_500.jsonl", - ) - parser.add_argument( - "--output", - type=Path, - default=ROOT / "evaluation/legacy_reference_500_diagnostics.csv", - ) - parser.add_argument( - "--summary-output", - type=Path, - default=ROOT / "evaluation/legacy_reference_500_failure_summary.json", - ) - args = parser.parse_args(argv) - - diagnostics = [diagnose_row(row) for row in _load_rows(args.data)] - args.output.parent.mkdir(parents=True, exist_ok=True) - with args.output.open("w", encoding="utf-8", newline="") as target: - writer = csv.DictWriter( - target, - fieldnames=list(diagnostics[0]), - lineterminator="\n", - ) - writer.writeheader() - writer.writerows(diagnostics) - - report = summarize(diagnostics) - rendered = json.dumps(report, ensure_ascii=False, indent=2) - args.summary_output.parent.mkdir(parents=True, exist_ok=True) - args.summary_output.write_text(f"{rendered}\n", encoding="utf-8") - print(rendered) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/evaluation/datamos_data.py b/evaluation/datamos_data.py deleted file mode 100644 index 22685d3..0000000 --- a/evaluation/datamos_data.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Pinned metadata and pure transformations for the Moscow address registry.""" - -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import re -from typing import Any -import uuid - - -ROOT = Path(__file__).resolve().parents[1] -DATASET_ID = 60_562 -DATASET_VERSION = "3.630" -DATASET_DATE = "2021-10-15" -SOURCE_ROWS = 440_399 -ARCHIVE_SHA256 = ( - "a272480189bf1e17e70b0e0b2e115520896ff1d168681d452697c3908ad75a66" -) -INNER_DATA_SHA256 = ( - "c253335e8b25fce0264837c026536689b33ff9fc5ac9d305158685d4528d95bb" -) -ARCHIVE_URL = ( - "https://data2.apicrafter.ru/packages/datamos-addressreestr/build/" - "datamos-7705031674-AddressReestr-2021-10-23-7-55/get" -) -DEFAULT_ARCHIVE = ( - ROOT / ".cache" / "external" / "datamos-addressreestr-2021-10-23.zip" -) -DEFAULT_FILTERED = ( - ROOT - / ".cache" - / "external" - / "datamos-addressreestr-usable-2021-10-23.jsonl.gz" -) -DEFAULT_MANIFEST = ROOT / "evaluation" / "datamos_manifest.json" -FIELDS = ("street", "house_num", "corpus", "structure") -_SPACE_RE = re.compile(r"\s+") - - -def fold(value: str) -> str: - return _SPACE_RE.sub(" ", value.casefold().replace("ё", "е")).strip(" ,.;") - - -def rejection_reason(row: dict[str, Any]) -> str | None: - if row.get("OnTerritoryOfMoscow") != "да": - return "outside_moscow" - if row.get("ADR_TYPE") != "Официальный": - return "not_official" - if row.get("SOSTAD") != "Зарегистрирован в АР": - return "not_registered" - if row.get("STATUS") != "Внесён в ГКН": - return "not_in_gkn" - for field in ("SIMPLE_ADDRESS", "P7", "L1_VALUE"): - if not str(row.get(field, "")).strip(): - return f"missing_{field.lower()}" - try: - uuid.UUID(str(row.get("N_FIAS", ""))) - except (ValueError, AttributeError): - return "invalid_fias_id" - return None - - -def expected_components(row: dict[str, Any]) -> dict[str, str | None]: - return { - "street": str(row["P7"]).strip(), - "house_num": str(row["L1_VALUE"]).strip(), - "corpus": str(row.get("L2_VALUE", "")).strip() or None, - "structure": str(row.get("L3_VALUE", "")).strip() or None, - } - - -def quality_tier(row: dict[str, Any]) -> str: - has_corpus = bool(str(row.get("L2_VALUE", "")).strip()) - has_structure = bool(str(row.get("L3_VALUE", "")).strip()) - if has_corpus and has_structure: - return "house_corpus_structure" - if has_corpus: - return "house_corpus" - if has_structure: - return "house_structure" - return "house_only" - - -def group_id_and_split(row: dict[str, Any]) -> tuple[str, str]: - identity = [ - fold(str(row.get(field, ""))) - for field in ("P7", "L1_VALUE", "L2_VALUE", "L3_VALUE") - ] - key = json.dumps( - identity, - ensure_ascii=False, - separators=(",", ":"), - ).encode("utf-8") - digest = hashlib.sha256(b"datamos-building-v1\0" + key).digest() - bucket = int.from_bytes(digest[:8], "big") % 10_000 - split = ( - "train" - if bucket < 9_000 - else "validation" - if bucket < 9_500 - else "test" - ) - return digest[:16].hex(), split diff --git a/evaluation/datamos_manifest.json b/evaluation/datamos_manifest.json deleted file mode 100644 index 5f6c918..0000000 --- a/evaluation/datamos_manifest.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "format_version": 1, - "source": { - "title": "Адресный реестр объектов недвижимости города Москвы", - "publisher": "Департамент городского имущества города Москвы", - "original_portal": "https://data.mos.ru", - "dataset_id": 60562, - "version": "3.630", - "release_date": "2021-10-15", - "mirror": "https://data2.apicrafter.ru/packages/datamos-addressreestr", - "archive_url": "https://data2.apicrafter.ru/packages/datamos-addressreestr/build/datamos-7705031674-AddressReestr-2021-10-23-7-55/get", - "archive_bytes": 121581813, - "archive_sha256": "a272480189bf1e17e70b0e0b2e115520896ff1d168681d452697c3908ad75a66", - "inner_data_sha256": "c253335e8b25fce0264837c026536689b33ff9fc5ac9d305158685d4528d95bb", - "source_rows": 440399, - "embedded_terms": "Типовые условия доступа к открытым данным органов власти в РФ", - "mirror_terms": "CC-BY-SA" - }, - "policy": { - "purpose": "Moscow-only official clean-address training/evaluation corpus; not bundled in the runtime package", - "filter": [ - "OnTerritoryOfMoscow == да", - "ADR_TYPE == Официальный", - "SOSTAD == Зарегистрирован в АР", - "STATUS == Внесён в ГКН", - "non-empty SIMPLE_ADDRESS, P7 street, and L1_VALUE house", - "valid N_FIAS UUID" - ], - "deduplication": "first case-folded, ё/е-folded, whitespace-normalized SIMPLE_ADDRESS", - "grouping": "street/house/corpus/structure identity; SHA-256 groups are assigned 90% train, 5% validation, 5% test" - }, - "counts": { - "duplicate_rows_removed": 893, - "object_Здание": 205838, - "object_Земельный участок": 97828, - "object_Сооружение": 3375, - "object_объект незавершенного строительства": 233, - "rejected_invalid_fias_id": 8245, - "rejected_missing_l1_value": 434, - "rejected_missing_p7": 84404, - "rejected_not_in_gkn": 23747, - "rejected_not_official": 14430, - "rejected_not_registered": 808, - "rejected_outside_moscow": 164, - "source_rows": 440399, - "split_test": 15196, - "split_train": 276368, - "split_validation": 15710, - "tier_house_corpus": 26072, - "tier_house_corpus_structure": 8178, - "tier_house_only": 158680, - "tier_house_structure": 114344, - "unique_usable_rows": 307274 - }, - "artifact": { - "filename": "datamos-addressreestr-usable-2021-10-23.jsonl.gz", - "rows": 307274, - "bytes": 26050795, - "sha256": "bba7849a479f2169d2f438032c65e069badb76f3f19ee590041267de27f84c63" - }, - "limitations": [ - "The snapshot is from October 2021 and is not a current registry.", - "The corpus is Moscow-only and consists of clean legal formatting.", - "Only street, house, corpus, and structure are scored from SIMPLE_ADDRESS; administrative fields are intentionally out of scope for this view.", - "The archive is obtained from an attributed mirror; retain its embedded metadata and confirm redistribution terms before publishing derived rows." - ] -} diff --git a/evaluation/datamos_report.json b/evaluation/datamos_report.json deleted file mode 100644 index b656d93..0000000 --- a/evaluation/datamos_report.json +++ /dev/null @@ -1,776 +0,0 @@ -{ - "scope": "untuned exact-value evaluation on a group-disjoint test split of active official Moscow registry building addresses", - "source": { - "dataset_id": 60562, - "version": "3.630", - "release_date": "2021-10-15" - }, - "limitations": [ - "October 2021 snapshot; not current FIAS/GAR truth", - "Moscow-only clean legal/simplified address formatting", - "administrative fields and address existence resolution are unscored" - ], - "rows": 15196, - "tiers": { - "house_corpus": 1326, - "house_corpus_structure": 427, - "house_only": 7801, - "house_structure": 5642 - }, - "matching": "case-insensitive exact component value after whitespace and ё/е folding; street includes its source type marker", - "metric_definitions": { - "exact_component_value_micro": "micro precision, recall, and F1 over case-insensitive exact component values after whitespace and ё/е folding", - "exact_address_rate": "fraction of rows where every scored component value matches", - "no_unparsed_rate": "fraction of rows with no residual word or number spans", - "fields": "per-field exact component-value metrics" - }, - "exact_component_value_micro": { - "tp": 32604, - "fp": 5572, - "fn": 5610, - "support": 38214, - "precision": 0.854044, - "recall": 0.853195, - "f1": 0.85362 - }, - "micro": { - "tp": 32604, - "fp": 5572, - "fn": 5610, - "support": 38214, - "precision": 0.854044, - "recall": 0.853195, - "f1": 0.85362 - }, - "macro_field_f1": 0.902612, - "exact_address_rate": 0.649316, - "no_unparsed_rate": 0.399052, - "fields": { - "street": { - "tp": 10067, - "fp": 5129, - "fn": 5129, - "support": 15196, - "precision": 0.662477, - "recall": 0.662477, - "f1": 0.662477 - }, - "house_num": { - "tp": 14910, - "fp": 259, - "fn": 286, - "support": 15196, - "precision": 0.982926, - "recall": 0.981179, - "f1": 0.982052 - }, - "corpus": { - "tp": 1742, - "fp": 3, - "fn": 11, - "support": 1753, - "precision": 0.998281, - "recall": 0.993725, - "f1": 0.995998 - }, - "structure": { - "tp": 5885, - "fp": 181, - "fn": 184, - "support": 6069, - "precision": 0.970162, - "recall": 0.969682, - "f1": 0.969922 - } - }, - "elapsed_seconds": 4.568, - "rows_per_second": 3326.9, - "failure_sample": [ - { - "source_row": 1796, - "fias_id": "30f3b4ea-773f-467c-8e8c-6c31c6547d1e", - "tier": "house_corpus", - "raw": "шоссе Энтузиастов, дом 29, корпус 4-5", - "mismatches": { - "corpus": { - "expected": "4-5", - "actual": "4" - } - } - }, - { - "source_row": 2673, - "fias_id": "799d5735-0711-4568-a2fc-43dd9cac49f0", - "tier": "house_structure", - "raw": "улица Сергия Радонежского, дом 15-17, строение 1", - "mismatches": { - "house_num": { - "expected": "15-17", - "actual": "15" - } - } - }, - { - "source_row": 4460, - "fias_id": "689da5c9-fec1-4b99-859d-de4e3d96a0ce", - "tier": "house_corpus_structure", - "raw": "улица Ивана Франко, дом 48, корпус Г, строение 4", - "mismatches": { - "corpus": { - "expected": "Г", - "actual": null - } - } - }, - { - "source_row": 5693, - "fias_id": "167afc58-12ef-46c4-a920-45761736fea6", - "tier": "house_corpus", - "raw": "Нижняя Сыромятническая улица, дом 11, корпус Б", - "mismatches": { - "corpus": { - "expected": "Б", - "actual": null - } - } - }, - { - "source_row": 5697, - "fias_id": "e4b09246-6937-434f-93a7-9fcb2a6501ac", - "tier": "house_structure", - "raw": "проспект Вернадского, владение 6-В, строение 1", - "mismatches": { - "house_num": { - "expected": "6-В", - "actual": "6" - } - } - }, - { - "source_row": 6580, - "fias_id": "c08786fa-400b-43a1-a1ef-5e95f7ad08ab", - "tier": "house_structure", - "raw": "2-я улица Новосёлки, дом 11А, строение 1", - "mismatches": { - "street": { - "expected": "2-я улица Новосёлки", - "actual": "улица Новосёлки" - } - } - }, - { - "source_row": 7409, - "fias_id": "ac5f4612-0e9f-4341-9980-d801b4d35c0b", - "tier": "house_structure", - "raw": "3-й Лучевой просек, дом 12, строение 6", - "mismatches": { - "street": { - "expected": "3-й Лучевой просек", - "actual": "й Лучевой" - } - } - }, - { - "source_row": 7531, - "fias_id": "1e791410-8ef4-4ab4-ad26-b316d57c7899", - "tier": "house_only", - "raw": "3-я улица Ямского Поля, дом 15", - "mismatches": { - "street": { - "expected": "3-я улица Ямского Поля", - "actual": "улица Ямского Поля" - } - } - }, - { - "source_row": 7870, - "fias_id": "15b0092f-c9ca-430a-a8fd-e288489d6aa9", - "tier": "house_only", - "raw": "4-я улица Новосёлки, дом 11", - "mismatches": { - "street": { - "expected": "4-я улица Новосёлки", - "actual": "улица Новосёлки" - } - } - }, - { - "source_row": 8043, - "fias_id": "ce9ef716-fc4c-422b-9ac3-a0e61bc8f4b7", - "tier": "house_structure", - "raw": "1-й переулок Тружеников, дом 14, строение 10", - "mismatches": { - "street": { - "expected": "1-й переулок Тружеников", - "actual": "1-й переулок" - } - } - }, - { - "source_row": 9140, - "fias_id": "b404108e-a463-446f-9284-2b3f58e7dc79", - "tier": "house_structure", - "raw": "Большая Грузинская улица, дом 4-6, строение 9", - "mismatches": { - "house_num": { - "expected": "4-6", - "actual": "4" - } - } - }, - { - "source_row": 10350, - "fias_id": "5f99e9de-5042-4d63-a0a0-a303ee7ea32a", - "tier": "house_structure", - "raw": "2-я улица Новые Сады, дом 18, строение 4", - "mismatches": { - "street": { - "expected": "2-я улица Новые Сады", - "actual": "улица Новые Сады" - } - } - }, - { - "source_row": 10758, - "fias_id": "46be4353-2bd1-4790-ad0c-5874bf4434df", - "tier": "house_structure", - "raw": "Пулковская улица, дом 4, строение 9-10", - "mismatches": { - "structure": { - "expected": "9-10", - "actual": "9" - } - } - }, - { - "source_row": 12286, - "fias_id": "c42af633-132f-4fce-bb98-a084fedb7711", - "tier": "house_corpus", - "raw": "1-я Северная линия, дом 1, корпус 8", - "mismatches": { - "street": { - "expected": "1-я Северная линия", - "actual": "линия" - } - } - }, - { - "source_row": 12293, - "fias_id": "cdc8ff56-5542-4a0e-abf8-53ec1799a2a3", - "tier": "house_structure", - "raw": "1-я Северная линия, дом 1, строение 57", - "mismatches": { - "street": { - "expected": "1-я Северная линия", - "actual": "линия" - } - } - }, - { - "source_row": 12518, - "fias_id": "d54c749d-55d6-4ee4-8ad1-84ae055ff1c5", - "tier": "house_only", - "raw": "3-я Северная линия, дом 3", - "mismatches": { - "street": { - "expected": "3-я Северная линия", - "actual": "линия" - } - } - }, - { - "source_row": 12854, - "fias_id": "f183b356-3be9-489c-9229-08e0825f0467", - "tier": "house_only", - "raw": "8-я улица Текстильщиков, дом 8", - "mismatches": { - "street": { - "expected": "8-я улица Текстильщиков", - "actual": "улица Текстильщиков" - } - } - }, - { - "source_row": 13058, - "fias_id": "dc5c9b19-500b-4ace-8357-bb6f5db3c14d", - "tier": "house_only", - "raw": "8-я улица Новые Сады, дом 8", - "mismatches": { - "street": { - "expected": "8-я улица Новые Сады", - "actual": "улица Новые Сады" - } - } - }, - { - "source_row": 13807, - "fias_id": "b23db3e9-6909-4abb-8f29-08413bd256e0", - "tier": "house_only", - "raw": "улица Мичуринский Проспект, Олимпийская Деревня, дом 9", - "mismatches": { - "street": { - "expected": "улица Мичуринский Проспект, Олимпийская Деревня", - "actual": "улица Мичуринский" - } - } - }, - { - "source_row": 14627, - "fias_id": "35c43eca-cf6b-4eca-ac22-3d73189cf778", - "tier": "house_only", - "raw": "улица Большая Молчановка, дом 26-28", - "mismatches": { - "house_num": { - "expected": "26-28", - "actual": "26" - } - } - }, - { - "source_row": 15270, - "fias_id": "2ab4df2d-7409-48d2-bdd0-28182cc243e5", - "tier": "house_structure", - "raw": "2-й квартал Капотня, дом 11, строение 1", - "mismatches": { - "street": { - "expected": "2-й квартал Капотня", - "actual": "й квартал Капотня" - } - } - }, - { - "source_row": 15392, - "fias_id": "9b6a0ea4-4813-4c03-8d5c-bfe5303b7c2d", - "tier": "house_only", - "raw": "92-й километр Московской Кольцевой Автодороги, владение 3", - "mismatches": { - "street": { - "expected": "92-й километр Московской Кольцевой Автодороги", - "actual": "й" - } - } - }, - { - "source_row": 15527, - "fias_id": "1f123b98-cf51-4af4-bd63-6cee1b8f149e", - "tier": "house_only", - "raw": "7-я улица Текстильщиков, дом 3А", - "mismatches": { - "street": { - "expected": "7-я улица Текстильщиков", - "actual": "улица Текстильщиков" - } - } - }, - { - "source_row": 15824, - "fias_id": "1e0b6f18-33b8-4831-a6ed-910b5171e2e0", - "tier": "house_only", - "raw": "1-й переулок Тружеников, дом 13", - "mismatches": { - "street": { - "expected": "1-й переулок Тружеников", - "actual": "1-й переулок" - } - } - }, - { - "source_row": 16460, - "fias_id": "38ac0703-f4c3-42e8-8379-9c8ab0838ca2", - "tier": "house_structure", - "raw": "1-я Северная линия, дом 1, строение 25", - "mismatches": { - "street": { - "expected": "1-я Северная линия", - "actual": "линия" - } - } - }, - { - "source_row": 17035, - "fias_id": "33b42a72-eec5-4320-a76e-0e7f39b994eb", - "tier": "house_only", - "raw": "53-й километр Московской Кольцевой Автодороги, дом 6", - "mismatches": { - "street": { - "expected": "53-й километр Московской Кольцевой Автодороги", - "actual": "й" - } - } - }, - { - "source_row": 17599, - "fias_id": "1010ba97-452b-4e8f-954d-af0d30e355f6", - "tier": "house_structure", - "raw": "проектируемый проезд № 4062, дом 6, строение 13", - "mismatches": { - "street": { - "expected": "проектируемый проезд № 4062", - "actual": "проектируемый проезд" - } - } - }, - { - "source_row": 18632, - "fias_id": "63de66e8-93dd-43d8-8cec-4b5ecb70d6df", - "tier": "house_structure", - "raw": "2-я линия Хорошёвского Серебряного Бора, домовладение 16, строение 3", - "mismatches": { - "street": { - "expected": "2-я линия Хорошёвского Серебряного Бора", - "actual": "я линия Хорошёвского Серебряного Бора, домовладение" - } - } - }, - { - "source_row": 18687, - "fias_id": "9be0d9d1-1f10-45ed-872f-c7c21594f33f", - "tier": "house_only", - "raw": "МЖД, Киевское, 2-й километр, дом 9", - "mismatches": { - "street": { - "expected": "МЖД, Киевское, 2-й километр", - "actual": "Киевское" - } - } - }, - { - "source_row": 19022, - "fias_id": "b3558a04-8ef9-4d17-afed-34d0ea5a3e0e", - "tier": "house_structure", - "raw": "Никольская улица, дом 11-13, строение 2", - "mismatches": { - "house_num": { - "expected": "11-13", - "actual": "11" - } - } - }, - { - "source_row": 20406, - "fias_id": "04bc1903-12ed-4297-a897-c8fd2547e196", - "tier": "house_corpus", - "raw": "9-я Северная линия, дом 15, корпус 3", - "mismatches": { - "street": { - "expected": "9-я Северная линия", - "actual": "линия" - } - } - }, - { - "source_row": 20711, - "fias_id": "0f98146a-2fa6-451c-831e-86ebd3d44968", - "tier": "house_only", - "raw": "17-й проезд Марьиной Рощи, дом 6А", - "mismatches": { - "street": { - "expected": "17-й проезд Марьиной Рощи", - "actual": "17-й проезд" - } - } - }, - { - "source_row": 21116, - "fias_id": "dd8f3a1f-6734-4e73-91a7-8c60832b51db", - "tier": "house_structure", - "raw": "2-я линия Хорошёвского Серебряного Бора, дом 47, строение 13", - "mismatches": { - "street": { - "expected": "2-я линия Хорошёвского Серебряного Бора", - "actual": "я линия Хорошёвского Серебряного Бора" - } - } - }, - { - "source_row": 21408, - "fias_id": "219ccd89-d02d-4ef1-9c3b-833b5619f503", - "tier": "house_structure", - "raw": "Ленинградский проспект, дом 47, строение А", - "mismatches": { - "structure": { - "expected": "А", - "actual": null - } - } - }, - { - "source_row": 21961, - "fias_id": "611dcb5a-4bd1-40c8-a037-1df4c121efaa", - "tier": "house_structure", - "raw": "Бакунинская улица, дом 62-68, строение 1", - "mismatches": { - "house_num": { - "expected": "62-68", - "actual": "62" - } - } - }, - { - "source_row": 25059, - "fias_id": "17d52287-c6e6-4f94-a4ef-951343b61698", - "tier": "house_structure", - "raw": "6-я улица Лазенки, дом 2, строение 16", - "mismatches": { - "street": { - "expected": "6-я улица Лазенки", - "actual": "улица Лазенки" - } - } - }, - { - "source_row": 25157, - "fias_id": "d34fd4f8-3c08-4fa2-b222-838ae30f486c", - "tier": "house_structure", - "raw": "1-я линия Хорошёвского Серебряного Бора, домовладение 4, строение 1", - "mismatches": { - "street": { - "expected": "1-я линия Хорошёвского Серебряного Бора", - "actual": "я линия Хорошёвского Серебряного Бора, домовладение" - } - } - }, - { - "source_row": 26886, - "fias_id": "681791f2-daeb-4c1f-86c0-7e302864531c", - "tier": "house_structure", - "raw": "1-й Лучевой просек, дом 7, строение 3", - "mismatches": { - "street": { - "expected": "1-й Лучевой просек", - "actual": "й Лучевой" - } - } - }, - { - "source_row": 27635, - "fias_id": "edd3b776-9e4c-49e8-8495-27a906b51d82", - "tier": "house_structure", - "raw": "5-й Лучевой просек, дом 3, строение 1", - "mismatches": { - "street": { - "expected": "5-й Лучевой просек", - "actual": "й Лучевой" - } - } - }, - { - "source_row": 29730, - "fias_id": "3e7116b5-f6a3-4581-a473-b17853bab618", - "tier": "house_structure", - "raw": "1-я улица Измайловского Зверинца, дом 19, строение 22", - "mismatches": { - "street": { - "expected": "1-я улица Измайловского Зверинца", - "actual": "улица Измайловского Зверинца" - } - } - }, - { - "source_row": 30057, - "fias_id": "e6e92639-57ea-47d8-a687-048e24ac626c", - "tier": "house_structure", - "raw": "МЖД, Киевское, 1-й километр, дом 3, строение 5", - "mismatches": { - "street": { - "expected": "МЖД, Киевское, 1-й километр", - "actual": "Киевское" - } - } - }, - { - "source_row": 30291, - "fias_id": "2f1dc8bc-0410-4e4b-b9b7-5193e82e17aa", - "tier": "house_only", - "raw": "6-я улица Лазенки, дом 30", - "mismatches": { - "street": { - "expected": "6-я улица Лазенки", - "actual": "улица Лазенки" - } - } - }, - { - "source_row": 30712, - "fias_id": "856d5b9f-a623-4772-9be0-38bfdcf2e996", - "tier": "house_only", - "raw": "5-й квартал Капотня, дом 13", - "mismatches": { - "street": { - "expected": "5-й квартал Капотня", - "actual": "й квартал Капотня" - } - } - }, - { - "source_row": 31192, - "fias_id": "be5bdb1f-916a-4b3c-a39b-bbd96aae2b3b", - "tier": "house_only", - "raw": "1-й проезд Марьиной Рощи, дом 7/9", - "mismatches": { - "street": { - "expected": "1-й проезд Марьиной Рощи", - "actual": "1-й проезд" - } - } - }, - { - "source_row": 31424, - "fias_id": "4f9042fc-77a6-4cec-b575-e42198e9e034", - "tier": "house_only", - "raw": "1-й Лучевой просек, дом 2А", - "mismatches": { - "street": { - "expected": "1-й Лучевой просек", - "actual": "й Лучевой" - } - } - }, - { - "source_row": 32210, - "fias_id": "9b84618c-68ce-444f-98e5-2dcdec402386", - "tier": "house_only", - "raw": "город Московский, 1-й микрорайон, дом 5А", - "mismatches": { - "street": { - "expected": "1-й микрорайон", - "actual": "й микрорайон" - } - } - }, - { - "source_row": 32494, - "fias_id": "d7b73a4d-7076-483e-90a1-a4e848a172bd", - "tier": "house_only", - "raw": "улица Мичуринский Проспект, Олимпийская Деревня, дом 17", - "mismatches": { - "street": { - "expected": "улица Мичуринский Проспект, Олимпийская Деревня", - "actual": "улица Мичуринский" - } - } - }, - { - "source_row": 33422, - "fias_id": "af57bd99-98ad-476e-b0f1-81a984c7214c", - "tier": "house_structure", - "raw": "4-й проезд Подбельского, дом 3, строение 11", - "mismatches": { - "street": { - "expected": "4-й проезд Подбельского", - "actual": "4-й проезд" - } - } - }, - { - "source_row": 33696, - "fias_id": "f64f6dc4-bf59-41d7-a8e3-d9c16bae14b0", - "tier": "house_structure", - "raw": "Майский просек, дом 7, строение 1", - "mismatches": { - "street": { - "expected": "Майский просек", - "actual": "Майский" - } - } - }, - { - "source_row": 34304, - "fias_id": "7f0f4f05-271b-4ba9-aa34-b77bdcf62805", - "tier": "house_only", - "raw": "поселение Рязановское, деревня Никульское, микрорайон \"Петровская Слобода\", дом 7", - "mismatches": { - "street": { - "expected": "микрорайон \"Петровская Слобода\"", - "actual": "микрорайон" - } - } - } - ], - "slices": { - "house_only": { - "rows": 7801, - "tiers": { - "house_only": 7801 - }, - "matching": "case-insensitive exact component value after whitespace and ё/е folding; street includes its source type marker", - "micro": { - "tp": 12715, - "fp": 2862, - "fn": 2887, - "support": 15602, - "precision": 0.816268, - "recall": 0.81496, - "f1": 0.815613 - }, - "macro_field_f1": 0.407909, - "exact_address_rate": 0.630304, - "no_unparsed_rate": 0.237021, - "elapsed_seconds": 3.465, - "rows_per_second": 2251.5 - }, - "house_corpus": { - "rows": 1326, - "tiers": { - "house_corpus": 1326 - }, - "matching": "case-insensitive exact component value after whitespace and ё/е folding; street includes its source type marker", - "micro": { - "tp": 3921, - "fp": 51, - "fn": 57, - "support": 3978, - "precision": 0.98716, - "recall": 0.985671, - "f1": 0.986415 - }, - "macro_field_f1": 0.739816, - "exact_address_rate": 0.957768, - "no_unparsed_rate": 0.846154, - "elapsed_seconds": 1.51, - "rows_per_second": 877.9 - }, - "house_structure": { - "rows": 5642, - "tiers": { - "house_structure": 5642 - }, - "matching": "case-insensitive exact component value after whitespace and ё/е folding; street includes its source type marker", - "micro": { - "tp": 14273, - "fp": 2648, - "fn": 2653, - "support": 16926, - "precision": 0.843508, - "recall": 0.843259, - "f1": 0.843383 - }, - "macro_field_f1": 0.632551, - "exact_address_rate": 0.578341, - "no_unparsed_rate": 0.49486, - "elapsed_seconds": 2.356, - "rows_per_second": 2394.8 - }, - "house_corpus_structure": { - "rows": 427, - "tiers": { - "house_corpus_structure": 427 - }, - "matching": "case-insensitive exact component value after whitespace and ё/е folding; street includes its source type marker", - "micro": { - "tp": 1695, - "fp": 11, - "fn": 13, - "support": 1708, - "precision": 0.993552, - "recall": 0.992389, - "f1": 0.99297 - }, - "macro_field_f1": 0.992973, - "exact_address_rate": 0.976581, - "no_unparsed_rate": 0.704918, - "elapsed_seconds": 1.473, - "rows_per_second": 289.9 - } - } -} diff --git a/evaluation/deepparse_data.py b/evaluation/deepparse_data.py deleted file mode 100644 index d6a76d6..0000000 --- a/evaluation/deepparse_data.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Pure helpers and pinned metadata for the Deepparse Russian address shard.""" - -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -import re -from typing import Sequence - - -ROOT = Path(__file__).resolve().parents[1] -DATASET_REVISION = "cb61e5e49db87f8c3586b5494149f612460f8992" -DATASET_SHA256 = ( - "e61981a059967a1062fe445fa5ffe745a661f8b4afcec59a60c3cbb4f1444110" -) -DATASET_ROWS = 13_152_918 -DATASET_URL = ( - "https://huggingface.co/datasets/deepparse/worldwide-addresses/resolve/" - f"{DATASET_REVISION}/ru/chunk-0.parquet" -) -DEFAULT_SOURCE = ( - ROOT - / ".cache" - / "external" - / f"deepparse-worldwide-ru-{DATASET_REVISION[:8]}.parquet" -) -DEFAULT_FILTERED = ( - ROOT - / ".cache" - / "external" - / f"deepparse-ru-usable-{DATASET_REVISION[:8]}.parquet" -) -DEFAULT_SAMPLE = ( - ROOT - / ".cache" - / "external" - / f"deepparse-ru-test-100k-{DATASET_REVISION[:8]}.jsonl.gz" -) -DEFAULT_MANIFEST = ROOT / "evaluation" / "deepparse_manifest.json" - -SOURCE_TAGS = frozenset( - { - "Country", - "Province", - "County", - "District", - "Municipality", - "Suburb", - "PostalCode", - "StreetName", - "StreetNumber", - "Unit", - } -) -USEFUL_TAGS = frozenset({"StreetName", "StreetNumber", "Unit"}) -LABEL_MAP = { - "PostalCode": "POSTAL_CODE", - "Province": "REGION", - "County": "DISTRICT", - "District": "DISTRICT", - "Municipality": "CITY", - "StreetName": "STREET", - "StreetNumber": "HOUSE", - "Unit": "APARTMENT", -} -SCORED_LABELS = ( - "POSTAL_CODE", - "REGION", - "DISTRICT", - "CITY", - "STREET", - "HOUSE", - "APARTMENT", -) -EXPECTED_FIELDS = ( - "postal_code", - "region", - "district", - "city", - "street", - "house_num", - "apartment", -) -LABEL_FIELDS = { - "POSTAL_CODE": "postal_code", - "REGION": "region", - "DISTRICT": "district", - "CITY": "city", - "STREET": "street", - "HOUSE": "house_num", - "APARTMENT": "apartment", -} -_IDENTITY_TAGS = ( - "Province", - "County", - "District", - "Municipality", - "Suburb", - "StreetName", - "StreetNumber", -) -_SPACE_RE = re.compile(r"\s+") - - -def fold(value: str) -> str: - """Normalize only distinctions that are not useful address evidence.""" - - return _SPACE_RE.sub(" ", value.casefold().replace("ё", "е")).strip() - - -def normalized_address_id(address: str) -> bytes: - """Return a compact deterministic ID used for exact-text deduplication.""" - - return hashlib.blake2b( - fold(address).encode("utf-8"), - digest_size=16, - person=b"addr-example-v1", - ).digest() - - -def mapped_labels(tags: Sequence[str]) -> tuple[str, ...]: - return tuple(LABEL_MAP.get(tag, "O") for tag in tags) - - -def expected_components( - tokens: Sequence[str], - labels: Sequence[str], -) -> dict[str, str | None]: - values: dict[str, list[str]] = { - field: [] for field in EXPECTED_FIELDS - } - for token, label in zip(tokens, labels): - field = LABEL_FIELDS.get(label) - if field is not None: - values[field].append(token) - return { - field: " ".join(parts) if parts else None - for field, parts in values.items() - } - - -def quality_tier(tags: Sequence[str]) -> str: - present = set(tags) - if {"StreetName", "StreetNumber", "Unit"} <= present: - return "street_house_unit" - if {"StreetName", "StreetNumber"} <= present: - return "street_house" - if "StreetName" in present: - return "street_only" - return "number_or_unit_only" - - -def canonical_group_key( - tokens: Sequence[str], - tags: Sequence[str], -) -> bytes: - """Group formatting variants without leaking one building across splits.""" - - by_tag: dict[str, list[str]] = {tag: [] for tag in _IDENTITY_TAGS} - for token, tag in zip(tokens, tags): - if tag in by_tag: - by_tag[tag].append(token) - identity = [ - [tag, fold(" ".join(by_tag[tag]))] - for tag in _IDENTITY_TAGS - if by_tag[tag] - ] - if not identity: - identity = [["fallback", fold(" ".join(tokens))]] - return json.dumps( - identity, - ensure_ascii=False, - separators=(",", ":"), - ).encode("utf-8") - - -def group_id_and_split( - tokens: Sequence[str], - tags: Sequence[str], -) -> tuple[bytes, str]: - key = canonical_group_key(tokens, tags) - digest = hashlib.sha256(b"address-group-v1\0" + key).digest() - bucket = int.from_bytes(digest[:8], "big") % 10_000 - split = ( - "train" - if bucket < 9_000 - else "validation" - if bucket < 9_500 - else "test" - ) - return digest[:16], split - - -def sample_rank(example_id: bytes) -> int: - return int.from_bytes( - hashlib.sha256(b"benchmark-sample-v1\0" + example_id).digest()[:8], - "big", - ) diff --git a/evaluation/deepparse_manifest.json b/evaluation/deepparse_manifest.json deleted file mode 100644 index 864fe7b..0000000 --- a/evaluation/deepparse_manifest.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "format_version": 1, - "source": { - "repository": "deepparse/worldwide-addresses", - "configuration": "ru", - "license": "CC BY 4.0", - "revision": "cb61e5e49db87f8c3586b5494149f612460f8992", - "url": "https://huggingface.co/datasets/deepparse/worldwide-addresses/resolve/cb61e5e49db87f8c3586b5494149f612460f8992/ru/chunk-0.parquet", - "rows": 13152918, - "bytes": 371595309, - "sha256": "e61981a059967a1062fe445fa5ffe745a661f8b4afcec59a60c3cbb4f1444110" - }, - "policy": { - "purpose": "external clean-address training/evaluation corpus; not bundled in the runtime package", - "filter": "Russian rows with 1-64 whitespace tokens, known tags, matching token/tag lengths, and at least one StreetName, StreetNumber, or Unit tag", - "deduplication": "first row for each BLAKE2b-128 hash of case-folded, ё/е-folded, whitespace-normalized address text", - "grouping": "SHA-256 of structured province/county/district/municipality/suburb/street/house identity; unit and presentation fields are excluded so one building cannot cross splits", - "split": "group hash buckets: train 90%, validation 5%, test 5%", - "sample": "lowest deterministic SHA-256 ranks from the sealed test split", - "ignored_labels": [ - "Country", - "Suburb" - ] - }, - "filter_counts": { - "source_rows": 13152918, - "rejected_administrative_only": 5403275, - "eligible_rows": 7749643, - "unique_usable_rows": 6314158, - "duplicate_rows_removed": 1435485 - }, - "split_counts": { - "train": 5681842, - "validation": 316586, - "test": 315730 - }, - "tier_counts": { - "street_house_unit": 679076, - "street_house": 4614613, - "street_only": 984676, - "number_or_unit_only": 35793 - }, - "split_tier_counts": { - "test/number_or_unit_only": 1767, - "test/street_house": 230504, - "test/street_house_unit": 33894, - "test/street_only": 49565, - "train/number_or_unit_only": 32226, - "train/street_house": 4152669, - "train/street_house_unit": 611003, - "train/street_only": 885944, - "validation/number_or_unit_only": 1800, - "validation/street_house": 231440, - "validation/street_house_unit": 34179, - "validation/street_only": 49167 - }, - "artifacts": { - "filtered_parquet": { - "filename": "deepparse-ru-usable-cb61e5e4.parquet", - "rows": 6314158, - "bytes": 557321339, - "sha256": "2ca55950ea5a4d08fbfeb55d2f798adb1dcae1d4a6dd61372f6c707928e625e8" - }, - "test_sample_jsonl_gz": { - "filename": "deepparse-ru-test-100k-cb61e5e4.jsonl.gz", - "rows": 100000, - "bytes": 10981774, - "sha256": "349140469f42a73f95fb58c9c37b2148c57e3373fed5c0fc265397800c72022b" - } - }, - "limitations": [ - "The source is curated from open geographic address data and does not reproduce misspellings or punctuation-heavy user input.", - "Country and Suburb have no direct public package field and are kept as source context but mapped to O.", - "This deterministic test split becomes tuning data after its failures are used to change the parser; reserve another test source before making a final production claim." - ] -} diff --git a/evaluation/deepparse_report.json b/evaluation/deepparse_report.json deleted file mode 100644 index b2645ca..0000000 --- a/evaluation/deepparse_report.json +++ /dev/null @@ -1,2226 +0,0 @@ -{ - "scope": "untuned external evaluation on clean Russian open-geographic address strings from the sealed Deepparse test split", - "source": { - "repository": "deepparse/worldwide-addresses", - "configuration": "ru", - "license": "CC BY 4.0", - "revision": "cb61e5e49db87f8c3586b5494149f612460f8992" - }, - "limitations": [ - "clean registry-derived strings are easier than user-entered text", - "Country and Suburb source tags are retained as context but unscored", - "span overlap gives partial credit and is not exact-value accuracy" - ], - "rows": 100000, - "tiers": { - "number_or_unit_only": 543, - "street_house": 72789, - "street_house_unit": 10915, - "street_only": 15753 - }, - "matching": "one-to-one same-label character-span overlap", - "metric_definitions": { - "span_overlap_micro": "binary micro precision, recall, and F1 for one-to-one same-label spans with any character overlap", - "character_overlap_micro": "micro precision, recall, and F1 over same-label overlapping characters", - "token_label_micro": "micro precision, recall, and F1 over aligned source-token labels", - "exact_address_rate": "fraction of rows with identical labels and exact span boundaries", - "exact_token_sequence_rate": "fraction of rows whose complete aligned token-label sequence matches", - "fields": "per-field binary span-overlap metrics", - "character_fields": "per-field character-overlap metrics", - "token_fields": "per-field aligned token-label metrics" - }, - "span_overlap_micro": { - "tp": 222353, - "fp": 8494, - "fn": 73735, - "support": 296088, - "precision": 0.963205, - "recall": 0.750969, - "f1": 0.843948 - }, - "character_overlap_micro": { - "tp": 1932109, - "fp": 723321, - "fn": 1247100, - "support": 3179209, - "precision": 0.727607, - "recall": 0.607733, - "f1": 0.662289 - }, - "token_label_micro": { - "tp": 323165, - "fp": 104063, - "fn": 221611, - "support": 544776, - "precision": 0.756423, - "recall": 0.593207, - "f1": 0.664946 - }, - "micro": { - "tp": 222353, - "fp": 8494, - "fn": 73735, - "support": 296088, - "precision": 0.963205, - "recall": 0.750969, - "f1": 0.843948 - }, - "character_micro": { - "tp": 1932109, - "fp": 723321, - "fn": 1247100, - "support": 3179209, - "precision": 0.727607, - "recall": 0.607733, - "f1": 0.662289 - }, - "token_micro": { - "tp": 323165, - "fp": 104063, - "fn": 221611, - "support": 544776, - "precision": 0.756423, - "recall": 0.593207, - "f1": 0.664946 - }, - "macro_field_f1": 0.720482, - "token_macro_field_f1": 0.544346, - "character_macro_field_f1": 0.542689, - "exact_address_rate": 0.07826, - "exact_token_sequence_rate": 0.07958, - "exact_span_recall": 0.319189, - "overlap_spans": 222353, - "exact_spans": 94508, - "fields": { - "POSTAL_CODE": { - "tp": 13089, - "fp": 3, - "fn": 22, - "support": 13111, - "precision": 0.999771, - "recall": 0.998322, - "f1": 0.999046 - }, - "REGION": { - "tp": 1783, - "fp": 909, - "fn": 5646, - "support": 7429, - "precision": 0.662333, - "recall": 0.240005, - "f1": 0.352337 - }, - "DISTRICT": { - "tp": 1855, - "fp": 1687, - "fn": 5286, - "support": 7141, - "precision": 0.523715, - "recall": 0.259768, - "f1": 0.347281 - }, - "CITY": { - "tp": 33472, - "fp": 2985, - "fn": 40316, - "support": 73788, - "precision": 0.918123, - "recall": 0.453624, - "f1": 0.607229 - }, - "STREET": { - "tp": 97310, - "fp": 1227, - "fn": 2147, - "support": 99457, - "precision": 0.987548, - "recall": 0.978413, - "f1": 0.982959 - }, - "HOUSE": { - "tp": 65006, - "fp": 192, - "fn": 19241, - "support": 84247, - "precision": 0.997055, - "recall": 0.771612, - "f1": 0.869966 - }, - "APARTMENT": { - "tp": 9838, - "fp": 1491, - "fn": 1077, - "support": 10915, - "precision": 0.868391, - "recall": 0.901328, - "f1": 0.884553 - } - }, - "character_fields": { - "POSTAL_CODE": { - "tp": 78534, - "fp": 18, - "fn": 134, - "support": 78668, - "precision": 0.999771, - "recall": 0.998297, - "f1": 0.999033 - }, - "REGION": { - "tp": 18798, - "fp": 20807, - "fn": 110481, - "support": 129279, - "precision": 0.474637, - "recall": 0.145406, - "f1": 0.222614 - }, - "DISTRICT": { - "tp": 22816, - "fp": 30971, - "fn": 124975, - "support": 147791, - "precision": 0.424192, - "recall": 0.15438, - "f1": 0.226374 - }, - "CITY": { - "tp": 344460, - "fp": 108504, - "fn": 522138, - "support": 866598, - "precision": 0.760458, - "recall": 0.397485, - "f1": 0.522082 - }, - "STREET": { - "tp": 1289132, - "fp": 560389, - "fn": 222117, - "support": 1511249, - "precision": 0.697009, - "recall": 0.853024, - "f1": 0.767165 - }, - "HOUSE": { - "tp": 152417, - "fp": 561, - "fn": 211120, - "support": 363537, - "precision": 0.996333, - "recall": 0.419261, - "f1": 0.590175 - }, - "APARTMENT": { - "tp": 25952, - "fp": 2071, - "fn": 56135, - "support": 82087, - "precision": 0.926096, - "recall": 0.316152, - "f1": 0.471383 - } - }, - "token_fields": { - "POSTAL_CODE": { - "tp": 13066, - "fp": 3, - "fn": 46, - "support": 13112, - "precision": 0.99977, - "recall": 0.996492, - "f1": 0.998128 - }, - "REGION": { - "tp": 1783, - "fp": 2457, - "fn": 12638, - "support": 14421, - "precision": 0.420519, - "recall": 0.123639, - "f1": 0.191094 - }, - "DISTRICT": { - "tp": 2183, - "fp": 3789, - "fn": 14731, - "support": 16914, - "precision": 0.365539, - "recall": 0.129065, - "f1": 0.190772 - }, - "CITY": { - "tp": 42429, - "fp": 13150, - "fn": 77682, - "support": 120111, - "precision": 0.7634, - "recall": 0.353248, - "f1": 0.482998 - }, - "STREET": { - "tp": 189444, - "fp": 83066, - "fn": 24055, - "support": 213499, - "precision": 0.695182, - "recall": 0.88733, - "f1": 0.779591 - }, - "HOUSE": { - "tp": 65321, - "fp": 219, - "fn": 78917, - "support": 144238, - "precision": 0.996659, - "recall": 0.45287, - "f1": 0.622763 - }, - "APARTMENT": { - "tp": 8939, - "fp": 1379, - "fn": 13542, - "support": 22481, - "precision": 0.86635, - "recall": 0.397625, - "f1": 0.545078 - } - }, - "elapsed_seconds": 13.802, - "rows_per_second": 7245.4, - "failure_sample": [ - { - "source_row": 25, - "example_id": "0ef2d0933ea8999a251e4997bb399d73", - "tier": "street_house", - "raw": "Ярославская ул 74 Тутаев Россия", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 14 - }, - { - "label": "HOUSE", - "start": 15, - "end": 17 - }, - { - "label": "CITY", - "start": 18, - "end": 24 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 12, - "end": 24 - } - ] - }, - { - "source_row": 75, - "example_id": "fb85bcd05b3f82cf0e9779a0193a12fc", - "tier": "street_house", - "raw": "Заречная ул Дом 12", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 11 - }, - { - "label": "HOUSE", - "start": 12, - "end": 18 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 16, - "end": 18 - }, - { - "label": "STREET", - "start": 0, - "end": 11 - } - ] - }, - { - "source_row": 112, - "example_id": "753f4f717564efc1a557d3c5d5dc3a10", - "tier": "street_house_unit", - "raw": "Российская Федерация Ахуново переулок Хади Такташа Д 19 Квартира 255", - "gold": [ - { - "label": "CITY", - "start": 21, - "end": 28 - }, - { - "label": "STREET", - "start": 29, - "end": 50 - }, - { - "label": "HOUSE", - "start": 51, - "end": 55 - }, - { - "label": "APARTMENT", - "start": 56, - "end": 68 - } - ], - "predicted": [ - { - "label": "APARTMENT", - "start": 65, - "end": 68 - }, - { - "label": "CITY", - "start": 38, - "end": 42 - }, - { - "label": "HOUSE", - "start": 53, - "end": 55 - }, - { - "label": "STREET", - "start": 21, - "end": 37 - } - ] - }, - { - "source_row": 185, - "example_id": "14c22a474ec7770eb4151cea26b1dd72", - "tier": "street_house", - "raw": "Заречная улица 20 Селихино 681085", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 14 - }, - { - "label": "HOUSE", - "start": 15, - "end": 17 - }, - { - "label": "CITY", - "start": 18, - "end": 26 - }, - { - "label": "POSTAL_CODE", - "start": 27, - "end": 33 - } - ], - "predicted": [ - { - "label": "POSTAL_CODE", - "start": 27, - "end": 33 - }, - { - "label": "STREET", - "start": 9, - "end": 26 - } - ] - }, - { - "source_row": 351, - "example_id": "d6815ec47a560809a0750e874ffcf5fa", - "tier": "street_house", - "raw": "улица Карла Маркса Дом 22", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 18 - }, - { - "label": "HOUSE", - "start": 19, - "end": 25 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 23, - "end": 25 - }, - { - "label": "STREET", - "start": 0, - "end": 18 - } - ] - }, - { - "source_row": 459, - "example_id": "53fe8f4192061f5aa228c788ae05396a", - "tier": "street_house", - "raw": "ул 50 лет НЛМК 13 Липецк Российская Федерация", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 14 - }, - { - "label": "HOUSE", - "start": 15, - "end": 17 - }, - { - "label": "CITY", - "start": 18, - "end": 24 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 24 - } - ] - }, - { - "source_row": 554, - "example_id": "e3044af36d0e4d29592bb9b3891c8e48", - "tier": "street_house", - "raw": "Гобзянско-Наб улица Дом 64 Демидовское городское поселение", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 19 - }, - { - "label": "HOUSE", - "start": 20, - "end": 26 - }, - { - "label": "CITY", - "start": 27, - "end": 58 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 14, - "end": 19 - }, - { - "label": "HOUSE", - "start": 24, - "end": 26 - }, - { - "label": "STREET", - "start": 0, - "end": 13 - } - ] - }, - { - "source_row": 719, - "example_id": "0a1028a39fbb93973b3213678275b4cf", - "tier": "street_house", - "raw": "улица Собинова Дом 6", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 14 - }, - { - "label": "HOUSE", - "start": 15, - "end": 20 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 19, - "end": 20 - }, - { - "label": "STREET", - "start": 0, - "end": 14 - } - ] - }, - { - "source_row": 935, - "example_id": "7346a7b84cf7b13b7820e01d03d0ed71", - "tier": "street_house", - "raw": "ул Тимирязева Дом 41", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 13 - }, - { - "label": "HOUSE", - "start": 14, - "end": 20 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 18, - "end": 20 - }, - { - "label": "STREET", - "start": 0, - "end": 13 - } - ] - }, - { - "source_row": 972, - "example_id": "88de18541c9c213ed87bdd0b9f89ccff", - "tier": "street_house", - "raw": "ул Герцена 5 Надёжная Россия", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 10 - }, - { - "label": "HOUSE", - "start": 11, - "end": 12 - }, - { - "label": "CITY", - "start": 13, - "end": 21 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 21 - } - ] - }, - { - "source_row": 1048, - "example_id": "b253a343fe06d1e407120c5826531e18", - "tier": "street_house", - "raw": "Верхняя Казацкая улица Д 321 Курск Россия", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 22 - }, - { - "label": "HOUSE", - "start": 23, - "end": 28 - }, - { - "label": "CITY", - "start": 29, - "end": 34 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 29, - "end": 34 - }, - { - "label": "HOUSE", - "start": 25, - "end": 28 - }, - { - "label": "STREET", - "start": 0, - "end": 22 - } - ] - }, - { - "source_row": 1203, - "example_id": "393afc4700b66a31ea665c1aa1d30bf7", - "tier": "street_house", - "raw": "Российская Федерация Новосибирск Станционная улица Дом 30А к5", - "gold": [ - { - "label": "CITY", - "start": 21, - "end": 32 - }, - { - "label": "STREET", - "start": 33, - "end": 50 - }, - { - "label": "HOUSE", - "start": 51, - "end": 61 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 55, - "end": 61 - }, - { - "label": "STREET", - "start": 21, - "end": 50 - } - ] - }, - { - "source_row": 1402, - "example_id": "64be9ddaeea089712d5702e7e1f04113", - "tier": "street_house", - "raw": "Российская Федерация г. Новосибирск улица Дуси Ковальчук 77", - "gold": [ - { - "label": "CITY", - "start": 21, - "end": 35 - }, - { - "label": "STREET", - "start": 36, - "end": 56 - }, - { - "label": "HOUSE", - "start": 57, - "end": 59 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 24, - "end": 35 - }, - { - "label": "HOUSE", - "start": 57, - "end": 59 - }, - { - "label": "STREET", - "start": 36, - "end": 56 - } - ] - }, - { - "source_row": 1502, - "example_id": "dd0bdafc5ff93e81488b2d2cca8daee0", - "tier": "street_only", - "raw": "улица Чернышевского Усольское городское поселение", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 19 - }, - { - "label": "CITY", - "start": 20, - "end": 49 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 49 - } - ] - }, - { - "source_row": 1922, - "example_id": "8298dd67cefc682b5ed0f6f8793ec97a", - "tier": "street_house", - "raw": "5-я линия В.О. Д 18", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 14 - }, - { - "label": "HOUSE", - "start": 15, - "end": 19 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 10, - "end": 11 - }, - { - "label": "HOUSE", - "start": 17, - "end": 19 - }, - { - "label": "REGION", - "start": 4, - "end": 9 - }, - { - "label": "STREET", - "start": 12, - "end": 13 - } - ] - }, - { - "source_row": 1991, - "example_id": "7eb8588da250affca540b721bbb04dcf", - "tier": "street_only", - "raw": "1-й проезд улицы Сулакской Махачкала RU", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 26 - }, - { - "label": "CITY", - "start": 27, - "end": 36 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 27, - "end": 36 - }, - { - "label": "STREET", - "start": 0, - "end": 10 - } - ] - }, - { - "source_row": 2056, - "example_id": "0ac07b06a18a203bdc1688505e036845", - "tier": "street_house", - "raw": "Трубчевская улица Д 13", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 17 - }, - { - "label": "HOUSE", - "start": 18, - "end": 22 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 20, - "end": 22 - }, - { - "label": "STREET", - "start": 0, - "end": 17 - } - ] - }, - { - "source_row": 2092, - "example_id": "1d9c21906da8a733403f456c25ec698b", - "tier": "street_house", - "raw": "RU Ворошнево Тепличная улица Дом 29", - "gold": [ - { - "label": "CITY", - "start": 3, - "end": 12 - }, - { - "label": "STREET", - "start": 13, - "end": 28 - }, - { - "label": "HOUSE", - "start": 29, - "end": 35 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 33, - "end": 35 - }, - { - "label": "STREET", - "start": 0, - "end": 28 - } - ] - }, - { - "source_row": 2153, - "example_id": "094108d4c64d61693c22177d85dffd41", - "tier": "street_house", - "raw": "603010 г. Нижний Новгород Зелёная улица 39", - "gold": [ - { - "label": "POSTAL_CODE", - "start": 0, - "end": 6 - }, - { - "label": "CITY", - "start": 7, - "end": 25 - }, - { - "label": "STREET", - "start": 26, - "end": 39 - }, - { - "label": "HOUSE", - "start": 40, - "end": 42 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 10, - "end": 33 - }, - { - "label": "HOUSE", - "start": 40, - "end": 42 - }, - { - "label": "POSTAL_CODE", - "start": 0, - "end": 6 - }, - { - "label": "STREET", - "start": 34, - "end": 39 - } - ] - }, - { - "source_row": 2566, - "example_id": "b4f5840d7bf952bf5708985b3c75e2e7", - "tier": "street_only", - "raw": "Российская Федерация г. Новый Завод Нагорная улица", - "gold": [ - { - "label": "CITY", - "start": 21, - "end": 35 - }, - { - "label": "STREET", - "start": 36, - "end": 50 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 24, - "end": 44 - } - ] - }, - { - "source_row": 2711, - "example_id": "40e0f642423d104c8ba6ca75185d8170", - "tier": "street_house", - "raw": "Россия городской округ Кулебаки Кулебаки улица Рекордов Дом 6", - "gold": [ - { - "label": "DISTRICT", - "start": 7, - "end": 31 - }, - { - "label": "CITY", - "start": 32, - "end": 40 - }, - { - "label": "STREET", - "start": 41, - "end": 55 - }, - { - "label": "HOUSE", - "start": 56, - "end": 61 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 23, - "end": 31 - }, - { - "label": "HOUSE", - "start": 60, - "end": 61 - }, - { - "label": "STREET", - "start": 41, - "end": 55 - } - ] - }, - { - "source_row": 2727, - "example_id": "67cebe6ccbad6123d0ade16294adb22d", - "tier": "street_only", - "raw": "Школьная улица Ахмановское сельское поселение кировской области", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 14 - }, - { - "label": "CITY", - "start": 15, - "end": 45 - }, - { - "label": "REGION", - "start": 46, - "end": 63 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 9, - "end": 63 - } - ] - }, - { - "source_row": 2734, - "example_id": "5bf8d8a777b4c48f6f618650f9fbeec5", - "tier": "street_only", - "raw": "ул Николаева Грязи Россия", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 12 - }, - { - "label": "CITY", - "start": 13, - "end": 18 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 18 - } - ] - }, - { - "source_row": 2740, - "example_id": "dc00252f73378e1ba52b1ae537f316ca", - "tier": "street_only", - "raw": "ул Октябрьской Революции Куса Россия 456940", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 24 - }, - { - "label": "CITY", - "start": 25, - "end": 29 - }, - { - "label": "POSTAL_CODE", - "start": 37, - "end": 43 - } - ], - "predicted": [ - { - "label": "POSTAL_CODE", - "start": 37, - "end": 43 - }, - { - "label": "STREET", - "start": 0, - "end": 29 - } - ] - }, - { - "source_row": 2914, - "example_id": "86f1bd6f336ee2db3e259f573e4e6415", - "tier": "street_house", - "raw": "Пионерский переулок Д 9", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 19 - }, - { - "label": "HOUSE", - "start": 20, - "end": 23 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 22, - "end": 23 - }, - { - "label": "STREET", - "start": 0, - "end": 19 - } - ] - }, - { - "source_row": 2920, - "example_id": "1ae9fdf582a580ce6512c4c295425c4e", - "tier": "street_house", - "raw": "ул Красина Дом 7 Махачкала Россия", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 10 - }, - { - "label": "HOUSE", - "start": 11, - "end": 16 - }, - { - "label": "CITY", - "start": 17, - "end": 26 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 15, - "end": 16 - }, - { - "label": "STREET", - "start": 0, - "end": 10 - } - ] - }, - { - "source_row": 2975, - "example_id": "85c023512fde9f1fac7ce2920872f94f", - "tier": "street_house", - "raw": "ул Гагарина Дом 34", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 11 - }, - { - "label": "HOUSE", - "start": 12, - "end": 18 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 16, - "end": 18 - }, - { - "label": "STREET", - "start": 0, - "end": 11 - } - ] - }, - { - "source_row": 3184, - "example_id": "ad88d01645f48170c9aee1ee0624abe3", - "tier": "street_only", - "raw": "403533 Россия г Фролово улица Красина", - "gold": [ - { - "label": "POSTAL_CODE", - "start": 0, - "end": 6 - }, - { - "label": "CITY", - "start": 14, - "end": 23 - }, - { - "label": "STREET", - "start": 24, - "end": 37 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 16, - "end": 23 - }, - { - "label": "POSTAL_CODE", - "start": 0, - "end": 6 - }, - { - "label": "STREET", - "start": 24, - "end": 37 - } - ] - }, - { - "source_row": 3518, - "example_id": "fc9a48db713540e8d5001fdfbe1a72b2", - "tier": "street_house", - "raw": "улица Физкультурников Д 5", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 21 - }, - { - "label": "HOUSE", - "start": 22, - "end": 25 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 24, - "end": 25 - }, - { - "label": "STREET", - "start": 0, - "end": 21 - } - ] - }, - { - "source_row": 3600, - "example_id": "fbc2cdb8c2cfa5f643a73769a5284288", - "tier": "street_house", - "raw": "Паровозный переулок Дом 32", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 19 - }, - { - "label": "HOUSE", - "start": 20, - "end": 26 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 24, - "end": 26 - }, - { - "label": "STREET", - "start": 0, - "end": 19 - } - ] - }, - { - "source_row": 3690, - "example_id": "82638e809f6ce79661ab19d7b4b905f4", - "tier": "street_only", - "raw": "Россия городское поселение Щёлково проспект 60 лет Октября", - "gold": [ - { - "label": "CITY", - "start": 7, - "end": 34 - }, - { - "label": "STREET", - "start": 35, - "end": 58 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 47, - "end": 50 - }, - { - "label": "STREET", - "start": 7, - "end": 43 - } - ] - }, - { - "source_row": 3717, - "example_id": "87f7c5162817c674fcdaaa93de560ddd", - "tier": "street_house", - "raw": "Октябрьская улица 37 г. Архангельск", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 17 - }, - { - "label": "HOUSE", - "start": 18, - "end": 20 - }, - { - "label": "CITY", - "start": 21, - "end": 35 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 24, - "end": 35 - }, - { - "label": "STREET", - "start": 0, - "end": 17 - } - ] - }, - { - "source_row": 3932, - "example_id": "308f25b2aaf7dc1acfd6dd21b5f89083", - "tier": "street_only", - "raw": "Мира Чуровское сельское поселение Россия", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 4 - }, - { - "label": "CITY", - "start": 5, - "end": 33 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 15, - "end": 23 - }, - { - "label": "STREET", - "start": 0, - "end": 14 - } - ] - }, - { - "source_row": 4307, - "example_id": "340ada078479e858b9d164ffb59fc334", - "tier": "street_house", - "raw": "Покровская улица Дом 149 Старый Оскол Россия", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 16 - }, - { - "label": "HOUSE", - "start": 17, - "end": 24 - }, - { - "label": "CITY", - "start": 25, - "end": 37 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 21, - "end": 24 - }, - { - "label": "STREET", - "start": 11, - "end": 37 - } - ] - }, - { - "source_row": 4428, - "example_id": "643fc58bf2ef03548600e3c697038239", - "tier": "street_house", - "raw": "Российская Федерация городского округа химок г. Химки улица Горшина 6 к2", - "gold": [ - { - "label": "DISTRICT", - "start": 21, - "end": 44 - }, - { - "label": "CITY", - "start": 45, - "end": 53 - }, - { - "label": "STREET", - "start": 54, - "end": 67 - }, - { - "label": "HOUSE", - "start": 68, - "end": 72 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 48, - "end": 53 - }, - { - "label": "HOUSE", - "start": 68, - "end": 72 - }, - { - "label": "STREET", - "start": 54, - "end": 67 - } - ] - }, - { - "source_row": 4705, - "example_id": "5916d39af436559b3a3938d7a6bf0ee2", - "tier": "street_house", - "raw": "Северограничная 28А Струги Красные Россия 181110", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 15 - }, - { - "label": "HOUSE", - "start": 16, - "end": 19 - }, - { - "label": "CITY", - "start": 20, - "end": 34 - }, - { - "label": "POSTAL_CODE", - "start": 42, - "end": 48 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 20, - "end": 26 - }, - { - "label": "POSTAL_CODE", - "start": 42, - "end": 48 - }, - { - "label": "REGION", - "start": 0, - "end": 15 - }, - { - "label": "STREET", - "start": 27, - "end": 34 - } - ] - }, - { - "source_row": 4805, - "example_id": "3890e9e2616a84d7fefb2d9d4abbc86e", - "tier": "street_house", - "raw": "Благовещенская ул Дом 4", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 17 - }, - { - "label": "HOUSE", - "start": 18, - "end": 23 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 22, - "end": 23 - }, - { - "label": "STREET", - "start": 0, - "end": 17 - } - ] - }, - { - "source_row": 4809, - "example_id": "27c38d8b1e3f5adb1260101ca4137fc2", - "tier": "street_house_unit", - "raw": "Парижский переулок 24 Кв 504 Троицкий Российская Федерация", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 18 - }, - { - "label": "HOUSE", - "start": 19, - "end": 21 - }, - { - "label": "APARTMENT", - "start": 22, - "end": 28 - }, - { - "label": "CITY", - "start": 29, - "end": 37 - } - ], - "predicted": [ - { - "label": "APARTMENT", - "start": 25, - "end": 28 - }, - { - "label": "HOUSE", - "start": 19, - "end": 21 - }, - { - "label": "STREET", - "start": 0, - "end": 18 - } - ] - }, - { - "source_row": 4861, - "example_id": "e003bd1043f64461b530b51479f770c8", - "tier": "street_house", - "raw": "Российская Федерация Новокузнецк улица Клименко Дом 27", - "gold": [ - { - "label": "CITY", - "start": 21, - "end": 32 - }, - { - "label": "STREET", - "start": 33, - "end": 47 - }, - { - "label": "HOUSE", - "start": 48, - "end": 54 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 21, - "end": 32 - }, - { - "label": "HOUSE", - "start": 52, - "end": 54 - }, - { - "label": "STREET", - "start": 33, - "end": 47 - } - ] - }, - { - "source_row": 5001, - "example_id": "5591203637c0dcab933867269287e3a9", - "tier": "street_house", - "raw": "Шарташская улица 4 Екатеринбург Российская Федерация", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 16 - }, - { - "label": "HOUSE", - "start": 17, - "end": 18 - }, - { - "label": "CITY", - "start": 19, - "end": 31 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 11, - "end": 31 - } - ] - }, - { - "source_row": 5067, - "example_id": "8a54ea3bfea9a8df538a54ad325cd23c", - "tier": "street_house", - "raw": "Крестьянская улица Дом 47 г Михайловское сельское поселение Россия", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 18 - }, - { - "label": "HOUSE", - "start": 19, - "end": 25 - }, - { - "label": "CITY", - "start": 26, - "end": 59 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 28, - "end": 59 - }, - { - "label": "HOUSE", - "start": 23, - "end": 25 - }, - { - "label": "STREET", - "start": 0, - "end": 18 - } - ] - }, - { - "source_row": 5192, - "example_id": "9e8970e3c3908da4dcf78519464dfe5c", - "tier": "street_house", - "raw": "Светлоярская улица Д 50 г. Волгоград 400029", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 18 - }, - { - "label": "HOUSE", - "start": 19, - "end": 23 - }, - { - "label": "CITY", - "start": 24, - "end": 36 - }, - { - "label": "POSTAL_CODE", - "start": 37, - "end": 43 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 27, - "end": 36 - }, - { - "label": "HOUSE", - "start": 21, - "end": 23 - }, - { - "label": "POSTAL_CODE", - "start": 37, - "end": 43 - }, - { - "label": "STREET", - "start": 0, - "end": 18 - } - ] - }, - { - "source_row": 5276, - "example_id": "c42b3e01f1a8b2b9bad5226dc0f4aa45", - "tier": "street_house", - "raw": "ул Салтыкова-Щедрина 3А Ярославль Российская Федерация 150000", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 20 - }, - { - "label": "HOUSE", - "start": 21, - "end": 23 - }, - { - "label": "CITY", - "start": 24, - "end": 33 - }, - { - "label": "POSTAL_CODE", - "start": 55, - "end": 61 - } - ], - "predicted": [ - { - "label": "POSTAL_CODE", - "start": 55, - "end": 61 - }, - { - "label": "STREET", - "start": 0, - "end": 33 - } - ] - }, - { - "source_row": 5452, - "example_id": "349e2410b67271cc34a4ee1c04fe825b", - "tier": "street_house", - "raw": "Советский проспект Д 46 Березники", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 18 - }, - { - "label": "HOUSE", - "start": 19, - "end": 23 - }, - { - "label": "CITY", - "start": 24, - "end": 33 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 24, - "end": 33 - }, - { - "label": "HOUSE", - "start": 21, - "end": 23 - }, - { - "label": "STREET", - "start": 0, - "end": 18 - } - ] - }, - { - "source_row": 5580, - "example_id": "621efcf5e26f100c815da93bb11b9319", - "tier": "street_house", - "raw": "Солнечная улица Д 224", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 15 - }, - { - "label": "HOUSE", - "start": 16, - "end": 21 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 18, - "end": 21 - }, - { - "label": "STREET", - "start": 0, - "end": 15 - } - ] - }, - { - "source_row": 5710, - "example_id": "9d135105352d7ccd6ad7b445ab87b52c", - "tier": "street_only", - "raw": "Российская Федерация г Смоленск Ремесленная улица", - "gold": [ - { - "label": "CITY", - "start": 21, - "end": 31 - }, - { - "label": "STREET", - "start": 32, - "end": 49 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 23, - "end": 43 - } - ] - }, - { - "source_row": 5820, - "example_id": "582df1c353aff489aa200010c61bd0a3", - "tier": "street_house", - "raw": "улица Гайдара Д 20 г Сафоновское сельское поселение", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 13 - }, - { - "label": "HOUSE", - "start": 14, - "end": 18 - }, - { - "label": "CITY", - "start": 19, - "end": 51 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 21, - "end": 51 - }, - { - "label": "HOUSE", - "start": 16, - "end": 18 - }, - { - "label": "STREET", - "start": 0, - "end": 13 - } - ] - }, - { - "source_row": 5945, - "example_id": "2c69b4beba1a71bff31f83ea0f60cfee", - "tier": "street_house_unit", - "raw": "Лесная ул 1 Кв 1", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 9 - }, - { - "label": "HOUSE", - "start": 10, - "end": 11 - }, - { - "label": "APARTMENT", - "start": 12, - "end": 16 - } - ], - "predicted": [ - { - "label": "APARTMENT", - "start": 15, - "end": 16 - }, - { - "label": "HOUSE", - "start": 10, - "end": 11 - }, - { - "label": "STREET", - "start": 0, - "end": 9 - } - ] - }, - { - "source_row": 6302, - "example_id": "93f6085dba835019900b9236f3aa365d", - "tier": "street_only", - "raw": "площадь Обороны г Санкт Петербург", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 15 - }, - { - "label": "CITY", - "start": 16, - "end": 33 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 18, - "end": 33 - }, - { - "label": "STREET", - "start": 0, - "end": 15 - } - ] - }, - { - "source_row": 6329, - "example_id": "55a13171dbddd70785ab3220c4316246", - "tier": "street_house", - "raw": "Дом 18 1-я ул Володарского", - "gold": [ - { - "label": "HOUSE", - "start": 0, - "end": 6 - }, - { - "label": "STREET", - "start": 7, - "end": 26 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 4, - "end": 6 - }, - { - "label": "STREET", - "start": 11, - "end": 26 - } - ] - } - ], - "slices": { - "street_house_unit": { - "rows": 10915, - "tiers": { - "street_house_unit": 10915 - }, - "matching": "one-to-one same-label character-span overlap", - "micro": { - "tp": 35073, - "fp": 1018, - "fn": 5279, - "support": 40352, - "precision": 0.971794, - "recall": 0.869176, - "f1": 0.917625 - }, - "character_micro": { - "tp": 227885, - "fp": 62424, - "fn": 160582, - "support": 388467, - "precision": 0.784974, - "recall": 0.586626, - "f1": 0.671459 - }, - "token_micro": { - "tp": 43996, - "fp": 10864, - "fn": 32490, - "support": 76486, - "precision": 0.801969, - "recall": 0.575216, - "f1": 0.669925 - }, - "macro_field_f1": 0.777179, - "token_macro_field_f1": 0.573404, - "character_macro_field_f1": 0.574805, - "exact_address_rate": 0.0, - "exact_token_sequence_rate": 0.0, - "exact_span_recall": 0.318002, - "overlap_spans": 35073, - "exact_spans": 12832, - "elapsed_seconds": 2.437, - "rows_per_second": 4479.5 - }, - "street_house": { - "rows": 72789, - "tiers": { - "street_house": 72789 - }, - "matching": "one-to-one same-label character-span overlap", - "micro": { - "tp": 162379, - "fp": 5852, - "fn": 54440, - "support": 216819, - "precision": 0.965214, - "recall": 0.748915, - "f1": 0.843418 - }, - "character_micro": { - "tp": 1427011, - "fp": 472848, - "fn": 835135, - "support": 2262146, - "precision": 0.751114, - "recall": 0.630822, - "f1": 0.685732 - }, - "token_micro": { - "tp": 239188, - "fp": 72217, - "fn": 155801, - "support": 394989, - "precision": 0.768093, - "recall": 0.605556, - "f1": 0.677208 - }, - "macro_field_f1": 0.602626, - "token_macro_field_f1": 0.472866, - "character_macro_field_f1": 0.484751, - "exact_address_rate": 0.098215, - "exact_token_sequence_rate": 0.099452, - "exact_span_recall": 0.334094, - "overlap_spans": 162379, - "exact_spans": 72438, - "elapsed_seconds": 10.453, - "rows_per_second": 6963.4 - }, - "street_only": { - "rows": 15753, - "tiers": { - "street_only": 15753 - }, - "matching": "one-to-one same-label character-span overlap", - "micro": { - "tp": 24041, - "fp": 1359, - "fn": 13590, - "support": 37631, - "precision": 0.946496, - "recall": 0.638862, - "f1": 0.762831 - }, - "character_micro": { - "tp": 272138, - "fp": 185047, - "fn": 244701, - "support": 516839, - "precision": 0.595247, - "recall": 0.526543, - "f1": 0.558791 - }, - "token_micro": { - "tp": 39026, - "fp": 20541, - "fn": 31920, - "support": 70946, - "precision": 0.655161, - "recall": 0.55008, - "f1": 0.59804 - }, - "macro_field_f1": 0.427581, - "token_macro_field_f1": 0.347224, - "character_macro_field_f1": 0.348103, - "exact_address_rate": 0.042976, - "exact_token_sequence_rate": 0.045642, - "exact_span_recall": 0.241184, - "overlap_spans": 24041, - "exact_spans": 9076, - "elapsed_seconds": 2.995, - "rows_per_second": 5259.3 - }, - "number_or_unit_only": { - "rows": 543, - "tiers": { - "number_or_unit_only": 543 - }, - "matching": "one-to-one same-label character-span overlap", - "micro": { - "tp": 860, - "fp": 265, - "fn": 426, - "support": 1286, - "precision": 0.764444, - "recall": 0.66874, - "f1": 0.713397 - }, - "character_micro": { - "tp": 5075, - "fp": 3002, - "fn": 6682, - "support": 11757, - "precision": 0.628327, - "recall": 0.431658, - "f1": 0.511748 - }, - "token_micro": { - "tp": 955, - "fp": 441, - "fn": 1400, - "support": 2355, - "precision": 0.684097, - "recall": 0.40552, - "f1": 0.509198 - }, - "macro_field_f1": 0.526859, - "token_macro_field_f1": 0.39145, - "character_macro_field_f1": 0.407957, - "exact_address_rate": 0.0, - "exact_token_sequence_rate": 0.0, - "exact_span_recall": 0.125972, - "overlap_spans": 860, - "exact_spans": 162, - "elapsed_seconds": 0.718, - "rows_per_second": 755.8 - } - } -} diff --git a/evaluation/detection_reference.jsonl b/evaluation/detection_reference.jsonl deleted file mode 100644 index e22edc6..0000000 --- a/evaluation/detection_reference.jsonl +++ /dev/null @@ -1,30 +0,0 @@ -{"id":"detect-001","message":"Курьер приедет по адресу: Москва, ул. Тверская, д. 13, кв. 4. Позвоните заранее.","expected":["Москва, ул. Тверская, д. 13, кв. 4"],"scenario_family":"positive_explicit","context_style":"sentence_with_address_cue","address_style":"city_street_house_unit","boundary_style":"period_after_number","polarity":"positive","ambiguity":"low","notes":"Cue should be excluded; city should be retained."} -{"id":"detect-002","message":"Напиши мне, когда будешь на ул. Тверской, д. 13.","expected":["ул. Тверской, д. 13"],"scenario_family":"positive_explicit","context_style":"prose_prefix","address_style":"street_house","boundary_style":"terminal_period","polarity":"positive","ambiguity":"low","notes":"Leading prose must not enter the span."} -{"id":"detect-003","message":"Встречаемся: Тверская улица, дом 13 завтра после шести.","expected":["Тверская улица, дом 13"],"scenario_family":"positive_suffix_marker","context_style":"colon_prefix_and_trailing_prose","address_style":"suffix_street_house","boundary_style":"house_before_trailing_words","polarity":"positive","ambiguity":"medium","notes":"Stop at the house value even without sentence punctuation."} -{"id":"detect-004","message":"Первый: ул. Ленина, д. 1; второй: ул. Мира, д. 2.","expected":["ул. Ленина, д. 1","ул. Мира, д. 2"],"scenario_family":"positive_multiple","context_style":"two_addresses","address_style":"street_house","boundary_style":"semicolon","polarity":"positive","ambiguity":"low","notes":"Return two ordered, non-overlapping spans."} -{"id":"detect-005","message":"Адрес: Ополченская 5-30","expected":["Ополченская 5-30"],"scenario_family":"positive_cue_unmarked","context_style":"address_cue","address_style":"unmarked_street_numeric_tail","boundary_style":"end_of_message","polarity":"positive","ambiguity":"high","notes":"The parser retains the compound-house alternative."} -{"id":"detect-006","message":"Доставка — ул.Тверская,д.13,кв.4, подъезд со двора","expected":["ул.Тверская,д.13,кв.4"],"scenario_family":"positive_compact","context_style":"label_and_trailing_instruction","address_style":"compact_punctuation","boundary_style":"comma_before_instruction","polarity":"positive","ambiguity":"low","notes":"No spaces after punctuation."} -{"id":"detect-007","message":"Отправьте документы: 300000, г. Тула, ул. Советская, д. 7. Спасибо.","expected":["300000, г. Тула, ул. Советская, д. 7"],"scenario_family":"positive_full","context_style":"sentence","address_style":"postal_city_street_house","boundary_style":"period_after_number","polarity":"positive","ambiguity":"low","notes":"Postal and city prefix belong to the address."} -{"id":"detect-008","message":"Точка выдачи:\nул.\u00a0Мира,\u00a0д.\u00a05\nработает до 20:00","expected":["ул.\u00a0Мира,\u00a0д.\u00a05"],"scenario_family":"positive_unicode","context_style":"multiline","address_style":"unicode_whitespace","boundary_style":"newline","polarity":"positive","ambiguity":"low","notes":"NBSP offsets must remain exact."} -{"id":"detect-009","message":"Офис находится здесь (проспект Мира, дом 10, офис 12), вход справа.","expected":["проспект Мира, дом 10, офис 12"],"scenario_family":"positive_parenthesized","context_style":"parentheses","address_style":"street_house_unit","boundary_style":"closing_parenthesis","polarity":"positive","ambiguity":"low","notes":"Parentheses are not part of the span."} -{"id":"detect-010","message":"Склад: ул. Южная, д. 2\nОфис: ул. Северная, д. 8","expected":["ул. Южная, д. 2","ул. Северная, д. 8"],"scenario_family":"positive_multiple","context_style":"multiline_two_addresses","address_style":"street_house","boundary_style":"newline","polarity":"positive","ambiguity":"low","notes":"Each line contains one address."} -{"id":"detect-011","message":"Заберите заказ на ул. Красной, д. 5-7.","expected":["ул. Красной, д. 5-7"],"scenario_family":"positive_compound_number","context_style":"prose_prefix","address_style":"hyphenated_house","boundary_style":"terminal_period","polarity":"positive","ambiguity":"medium","notes":"Compound number stays inside the detected span."} -{"id":"detect-012","message":"Мы находимся по адресу: ул. Полевая, д. 5/1, корп. 2.","expected":["ул. Полевая, д. 5/1, корп. 2"],"scenario_family":"positive_compound_number","context_style":"address_cue","address_style":"slash_house_corpus","boundary_style":"terminal_period","polarity":"positive","ambiguity":"low","notes":"Include slash house and корпус."} -{"id":"detect-013","message":"Проезд возможен до шоссе Энтузиастов, дом 29, корпус 4.","expected":["шоссе Энтузиастов, дом 29, корпус 4"],"scenario_family":"positive_street_type","context_style":"prose_prefix","address_style":"highway_house_corpus","boundary_style":"terminal_period","polarity":"positive","ambiguity":"low","notes":"Long street marker."} -{"id":"detect-014","message":"Жду у 2-й улицы Новосёлки, дом 11.","expected":["2-й улицы Новосёлки, дом 11"],"scenario_family":"positive_ordinal","context_style":"prose_prefix","address_style":"ordinal_suffix_street_house","boundary_style":"terminal_period","polarity":"positive","ambiguity":"medium","notes":"Ordinal prefix belongs to the street span."} -{"id":"detect-015","message":"Новый адрес офиса: г. Казань, улица Баумана, дом 3.","expected":["г. Казань, улица Баумана, дом 3"],"scenario_family":"positive_full","context_style":"address_cue","address_style":"city_street_house","boundary_style":"terminal_period","polarity":"positive","ambiguity":"low","notes":"Generic marked city prefix."} -{"id":"detect-016","message":"Оставьте у охраны: ул. 8 Марта, 5.","expected":["ул. 8 Марта, 5"],"scenario_family":"positive_bare_house","context_style":"prose_prefix","address_style":"numeric_street_bare_house","boundary_style":"terminal_period","polarity":"positive","ambiguity":"medium","notes":"Do not mistake 8 in the street name for the house."} -{"id":"detect-017","message":"В доме 13 квартир и два подъезда.","expected":[],"scenario_family":"negative_house_only","context_style":"ordinary_sentence","address_style":"none","boundary_style":"terminal_period","polarity":"negative","ambiguity":"low","notes":"House word and number are insufficient."} -{"id":"detect-018","message":"Встреча 13.05.2027 в 18:30.","expected":[],"scenario_family":"negative_datetime","context_style":"ordinary_sentence","address_style":"none","boundary_style":"terminal_period","polarity":"negative","ambiguity":"low","notes":"Dates and times are not addresses."} -{"id":"detect-019","message":"Я живу на улице Науки.","expected":[],"scenario_family":"negative_street_without_building","context_style":"ordinary_sentence","address_style":"street_without_house","boundary_style":"terminal_period","polarity":"negative","ambiguity":"medium","notes":"Street without a building is deliberately not detected."} -{"id":"detect-020","message":"Заказ № 4815 уже передан курьеру.","expected":[],"scenario_family":"negative_order_number","context_style":"ordinary_sentence","address_style":"none","boundary_style":"terminal_period","polarity":"negative","ambiguity":"low","notes":"Order numbers are not houses."} -{"id":"detect-021","message":"Заполните поле «Адрес» в форме.","expected":[],"scenario_family":"negative_cue_without_value","context_style":"instruction","address_style":"none","boundary_style":"terminal_period","polarity":"negative","ambiguity":"low","notes":"A cue without a parseable value is insufficient."} -{"id":"detect-022","message":"Нужно улучшить дом 13 и покрасить фасад.","expected":[],"scenario_family":"negative_lexical_overlap","context_style":"ordinary_sentence","address_style":"none","boundary_style":"terminal_period","polarity":"negative","ambiguity":"low","notes":"The substring «ул» inside a verb is not a marker."} -{"id":"detect-023","message":"Диапазон значений: 5-30, результат сохранён.","expected":[],"scenario_family":"negative_numeric_range","context_style":"ordinary_sentence","address_style":"none","boundary_style":"comma","polarity":"negative","ambiguity":"low","notes":"A numeric range without address evidence is insufficient."} -{"id":"detect-024","message":"Ополченская 5-30 указана в старом справочнике.","expected":[],"scenario_family":"negative_unmarked_without_cue","context_style":"ordinary_sentence","address_style":"unmarked_address_like","boundary_style":"terminal_period","polarity":"negative","ambiguity":"high","notes":"Conservative policy requires a cue or markers for unmarked text."} -{"id":"detect-025","message":"Реквизиты: УЛ . РОКОССОВСКОГО , дом 10 , офис 15 .","expected":["УЛ . РОКОССОВСКОГО , дом 10 , офис 15"],"scenario_family":"positive_tokenized_punctuation","context_style":"label_prefix","address_style":"spaces_around_periods","boundary_style":"terminal_period","polarity":"positive","ambiguity":"low","notes":"Spaces around abbreviation periods must not create sentence boundaries."} -{"id":"detect-026","message":"Администратор проверил IP-адресом d128:3c31:10d8:8ebf.","expected":[],"scenario_family":"negative_network_address","context_style":"technical_sentence","address_style":"ipv6","boundary_style":"terminal_period","polarity":"negative","ambiguity":"low","notes":"The lexical suffix in IP-адресом is not a postal address cue."} -{"id":"detect-027","message":"Ошибка соединения с IPv6 адресом 844e:4b9f:37c8:994e:b889.","expected":[],"scenario_family":"negative_network_address","context_style":"technical_sentence","address_style":"ipv6","boundary_style":"terminal_period","polarity":"negative","ambiguity":"low","notes":"A network-address cue must not activate postal detection."} -{"id":"detect-028","message":"Заявитель указал адрес, а также идентификационный номер 500100732259.","expected":[],"scenario_family":"negative_distant_number_after_cue","context_style":"ordinary_sentence","address_style":"none","boundary_style":"terminal_period","polarity":"negative","ambiguity":"medium","notes":"A later business identifier is not an unmarked address value."} -{"id":"detect-029","message":"Перерасход на проезд в командировки составил 666 тыс. руб.","expected":[],"scenario_family":"negative_street_marker_homonym","context_style":"ordinary_sentence","address_style":"none","boundary_style":"terminal_period","polarity":"negative","ambiguity":"medium","notes":"The ordinary noun «проезд» is not a street marker in this context."} -{"id":"detect-030","message":"Адрес: г . Чапаевск , ш . Ярцевская , д . 32 , кв . 4 .","expected":["г . Чапаевск , ш . Ярцевская , д . 32 , кв . 4"],"scenario_family":"positive_tokenized_punctuation","context_style":"address_cue","address_style":"spaces_around_all_periods","boundary_style":"terminal_period","polarity":"positive","ambiguity":"low","notes":"Separated abbreviation periods must remain inside one address clause."} diff --git a/evaluation/detection_report.json b/evaluation/detection_report.json deleted file mode 100644 index 39602eb..0000000 --- a/evaluation/detection_report.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "rows": 30, - "positive_rows": 18, - "negative_rows": 12, - "scope": "small, manually authored detection-behavior fixture; suitable for regression, not a production accuracy claim", - "metric_definitions": { - "exact_span_micro": "one-to-one precision, recall, and F1 requiring exact message boundaries", - "overlap_span_micro": "one-to-one precision, recall, and F1 requiring any character overlap; reported separately because it is lenient", - "exact_message_rate": "fraction of messages where the complete ordered span list is exact", - "negative_message_specificity": "fraction of annotated negative messages returning no spans" - }, - "exact_span_micro": { - "tp": 20, - "fp": 0, - "fn": 0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "overlap_span_micro": { - "tp": 20, - "fp": 0, - "fn": 0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "exact_message_rate": 1.0, - "negative_message_specificity": 1.0, - "scenarios": { - "negative_cue_without_value": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "negative_datetime": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "negative_distant_number_after_cue": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "negative_house_only": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "negative_lexical_overlap": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "negative_network_address": { - "rows": 2, - "exact_messages": 2, - "exact_message_rate": 1.0 - }, - "negative_numeric_range": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "negative_order_number": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "negative_street_marker_homonym": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "negative_street_without_building": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "negative_unmarked_without_cue": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "positive_bare_house": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "positive_compact": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "positive_compound_number": { - "rows": 2, - "exact_messages": 2, - "exact_message_rate": 1.0 - }, - "positive_cue_unmarked": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "positive_explicit": { - "rows": 2, - "exact_messages": 2, - "exact_message_rate": 1.0 - }, - "positive_full": { - "rows": 2, - "exact_messages": 2, - "exact_message_rate": 1.0 - }, - "positive_multiple": { - "rows": 2, - "exact_messages": 2, - "exact_message_rate": 1.0 - }, - "positive_ordinal": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "positive_parenthesized": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "positive_street_type": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "positive_suffix_marker": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - }, - "positive_tokenized_punctuation": { - "rows": 2, - "exact_messages": 2, - "exact_message_rate": 1.0 - }, - "positive_unicode": { - "rows": 1, - "exact_messages": 1, - "exact_message_rate": 1.0 - } - }, - "failure_sample": [] -} diff --git a/evaluation/evaluate_datamos.py b/evaluation/evaluate_datamos.py deleted file mode 100644 index 44c3110..0000000 --- a/evaluation/evaluate_datamos.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Score exact building components on the filtered Moscow registry test split.""" - -from __future__ import annotations - -import argparse -from collections import Counter -import gzip -import json -from pathlib import Path -import sys -import time -from typing import Any, Callable, Iterable - - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from address_normalizer import parse -from address_normalizer.types import ParsedAddress - -from datamos_data import ( - DATASET_DATE, - DATASET_ID, - DATASET_VERSION, - DEFAULT_FILTERED, - FIELDS, - fold, -) - - -def _predicted_values(result: ParsedAddress) -> dict[str, str | None]: - street_parts = [ - part - for part in (result.street, result.street_type) - if part is not None - ] - street = ( - result.raw[ - min(part.start for part in street_parts) : - max(part.end for part in street_parts) - ] - if street_parts - else None - ) - return { - "street": street, - "house_num": result.house_num.value if result.house_num else None, - "corpus": result.corpus.value if result.corpus else None, - "structure": result.structure.value if result.structure else None, - } - - -def _metrics(counts: dict[str, int]) -> dict[str, int | float]: - precision = ( - counts["tp"] / (counts["tp"] + counts["fp"]) - if counts["tp"] + counts["fp"] - else 0.0 - ) - recall = ( - counts["tp"] / (counts["tp"] + counts["fn"]) - if counts["tp"] + counts["fn"] - else 0.0 - ) - f1 = ( - 2 * precision * recall / (precision + recall) - if precision + recall - else 0.0 - ) - return { - **counts, - "precision": round(precision, 6), - "recall": round(recall, 6), - "f1": round(f1, 6), - } - - -def score( - rows: Iterable[dict[str, Any]], - parse_address: Callable[[str], ParsedAddress] = parse, -) -> dict[str, Any]: - counts = { - field: {"tp": 0, "fp": 0, "fn": 0, "support": 0} - for field in FIELDS - } - failures: list[dict[str, Any]] = [] - tiers: Counter[str] = Counter() - row_count = exact_rows = no_unparsed_rows = 0 - started = time.monotonic() - for row in rows: - row_count += 1 - tiers[row["tier"]] += 1 - result = parse_address(row["raw"]) - predicted = _predicted_values(result) - mismatches: dict[str, dict[str, str | None]] = {} - for field in FIELDS: - wanted_value = row["expected"].get(field) - actual_value = predicted[field] - wanted = fold(wanted_value) if wanted_value is not None else None - actual = fold(actual_value) if actual_value is not None else None - field_counts = counts[field] - if wanted is not None: - field_counts["support"] += 1 - if wanted == actual: - if wanted is not None: - field_counts["tp"] += 1 - else: - if actual is not None: - field_counts["fp"] += 1 - if wanted is not None: - field_counts["fn"] += 1 - mismatches[field] = { - "expected": wanted_value, - "actual": actual_value, - } - if not mismatches: - exact_rows += 1 - elif len(failures) < 50: - failures.append( - { - "source_row": row["source_row"], - "fias_id": row["fias_id"], - "tier": row["tier"], - "raw": row["raw"], - "mismatches": mismatches, - } - ) - if not result.unparsed: - no_unparsed_rows += 1 - - fields = {field: _metrics(values) for field, values in counts.items()} - totals = { - key: sum(values[key] for values in counts.values()) - for key in ("tp", "fp", "fn", "support") - } - elapsed = time.monotonic() - started - exact_component_value_micro = _metrics(totals) - return { - "rows": row_count, - "tiers": dict(sorted(tiers.items())), - "matching": ( - "case-insensitive exact component value after whitespace and ё/е " - "folding; street includes its source type marker" - ), - "metric_definitions": { - "exact_component_value_micro": ( - "micro precision, recall, and F1 over case-insensitive exact " - "component values after whitespace and ё/е folding" - ), - "exact_address_rate": ( - "fraction of rows where every scored component value matches" - ), - "no_unparsed_rate": ( - "fraction of rows with no residual word or number spans" - ), - "fields": "per-field exact component-value metrics", - }, - "exact_component_value_micro": exact_component_value_micro, - # Retained for compatibility with the first published report. - "micro": exact_component_value_micro, - "macro_field_f1": round( - sum(float(value["f1"]) for value in fields.values()) - / len(fields), - 6, - ), - "exact_address_rate": round( - exact_rows / row_count if row_count else 0.0, - 6, - ), - "no_unparsed_rate": round( - no_unparsed_rows / row_count if row_count else 0.0, - 6, - ), - "fields": fields, - "elapsed_seconds": round(elapsed, 3), - "rows_per_second": round(row_count / elapsed, 1) if elapsed else None, - "failure_sample": failures, - } - - -def load_rows( - path: Path, - *, - split: str = "test", - tier: str | None = None, - limit: int | None = None, -) -> Iterable[dict[str, Any]]: - yielded = 0 - with gzip.open(path, "rt", encoding="utf-8") as source: - for line in source: - row = json.loads(line) - if row.get("split") != split: - continue - if tier is not None and row.get("tier") != tier: - continue - yield row - yielded += 1 - if limit is not None and yielded >= limit: - break - - -def _summary(report: dict[str, Any]) -> dict[str, Any]: - return { - key: value - for key, value in report.items() - if key - not in { - "failure_sample", - "fields", - "metric_definitions", - "exact_component_value_micro", - } - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--data", type=Path, default=DEFAULT_FILTERED) - parser.add_argument("--limit", type=int) - parser.add_argument("--output", type=Path) - args = parser.parse_args(argv) - if not args.data.exists(): - parser.error(f"{args.data} does not exist; run prepare_datamos.py first") - if args.limit is not None and args.limit <= 0: - parser.error("--limit must be positive") - - report = { - "scope": ( - "untuned exact-value evaluation on a group-disjoint test split of " - "active official Moscow registry building addresses" - ), - "source": { - "dataset_id": DATASET_ID, - "version": DATASET_VERSION, - "release_date": DATASET_DATE, - }, - "limitations": [ - "October 2021 snapshot; not current FIAS/GAR truth", - "Moscow-only clean legal/simplified address formatting", - "administrative fields and address existence resolution are unscored", - ], - **score(load_rows(args.data, limit=args.limit)), - "slices": { - tier: _summary( - score(load_rows(args.data, tier=tier, limit=args.limit)) - ) - for tier in ( - "house_only", - "house_corpus", - "house_structure", - "house_corpus_structure", - ) - }, - } - rendered = f"{json.dumps(report, ensure_ascii=False, indent=2)}\n" - if args.output: - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(rendered, encoding="utf-8") - print(rendered, end="") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/evaluation/evaluate_deepparse.py b/evaluation/evaluate_deepparse.py deleted file mode 100644 index 02e8f95..0000000 --- a/evaluation/evaluate_deepparse.py +++ /dev/null @@ -1,462 +0,0 @@ -"""Evaluate the parser on a prepared Deepparse Russian test sample.""" - -from __future__ import annotations - -import argparse -from collections import Counter -from dataclasses import asdict, dataclass -import gzip -import json -from pathlib import Path -import sys -import time -from typing import Any, Callable, Iterable, Sequence - - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from address_normalizer import parse -from address_normalizer.types import ParsedAddress - -from deepparse_data import ( - DATASET_REVISION, - DEFAULT_SAMPLE, - SCORED_LABELS, -) - - -@dataclass(frozen=True, slots=True) -class Span: - label: str - start: int - end: int - - -PREDICTED_FIELDS = { - "postal_code": "POSTAL_CODE", - "region": "REGION", - "district": "DISTRICT", - "city": "CITY", - "settlement": "CITY", - "street": "STREET", - "street_type": "STREET", - "house_num": "HOUSE", - "corpus": "HOUSE", - "structure": "HOUSE", - "apartment": "APARTMENT", -} - - -def _offsets(text: str, tokens: Sequence[str]) -> tuple[tuple[int, int], ...]: - offsets: list[tuple[int, int]] = [] - cursor = 0 - for token in tokens: - start = text.find(token, cursor) - if start < 0: - raise ValueError(f"token {token!r} cannot be aligned in {text!r}") - end = start + len(token) - offsets.append((start, end)) - cursor = end - return tuple(offsets) - - -def gold_spans(row: dict[str, Any]) -> list[Span]: - tokens = row["tokens"] - labels = row["labels"] - if len(tokens) != len(labels): - raise ValueError("prepared row has mismatched tokens and labels") - offsets = _offsets(row["raw"], tokens) - spans: list[Span] = [] - active_label: str | None = None - active_start = active_end = 0 - for label, (start, end) in zip(labels, offsets): - if label != active_label: - if active_label in SCORED_LABELS: - spans.append(Span(active_label, active_start, active_end)) - active_label = label if label in SCORED_LABELS else None - active_start = start - if active_label is not None: - active_end = end - if active_label in SCORED_LABELS: - spans.append(Span(active_label, active_start, active_end)) - return spans - - -def predicted_spans(result: ParsedAddress) -> list[Span]: - grouped: dict[str, list[tuple[int, int]]] = {} - for field, label in PREDICTED_FIELDS.items(): - part = getattr(result, field) - if part is not None: - grouped.setdefault(label, []).append((part.start, part.end)) - return [ - Span( - label, - min(start for start, _ in field_offsets), - max(end for _, end in field_offsets), - ) - for label, field_offsets in sorted(grouped.items()) - ] - - -def _overlap(left: Span, right: Span) -> int: - return max(0, min(left.end, right.end) - max(left.start, right.start)) - - -def _match( - gold: Sequence[Span], - predicted: Sequence[Span], -) -> tuple[list[tuple[int, int]], set[int], set[int]]: - candidates = sorted( - ( - (_overlap(wanted, actual), gold_index, predicted_index) - for gold_index, wanted in enumerate(gold) - for predicted_index, actual in enumerate(predicted) - if wanted.label == actual.label and _overlap(wanted, actual) - ), - reverse=True, - ) - matched_gold: set[int] = set() - matched_predicted: set[int] = set() - matches: list[tuple[int, int]] = [] - for _, gold_index, predicted_index in candidates: - if gold_index in matched_gold or predicted_index in matched_predicted: - continue - matched_gold.add(gold_index) - matched_predicted.add(predicted_index) - matches.append((gold_index, predicted_index)) - return matches, matched_gold, matched_predicted - - -def _metrics(counts: dict[str, int]) -> dict[str, int | float]: - precision = ( - counts["tp"] / (counts["tp"] + counts["fp"]) - if counts["tp"] + counts["fp"] - else 0.0 - ) - recall = ( - counts["tp"] / (counts["tp"] + counts["fn"]) - if counts["tp"] + counts["fn"] - else 0.0 - ) - f1 = ( - 2 * precision * recall / (precision + recall) - if precision + recall - else 0.0 - ) - return { - **counts, - "precision": round(precision, 6), - "recall": round(recall, 6), - "f1": round(f1, 6), - } - - -def _score(rows: Iterable[dict[str, Any]]) -> dict[str, Any]: - counts = { - label: {"tp": 0, "fp": 0, "fn": 0, "support": 0} - for label in SCORED_LABELS - } - token_counts = { - label: {"tp": 0, "fp": 0, "fn": 0, "support": 0} - for label in SCORED_LABELS - } - character_counts = { - label: {"tp": 0, "fp": 0, "fn": 0, "support": 0} - for label in SCORED_LABELS - } - failures: list[dict[str, Any]] = [] - row_count = exact_rows = exact_token_rows = exact_spans = overlap_spans = 0 - tiers: Counter[str] = Counter() - started = time.monotonic() - for row in rows: - row_count += 1 - tiers[row["tier"]] += 1 - gold = gold_spans(row) - predicted = predicted_spans(parse(row["raw"])) - matches, matched_gold, matched_predicted = _match(gold, predicted) - overlap_spans += len(matches) - exact_spans += sum( - gold[gold_index] == predicted[predicted_index] - for gold_index, predicted_index in matches - ) - exact_row = ( - len(matches) == len(gold) == len(predicted) - and all( - gold[gold_index] == predicted[predicted_index] - for gold_index, predicted_index in matches - ) - ) - if exact_row: - exact_rows += 1 - - token_offsets = _offsets(row["raw"], row["tokens"]) - actual_token_labels: list[str] = [] - for start, end in token_offsets: - token_span = Span("", start, end) - candidates = [ - (_overlap(token_span, span), span.label) - for span in predicted - if _overlap(token_span, span) - ] - actual_token_labels.append( - max(candidates)[1] if candidates else "O" - ) - wanted_token_labels = [ - label if label in SCORED_LABELS else "O" - for label in row["labels"] - ] - if actual_token_labels == wanted_token_labels: - exact_token_rows += 1 - for wanted, actual in zip(wanted_token_labels, actual_token_labels): - if wanted in SCORED_LABELS: - token_counts[wanted]["support"] += 1 - if wanted == actual: - if wanted in SCORED_LABELS: - token_counts[wanted]["tp"] += 1 - continue - if actual in SCORED_LABELS: - token_counts[actual]["fp"] += 1 - if wanted in SCORED_LABELS: - token_counts[wanted]["fn"] += 1 - - for label in SCORED_LABELS: - gold_indices = { - index for index, span in enumerate(gold) if span.label == label - } - predicted_indices = { - index - for index, span in enumerate(predicted) - if span.label == label - } - label_matches = sum( - gold_index in gold_indices - and predicted_index in predicted_indices - for gold_index, predicted_index in matches - ) - counts[label]["tp"] += label_matches - counts[label]["fp"] += len(predicted_indices - matched_predicted) - counts[label]["fn"] += len(gold_indices - matched_gold) - counts[label]["support"] += len(gold_indices) - gold_characters = sum( - gold[index].end - gold[index].start for index in gold_indices - ) - predicted_characters = sum( - predicted[index].end - predicted[index].start - for index in predicted_indices - ) - overlapping_characters = sum( - _overlap(gold[gold_index], predicted[predicted_index]) - for gold_index, predicted_index in matches - if gold_index in gold_indices - and predicted_index in predicted_indices - ) - character_counts[label]["tp"] += overlapping_characters - character_counts[label]["fp"] += ( - predicted_characters - overlapping_characters - ) - character_counts[label]["fn"] += ( - gold_characters - overlapping_characters - ) - character_counts[label]["support"] += gold_characters - - if not exact_row and len(failures) < 50: - failures.append( - { - "source_row": row["source_row"], - "example_id": row["example_id"], - "tier": row["tier"], - "raw": row["raw"], - "gold": [asdict(span) for span in gold], - "predicted": [asdict(span) for span in predicted], - } - ) - - fields = {label: _metrics(values) for label, values in counts.items()} - token_fields = { - label: _metrics(values) for label, values in token_counts.items() - } - character_fields = { - label: _metrics(values) for label, values in character_counts.items() - } - totals = { - key: sum(values[key] for values in counts.values()) - for key in ("tp", "fp", "fn", "support") - } - token_totals = { - key: sum(values[key] for values in token_counts.values()) - for key in ("tp", "fp", "fn", "support") - } - character_totals = { - key: sum(values[key] for values in character_counts.values()) - for key in ("tp", "fp", "fn", "support") - } - elapsed = time.monotonic() - started - span_overlap_micro = _metrics(totals) - character_overlap_micro = _metrics(character_totals) - token_label_micro = _metrics(token_totals) - return { - "rows": row_count, - "tiers": dict(sorted(tiers.items())), - "matching": "one-to-one same-label character-span overlap", - "metric_definitions": { - "span_overlap_micro": ( - "binary micro precision, recall, and F1 for one-to-one " - "same-label spans with any character overlap" - ), - "character_overlap_micro": ( - "micro precision, recall, and F1 over same-label overlapping " - "characters" - ), - "token_label_micro": ( - "micro precision, recall, and F1 over aligned source-token " - "labels" - ), - "exact_address_rate": ( - "fraction of rows with identical labels and exact span " - "boundaries" - ), - "exact_token_sequence_rate": ( - "fraction of rows whose complete aligned token-label sequence " - "matches" - ), - "fields": "per-field binary span-overlap metrics", - "character_fields": "per-field character-overlap metrics", - "token_fields": "per-field aligned token-label metrics", - }, - "span_overlap_micro": span_overlap_micro, - "character_overlap_micro": character_overlap_micro, - "token_label_micro": token_label_micro, - # Retained for compatibility with the first published report. - "micro": span_overlap_micro, - "character_micro": character_overlap_micro, - "token_micro": token_label_micro, - "macro_field_f1": round( - sum(float(value["f1"]) for value in fields.values()) - / len(fields), - 6, - ), - "token_macro_field_f1": round( - sum(float(value["f1"]) for value in token_fields.values()) - / len(token_fields), - 6, - ), - "character_macro_field_f1": round( - sum(float(value["f1"]) for value in character_fields.values()) - / len(character_fields), - 6, - ), - "exact_address_rate": round( - exact_rows / row_count if row_count else 0.0, - 6, - ), - "exact_token_sequence_rate": round( - exact_token_rows / row_count if row_count else 0.0, - 6, - ), - "exact_span_recall": round( - exact_spans / totals["support"] if totals["support"] else 0.0, - 6, - ), - "overlap_spans": overlap_spans, - "exact_spans": exact_spans, - "fields": fields, - "character_fields": character_fields, - "token_fields": token_fields, - "elapsed_seconds": round(elapsed, 3), - "rows_per_second": round(row_count / elapsed, 1) if elapsed else None, - "failure_sample": failures, - } - - -def load_rows( - path: Path, - *, - limit: int | None = None, - tier: str | None = None, -) -> Iterable[dict[str, Any]]: - with gzip.open(path, "rt", encoding="utf-8") as source: - yielded = 0 - for line in source: - row = json.loads(line) - if row.get("split") != "test": - raise ValueError("prepared benchmark contains a non-test row") - if tier is not None and row.get("tier") != tier: - continue - yield row - yielded += 1 - if limit is not None and yielded >= limit: - break - - -def _summary(report: dict[str, Any]) -> dict[str, Any]: - return { - key: value - for key, value in report.items() - if key - not in { - "failure_sample", - "fields", - "character_fields", - "token_fields", - "metric_definitions", - "span_overlap_micro", - "character_overlap_micro", - "token_label_micro", - } - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--data", type=Path, default=DEFAULT_SAMPLE) - parser.add_argument("--limit", type=int) - parser.add_argument("--output", type=Path) - args = parser.parse_args(argv) - if not args.data.exists(): - parser.error( - f"{args.data} does not exist; run prepare_deepparse.py first" - ) - if args.limit is not None and args.limit <= 0: - parser.error("--limit must be positive") - - score = _score(load_rows(args.data, limit=args.limit)) - report = { - "scope": ( - "untuned external evaluation on clean Russian open-geographic " - "address strings from the sealed Deepparse test split" - ), - "source": { - "repository": "deepparse/worldwide-addresses", - "configuration": "ru", - "license": "CC BY 4.0", - "revision": DATASET_REVISION, - }, - "limitations": [ - "clean registry-derived strings are easier than user-entered text", - "Country and Suburb source tags are retained as context but unscored", - "span overlap gives partial credit and is not exact-value accuracy", - ], - **score, - "slices": { - tier: _summary( - _score(load_rows(args.data, limit=args.limit, tier=tier)) - ) - for tier in ( - "street_house_unit", - "street_house", - "street_only", - "number_or_unit_only", - ) - }, - } - rendered = f"{json.dumps(report, ensure_ascii=False, indent=2)}\n" - if args.output: - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(rendered, encoding="utf-8") - print(rendered, end="") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/evaluation/evaluate_detection.py b/evaluation/evaluate_detection.py deleted file mode 100644 index 4c3ab31..0000000 --- a/evaluation/evaluate_detection.py +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env python3 -"""Evaluate free-form message address detection on a transparent fixture.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -import sys -from typing import Any, Callable, Iterable, Sequence - - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from address_normalizer import detect_addresses -from address_normalizer.types import DetectedAddress - - -Span = tuple[int, int] - - -def _ratio(numerator: int, denominator: int) -> float: - return numerator / denominator if denominator else 0.0 - - -def _prf(tp: int, fp: int, fn: int) -> dict[str, int | float]: - precision = _ratio(tp, tp + fp) - recall = _ratio(tp, tp + fn) - f1 = _ratio(2 * precision * recall, precision + recall) - return { - "tp": tp, - "fp": fp, - "fn": fn, - "precision": round(precision, 6), - "recall": round(recall, 6), - "f1": round(f1, 6), - } - - -def load_rows(path: Path) -> list[dict[str, Any]]: - """Load and validate the annotated JSONL fixture.""" - - rows: list[dict[str, Any]] = [] - for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): - if not line.strip(): - continue - row = json.loads(line) - if not isinstance(row, dict) or not isinstance(row.get("message"), str): - raise ValueError(f"{path}:{line_number}: message must be a string") - expected = row.get("expected") - if not isinstance(expected, list) or not all( - isinstance(value, str) for value in expected - ): - raise ValueError(f"{path}:{line_number}: expected must be strings") - rows.append(row) - if not rows: - raise ValueError(f"{path}: no rows") - return rows - - -def _expected_spans(message: str, expected: Sequence[str]) -> list[Span]: - spans: list[Span] = [] - search_start = 0 - for value in expected: - start = message.find(value, search_start) - if start < 0: - raise ValueError(f"expected text is absent from message: {value!r}") - spans.append((start, start + len(value))) - search_start = start + len(value) - return spans - - -def _overlap(left: Span, right: Span) -> int: - return max(0, min(left[1], right[1]) - max(left[0], right[0])) - - -def _match( - expected: Sequence[Span], - actual: Sequence[Span], - *, - exact: bool, -) -> tuple[list[tuple[int, int]], set[int], set[int]]: - candidates = sorted( - ( - ( - _overlap(wanted, observed), - expected_index, - actual_index, - ) - for expected_index, wanted in enumerate(expected) - for actual_index, observed in enumerate(actual) - if (wanted == observed if exact else _overlap(wanted, observed) > 0) - ), - reverse=True, - ) - matched_expected: set[int] = set() - matched_actual: set[int] = set() - matches: list[tuple[int, int]] = [] - for _, expected_index, actual_index in candidates: - if ( - expected_index in matched_expected - or actual_index in matched_actual - ): - continue - matched_expected.add(expected_index) - matched_actual.add(actual_index) - matches.append((expected_index, actual_index)) - return matches, matched_expected, matched_actual - - -def _failure_reasons( - expected: Sequence[Span], - actual: Sequence[Span], -) -> list[str]: - _, matched_expected, matched_actual = _match(expected, actual, exact=False) - reasons: set[str] = set() - for expected_index, wanted in enumerate(expected): - if expected_index not in matched_expected: - reasons.add("missed_address") - continue - overlaps = [ - observed for observed in actual if _overlap(wanted, observed) - ] - for observed in overlaps: - if observed[0] < wanted[0] or observed[1] > wanted[1]: - reasons.add("span_includes_context") - if observed[0] > wanted[0] or observed[1] < wanted[1]: - reasons.add("span_drops_address_text") - if len(matched_actual) != len(actual): - reasons.add("spurious_address") - return sorted(reasons) - - -def evaluate( - rows: Iterable[dict[str, Any]], - detector: Callable[[str], Sequence[DetectedAddress]] = detect_addresses, -) -> dict[str, Any]: - """Score exact and overlap spans without treating overlap as exact.""" - - row_count = positive_rows = negative_rows = exact_messages = 0 - correctly_empty = 0 - exact_tp = exact_fp = exact_fn = 0 - overlap_tp = overlap_fp = overlap_fn = 0 - failures: list[dict[str, Any]] = [] - scenario_counts: dict[str, dict[str, int]] = {} - - for row in rows: - row_count += 1 - message = str(row["message"]) - expected = _expected_spans(message, row["expected"]) - detected = tuple(detector(message)) - actual = [item.span for item in detected] - if expected: - positive_rows += 1 - else: - negative_rows += 1 - if not actual: - correctly_empty += 1 - - exact_matches, _, _ = _match(expected, actual, exact=True) - overlap_matches, _, _ = _match(expected, actual, exact=False) - exact_tp += len(exact_matches) - exact_fp += len(actual) - len(exact_matches) - exact_fn += len(expected) - len(exact_matches) - overlap_tp += len(overlap_matches) - overlap_fp += len(actual) - len(overlap_matches) - overlap_fn += len(expected) - len(overlap_matches) - message_exact = expected == actual - exact_messages += int(message_exact) - - scenario = str(row.get("scenario_family", "unspecified")) - scenario_result = scenario_counts.setdefault( - scenario, - {"rows": 0, "exact_messages": 0}, - ) - scenario_result["rows"] += 1 - scenario_result["exact_messages"] += int(message_exact) - - if not message_exact: - failures.append( - { - "id": row.get("id"), - "message": message, - "scenario_family": scenario, - "context_style": row.get("context_style"), - "address_style": row.get("address_style"), - "boundary_style": row.get("boundary_style"), - "ambiguity": row.get("ambiguity"), - "expected": [ - {"text": message[start:end], "span": [start, end]} - for start, end in expected - ], - "actual": [ - { - "text": item.text, - "span": [item.start, item.end], - "confidence": round(item.confidence, 4), - "signals": list(item.signals), - } - for item in detected - ], - "failure_reasons": _failure_reasons(expected, actual), - "notes": row.get("notes"), - } - ) - - scenario_report = { - name: { - **values, - "exact_message_rate": round( - _ratio(values["exact_messages"], values["rows"]), - 6, - ), - } - for name, values in sorted(scenario_counts.items()) - } - return { - "rows": row_count, - "positive_rows": positive_rows, - "negative_rows": negative_rows, - "scope": ( - "small, manually authored detection-behavior fixture; suitable for " - "regression, not a production accuracy claim" - ), - "metric_definitions": { - "exact_span_micro": ( - "one-to-one precision, recall, and F1 requiring exact message " - "boundaries" - ), - "overlap_span_micro": ( - "one-to-one precision, recall, and F1 requiring any character " - "overlap; reported separately because it is lenient" - ), - "exact_message_rate": ( - "fraction of messages where the complete ordered span list is " - "exact" - ), - "negative_message_specificity": ( - "fraction of annotated negative messages returning no spans" - ), - }, - "exact_span_micro": _prf(exact_tp, exact_fp, exact_fn), - "overlap_span_micro": _prf(overlap_tp, overlap_fp, overlap_fn), - "exact_message_rate": round(_ratio(exact_messages, row_count), 6), - "negative_message_specificity": round( - _ratio(correctly_empty, negative_rows), - 6, - ), - "scenarios": scenario_report, - "failure_sample": failures, - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument( - "--data", - type=Path, - default=ROOT / "evaluation/detection_reference.jsonl", - ) - parser.add_argument( - "--output", - type=Path, - default=ROOT / "evaluation/detection_report.json", - ) - args = parser.parse_args(argv) - - report = evaluate(load_rows(args.data)) - rendered = json.dumps(report, ensure_ascii=False, indent=2) - print(rendered) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(f"{rendered}\n", encoding="utf-8") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/evaluation/evaluate_redmadrobot.py b/evaluation/evaluate_redmadrobot.py deleted file mode 100644 index ddcf3d4..0000000 --- a/evaluation/evaluate_redmadrobot.py +++ /dev/null @@ -1,456 +0,0 @@ -"""Evaluate address extraction on an independent Russian NER benchmark.""" - -from __future__ import annotations - -import argparse -import csv -from dataclasses import asdict, dataclass -import hashlib -import json -from pathlib import Path -import sys -from typing import Any, Callable, Iterable, Sequence -from urllib.request import Request, urlopen - - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from address_normalizer import parse -from address_normalizer.types import ParsedAddress - - -DATASET_REVISION = "f77ea831274daf980cc45c61a93c226be9d978d6" -DATASET_SHA256 = "6bf544a380a3ee5bec94b946124bea3afaecce49e734679ad0f0c0e7c12977bb" -DATASET_URL = ( - "https://huggingface.co/datasets/redmadrobot-rnd/pii_benchmark/resolve/" - f"{DATASET_REVISION}/test.csv" -) -DEFAULT_DATA = ( - ROOT - / ".cache" - / f"redmadrobot-pii-benchmark-{DATASET_REVISION[:8]}.csv" -) -LOCATION_LABELS = { - "COUNTRY", - "REGION", - "DISTRICT", - "CITY", - "STREET", - "HOUSE", -} -SCORED_LABELS = ("REGION", "DISTRICT", "CITY", "STREET", "HOUSE") -PART_LABELS = { - "region": "REGION", - "district": "DISTRICT", - "city": "CITY", - "settlement": "CITY", - "street": "STREET", - "street_type": "STREET", - "house_num": "HOUSE", - "corpus": "HOUSE", - "structure": "HOUSE", - "apartment": "HOUSE", -} - - -@dataclass(frozen=True, slots=True) -class Span: - label: str - start: int - end: int - - -@dataclass(frozen=True, slots=True) -class AddressSnippet: - source_row: int - text: str - tokens: tuple[str, ...] - tags: tuple[str, ...] - offsets: tuple[tuple[int, int], ...] - - -def _base_label(tag: str) -> str: - return tag[2:] if tag.startswith(("B-", "I-")) else tag - - -def _reconstruct(tokens: Sequence[str]) -> tuple[str, tuple[tuple[int, int], ...]]: - text_parts: list[str] = [] - offsets: list[tuple[int, int]] = [] - cursor = 0 - for index, token in enumerate(tokens): - if index: - text_parts.append(" ") - cursor += 1 - start = cursor - text_parts.append(token) - cursor += len(token) - offsets.append((start, cursor)) - return "".join(text_parts), tuple(offsets) - - -def _clusters(tags: Sequence[str], max_gap: int) -> list[tuple[int, int]]: - location_indices = [ - index - for index, tag in enumerate(tags) - if _base_label(tag) in LOCATION_LABELS - ] - if not location_indices: - return [] - clusters: list[tuple[int, int]] = [] - start = previous = location_indices[0] - for index in location_indices[1:]: - if index - previous - 1 > max_gap: - clusters.append((start, previous + 1)) - start = index - previous = index - clusters.append((start, previous + 1)) - return clusters - - -def load_snippets(path: Path, *, max_gap: int = 3) -> list[AddressSnippet]: - snippets: list[AddressSnippet] = [] - with path.open(encoding="utf-8", newline="") as source: - for row_number, row in enumerate(csv.DictReader(source)): - tokens = tuple(json.loads(row["tokens"])) - tags = tuple(json.loads(row["ner_tags"])) - if len(tokens) != len(tags): - raise ValueError(f"row {row_number}: token/tag length mismatch") - for start, end in _clusters(tags, max_gap): - selected_tags = tags[start:end] - if not any( - _base_label(tag) in SCORED_LABELS - for tag in selected_tags - ): - continue - selected_tokens = tokens[start:end] - text, offsets = _reconstruct(selected_tokens) - snippets.append( - AddressSnippet( - source_row=row_number, - text=text, - tokens=selected_tokens, - tags=selected_tags, - offsets=offsets, - ) - ) - return snippets - - -def _gold_spans(snippet: AddressSnippet) -> list[Span]: - spans: list[Span] = [] - active_label: str | None = None - active_start = 0 - active_end = 0 - for tag, (start, end) in zip(snippet.tags, snippet.offsets): - label = _base_label(tag) - continues = tag.startswith("I-") and label == active_label - if active_label is not None and not continues: - if active_label in SCORED_LABELS: - spans.append(Span(active_label, active_start, active_end)) - active_label = None - if label not in LOCATION_LABELS: - continue - if active_label is None: - active_label = label - active_start = start - active_end = end - if active_label in SCORED_LABELS: - spans.append(Span(active_label, active_start, active_end)) - return spans - - -def _predicted_spans(result: ParsedAddress) -> list[Span]: - grouped: dict[str, list[tuple[int, int]]] = {} - for field, label in PART_LABELS.items(): - part = getattr(result, field) - if part is not None: - grouped.setdefault(label, []).append((part.start, part.end)) - return [ - Span( - label=label, - start=min(start for start, _ in offsets), - end=max(end for _, end in offsets), - ) - for label, offsets in sorted(grouped.items()) - ] - - -def _overlap(left: Span, right: Span) -> int: - return max(0, min(left.end, right.end) - max(left.start, right.start)) - - -def _match( - gold: Sequence[Span], - predicted: Sequence[Span], -) -> tuple[list[tuple[int, int]], set[int], set[int]]: - candidates = sorted( - ( - (_overlap(wanted, actual), gold_index, predicted_index) - for gold_index, wanted in enumerate(gold) - for predicted_index, actual in enumerate(predicted) - if wanted.label == actual.label and _overlap(wanted, actual) - ), - reverse=True, - ) - matched_gold: set[int] = set() - matched_predicted: set[int] = set() - matches: list[tuple[int, int]] = [] - for _, gold_index, predicted_index in candidates: - if gold_index in matched_gold or predicted_index in matched_predicted: - continue - matched_gold.add(gold_index) - matched_predicted.add(predicted_index) - matches.append((gold_index, predicted_index)) - return matches, matched_gold, matched_predicted - - -def _prf(values: dict[str, int]) -> dict[str, int | float]: - precision = ( - values["tp"] / (values["tp"] + values["fp"]) - if values["tp"] + values["fp"] - else 0.0 - ) - recall = ( - values["tp"] / (values["tp"] + values["fn"]) - if values["tp"] + values["fn"] - else 0.0 - ) - f1 = ( - 2 * precision * recall / (precision + recall) - if precision + recall - else 0.0 - ) - return { - **values, - "precision": round(precision, 6), - "recall": round(recall, 6), - "f1": round(f1, 6), - } - - -def evaluate( - snippets: Iterable[AddressSnippet], - parse_address: Callable[[str], ParsedAddress] = parse, -) -> dict[str, Any]: - counts = { - label: {"tp": 0, "fp": 0, "fn": 0, "support": 0} - for label in SCORED_LABELS - } - failures: list[dict[str, Any]] = [] - snippet_count = exact_span_matches = overlap_matches = 0 - source_rows: set[int] = set() - - for snippet in snippets: - snippet_count += 1 - source_rows.add(snippet.source_row) - gold = _gold_spans(snippet) - predicted = _predicted_spans(parse_address(snippet.text)) - matches, matched_gold, matched_predicted = _match(gold, predicted) - overlap_matches += len(matches) - exact_span_matches += sum( - gold[gold_index] == predicted[predicted_index] - for gold_index, predicted_index in matches - ) - - for label in SCORED_LABELS: - gold_indices = { - index for index, span in enumerate(gold) if span.label == label - } - predicted_indices = { - index - for index, span in enumerate(predicted) - if span.label == label - } - label_matches = sum( - gold_index in gold_indices - and predicted_index in predicted_indices - for gold_index, predicted_index in matches - ) - counts[label]["tp"] += label_matches - counts[label]["fp"] += len(predicted_indices - matched_predicted) - counts[label]["fn"] += len(gold_indices - matched_gold) - counts[label]["support"] += len(gold_indices) - - if ( - len(matched_gold) != len(gold) - or len(matched_predicted) != len(predicted) - ) and len(failures) < 50: - failures.append( - { - "source_row": snippet.source_row, - "text": snippet.text, - "gold": [asdict(span) for span in gold], - "predicted": [asdict(span) for span in predicted], - } - ) - - fields = {label: _prf(values) for label, values in counts.items()} - totals = { - name: sum(values[name] for values in counts.values()) - for name in ("tp", "fp", "fn", "support") - } - supported_f1 = [ - float(values["f1"]) - for values in fields.values() - if values["support"] - ] - span_overlap_micro = _prf(totals) - return { - "source_rows": len(source_rows), - "address_snippets": snippet_count, - "matching": ( - "one-to-one same-label span overlap; address windows are oracle-" - "cropped from the benchmark's BIO annotations" - ), - "metric_definitions": { - "span_overlap_micro": ( - "micro precision, recall, and F1 for one-to-one same-label " - "spans with any character overlap" - ), - "exact_span_recall": ( - "exact-boundary same-label matches divided by gold span count" - ), - "fields": "per-field span-overlap precision, recall, and F1", - }, - "span_overlap_micro": span_overlap_micro, - # Retained for compatibility with the first published report. - "micro": span_overlap_micro, - "macro_field_f1": round( - sum(supported_f1) / len(supported_f1) if supported_f1 else 0.0, - 6, - ), - "overlap_matches": overlap_matches, - "exact_span_matches": exact_span_matches, - "exact_span_recall": round( - exact_span_matches / totals["support"] if totals["support"] else 0.0, - 6, - ), - "fields": fields, - "failure_sample": failures, - } - - -def _without_failures(report: dict[str, Any]) -> dict[str, Any]: - return { - key: value - for key, value in report.items() - if key - not in { - "failure_sample", - "metric_definitions", - "span_overlap_micro", - } - } - - -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def download_dataset(path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_suffix(f"{path.suffix}.part") - request = Request(DATASET_URL, headers={"User-Agent": "address-normalizer/2"}) - with urlopen(request, timeout=60) as response, temporary.open("wb") as output: - while chunk := response.read(1024 * 1024): - output.write(chunk) - actual_sha256 = _sha256(temporary) - if actual_sha256 != DATASET_SHA256: - temporary.unlink(missing_ok=True) - raise ValueError( - "external benchmark checksum mismatch: " - f"expected {DATASET_SHA256}, got {actual_sha256}" - ) - temporary.replace(path) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--data", type=Path, default=DEFAULT_DATA) - parser.add_argument( - "--download", - action="store_true", - help="download the pinned benchmark revision before evaluation", - ) - parser.add_argument("--max-gap", type=int, default=3) - parser.add_argument("--output", type=Path) - args = parser.parse_args(argv) - - if args.download: - download_dataset(args.data) - if not args.data.exists(): - parser.error( - f"{args.data} does not exist; provide --data or run with --download" - ) - actual_sha256 = _sha256(args.data) - if actual_sha256 != DATASET_SHA256: - parser.error( - "benchmark checksum mismatch: " - f"expected {DATASET_SHA256}, got {actual_sha256}" - ) - if args.max_gap < 0: - parser.error("--max-gap must be non-negative") - - snippets = load_snippets(args.data, max_gap=args.max_gap) - multi_field = [ - snippet - for snippet in snippets - if len({span.label for span in _gold_spans(snippet)}) >= 2 - ] - street_and_house = [ - snippet - for snippet in snippets - if {"STREET", "HOUSE"} - <= {span.label for span in _gold_spans(snippet)} - ] - administrative_only = [ - snippet - for snippet in snippets - if {span.label for span in _gold_spans(snippet)} - <= {"REGION", "DISTRICT", "CITY"} - ] - report = { - "scope": ( - "independent, untuned external evaluation on address snippets from " - "the RedMadRobot Russian PII NER benchmark" - ), - "source": { - "repository": "redmadrobot-rnd/pii_benchmark", - "license": "MIT", - "revision": DATASET_REVISION, - "sha256": actual_sha256, - "url": DATASET_URL, - "limitations": ( - "production-log-shaped and manually annotated, with real " - "personal values replaced; includes synthetic document-style " - "examples and hard negatives" - ), - }, - "windowing": { - "max_non_location_tokens_between_spans": args.max_gap, - "country_is_context_only": True, - }, - **evaluate(snippets), - "slices": { - "multi_field": _without_failures(evaluate(multi_field)), - "street_and_house": _without_failures(evaluate(street_and_house)), - "administrative_only": _without_failures( - evaluate(administrative_only) - ), - }, - } - rendered = f"{json.dumps(report, ensure_ascii=False, indent=2)}\n" - if args.output: - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(rendered, encoding="utf-8") - print(rendered, end="") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/evaluation/evaluate_redmadrobot_detection.py b/evaluation/evaluate_redmadrobot_detection.py deleted file mode 100644 index cb19670..0000000 --- a/evaluation/evaluate_redmadrobot_detection.py +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env python3 -"""Evaluate message-level address detection on complete RedMadRobot rows.""" - -from __future__ import annotations - -import argparse -from collections import Counter -import csv -from dataclasses import asdict, dataclass -import json -from pathlib import Path -import sys -from typing import Any, Callable, Iterable, Sequence - - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from address_normalizer import detect_addresses -from address_normalizer.types import DetectedAddress - -from evaluate_redmadrobot import ( - DATASET_REVISION, - DATASET_SHA256, - _base_label, - _clusters, - _reconstruct, - _sha256, -) - - -DEFAULT_DATA = ( - ROOT - / ".cache" - / "external" - / f"redmadrobot-pii-benchmark-{DATASET_REVISION[:8]}.csv" -) - - -@dataclass(frozen=True, slots=True) -class Span: - start: int - end: int - - -@dataclass(frozen=True, slots=True) -class Message: - source_row: int - text: str - gold: tuple[Span, ...] - - -def _ratio(numerator: int, denominator: int) -> float: - return numerator / denominator if denominator else 0.0 - - -def _prf(tp: int, fp: int, fn: int) -> dict[str, int | float]: - precision = _ratio(tp, tp + fp) - recall = _ratio(tp, tp + fn) - f1 = _ratio(2 * precision * recall, precision + recall) - return { - "tp": tp, - "fp": fp, - "fn": fn, - "precision": round(precision, 6), - "recall": round(recall, 6), - "f1": round(f1, 6), - } - - -def load_messages(path: Path, *, max_gap: int = 3) -> list[Message]: - """Reconstruct complete messages and STREET+HOUSE gold windows.""" - - messages: list[Message] = [] - with path.open(encoding="utf-8", newline="") as source: - for row_number, row in enumerate(csv.DictReader(source)): - tokens = tuple(json.loads(row["tokens"])) - tags = tuple(json.loads(row["ner_tags"])) - if len(tokens) != len(tags): - raise ValueError(f"row {row_number}: token/tag length mismatch") - text, offsets = _reconstruct(tokens) - spans: list[Span] = [] - for start, end in _clusters(tags, max_gap): - labels = { - _base_label(tag) - for tag in tags[start:end] - } - if not {"STREET", "HOUSE"} <= labels: - continue - spans.append( - Span( - start=offsets[start][0], - end=offsets[end - 1][1], - ) - ) - messages.append( - Message( - source_row=row_number, - text=text, - gold=tuple(spans), - ) - ) - return messages - - -def _overlap(left: Span, right: Span) -> int: - return max(0, min(left.end, right.end) - max(left.start, right.start)) - - -def _match( - expected: Sequence[Span], - actual: Sequence[Span], - *, - exact: bool, -) -> tuple[list[tuple[int, int]], set[int], set[int]]: - candidates = sorted( - ( - ( - _overlap(wanted, observed), - expected_index, - actual_index, - ) - for expected_index, wanted in enumerate(expected) - for actual_index, observed in enumerate(actual) - if ( - wanted == observed - if exact - else _overlap(wanted, observed) > 0 - ) - ), - reverse=True, - ) - matched_expected: set[int] = set() - matched_actual: set[int] = set() - matches: list[tuple[int, int]] = [] - for _, expected_index, actual_index in candidates: - if ( - expected_index in matched_expected - or actual_index in matched_actual - ): - continue - matched_expected.add(expected_index) - matched_actual.add(actual_index) - matches.append((expected_index, actual_index)) - return matches, matched_expected, matched_actual - - -def _failure_reasons( - expected: Sequence[Span], - actual: Sequence[Span], -) -> list[str]: - matches, matched_expected, matched_actual = _match( - expected, - actual, - exact=False, - ) - reasons: set[str] = set() - for expected_index, actual_index in matches: - wanted = expected[expected_index] - observed = actual[actual_index] - if observed.start < wanted.start or observed.end > wanted.end: - reasons.add("span_includes_context") - if observed.start > wanted.start or observed.end < wanted.end: - reasons.add("span_drops_gold_text") - if len(matched_expected) != len(expected): - reasons.add("missed_address") - if len(matched_actual) != len(actual): - reasons.add("spurious_address") - return sorted(reasons) - - -def evaluate( - messages: Iterable[Message], - detector: Callable[[str], Sequence[DetectedAddress]] = detect_addresses, -) -> dict[str, Any]: - """Score complete messages without oracle-cropping detector input.""" - - rows = positive_rows = negative_rows = exact_messages = correctly_empty = 0 - exact_tp = exact_fp = exact_fn = 0 - overlap_tp = overlap_fp = overlap_fn = 0 - failures: list[dict[str, Any]] = [] - failure_reasons: Counter[str] = Counter() - - for message in messages: - rows += 1 - if message.gold: - positive_rows += 1 - else: - negative_rows += 1 - detected = tuple(detector(message.text)) - actual = tuple(Span(item.start, item.end) for item in detected) - if not message.gold and not actual: - correctly_empty += 1 - - exact_matches, _, _ = _match(message.gold, actual, exact=True) - overlap_matches, _, _ = _match(message.gold, actual, exact=False) - exact_tp += len(exact_matches) - exact_fp += len(actual) - len(exact_matches) - exact_fn += len(message.gold) - len(exact_matches) - overlap_tp += len(overlap_matches) - overlap_fp += len(actual) - len(overlap_matches) - overlap_fn += len(message.gold) - len(overlap_matches) - - message_exact = list(message.gold) == list(actual) - exact_messages += int(message_exact) - if not message_exact: - reasons = _failure_reasons(message.gold, actual) - failure_reasons.update(reasons) - failures.append( - { - "source_row": message.source_row, - "text": message.text, - "gold": [ - { - **asdict(span), - "text": message.text[span.start : span.end], - } - for span in message.gold - ], - "predicted": [ - { - "start": item.start, - "end": item.end, - "text": item.text, - "confidence": round(item.confidence, 4), - "signals": list(item.signals), - } - for item in detected - ], - "failure_reasons": reasons, - } - ) - - return { - "rows": rows, - "positive_rows": positive_rows, - "negative_rows": negative_rows, - "scope": ( - "complete reconstructed benchmark messages; gold address windows " - "must contain both STREET and HOUSE labels" - ), - "limitations": [ - ( - "The source is a PII NER benchmark, not a detector-specific " - "Russian message sample." - ), - ( - "Gold spans follow BIO annotation boundaries while predicted " - "spans intentionally include parseable markers and units." - ), - ( - "Rows without a STREET+HOUSE cluster are treated as detection " - "negatives even when they contain isolated location entities." - ), - ], - "metric_definitions": { - "span_overlap_micro": ( - "one-to-one precision, recall, and F1 for any character " - "overlap between a detected span and a STREET+HOUSE gold window" - ), - "exact_span_micro": ( - "one-to-one precision, recall, and F1 requiring identical " - "half-open boundaries" - ), - "exact_message_rate": ( - "fraction of complete messages whose ordered span lists match" - ), - "negative_message_specificity": ( - "fraction of messages without a STREET+HOUSE gold window where " - "the detector returns no span" - ), - }, - "span_overlap_micro": _prf(overlap_tp, overlap_fp, overlap_fn), - "exact_span_micro": _prf(exact_tp, exact_fp, exact_fn), - "exact_message_rate": round(_ratio(exact_messages, rows), 6), - "negative_message_specificity": round( - _ratio(correctly_empty, negative_rows), - 6, - ), - "failure_case_count": len(failures), - "failure_rows_by_reason": dict(failure_reasons.most_common()), - "failure_cases": failures, - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--data", type=Path, default=DEFAULT_DATA) - parser.add_argument("--max-gap", type=int, default=3) - parser.add_argument( - "--output", - type=Path, - default=ROOT / "evaluation/redmadrobot_detection_report.json", - ) - args = parser.parse_args(argv) - if not args.data.exists(): - raise SystemExit( - f"dataset not found: {args.data}; run the RedMadRobot download " - "command documented in evaluation/README.md" - ) - if _sha256(args.data) != DATASET_SHA256: - raise SystemExit("RedMadRobot dataset checksum mismatch") - - report = evaluate(load_messages(args.data, max_gap=args.max_gap)) - report["source"] = { - "dataset": "redmadrobot-rnd/pii_benchmark", - "revision": DATASET_REVISION, - "sha256": DATASET_SHA256, - } - rendered = json.dumps(report, ensure_ascii=False, indent=2) - print(rendered) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(f"{rendered}\n", encoding="utf-8") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/evaluation/legacy_reference_500_failure_summary.json b/evaluation/legacy_reference_500_failure_summary.json deleted file mode 100644 index 01bf1b0..0000000 --- a/evaluation/legacy_reference_500_failure_summary.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - "dataset": "legacy_reference_500", - "rows": 500, - "exact_rows": 402, - "failed_rows": 98, - "diagnostic_semantics": "likely_causes are deterministic triage hypotheses and require human review; they are not causal ground truth", - "failure_rows_by_likely_cause": { - "component_label_confusion": 25, - "numeric_component_not_recognized": 24, - "conflicting_street_markers": 22, - "reference_infers_missing_street_type": 20, - "ambiguous_or_unsupported_abbreviation": 15, - "unmarked_numeric_role_ambiguity": 14, - "compound_or_letter_number_boundary": 14, - "numeric_value_or_role": 13, - "administrative_label_or_boundary": 11, - "normalization_only_difference": 7, - "street_label_or_value": 7, - "street_type_recognition": 7, - "street_span_boundary": 6, - "administrative_component_missed": 5, - "spurious_numeric_component": 5, - "reference_conflicts_with_explicit_numeric_marker": 3, - "spurious_administrative_component": 2, - "reference_conflicts_with_explicit_street_type": 1 - }, - "failure_rows_by_primary_likely_cause": { - "conflicting_street_markers": 22, - "reference_infers_missing_street_type": 20, - "ambiguous_or_unsupported_abbreviation": 15, - "unmarked_numeric_role_ambiguity": 14, - "compound_or_letter_number_boundary": 7, - "administrative_label_or_boundary": 6, - "street_type_recognition": 4, - "reference_conflicts_with_explicit_numeric_marker": 3, - "numeric_component_not_recognized": 3, - "component_label_confusion": 1, - "reference_conflicts_with_explicit_street_type": 1, - "spurious_administrative_component": 1, - "street_span_boundary": 1 - }, - "failure_rows_by_mismatch_field": { - "street_type": 61, - "house_num": 25, - "apartment": 21, - "street": 20, - "structure": 7, - "corpus": 6, - "region": 6, - "settlement": 5, - "city": 5, - "district": 2 - }, - "scenario_tags_on_failure_rows": { - "administrative_expected": 98, - "postal_code": 97, - "street_marker": 68, - "unit_expected": 57, - "house_marker": 56, - "compact_punctuation": 45, - "unit_marker": 42, - "street_marker_absent": 30, - "street_marker_prefix": 30, - "country_phrase": 26, - "street_marker_multiple": 22, - "street_marker_suffix": 15, - "ambiguous_abbreviation": 15, - "compound_number": 15, - "unmarked_numeric_sequence": 14, - "multiword_street": 14, - "letter_suffix_number": 12, - "repeated_city": 10, - "hyphenated_number": 9, - "slash_number": 8, - "ordinal_street": 3, - "street_marker_present_street_not_literal": 1 - }, - "representative_failure_sample": [ - { - "id": "legacy-good-0005", - "raw": "109052, Москва г, ул.Рязанский проспект, д.2", - "mismatch_fields": "street_type", - "primary_likely_cause": "conflicting_street_markers", - "likely_causes": "conflicting_street_markers", - "failure_summary": "street_type: expected='пр-кт', actual='ул'", - "unparsed_spans": "[\"г\",\"проспект\"]" - }, - { - "id": "legacy-good-0359", - "raw": "109651, Российская Федерация, 77 г. Москва, Москва, Батайский проезд, 17, 294", - "mismatch_fields": "house_num|apartment", - "primary_likely_cause": "unmarked_numeric_role_ambiguity", - "likely_causes": "component_label_confusion|numeric_component_not_recognized|numeric_value_or_role|unmarked_numeric_role_ambiguity", - "failure_summary": "house_num: expected='17', actual='294'; apartment: expected='294', actual=None", - "unparsed_spans": "[\"77\",\"Москва\",\"17\"]" - }, - { - "id": "legacy-good-0165", - "raw": "142701, Московская область, Ленинский район, г.Видное, пр.Ленинского Комсомола, д.15, корп.2, офис 108", - "mismatch_fields": "street_type", - "primary_likely_cause": "ambiguous_or_unsupported_abbreviation", - "likely_causes": "ambiguous_or_unsupported_abbreviation", - "failure_summary": "street_type: expected='пр-кт', actual=None", - "unparsed_spans": "[\"пр\"]" - }, - { - "id": "legacy-good-0751", - "raw": "115682,г.Москва, ул.Ореховый бул.,д.26,корп.2", - "mismatch_fields": "street|street_type", - "primary_likely_cause": "conflicting_street_markers", - "likely_causes": "conflicting_street_markers|street_span_boundary", - "failure_summary": "street: expected='Ореховый', actual='Ореховый Бул'; street_type: expected='б-р', actual='ул'", - "unparsed_spans": "[]" - }, - { - "id": "legacy-good-0089", - "raw": "142140, Москва г, Михайлово-Ярцевское п, Исаково д, ул.Исаково-2, д.2", - "mismatch_fields": "district|settlement", - "primary_likely_cause": "component_label_confusion", - "likely_causes": "administrative_component_missed|administrative_label_or_boundary|component_label_confusion", - "failure_summary": "district: expected='Михайлово-Ярцевское', actual=None; settlement: expected='Исаково', actual='Михайлово-Ярцевское'", - "unparsed_spans": "[\"г\",\"Исаково д\"]" - }, - { - "id": "legacy-good-0795", - "raw": "119991, г. Москва, Ордынка Б., д. 25, с.1", - "mismatch_fields": "street|street_type|structure", - "primary_likely_cause": "ambiguous_or_unsupported_abbreviation", - "likely_causes": "ambiguous_or_unsupported_abbreviation|normalization_only_difference|numeric_component_not_recognized", - "failure_summary": "street: expected='Ордынка Б.', actual='Ордынка Б'; street_type: expected='ул', actual=None; structure: expected='1', actual=None", - "unparsed_spans": "[\"с.1\"]" - }, - { - "id": "legacy-good-0006", - "raw": "105066, г. Москва Денисовский переулок, д.9", - "mismatch_fields": "city|street|street_type", - "primary_likely_cause": "administrative_label_or_boundary", - "likely_causes": "administrative_label_or_boundary|street_label_or_value|street_type_recognition", - "failure_summary": "city: expected='Москва', actual='Москва Денисовский'; street: expected='Денисовский', actual='Переулок'; street_type: expected='пер', actual=None", - "unparsed_spans": "[]" - }, - { - "id": "legacy-good-0935", - "raw": "115201, г. Москва, Котляковская, 4", - "mismatch_fields": "street_type", - "primary_likely_cause": "reference_infers_missing_street_type", - "likely_causes": "reference_infers_missing_street_type", - "failure_summary": "street_type: expected='ул', actual=None", - "unparsed_spans": "[]" - }, - { - "id": "legacy-good-0029", - "raw": "117574, г. Москва, ул. Одоевского, д. 3, корп. 7", - "mismatch_fields": "street_type", - "primary_likely_cause": "reference_conflicts_with_explicit_street_type", - "likely_causes": "reference_conflicts_with_explicit_street_type", - "failure_summary": "street_type: expected='пр-д', actual='ул'", - "unparsed_spans": "[]" - }, - { - "id": "legacy-good-0036", - "raw": "140083, Московская область, г. Лыткарино, ул. Парковая, стр. 4А.", - "mismatch_fields": "house_num|structure", - "primary_likely_cause": "reference_conflicts_with_explicit_numeric_marker", - "likely_causes": "component_label_confusion|compound_or_letter_number_boundary|reference_conflicts_with_explicit_numeric_marker|spurious_numeric_component", - "failure_summary": "house_num: expected='4А', actual=None; structure: expected=None, actual='4А'", - "unparsed_spans": "[]" - } - ] -} diff --git a/evaluation/legacy_reference_500_report.json b/evaluation/legacy_reference_500_report.json deleted file mode 100644 index 66aa9ef..0000000 --- a/evaluation/legacy_reference_500_report.json +++ /dev/null @@ -1,958 +0,0 @@ -{ - "rows": 500, - "review_statuses": { - "legacy_reference_not_independently_rereviewed": 500 - }, - "metric_definitions": { - "exact_address_rate": "fraction of rows where every public component value matches", - "no_unparsed_rate": "fraction of rows with no residual word or number spans", - "exact_component_value_micro": "micro precision, recall, and F1 over case-insensitive exact component values after whitespace and ё/е folding" - }, - "exact_address_rate": 0.804, - "no_unparsed_rate": 0.768, - "exact_component_value_micro": { - "tp": 2728, - "fp": 85, - "fn": 151, - "precision": 0.969783, - "recall": 0.947551, - "f1": 0.958538 - }, - "micro": { - "tp": 2728, - "fp": 85, - "fn": 151, - "precision": 0.969783, - "recall": 0.947551, - "f1": 0.958538 - }, - "fields": { - "postal_code": { - "tp": 498, - "fp": 0, - "fn": 0, - "support": 498, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "region": { - "tp": 53, - "fp": 6, - "fn": 5, - "support": 58, - "precision": 0.898305, - "recall": 0.913793, - "f1": 0.905983 - }, - "district": { - "tp": 10, - "fp": 1, - "fn": 1, - "support": 11, - "precision": 0.909091, - "recall": 0.909091, - "f1": 0.909091 - }, - "city": { - "tp": 491, - "fp": 5, - "fn": 5, - "support": 496, - "precision": 0.989919, - "recall": 0.989919, - "f1": 0.989919 - }, - "settlement": { - "tp": 4, - "fp": 1, - "fn": 5, - "support": 9, - "precision": 0.8, - "recall": 0.444444, - "f1": 0.571429 - }, - "street": { - "tp": 480, - "fp": 20, - "fn": 20, - "support": 500, - "precision": 0.96, - "recall": 0.96, - "f1": 0.96 - }, - "street_type": { - "tp": 439, - "fp": 24, - "fn": 61, - "support": 500, - "precision": 0.948164, - "recall": 0.878, - "f1": 0.911734 - }, - "house_num": { - "tp": 475, - "fp": 21, - "fn": 25, - "support": 500, - "precision": 0.957661, - "recall": 0.95, - "f1": 0.953815 - }, - "corpus": { - "tp": 69, - "fp": 0, - "fn": 6, - "support": 75, - "precision": 1.0, - "recall": 0.92, - "f1": 0.958333 - }, - "structure": { - "tp": 133, - "fp": 4, - "fn": 4, - "support": 137, - "precision": 0.970803, - "recall": 0.970803, - "f1": 0.970803 - }, - "apartment": { - "tp": 76, - "fp": 3, - "fn": 19, - "support": 95, - "precision": 0.962025, - "recall": 0.8, - "f1": 0.873563 - } - }, - "failure_sample": [ - { - "id": "legacy-good-0005", - "raw": "109052, Москва г, ул.Рязанский проспект, д.2", - "mismatches": { - "street_type": { - "expected": "пр-кт", - "actual": "ул" - } - }, - "unparsed": [ - "г", - "проспект" - ] - }, - { - "id": "legacy-good-0359", - "raw": "109651, Российская Федерация, 77 г. Москва, Москва, Батайский проезд, 17, 294", - "mismatches": { - "house_num": { - "expected": "17", - "actual": "294" - }, - "apartment": { - "expected": "294", - "actual": null - } - }, - "unparsed": [ - "77", - "Москва", - "17" - ] - }, - { - "id": "legacy-good-0165", - "raw": "142701, Московская область, Ленинский район, г.Видное, пр.Ленинского Комсомола, д.15, корп.2, офис 108", - "mismatches": { - "street_type": { - "expected": "пр-кт", - "actual": null - } - }, - "unparsed": [ - "пр" - ] - }, - { - "id": "legacy-good-0083", - "raw": "111123, г. Москва, Электродный пр., д.1б", - "mismatches": { - "street_type": { - "expected": "пр-д", - "actual": null - } - }, - "unparsed": [ - "пр" - ] - }, - { - "id": "legacy-good-0751", - "raw": "115682,г.Москва, ул.Ореховый бул.,д.26,корп.2", - "mismatches": { - "street": { - "expected": "Ореховый", - "actual": "Ореховый Бул" - }, - "street_type": { - "expected": "б-р", - "actual": "ул" - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0089", - "raw": "142140, Москва г, Михайлово-Ярцевское п, Исаково д, ул.Исаково-2, д.2", - "mismatches": { - "district": { - "expected": "Михайлово-Ярцевское", - "actual": null - }, - "settlement": { - "expected": "Исаково", - "actual": "Михайлово-Ярцевское" - } - }, - "unparsed": [ - "г", - "Исаково д" - ] - }, - { - "id": "legacy-good-0079", - "raw": "119619, г Москва, ул Авиаторов, 9, 2, 50", - "mismatches": { - "house_num": { - "expected": "9", - "actual": "50" - }, - "corpus": { - "expected": "2", - "actual": null - }, - "apartment": { - "expected": "50", - "actual": null - } - }, - "unparsed": [ - "9, 2" - ] - }, - { - "id": "legacy-good-0749", - "raw": "109117, г. Москва, Волгоградский пр.,д.113,к.5", - "mismatches": { - "street_type": { - "expected": "пр-кт", - "actual": null - } - }, - "unparsed": [ - "пр" - ] - }, - { - "id": "legacy-good-0812", - "raw": "125009, Москва г, ул.Брюсов переулок, дом 21, строение 2", - "mismatches": { - "street_type": { - "expected": "пер", - "actual": "ул" - } - }, - "unparsed": [ - "г", - "переулок" - ] - }, - { - "id": "legacy-good-0795", - "raw": "119991, г. Москва, Ордынка Б., д. 25, с.1", - "mismatches": { - "street": { - "expected": "Ордынка Б.", - "actual": "Ордынка Б" - }, - "street_type": { - "expected": "ул", - "actual": null - }, - "structure": { - "expected": "1", - "actual": null - } - }, - "unparsed": [ - "с.1" - ] - }, - { - "id": "legacy-good-0006", - "raw": "105066, г. Москва Денисовский переулок, д.9", - "mismatches": { - "city": { - "expected": "Москва", - "actual": "Москва Денисовский" - }, - "street": { - "expected": "Денисовский", - "actual": "Переулок" - }, - "street_type": { - "expected": "пер", - "actual": null - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0483", - "raw": "428017, Чувашская Республика, г. Чебоксары, ул. Гузовского,11,офис 18", - "mismatches": { - "region": { - "expected": "Чувашская Республика", - "actual": "Чувашская" - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0935", - "raw": "115201, г. Москва, Котляковская, 4", - "mismatches": { - "street_type": { - "expected": "ул", - "actual": null - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0029", - "raw": "117574, г. Москва, ул. Одоевского, д. 3, корп. 7", - "mismatches": { - "street_type": { - "expected": "пр-д", - "actual": "ул" - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0566", - "raw": "142530, Российская Федерация, Московская область, Электрогорск г, Буденного, 1", - "mismatches": { - "street_type": { - "expected": "ул", - "actual": null - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0418", - "raw": "127521, Российская Федерация, Москва, Москва, пр. Марьиной Рощи 17-й, д. 13, стр. 5", - "mismatches": { - "street": { - "expected": "Марьиной Рощи 17-й", - "actual": "Марьиной Рощи" - }, - "street_type": { - "expected": "пр-д", - "actual": null - } - }, - "unparsed": [ - "Москва, пр", - "17-й" - ] - }, - { - "id": "legacy-good-0719", - "raw": "119200, Москва г, ул.Смоленская-Сенная пл., д.32/34", - "mismatches": { - "street_type": { - "expected": "пл", - "actual": "ул" - } - }, - "unparsed": [ - "г", - "пл" - ] - }, - { - "id": "legacy-good-0601", - "raw": "194064, г. Санкт-Петербург, пр. Тихорецкий, д. 21", - "mismatches": { - "street_type": { - "expected": "пр-кт", - "actual": null - } - }, - "unparsed": [ - "пр" - ] - }, - { - "id": "legacy-good-0036", - "raw": "140083, Московская область, г. Лыткарино, ул. Парковая, стр. 4А.", - "mismatches": { - "house_num": { - "expected": "4А", - "actual": null - }, - "structure": { - "expected": null, - "actual": "4А" - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0216", - "raw": "125445, г Москва, ул Смольная, 24А, помещение №8", - "mismatches": { - "house_num": { - "expected": "24А", - "actual": "24" - } - }, - "unparsed": [ - "А" - ] - }, - { - "id": "legacy-good-0095", - "raw": "127055, г Москва, пер Порядковый, 21, 401", - "mismatches": { - "house_num": { - "expected": "21", - "actual": "401" - }, - "apartment": { - "expected": "401", - "actual": null - } - }, - "unparsed": [ - "21" - ] - }, - { - "id": "legacy-good-0868", - "raw": "109144, , Российская Федерация, г.Москва, Новомарьинская, 4", - "mismatches": { - "street_type": { - "expected": "ул", - "actual": null - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0425", - "raw": "143960, Российская Федерация, 50 Московская область, Реутов, Фабричная, д.7", - "mismatches": { - "region": { - "expected": "Московская", - "actual": "50 Московская" - }, - "street_type": { - "expected": "ул", - "actual": null - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0264", - "raw": "117312, Москва, Вавилова, 9", - "mismatches": { - "street_type": { - "expected": "ул", - "actual": null - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0818", - "raw": "127055, Москва г, ул.Порядковый пер, д.21 - 401", - "mismatches": { - "street_type": { - "expected": "пер", - "actual": "ул" - } - }, - "unparsed": [ - "г", - "пер" - ] - }, - { - "id": "legacy-good-0242", - "raw": "142000, обл Московская, г Домодедово, мкр Северный, ш Каширское, 7", - "mismatches": { - "settlement": { - "expected": "Северный", - "actual": null - }, - "street": { - "expected": "Каширское", - "actual": "Северный" - }, - "street_type": { - "expected": "ш", - "actual": "мкр" - } - }, - "unparsed": [ - "ш Каширское" - ] - }, - { - "id": "legacy-good-0563", - "raw": "107076, г. Москва, Варшавское шоссе, д.1, стр.6, ком. 35", - "mismatches": { - "apartment": { - "expected": "35", - "actual": null - } - }, - "unparsed": [ - "ком. 35" - ] - }, - { - "id": "legacy-good-0988", - "raw": "248017, Российская Федерация, Калужская область, Калуга, Московская, 292", - "mismatches": { - "street_type": { - "expected": "ул", - "actual": null - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0065", - "raw": "105066, Москва г, ул.Денисовский переулок, д.9", - "mismatches": { - "street_type": { - "expected": "пер", - "actual": "ул" - } - }, - "unparsed": [ - "г", - "переулок" - ] - }, - { - "id": "legacy-good-0560", - "raw": "111672, Российская Федерация, 77 Москва г, Москва, Городецкая ул, дом 8, корп. 2", - "mismatches": { - "city": { - "expected": "Москва", - "actual": "77 Москва" - } - }, - "unparsed": [ - "Москва" - ] - }, - { - "id": "legacy-good-0208", - "raw": "142100, Московская обл., г. Подольск, Революционный пр-т, д. 45", - "mismatches": { - "street_type": { - "expected": "пр-кт", - "actual": null - } - }, - "unparsed": [ - "пр-т" - ] - }, - { - "id": "legacy-good-0224", - "raw": "105094, г МОСКВА, ул СЕМЕНОВСКАЯ Б., 42/2-4, 5", - "mismatches": { - "street": { - "expected": "Семёновская Б.", - "actual": "Семеновская Б" - }, - "house_num": { - "expected": "42/2", - "actual": "5" - }, - "corpus": { - "expected": "4", - "actual": null - }, - "apartment": { - "expected": "5", - "actual": null - } - }, - "unparsed": [ - "42/2-4" - ] - }, - { - "id": "legacy-good-0228", - "raw": "105062, г Москва, ул Чаплыгина, 8, 27", - "mismatches": { - "house_num": { - "expected": "8", - "actual": "27" - }, - "apartment": { - "expected": "27", - "actual": null - } - }, - "unparsed": [ - "8" - ] - }, - { - "id": "legacy-good-0128", - "raw": "109004, г. Москва, ул. Николоямская, 49/1-199", - "mismatches": { - "house_num": { - "expected": "49/1", - "actual": "199" - }, - "apartment": { - "expected": "199", - "actual": null - } - }, - "unparsed": [ - "49/1" - ] - }, - { - "id": "legacy-good-0666", - "raw": "119200. г.Москва, Смоленская -Сенная пл., д.32-34", - "mismatches": { - "street": { - "expected": "Смоленская-Сенная", - "actual": "Смоленская -Сенная" - }, - "house_num": { - "expected": "32-34", - "actual": "32" - }, - "apartment": { - "expected": null, - "actual": "34" - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0770", - "raw": "119991, Россия, г.Москва, Ордынка Б. ул., д.25, с.1", - "mismatches": { - "street": { - "expected": "Ордынка Б.", - "actual": "Ордынка Б" - }, - "structure": { - "expected": "1", - "actual": null - } - }, - "unparsed": [ - "с.1" - ] - }, - { - "id": "legacy-good-0340", - "raw": "142100, Российская Федерация, 50 Московская область, Подольск, Комсомольская, 1", - "mismatches": { - "region": { - "expected": "Московская", - "actual": "50 Московская" - }, - "street_type": { - "expected": "ул", - "actual": null - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0970", - "raw": "127055, г. Москва, ул. Новослободская , 14/19 стр.1", - "mismatches": { - "house_num": { - "expected": "14/19", - "actual": "19" - } - }, - "unparsed": [ - "14" - ] - }, - { - "id": "legacy-good-0715", - "raw": "119200, Москва г, ул.Смоленская-Сенная площадь, д.32/34", - "mismatches": { - "street_type": { - "expected": "пл", - "actual": "ул" - } - }, - "unparsed": [ - "г", - "площадь" - ] - }, - { - "id": "legacy-good-0502", - "raw": "123001 г. Москва, ул. Гранатный пер., д. 3, стр.1", - "mismatches": { - "street_type": { - "expected": "пер", - "actual": "ул" - } - }, - "unparsed": [ - "пер" - ] - }, - { - "id": "legacy-good-0086", - "raw": "105062, Москва г, ул.Фурманный переулок, д.10 стр.1", - "mismatches": { - "street_type": { - "expected": "пер", - "actual": "ул" - } - }, - "unparsed": [ - "г", - "переулок" - ] - }, - { - "id": "legacy-good-0160", - "raw": "113405, г.Москва, ул.Варшавское шоссе, д.125Д, корп.1.", - "mismatches": { - "street_type": { - "expected": "ш", - "actual": "ул" - } - }, - "unparsed": [ - "шоссе" - ] - }, - { - "id": "legacy-good-1029", - "raw": "109147, Российская Федерация, г. Москва, г. Москва, Марксистская, 20, строение 6", - "mismatches": { - "street_type": { - "expected": "ул", - "actual": null - } - }, - "unparsed": [ - "г. Москва" - ] - }, - { - "id": "legacy-good-0227", - "raw": "125373, г Москва, проезд Походный, 4, 1, офис 111", - "mismatches": { - "house_num": { - "expected": "4", - "actual": "1" - }, - "corpus": { - "expected": "1", - "actual": null - } - }, - "unparsed": [ - "4" - ] - }, - { - "id": "legacy-good-0753", - "raw": "125212, г.Москва,уп.Адмирала Макарова,д.10,стр.1", - "mismatches": { - "street": { - "expected": "Адмирала Макарова", - "actual": "Макарова" - }, - "street_type": { - "expected": "ул", - "actual": null - } - }, - "unparsed": [ - "уп.Адмирала" - ] - }, - { - "id": "legacy-good-0610", - "raw": "127238, г. Москва. Дмитровское шоссе, д.79", - "mismatches": { - "city": { - "expected": "Москва", - "actual": "Москва. Дмитровское" - }, - "street": { - "expected": "Дмитровское", - "actual": "Шоссе" - }, - "street_type": { - "expected": "ш", - "actual": null - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0908", - "raw": "105094, г. Москва, Гольяновская ул, 7а, стр.1", - "mismatches": { - "house_num": { - "expected": "7А", - "actual": "7" - } - }, - "unparsed": [ - "а" - ] - }, - { - "id": "legacy-good-0241", - "raw": "115573, г. Москва, ул. Ореховый Бульвар, влад. 22Д", - "mismatches": { - "street_type": { - "expected": "б-р", - "actual": "ул" - } - }, - "unparsed": [ - "Бульвар, влад" - ] - }, - { - "id": "legacy-good-0446", - "raw": ", Российская Федерация, Смоленская область, Смоленск, Ново-Ленинградская, 16", - "mismatches": { - "street_type": { - "expected": "ул", - "actual": null - } - }, - "unparsed": [] - }, - { - "id": "legacy-good-0947", - "raw": "125466, г. Москва, Соколово- Мещерская, 16/114", - "mismatches": { - "street": { - "expected": "Соколово-Мещерская", - "actual": "Соколово- Мещерская" - }, - "street_type": { - "expected": "ул", - "actual": null - } - }, - "unparsed": [] - } - ], - "gates": [ - { - "metric": "rows", - "actual": 500, - "minimum": 500, - "passed": true - }, - { - "metric": "exact_address_rate", - "actual": 0.804, - "minimum": 0.8, - "passed": true - }, - { - "metric": "no_unparsed_rate", - "actual": 0.768, - "minimum": 0.75, - "passed": true - }, - { - "metric": "micro.f1", - "actual": 0.958538, - "minimum": 0.955, - "passed": true - }, - { - "metric": "fields.postal_code.f1", - "actual": 1.0, - "minimum": 0.999, - "passed": true - }, - { - "metric": "fields.region.f1", - "actual": 0.905983, - "minimum": 0.9, - "passed": true - }, - { - "metric": "fields.city.f1", - "actual": 0.989919, - "minimum": 0.985, - "passed": true - }, - { - "metric": "fields.street.f1", - "actual": 0.96, - "minimum": 0.955, - "passed": true - }, - { - "metric": "fields.street_type.f1", - "actual": 0.911734, - "minimum": 0.9, - "passed": true - }, - { - "metric": "fields.house_num.f1", - "actual": 0.953815, - "minimum": 0.95, - "passed": true - }, - { - "metric": "fields.corpus.f1", - "actual": 0.958333, - "minimum": 0.95, - "passed": true - }, - { - "metric": "fields.structure.f1", - "actual": 0.970803, - "minimum": 0.96, - "passed": true - }, - { - "metric": "fields.apartment.f1", - "actual": 0.873563, - "minimum": 0.85, - "passed": true - } - ], - "release_gate_passed": true -} diff --git a/evaluation/prepare_datamos.py b/evaluation/prepare_datamos.py deleted file mode 100644 index be1ef1b..0000000 --- a/evaluation/prepare_datamos.py +++ /dev/null @@ -1,274 +0,0 @@ -"""Filter a pinned Moscow official-address snapshot into deterministic splits.""" - -from __future__ import annotations - -import argparse -from collections import Counter -import gzip -import hashlib -import io -import json -from pathlib import Path -import sys -from typing import Any, Iterable -from urllib.request import Request, urlopen -import zipfile - -from datamos_data import ( - ARCHIVE_SHA256, - ARCHIVE_URL, - DATASET_DATE, - DATASET_ID, - DATASET_VERSION, - DEFAULT_ARCHIVE, - DEFAULT_FILTERED, - DEFAULT_MANIFEST, - INNER_DATA_SHA256, - SOURCE_ROWS, - expected_components, - fold, - group_id_and_split, - quality_tier, - rejection_reason, -) - - -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _archive_member_sha256(path: Path, member: str) -> str: - digest = hashlib.sha256() - with zipfile.ZipFile(path) as archive: - with archive.open(member) as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def download_source(path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_suffix(f"{path.suffix}.part") - request = Request(ARCHIVE_URL, headers={"User-Agent": "address-normalizer/2"}) - with urlopen(request, timeout=120) as response, temporary.open("wb") as output: - while chunk := response.read(1024 * 1024): - output.write(chunk) - actual = _sha256(temporary) - if actual != ARCHIVE_SHA256: - temporary.unlink(missing_ok=True) - raise ValueError( - f"archive checksum mismatch: expected {ARCHIVE_SHA256}, got {actual}" - ) - temporary.replace(path) - - -def _documents(path: Path) -> Iterable[dict[str, Any]]: - try: - from bson import decode_file_iter - except ModuleNotFoundError as error: - raise SystemExit( - "PyMongo is required only for Moscow dataset preparation. Install " - "`requirements-evaluation.txt` in a separate environment." - ) from error - with zipfile.ZipFile(path) as archive: - with archive.open("data.bson.gz") as compressed_source: - with gzip.GzipFile(fileobj=compressed_source, mode="rb") as source: - yield from decode_file_iter(source) - - -def prepare( - source: Path, - filtered: Path, - manifest_path: Path, - *, - overwrite: bool, -) -> dict[str, Any]: - for path in (filtered, manifest_path): - if path.exists() and not overwrite: - raise FileExistsError(f"{path} already exists; pass --overwrite") - actual_archive_sha256 = _sha256(source) - if actual_archive_sha256 != ARCHIVE_SHA256: - raise ValueError( - f"archive checksum mismatch: expected {ARCHIVE_SHA256}, " - f"got {actual_archive_sha256}" - ) - actual_inner_sha256 = _archive_member_sha256(source, "data.bson.gz") - if actual_inner_sha256 != INNER_DATA_SHA256: - raise ValueError( - f"inner data checksum mismatch: expected {INNER_DATA_SHA256}, " - f"got {actual_inner_sha256}" - ) - - filtered.parent.mkdir(parents=True, exist_ok=True) - manifest_path.parent.mkdir(parents=True, exist_ok=True) - temporary = filtered.with_suffix(f"{filtered.suffix}.part") - counts: Counter[str] = Counter() - seen: set[str] = set() - with temporary.open("wb") as raw_output: - with gzip.GzipFile( - filename="", - mode="wb", - fileobj=raw_output, - mtime=0, - ) as compressed: - with io.TextIOWrapper(compressed, encoding="utf-8") as output: - for source_row, row in enumerate(_documents(source)): - counts["source_rows"] += 1 - reason = rejection_reason(row) - if reason is not None: - counts[f"rejected_{reason}"] += 1 - continue - raw = str(row["SIMPLE_ADDRESS"]).strip() - normalized = fold(raw) - if normalized in seen: - counts["duplicate_rows_removed"] += 1 - continue - seen.add(normalized) - group_id, split = group_id_and_split(row) - tier = quality_tier(row) - record = { - "source_row": source_row, - "group_id": group_id, - "split": split, - "tier": tier, - "raw": raw, - "legal_address": str(row["ADDRESS"]).strip(), - "expected": expected_components(row), - "fias_id": str(row["N_FIAS"]).lower(), - "unom": row.get("UNOM"), - "registry_id": row.get("NREG"), - "object_type": row.get("OBJ_TYPE"), - } - output.write( - json.dumps( - record, - ensure_ascii=False, - separators=(",", ":"), - ) - ) - output.write("\n") - counts["unique_usable_rows"] += 1 - counts[f"split_{split}"] += 1 - counts[f"tier_{tier}"] += 1 - counts[f"object_{row.get('OBJ_TYPE')}"] += 1 - if counts["source_rows"] != SOURCE_ROWS: - temporary.unlink(missing_ok=True) - raise ValueError( - f"unexpected source row count: expected {SOURCE_ROWS}, " - f"got {counts['source_rows']}" - ) - temporary.replace(filtered) - - manifest = { - "format_version": 1, - "source": { - "title": ( - "Адресный реестр объектов недвижимости города Москвы" - ), - "publisher": ( - "Департамент городского имущества города Москвы" - ), - "original_portal": "https://data.mos.ru", - "dataset_id": DATASET_ID, - "version": DATASET_VERSION, - "release_date": DATASET_DATE, - "mirror": ( - "https://data2.apicrafter.ru/packages/" - "datamos-addressreestr" - ), - "archive_url": ARCHIVE_URL, - "archive_bytes": source.stat().st_size, - "archive_sha256": actual_archive_sha256, - "inner_data_sha256": actual_inner_sha256, - "source_rows": SOURCE_ROWS, - "embedded_terms": ( - "Типовые условия доступа к открытым данным органов власти в РФ" - ), - "mirror_terms": "CC-BY-SA", - }, - "policy": { - "purpose": ( - "Moscow-only official clean-address training/evaluation corpus; " - "not bundled in the runtime package" - ), - "filter": [ - "OnTerritoryOfMoscow == да", - "ADR_TYPE == Официальный", - "SOSTAD == Зарегистрирован в АР", - "STATUS == Внесён в ГКН", - "non-empty SIMPLE_ADDRESS, P7 street, and L1_VALUE house", - "valid N_FIAS UUID", - ], - "deduplication": ( - "first case-folded, ё/е-folded, whitespace-normalized " - "SIMPLE_ADDRESS" - ), - "grouping": ( - "street/house/corpus/structure identity; SHA-256 groups are " - "assigned 90% train, 5% validation, 5% test" - ), - }, - "counts": dict(sorted(counts.items())), - "artifact": { - "filename": filtered.name, - "rows": counts["unique_usable_rows"], - "bytes": filtered.stat().st_size, - "sha256": _sha256(filtered), - }, - "limitations": [ - "The snapshot is from October 2021 and is not a current registry.", - "The corpus is Moscow-only and consists of clean legal formatting.", - ( - "Only street, house, corpus, and structure are scored from " - "SIMPLE_ADDRESS; administrative fields are intentionally out " - "of scope for this view." - ), - ( - "The archive is obtained from an attributed mirror; retain its " - "embedded metadata and confirm redistribution terms before " - "publishing derived rows." - ), - ], - } - rendered = f"{json.dumps(manifest, ensure_ascii=False, indent=2)}\n" - temporary_manifest = manifest_path.with_suffix( - f"{manifest_path.suffix}.part" - ) - temporary_manifest.write_text(rendered, encoding="utf-8") - temporary_manifest.replace(manifest_path) - return manifest - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--source", type=Path, default=DEFAULT_ARCHIVE) - parser.add_argument("--filtered", type=Path, default=DEFAULT_FILTERED) - parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) - parser.add_argument("--download", action="store_true") - parser.add_argument("--overwrite", action="store_true") - args = parser.parse_args(argv) - if args.download: - download_source(args.source) - if not args.source.exists(): - parser.error( - f"{args.source} does not exist; provide --source or use --download" - ) - try: - manifest = prepare( - args.source, - args.filtered, - args.manifest, - overwrite=args.overwrite, - ) - except (FileExistsError, ValueError) as error: - parser.error(str(error)) - print(json.dumps(manifest, ensure_ascii=False, indent=2)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/evaluation/prepare_deepparse.py b/evaluation/prepare_deepparse.py deleted file mode 100644 index 58d6e1c..0000000 --- a/evaluation/prepare_deepparse.py +++ /dev/null @@ -1,487 +0,0 @@ -"""Prepare the full Deepparse Russian shard for training and evaluation.""" - -from __future__ import annotations - -import argparse -from collections import Counter -import gzip -import hashlib -import heapq -import io -import json -from pathlib import Path -import sqlite3 -import sys -from typing import Any, Iterable -from urllib.request import Request, urlopen - -from deepparse_data import ( - DATASET_REVISION, - DATASET_ROWS, - DATASET_SHA256, - DATASET_URL, - DEFAULT_FILTERED, - DEFAULT_MANIFEST, - DEFAULT_SAMPLE, - DEFAULT_SOURCE, - EXPECTED_FIELDS, - SOURCE_TAGS, - USEFUL_TAGS, - expected_components, - group_id_and_split, - mapped_labels, - normalized_address_id, - quality_tier, - sample_rank, -) - - -BATCH_SIZE = 131_072 -WRITE_BATCH_SIZE = 65_536 - - -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def download_source(path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_suffix(f"{path.suffix}.part") - request = Request(DATASET_URL, headers={"User-Agent": "address-normalizer/2"}) - with urlopen(request, timeout=120) as response, temporary.open("wb") as output: - while chunk := response.read(1024 * 1024): - output.write(chunk) - actual = _sha256(temporary) - if actual != DATASET_SHA256: - temporary.unlink(missing_ok=True) - raise ValueError( - f"source checksum mismatch: expected {DATASET_SHA256}, got {actual}" - ) - temporary.replace(path) - - -def _require_pyarrow() -> tuple[Any, Any]: - try: - import pyarrow as pa - import pyarrow.parquet as pq - except ModuleNotFoundError as error: - raise SystemExit( - "PyArrow is required only for dataset preparation. Install " - "`requirements-evaluation.txt` in a separate environment." - ) from error - return pa, pq - - -def _schema(pa: Any) -> Any: - expected = pa.struct([(field, pa.string()) for field in EXPECTED_FIELDS]) - return pa.schema( - [ - ("source_row", pa.int64()), - ("example_id", pa.binary(16)), - ("group_id", pa.binary(16)), - ("split", pa.string()), - ("tier", pa.string()), - ("language", pa.string()), - ("raw", pa.string()), - ("tokens", pa.list_(pa.string())), - ("source_tags", pa.list_(pa.string())), - ("labels", pa.list_(pa.string())), - ("expected", expected), - ] - ) - - -def _source_batches(parquet_file: Any) -> Iterable[tuple[list[Any], ...]]: - for batch in parquet_file.iter_batches(batch_size=BATCH_SIZE): - yield tuple(column.to_pylist() for column in batch.columns) - - -def _validate_source(parquet_file: Any) -> None: - required = {"Address", "Tags", "Language"} - actual = set(parquet_file.schema_arrow.names) - if actual != required: - raise ValueError( - f"unexpected source columns: expected {sorted(required)}, " - f"got {sorted(actual)}" - ) - if parquet_file.metadata.num_rows != DATASET_ROWS: - raise ValueError( - f"unexpected row count: expected {DATASET_ROWS}, " - f"got {parquet_file.metadata.num_rows}" - ) - - -def _open_dedupe_database(path: Path) -> sqlite3.Connection: - connection = sqlite3.connect(path) - connection.execute("PRAGMA journal_mode=OFF") - connection.execute("PRAGMA synchronous=OFF") - connection.execute("PRAGMA temp_store=MEMORY") - connection.execute("PRAGMA locking_mode=EXCLUSIVE") - connection.execute( - "CREATE TABLE seen (example_id BLOB PRIMARY KEY, source_row INTEGER) " - "WITHOUT ROWID" - ) - return connection - - -def _mark_accepted_rows( - parquet_file: Any, - database: sqlite3.Connection, -) -> tuple[bytearray, dict[str, int]]: - stats: Counter[str] = Counter() - source_row = 0 - for addresses, tag_lists, languages in _source_batches(parquet_file): - candidates: list[tuple[bytes, int]] = [] - for address, tags, language in zip(addresses, tag_lists, languages): - stats["source_rows"] += 1 - tokens = address.split() if isinstance(address, str) else [] - if language != "rus": - stats["rejected_language"] += 1 - elif not tokens: - stats["rejected_empty"] += 1 - elif len(tokens) != len(tags): - stats["rejected_token_tag_mismatch"] += 1 - elif len(tokens) > 64: - stats["rejected_too_long"] += 1 - elif not set(tags) <= SOURCE_TAGS: - stats["rejected_unknown_tag"] += 1 - elif not set(tags) & USEFUL_TAGS: - stats["rejected_administrative_only"] += 1 - else: - stats["eligible_rows"] += 1 - candidates.append((normalized_address_id(address), source_row)) - source_row += 1 - database.executemany( - "INSERT OR IGNORE INTO seen(example_id, source_row) VALUES (?, ?)", - candidates, - ) - database.commit() - - unique_rows = int(database.execute("SELECT count(*) FROM seen").fetchone()[0]) - stats["unique_usable_rows"] = unique_rows - stats["duplicate_rows_removed"] = stats["eligible_rows"] - unique_rows - accepted = bytearray((stats["source_rows"] + 7) // 8) - for (row_number,) in database.execute("SELECT source_row FROM seen"): - accepted[row_number >> 3] |= 1 << (row_number & 7) - return accepted, dict(stats) - - -def _is_accepted(accepted: bytearray, source_row: int) -> bool: - return bool(accepted[source_row >> 3] & (1 << (source_row & 7))) - - -def _write_records( - writer: Any, - pa: Any, - schema: Any, - records: list[dict[str, Any]], -) -> None: - if records: - writer.write_table(pa.Table.from_pylist(records, schema=schema)) - records.clear() - - -def _write_filtered( - parquet_file: Any, - accepted: bytearray, - output: Path, - sample_size: int, - pa: Any, - pq: Any, -) -> tuple[Counter[tuple[str, str]], set[int]]: - schema = _schema(pa) - temporary = output.with_suffix(f"{output.suffix}.part") - writer = pq.ParquetWriter( - temporary, - schema, - compression="zstd", - compression_level=6, - use_dictionary=["split", "tier", "language", "source_tags", "labels"], - write_statistics=True, - ) - records: list[dict[str, Any]] = [] - sample_heap: list[tuple[int, int]] = [] - counts: Counter[tuple[str, str]] = Counter() - source_row = 0 - try: - for addresses, tag_lists, languages in _source_batches(parquet_file): - for address, tags, language in zip( - addresses, tag_lists, languages - ): - if not _is_accepted(accepted, source_row): - source_row += 1 - continue - tokens = address.split() - labels = mapped_labels(tags) - example_id = normalized_address_id(address) - group_id, split = group_id_and_split(tokens, tags) - tier = quality_tier(tags) - records.append( - { - "source_row": source_row, - "example_id": example_id, - "group_id": group_id, - "split": split, - "tier": tier, - "language": language, - "raw": address, - "tokens": tokens, - "source_tags": tags, - "labels": labels, - "expected": expected_components(tokens, labels), - } - ) - counts[(split, tier)] += 1 - if split == "test" and sample_size: - rank = sample_rank(example_id) - candidate = (-rank, source_row) - if len(sample_heap) < sample_size: - heapq.heappush(sample_heap, candidate) - elif candidate > sample_heap[0]: - heapq.heapreplace(sample_heap, candidate) - if len(records) >= WRITE_BATCH_SIZE: - _write_records(writer, pa, schema, records) - source_row += 1 - _write_records(writer, pa, schema, records) - except BaseException: - writer.close() - temporary.unlink(missing_ok=True) - raise - writer.close() - temporary.replace(output) - return counts, {source_row for _, source_row in sample_heap} - - -def _write_sample( - filtered: Path, - output: Path, - selected_rows: set[int], - pq: Any, -) -> int: - temporary = output.with_suffix(f"{output.suffix}.part") - written = 0 - with temporary.open("wb") as raw_output: - with gzip.GzipFile( - filename="", - mode="wb", - fileobj=raw_output, - mtime=0, - ) as compressed: - with io.TextIOWrapper(compressed, encoding="utf-8") as text: - parquet_file = pq.ParquetFile(filtered) - for batch in parquet_file.iter_batches(batch_size=BATCH_SIZE): - for row in batch.to_pylist(): - if row["source_row"] not in selected_rows: - continue - row["example_id"] = row["example_id"].hex() - row["group_id"] = row["group_id"].hex() - text.write( - json.dumps( - row, - ensure_ascii=False, - separators=(",", ":"), - ) - ) - text.write("\n") - written += 1 - temporary.replace(output) - return written - - -def prepare( - source: Path, - filtered: Path, - sample: Path, - manifest_path: Path, - *, - sample_size: int, - overwrite: bool, -) -> dict[str, Any]: - if sample_size < 0: - raise ValueError("sample_size must be non-negative") - for path in (filtered, sample, manifest_path): - if path.exists() and not overwrite: - raise FileExistsError(f"{path} already exists; pass --overwrite") - actual_sha256 = _sha256(source) - if actual_sha256 != DATASET_SHA256: - raise ValueError( - f"source checksum mismatch: expected {DATASET_SHA256}, " - f"got {actual_sha256}" - ) - - pa, pq = _require_pyarrow() - parquet_file = pq.ParquetFile(source) - _validate_source(parquet_file) - filtered.parent.mkdir(parents=True, exist_ok=True) - sample.parent.mkdir(parents=True, exist_ok=True) - manifest_path.parent.mkdir(parents=True, exist_ok=True) - database_path = filtered.with_suffix(".dedupe.sqlite3") - database_path.unlink(missing_ok=True) - database = _open_dedupe_database(database_path) - try: - accepted, filter_counts = _mark_accepted_rows(parquet_file, database) - finally: - database.close() - - parquet_file = pq.ParquetFile(source) - counts, selected_rows = _write_filtered( - parquet_file, - accepted, - filtered, - sample_size, - pa, - pq, - ) - written_sample_rows = _write_sample( - filtered, - sample, - selected_rows, - pq, - ) - if written_sample_rows != min( - sample_size, - sum(count for (split, _), count in counts.items() if split == "test"), - ): - raise RuntimeError("benchmark sample row count does not match selection") - database_path.unlink(missing_ok=True) - - split_counts = { - split: sum( - count for (candidate, _), count in counts.items() - if candidate == split - ) - for split in ("train", "validation", "test") - } - tier_counts = { - tier: sum( - count for (_, candidate), count in counts.items() - if candidate == tier - ) - for tier in ( - "street_house_unit", - "street_house", - "street_only", - "number_or_unit_only", - ) - } - manifest = { - "format_version": 1, - "source": { - "repository": "deepparse/worldwide-addresses", - "configuration": "ru", - "license": "CC BY 4.0", - "revision": DATASET_REVISION, - "url": DATASET_URL, - "rows": DATASET_ROWS, - "bytes": source.stat().st_size, - "sha256": actual_sha256, - }, - "policy": { - "purpose": ( - "external clean-address training/evaluation corpus; not bundled " - "in the runtime package" - ), - "filter": ( - "Russian rows with 1-64 whitespace tokens, known tags, matching " - "token/tag lengths, and at least one StreetName, StreetNumber, " - "or Unit tag" - ), - "deduplication": ( - "first row for each BLAKE2b-128 hash of case-folded, ё/е-folded, " - "whitespace-normalized address text" - ), - "grouping": ( - "SHA-256 of structured province/county/district/municipality/" - "suburb/street/house identity; unit and presentation fields are " - "excluded so one building cannot cross splits" - ), - "split": "group hash buckets: train 90%, validation 5%, test 5%", - "sample": ( - "lowest deterministic SHA-256 ranks from the sealed test split" - ), - "ignored_labels": ["Country", "Suburb"], - }, - "filter_counts": filter_counts, - "split_counts": split_counts, - "tier_counts": tier_counts, - "split_tier_counts": { - f"{split}/{tier}": count - for (split, tier), count in sorted(counts.items()) - }, - "artifacts": { - "filtered_parquet": { - "filename": filtered.name, - "rows": sum(split_counts.values()), - "bytes": filtered.stat().st_size, - "sha256": _sha256(filtered), - }, - "test_sample_jsonl_gz": { - "filename": sample.name, - "rows": written_sample_rows, - "bytes": sample.stat().st_size, - "sha256": _sha256(sample), - }, - }, - "limitations": [ - ( - "The source is curated from open geographic address data and " - "does not reproduce misspellings or punctuation-heavy user input." - ), - ( - "Country and Suburb have no direct public package field and are " - "kept as source context but mapped to O." - ), - ( - "This deterministic test split becomes tuning data after its " - "failures are used to change the parser; reserve another test " - "source before making a final production claim." - ), - ], - } - rendered = f"{json.dumps(manifest, ensure_ascii=False, indent=2)}\n" - temporary_manifest = manifest_path.with_suffix( - f"{manifest_path.suffix}.part" - ) - temporary_manifest.write_text(rendered, encoding="utf-8") - temporary_manifest.replace(manifest_path) - return manifest - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE) - parser.add_argument("--filtered", type=Path, default=DEFAULT_FILTERED) - parser.add_argument("--sample", type=Path, default=DEFAULT_SAMPLE) - parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) - parser.add_argument("--sample-size", type=int, default=100_000) - parser.add_argument("--download", action="store_true") - parser.add_argument("--overwrite", action="store_true") - args = parser.parse_args(argv) - - if args.download: - download_source(args.source) - if not args.source.exists(): - parser.error( - f"{args.source} does not exist; provide --source or use --download" - ) - try: - report = prepare( - args.source, - args.filtered, - args.sample, - args.manifest, - sample_size=args.sample_size, - overwrite=args.overwrite, - ) - except (FileExistsError, ValueError) as error: - parser.error(str(error)) - print(json.dumps(report, ensure_ascii=False, indent=2)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/evaluation/redmadrobot_detection_report.json b/evaluation/redmadrobot_detection_report.json deleted file mode 100644 index 0916650..0000000 --- a/evaluation/redmadrobot_detection_report.json +++ /dev/null @@ -1,2758 +0,0 @@ -{ - "rows": 2841, - "positive_rows": 135, - "negative_rows": 2706, - "scope": "complete reconstructed benchmark messages; gold address windows must contain both STREET and HOUSE labels", - "limitations": [ - "The source is a PII NER benchmark, not a detector-specific Russian message sample.", - "Gold spans follow BIO annotation boundaries while predicted spans intentionally include parseable markers and units.", - "Rows without a STREET+HOUSE cluster are treated as detection negatives even when they contain isolated location entities." - ], - "metric_definitions": { - "span_overlap_micro": "one-to-one precision, recall, and F1 for any character overlap between a detected span and a STREET+HOUSE gold window", - "exact_span_micro": "one-to-one precision, recall, and F1 requiring identical half-open boundaries", - "exact_message_rate": "fraction of complete messages whose ordered span lists match", - "negative_message_specificity": "fraction of messages without a STREET+HOUSE gold window where the detector returns no span" - }, - "span_overlap_micro": { - "tp": 98, - "fp": 2, - "fn": 46, - "precision": 0.98, - "recall": 0.680556, - "f1": 0.803279 - }, - "exact_span_micro": { - "tp": 29, - "fp": 71, - "fn": 115, - "precision": 0.29, - "recall": 0.201389, - "f1": 0.237705 - }, - "exact_message_rate": 0.962337, - "negative_message_specificity": 1.0, - "failure_case_count": 107, - "failure_rows_by_reason": { - "missed_address": 44, - "span_drops_gold_text": 33, - "span_includes_context": 33, - "spurious_address": 1 - }, - "failure_cases": [ - { - "source_row": 0, - "text": "ООО « Ремонт и Обслуживание » УЛ . РОКОССОВСКОГО , дом 10 , офис 15 , ВОЛЬСК , 191023 , Танзания ТЕЛ . : +7 ( 812 ) 987 6543 , WWW . REMONT-I-OBSLUZHIVANIE . RU", - "gold": [ - { - "start": 30, - "end": 96, - "text": "УЛ . РОКОССОВСКОГО , дом 10 , офис 15 , ВОЛЬСК , 191023 , Танзания" - } - ], - "predicted": [ - { - "start": 30, - "end": 67, - "text": "УЛ . РОКОССОВСКОГО , дом 10 , офис 15", - "confidence": 0.77, - "signals": [ - "street_marker", - "house_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 54, - "text": "ул . Крымский Вал , 10 , Бузулук , 119049 Форма заявки: на https://help . bitbucket . org/terms .", - "gold": [ - { - "start": 0, - "end": 32, - "text": "ул . Крымский Вал , 10 , Бузулук" - } - ], - "predicted": [ - { - "start": 0, - "end": 22, - "text": "ул . Крымский Вал , 10", - "confidence": 0.44, - "signals": [ - "street_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 74, - "text": "Адрес: Leonard Назаровна терехов , пер . Маршала Жукова , д . 181 .", - "gold": [ - { - "start": 35, - "end": 65, - "text": "пер . Маршала Жукова , д . 181" - } - ], - "predicted": [ - { - "start": 7, - "end": 65, - "text": "Leonard Назаровна терехов , пер . Маршала Жукова , д . 181", - "confidence": 0.85, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 118, - "text": "ООО \" Спортивный мир \" 630005 , Регион 63 , Невинномысск , улица Кирова , дом 25 , квартира 10 ИНН 6312345678 Место расчётов: sportivnyimir . ru", - "gold": [ - { - "start": 44, - "end": 94, - "text": "Невинномысск , улица Кирова , дом 25 , квартира 10" - } - ], - "predicted": [ - { - "start": 59, - "end": 94, - "text": "улица Кирова , дом 25 , квартира 10", - "confidence": 0.77, - "signals": [ - "street_marker", - "house_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 151, - "text": "Клиника: адрес пр . бауманская , д . 71 . Запись по email: timur . morozov@protonmail . com .", - "gold": [ - { - "start": 15, - "end": 39, - "text": "пр . бауманская , д . 71" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 218, - "text": "Адрес: 2 этаж , центр ТОРГОВЫЙ КВАРТАЛ 3-а , Каширское ш . , Вольск , Глазов , АМУРСКАЯ ОБЛ . , 142000", - "gold": [ - { - "start": 39, - "end": 91, - "text": "3-а , Каширское ш . , Вольск , Глазов , АМУРСКАЯ ОБЛ" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 223, - "text": "Заявитель: Пестов Reginald Демьяновна , адрес: г . Чапаевск , ш . Ярцевская , д . 32 , email: beth09@example . org .", - "gold": [ - { - "start": 51, - "end": 84, - "text": "Чапаевск , ш . Ярцевская , д . 32" - } - ], - "predicted": [ - { - "start": 47, - "end": 84, - "text": "г . Чапаевск , ш . Ярцевская , д . 32", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 232, - "text": "В соответствии с запросом котировки № 54321 , ООО « Луч » предоставляет следующие сведения : ИНН 7707083893 , адрес : г . Златоуст , ул . Примерная , д . 1 . Просим учесть наши предложения при рассмотрении заявки .", - "gold": [ - { - "start": 122, - "end": 155, - "text": "Златоуст , ул . Примерная , д . 1" - } - ], - "predicted": [ - { - "start": 118, - "end": 155, - "text": "г . Златоуст , ул . Примерная , д . 1", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 246, - "text": "MOY ADRES PROPISKI: АБХАЗИЯ , almatinskaya OBLAST G . TALDYKORGAN ul . abaya D . 10 KV . 4", - "gold": [ - { - "start": 20, - "end": 90, - "text": "АБХАЗИЯ , almatinskaya OBLAST G . TALDYKORGAN ul . abaya D . 10 KV . 4" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 262, - "text": "MOY ADRES PROPISKI: ROSSIYA , тАмБовсКАя OBLAST G . MYTISHCHI Стромынка D . 5 KV . 12", - "gold": [ - { - "start": 20, - "end": 85, - "text": "ROSSIYA , тАмБовсКАя OBLAST G . MYTISHCHI Стромынка D . 5 KV . 12" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 274, - "text": "УВЕДОМЛЕНИЕ об одностороннем отказе от исполнения договора возмездного оказания услуг от 04 . 03 . 2026 и окончательном расчёте Кому : Жолобовой Маргарите Петровне e - mail : golobovaantonina @ gmail . com От : Общества с ограниченной ответственностью « Юридическая группа ТАРАН » адрес : 655917 , Республика Хакасия , г . Абакан , пр . Ленина , д . 29А , пом . 8Н ИНН 1902029948 , ОГРН 1201900001885 Дата : 28 апреля 2026 г . Уважаемая Алина Ивановна ! Между Вами ( Исполнитель ) и ООО « Юридическая группа ТАРЗАН » ( Заказчик ) заключён договор возмездного оказания услуг от 04 марта 2026 г . ( далее — Договор ) . С 22 апреля 2021 г . Вы в одностор . . . . . . соответствует ранее направленному Акту от 21 апреля 2026 г . , составленному в связи с фактическим прекращением оказания услуг . Сумма налога на доходы физических лиц , подлежащая удержанию , — 4 334 руб . Таким образом , к перечислению на Ваш расчётный счёт причитается 28 999 ( Двадцать восемь тысяч девятьсот девяносто девять ) рублей 00 копеек . Указанная сумма будет перечислена в течение 10 ( десяти ) банковских дней с даты получения Вами настоящего уведомления . Иные выплаты , включая полное фиксированное вознаграждение за апрель , компенсации и неустойки , Заказчиком не производятся , поскольку услуги в полном объёме и надлежащим образом не оказывались , а Договор правомерно прекращён . Направление дополнительных актов или претензий со стороны Исполнителя не повлияет на изложенную позицию : Договор расторгнут Заказчиком в одностороннем порядке в соответствии с императивной нормой ст . 782 ГК РФ , и надлежащее прекращение обязательств не требует подписания двустороннего соглашения . В случае возникновения судебного спора Заказчик оставляет за собой право предъявить требования о взыскании убытков , вызванных некачественным оказанием услуг ( в том числе упущенной выгоды вследствие потери клиентов из - за пропущенных обращений ) , а также о возмещении всех судебных издержек . Проверить на безопасность по законодательству для заказчика", - "gold": [ - { - "start": 298, - "end": 353, - "text": "Республика Хакасия , г . Абакан , пр . Ленина , д . 29А" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 293, - "text": "kservice Кузовной сервис №1 в Невинномысск г . УСОЛЬЕ-СИБИРСКОЕ , УЛ . ДУБИНИНСКАЯ , 70 стр . 1 ( ~ 14000 без стоимости бампера и без дополнительных работ )", - "gold": [ - { - "start": 30, - "end": 95, - "text": "Невинномысск г . УСОЛЬЕ-СИБИРСКОЕ , УЛ . ДУБИНИНСКАЯ , 70 стр . 1" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 350, - "text": "+79772711425 , адрес: пр . Хавская , д . 149 , кв . 230 .", - "gold": [ - { - "start": 22, - "end": 55, - "text": "пр . Хавская , д . 149 , кв . 230" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 407, - "text": "Получатель: Greenwood Percy Зиновьевна , адрес: пр . Школьная , дом 101 , кв . 266 .", - "gold": [ - { - "start": 48, - "end": 82, - "text": "пр . Школьная , дом 101 , кв . 266" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 424, - "text": "Japan cars сервис - ТО и ремонт автомобилей 924 797 21 83 Авдотья - Старокоптевский пер . , 6 ( м . Войковская )", - "gold": [ - { - "start": 68, - "end": 93, - "text": "Старокоптевский пер . , 6" - } - ], - "predicted": [ - { - "start": 0, - "end": 93, - "text": "Japan cars сервис - ТО и ремонт автомобилей 924 797 21 83 Авдотья - Старокоптевский пер . , 6", - "confidence": 0.44, - "signals": [ - "street_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 467, - "text": "Ping to 26 . 204 . 205 . 243 successful . Facebook , Inc . 1 Hacker Way , MENLO PARK , УЛЬЯНОВСКАЯ 94025 БАНГЛАДЕШ", - "gold": [ - { - "start": 59, - "end": 114, - "text": "1 Hacker Way , MENLO PARK , УЛЬЯНОВСКАЯ 94025 БАНГЛАДЕШ" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 500, - "text": "братан 181 . 61 . 145 . 201:5432 это наш прод . Штрих-код 4607038490125 , доставить в Уфу , пер . Заводской , стр . 2", - "gold": [ - { - "start": 86, - "end": 117, - "text": "Уфу , пер . Заводской , стр . 2" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 533, - "text": "Проверили данные по контрагенту ООО \" Ромашка \" . Юридический адрес : 123456 , г . Невинномысск , ул . Ленина , д . 1 . ИНН : 7707083893 , КПП : 770101001 . Необходимо запросить у них свежую выписку из ЕГРЮЛ .", - "gold": [ - { - "start": 83, - "end": 117, - "text": "Невинномысск , ул . Ленина , д . 1" - } - ], - "predicted": [ - { - "start": 70, - "end": 117, - "text": "123456 , г . Невинномысск , ул . Ленина , д . 1", - "confidence": 0.99, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "postal_code", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 556, - "text": "г Алатырь , наб Новоданиловская , 10 , 3 . 318", - "gold": [ - { - "start": 2, - "end": 46, - "text": "Алатырь , наб Новоданиловская , 10 , 3 . 318" - } - ], - "predicted": [ - { - "start": 0, - "end": 36, - "text": "г Алатырь , наб Новоданиловская , 10", - "confidence": 0.5, - "signals": [ - "street_marker", - "location_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text", - "span_includes_context" - ] - }, - { - "source_row": 607, - "text": "ООО \" Яндекс . Маркет \" 119021 , Бузулук , УЛИЦА ЛЬВА ТОЛСТОГО , дом 16 , строение 2 ИНН 7705396987 Место расчётов: market . yandex . ru", - "gold": [ - { - "start": 33, - "end": 84, - "text": "Бузулук , УЛИЦА ЛЬВА ТОЛСТОГО , дом 16 , строение 2" - } - ], - "predicted": [ - { - "start": 43, - "end": 84, - "text": "УЛИЦА ЛЬВА ТОЛСТОГО , дом 16 , строение 2", - "confidence": 0.77, - "signals": [ - "street_marker", - "house_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 693, - "text": "Претензия направляется в адрес ООО \" Строительная Компания \" ИНН 7707083893 . Мы требуем немедленно устранить недоделки , выявленные при приемке объекта по адресу : г . Чапаевск , ул . Строителей , д . 1 . В противном случае мы будем вынуждены обратиться в суд .", - "gold": [ - { - "start": 169, - "end": 203, - "text": "Чапаевск , ул . Строителей , д . 1" - } - ], - "predicted": [ - { - "start": 165, - "end": 203, - "text": "г . Чапаевск , ул . Строителей , д . 1", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 724, - "text": "Автосервис « Лихач » Невинномысск , 2-Й ЛИХАЧЁВСКИЙ ПЕРЕУЛОК , 10с1 ( - )", - "gold": [ - { - "start": 21, - "end": 67, - "text": "Невинномысск , 2-Й ЛИХАЧЁВСКИЙ ПЕРЕУЛОК , 10с1" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 764, - "text": "MOY ADRES PROPISKI: ROSSIYA , LENINGRADSKAYA OBLAST G . SAINT-PETERSBURG UL . NEVSKYI PROSPEKT D . 20 KV . 7", - "gold": [ - { - "start": 20, - "end": 108, - "text": "ROSSIYA , LENINGRADSKAYA OBLAST G . SAINT-PETERSBURG UL . NEVSKYI PROSPEKT D . 20 KV . 7" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 774, - "text": "NOVOSIBIRSK , gagARiNA 7", - "gold": [ - { - "start": 0, - "end": 24, - "text": "NOVOSIBIRSK , gagARiNA 7" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 843, - "text": "Адрес: офис на просп . Комсомольская , д . 15 . Сайт: https://app . mchs . gov . ru .", - "gold": [ - { - "start": 15, - "end": 45, - "text": "просп . Комсомольская , д . 15" - } - ], - "predicted": [ - { - "start": 7, - "end": 45, - "text": "офис на просп . Комсомольская , д . 15", - "confidence": 0.85, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 871, - "text": "MOY ADRES PROPISKI: МАЛЬТА , ALMATINSKAYA OBLAST G . almaty UL . DOSTYK D . 15 KV . 20", - "gold": [ - { - "start": 20, - "end": 86, - "text": "МАЛЬТА , ALMATINSKAYA OBLAST G . almaty UL . DOSTYK D . 15 KV . 20" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 872, - "text": "ООО \" Рога и копыта \" 115487 , Регион 77 , Чапаевск , Дмитровское шоссе , дом 10 , строение 2 ИНН 7701234567 Место расчётов: rogaikopyta . ru", - "gold": [ - { - "start": 43, - "end": 93, - "text": "Чапаевск , Дмитровское шоссе , дом 10 , строение 2" - } - ], - "predicted": [ - { - "start": 54, - "end": 93, - "text": "Дмитровское шоссе , дом 10 , строение 2", - "confidence": 0.77, - "signals": [ - "street_marker", - "house_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 887, - "text": "По адресу УЛ . МАЯКОВСКОГО , д . 7 , г . Минск , Ботсвана", - "gold": [ - { - "start": 10, - "end": 57, - "text": "УЛ . МАЯКОВСКОГО , д . 7 , г . Минск , Ботсвана" - } - ], - "predicted": [ - { - "start": 10, - "end": 34, - "text": "УЛ . МАЯКОВСКОГО , д . 7", - "confidence": 0.85, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 951, - "text": "Офис в Южном федеральном округе , Краснодар , бульвар Платановый 7", - "gold": [ - { - "start": 7, - "end": 66, - "text": "Южном федеральном округе , Краснодар , бульвар Платановый 7" - } - ], - "predicted": [ - { - "start": 46, - "end": 66, - "text": "бульвар Платановый 7", - "confidence": 0.44, - "signals": [ - "street_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 953, - "text": "Уведомление о прибытии : ФИО : Устинов А . В . Регион : MYTISHCHI . Адрес : ул . Ленина , д . 1 . Паспортные данные : 4500 123456 . ИНН : 500100732259 .", - "gold": [ - { - "start": 56, - "end": 95, - "text": "MYTISHCHI . Адрес : ул . Ленина , д . 1" - } - ], - "predicted": [ - { - "start": 76, - "end": 95, - "text": "ул . Ленина , д . 1", - "confidence": 0.85, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 963, - "text": "Склад: Свердловская область , Берёзовский , Транспортная 8А", - "gold": [ - { - "start": 7, - "end": 59, - "text": "Свердловская область , Берёзовский , Транспортная 8А" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1012, - "text": "12 Crag South Avenue , Downtown Westside . He laughs , saying both are true . His phone number printed above the bar", - "gold": [ - { - "start": 0, - "end": 40, - "text": "12 Crag South Avenue , Downtown Westside" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1040, - "text": "Панама , Димитровград , 117312 , УЛ . ВАВИЛОВА , д . 19", - "gold": [ - { - "start": 0, - "end": 55, - "text": "Панама , Димитровград , 117312 , УЛ . ВАВИЛОВА , д . 19" - } - ], - "predicted": [ - { - "start": 24, - "end": 55, - "text": "117312 , УЛ . ВАВИЛОВА , д . 19", - "confidence": 0.75, - "signals": [ - "street_marker", - "house_marker", - "postal_code", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 1070, - "text": "Уважаемые господа , направляем вам наше коммерческое предложение на поставку комплектующих . Подробные условия и спецификацию вы найдете в приложении . Для заключения договора просим использовать следующие реквизиты : ООО « Строитель » , ИНН 7707083893 , КПП 770101001 , ОГРН 1027739123456 , адрес : 105064 , г . Невинномысск , ул . Земляной Вал , д . 26 , стр . 2 . Ожидаем вашего ответа .", - "gold": [ - { - "start": 313, - "end": 364, - "text": "Невинномысск , ул . Земляной Вал , д . 26 , стр . 2" - } - ], - "predicted": [ - { - "start": 300, - "end": 364, - "text": "105064 , г . Невинномысск , ул . Земляной Вал , д . 26 , стр . 2", - "confidence": 0.99, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "postal_code", - "location_marker", - "unit_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 1074, - "text": "Поставщик : ООО « Ромашка » , ИНН 7707083893 , юридический адрес : г . Чапаевск , ул . Ленина , д . 1 . Получатель : ООО « Василек » , ИНН 0987654321 , адрес : г . Димитровград , пр . Просвещения , д . 10 . Договор № 55 от 15 . 03 . 2024 .", - "gold": [ - { - "start": 71, - "end": 101, - "text": "Чапаевск , ул . Ленина , д . 1" - }, - { - "start": 164, - "end": 204, - "text": "Димитровград , пр . Просвещения , д . 10" - } - ], - "predicted": [ - { - "start": 67, - "end": 101, - "text": "г . Чапаевск , ул . Ленина , д . 1", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "missed_address", - "span_includes_context" - ] - }, - { - "source_row": 1101, - "text": "| Поставщик | Адрес | Email | | ООО Вектор | Тверь , ш . Московское , д . 2 , оф . 310 | timur@dev . global . com |", - "gold": [ - { - "start": 45, - "end": 86, - "text": "Тверь , ш . Московское , д . 2 , оф . 310" - } - ], - "predicted": [ - { - "start": 20, - "end": 86, - "text": "| Email | | ООО Вектор | Тверь , ш . Московское , д . 2 , оф . 310", - "confidence": 0.9, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "unit_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 1114, - "text": "Адрес: г . Невинномысск , ш . Люблинская , дом 177 . Email: natalya_vinogradov@rambler . ru .", - "gold": [ - { - "start": 11, - "end": 50, - "text": "Невинномысск , ш . Люблинская , дом 177" - } - ], - "predicted": [ - { - "start": 7, - "end": 50, - "text": "г . Невинномысск , ш . Люблинская , дом 177", - "confidence": 0.96, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 1148, - "text": "Адрес: г . Чапаевск , просп . Рокоссовского , д . 151 . Сайт: https://spotify . com/terms .", - "gold": [ - { - "start": 11, - "end": 53, - "text": "Чапаевск , просп . Рокоссовского , д . 151" - } - ], - "predicted": [ - { - "start": 7, - "end": 53, - "text": "г . Чапаевск , просп . Рокоссовского , д . 151", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 1161, - "text": "ВОЛЬСК , ПРОФСОЮЗНАЯ 15", - "gold": [ - { - "start": 0, - "end": 23, - "text": "ВОЛЬСК , ПРОФСОЮЗНАЯ 15" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1164, - "text": "Настоящим удостоверяю , что Лукин Питер Платонович , проживающий по адресу : г . Нижнем Новгород , ул . Ленина , д . 5 , кв . 10 , паспорт серии 4508 № 123456 , выдан ОВД \" Замоскворечье \" г . Crestville 10 . 05 . 2008 , ИНН 500100732259 , является надлежащим образом уполномоченным представителем по всем вопросам , связанным с управлением недвижимым имуществом , расположенным по адресу : г . Каменск-Уральский , ул . Садовая , д . 1 , к . 2 .", - "gold": [ - { - "start": 81, - "end": 128, - "text": "Нижнем Новгород , ул . Ленина , д . 5 , кв . 10" - }, - { - "start": 395, - "end": 443, - "text": "Каменск-Уральский , ул . Садовая , д . 1 , к . 2" - } - ], - "predicted": [ - { - "start": 77, - "end": 128, - "text": "г . Нижнем Новгород , ул . Ленина , д . 5 , кв . 10", - "confidence": 0.96, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "unit_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - }, - { - "start": 391, - "end": 443, - "text": "г . Каменск-Уральский , ул . Садовая , д . 1 , к . 2", - "confidence": 0.96, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "unit_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 1196, - "text": "Счет № 123 от 15 . 03 . 2023 . Поставщик : ООО \" СтройПрогресс \" , ИНН 7707083893 , КПП 772301001 . Адрес : г . Бузулук , ул . Лесная , д . 5 . Получатель : ЗАО \" Альфа \" , ИНН 7734567890 , КПП 773401001 . Адрес : г . Невинномысск , ул . Тверская , д . 10 . Сумма к оплате : 50 000 руб . Назначение платежа : Оплата по договору № 45 от 01 . 03 . 2023 .", - "gold": [ - { - "start": 112, - "end": 141, - "text": "Бузулук , ул . Лесная , д . 5" - }, - { - "start": 218, - "end": 255, - "text": "Невинномысск , ул . Тверская , д . 10" - } - ], - "predicted": [ - { - "start": 108, - "end": 141, - "text": "г . Бузулук , ул . Лесная , д . 5", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - }, - { - "start": 214, - "end": 255, - "text": "г . Невинномысск , ул . Тверская , д . 10", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 1206, - "text": "Договор № 123 / 2023 от 15 . 03 . 2023 . Поставщик : ООО « Ромашка » , ИНН 7707083893 , адрес : г . Kronshtadt , ул . Лесная , д . 5 . Покупатель : ИП Филат И . И . , ИНН 5001122334 , адрес : г . Обнински , ул . Мира , д . 10 . Настоящий договор составлен в двух экземплярах , имеющих одинаковую юридическую силу .", - "gold": [ - { - "start": 100, - "end": 132, - "text": "Kronshtadt , ул . Лесная , д . 5" - }, - { - "start": 196, - "end": 225, - "text": "Обнински , ул . Мира , д . 10" - } - ], - "predicted": [ - { - "start": 96, - "end": 132, - "text": "г . Kronshtadt , ул . Лесная , д . 5", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - }, - { - "start": 192, - "end": 225, - "text": "г . Обнински , ул . Мира , д . 10", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 1209, - "text": "Vernon живЕТ на улице ЛЕНинГрадской , в городе Ейск , в квартире № 12", - "gold": [ - { - "start": 16, - "end": 69, - "text": "улице ЛЕНинГрадской , в городе Ейск , в квартире № 12" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1216, - "text": "Нижний Новгород , пр . Хавская , 88 — 8-831-422-50-60", - "gold": [ - { - "start": 0, - "end": 35, - "text": "Нижний Новгород , пр . Хавская , 88" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1219, - "text": "Автосервис Автопилот на метро Войковская Димитровград , Старопетровский проезд , 9А Округ САО , Нижегородский Войковский ( - )", - "gold": [ - { - "start": 41, - "end": 120, - "text": "Димитровград , Старопетровский проезд , 9А Округ САО , Нижегородский Войковский" - } - ], - "predicted": [ - { - "start": 56, - "end": 83, - "text": "Старопетровский проезд , 9А", - "confidence": 0.44, - "signals": [ - "street_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 1295, - "text": "Адрес доставки: Новошахтинск ул . Нижегородская 9 , кв . 11", - "gold": [ - { - "start": 16, - "end": 59, - "text": "Новошахтинск ул . Нижегородская 9 , кв . 11" - } - ], - "predicted": [ - { - "start": 29, - "end": 59, - "text": "ул . Нижегородская 9 , кв . 11", - "confidence": 0.67, - "signals": [ - "address_cue", - "street_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 1310, - "text": "пр . Нижегородская , 12А , +44-8133-836688 Онлайн-запись на https://new . cian . ru/catalog . Cluster node: 172 . 22 . 197 . 235 .", - "gold": [ - { - "start": 0, - "end": 24, - "text": "пр . Нижегородская , 12А" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1339, - "text": "ООО \" Русские технологии \" 127051 , Регион 77 , ГлАзОВ , улица петровка , дом 38 , строение 1 ИНН 7706543210 Место расчётов: russian-technologies . ru", - "gold": [ - { - "start": 48, - "end": 93, - "text": "ГлАзОВ , улица петровка , дом 38 , строение 1" - } - ], - "predicted": [ - { - "start": 57, - "end": 93, - "text": "улица петровка , дом 38 , строение 1", - "confidence": 0.77, - "signals": [ - "street_marker", - "house_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 1346, - "text": "MOY ADRES PROPISKI: ЭСТОНИЯ , AKMOLINSKAYA OBLAST G . ASTANA UL . DOSTYK D . 5 KV . 1", - "gold": [ - { - "start": 20, - "end": 85, - "text": "ЭСТОНИЯ , AKMOLINSKAYA OBLAST G . ASTANA UL . DOSTYK D . 5 KV . 1" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1364, - "text": "Приёмная , пр . Генерала Карбышева , д . 7 , кв . 88 . 8 812 682 53 49 .", - "gold": [ - { - "start": 11, - "end": 52, - "text": "пр . Генерала Карбышева , д . 7 , кв . 88" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1420, - "text": "напиши подробный план равизтия учебного центра английского языка - помещение 2 класса , хол , первый этаж , парковка Бритландия Трофимова 7 А", - "gold": [ - { - "start": 128, - "end": 141, - "text": "Трофимова 7 А" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1450, - "text": "MOY ADRES PROPISKI: МАДАГАСКАР , KIEVSKAYA OBLAST G . IRPIN UL . KIEVSKAYA D . 20 KV . 2", - "gold": [ - { - "start": 20, - "end": 88, - "text": "МАДАГАСКАР , KIEVSKAYA OBLAST G . IRPIN UL . KIEVSKAYA D . 20 KV . 2" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1489, - "text": "ООО \" Mail . ru Group \" 125167 , Алатырь , Ленинградский проспект , дом 39 , строение 79 ИНН 7719488300 Место расчётов: mail . ru", - "gold": [ - { - "start": 33, - "end": 88, - "text": "Алатырь , Ленинградский проспект , дом 39 , строение 79" - } - ], - "predicted": [ - { - "start": 43, - "end": 88, - "text": "Ленинградский проспект , дом 39 , строение 79", - "confidence": 0.77, - "signals": [ - "street_marker", - "house_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 1504, - "text": "Адрес доставки: бузулук ул . Краснопролетарская 34 , кв . 78", - "gold": [ - { - "start": 16, - "end": 60, - "text": "бузулук ул . Краснопролетарская 34 , кв . 78" - } - ], - "predicted": [ - { - "start": 24, - "end": 60, - "text": "ул . Краснопролетарская 34 , кв . 78", - "confidence": 0.67, - "signals": [ - "address_cue", - "street_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 1578, - "text": "Для формирования счета на оплату , пожалуйста , предоставьте полный перечень реквизитов вашей организации . Идентификационный номер налогоплательщика : 7707083893 . Адрес доставки : г . Краснодар , Невский пр . , д . 10 .", - "gold": [ - { - "start": 186, - "end": 219, - "text": "Краснодар , Невский пр . , д . 10" - } - ], - "predicted": [ - { - "start": 182, - "end": 219, - "text": "г . Краснодар , Невский пр . , д . 10", - "confidence": 0.63, - "signals": [ - "address_cue", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 1596, - "text": "Склад: Промышленная ул . , стр . 7 , +7 926 301-44-58", - "gold": [ - { - "start": 7, - "end": 34, - "text": "Промышленная ул . , стр . 7" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1602, - "text": "SAMARA , LENINA 25", - "gold": [ - { - "start": 0, - "end": 18, - "text": "SAMARA , LENINA 25" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1614, - "text": "Apple Inc . 1 Infinite Loop , cupertino , ОРЛОВСКАЯ 95014 БОТСВАНА", - "gold": [ - { - "start": 12, - "end": 66, - "text": "1 Infinite Loop , cupertino , ОРЛОВСКАЯ 95014 БОТСВАНА" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1628, - "text": "АО « СБЕРБАНК катар » УЛ . БОЛЬШАЯ ДМИТРОВКА , 32 , СТР . 1 , УСОЛЬЕ-СИБИРСКОЕ , 107031 , ГРЕНАДА ТЕЛ . : +1-244-518-3825 , WWW . SBERBANK . RU", - "gold": [ - { - "start": 14, - "end": 97, - "text": "катар » УЛ . БОЛЬШАЯ ДМИТРОВКА , 32 , СТР . 1 , УСОЛЬЕ-СИБИРСКОЕ , 107031 , ГРЕНАДА" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1636, - "text": "Как добраться до пр . Толстого , д . 176 в Люберцы ?", - "gold": [ - { - "start": 17, - "end": 50, - "text": "пр . Толстого , д . 176 в Люберцы" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1668, - "text": "recipient_fio = Калашникова Инна recipient_addr = Вологда , ул . Герцена , дом 11 recipient_email = i . kalashnikova@vologda . ru", - "gold": [ - { - "start": 50, - "end": 81, - "text": "Вологда , ул . Герцена , дом 11" - } - ], - "predicted": [ - { - "start": 60, - "end": 81, - "text": "ул . Герцена , дом 11", - "confidence": 0.72, - "signals": [ - "street_marker", - "house_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 1693, - "text": "Ближайшее отделение: бул . Академика Королёва , д . 78 , кв . 7 . Почта: maksim@bedrijf . nl .", - "gold": [ - { - "start": 21, - "end": 63, - "text": "бул . Академика Королёва , д . 78 , кв . 7" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1739, - "text": "MOY ADRES PROPISKI: ROSSIYA , КИРОВСКАЯ OBLAST G . MOSKVA UL . ARBAT D . 15 KV . 50", - "gold": [ - { - "start": 20, - "end": 83, - "text": "ROSSIYA , КИРОВСКАЯ OBLAST G . MOSKVA UL . ARBAT D . 15 KV . 50" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1777, - "text": "Счет № 123 от 15 . 03 . 2024 Поставщик : ООО \" Вектор \" , ИНН 7707083893 , Юридический адрес : 190000 , г . ASTANA , ул . Невская , д . 5 . Покупатель : ЗАО \" Горизонт \" , ИНН : 4321098765 . Адрес : 125009 , г . Пермь , ул . Тверская , д . 10 .", - "gold": [ - { - "start": 108, - "end": 137, - "text": "ASTANA , ул . Невская , д . 5" - }, - { - "start": 212, - "end": 242, - "text": "Пермь , ул . Тверская , д . 10" - } - ], - "predicted": [ - { - "start": 95, - "end": 137, - "text": "190000 , г . ASTANA , ул . Невская , д . 5", - "confidence": 0.99, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "postal_code", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - }, - { - "start": 199, - "end": 242, - "text": "125009 , г . Пермь , ул . Тверская , д . 10", - "confidence": 0.99, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "postal_code", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 1883, - "text": "Завещание . Я , Торопин Ермолай Германович , residing at : ромнах , Tverskaya St . 20 , apt . 5 , my INN is 500100732259 , do hereby declare my last will and testament .", - "gold": [ - { - "start": 59, - "end": 95, - "text": "ромнах , Tverskaya St . 20 , apt . 5" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 1947, - "text": "Глазов , 117997 , ул . Вавилова , д . 19", - "gold": [ - { - "start": 0, - "end": 40, - "text": "Глазов , 117997 , ул . Вавилова , д . 19" - } - ], - "predicted": [ - { - "start": 9, - "end": 40, - "text": "117997 , ул . Вавилова , д . 19", - "confidence": 0.75, - "signals": [ - "street_marker", - "house_marker", - "postal_code", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 1951, - "text": "Здравствуйте , хочу заказать товары из вашего интернет-магазина . Доставка в город аЛАТЫРЬ , улица Советская , дом 8 , квартира 74 .", - "gold": [ - { - "start": 83, - "end": 130, - "text": "аЛАТЫРЬ , улица Советская , дом 8 , квартира 74" - } - ], - "predicted": [ - { - "start": 66, - "end": 130, - "text": "Доставка в город аЛАТЫРЬ , улица Советская , дом 8 , квартира 74", - "confidence": 0.78, - "signals": [ - "street_marker", - "house_marker", - "location_marker", - "unit_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 1978, - "text": "MOY ADRES PROPISKI: АБХАЗИЯ , GRODNENSKAYA OBLAST G . GRODNO Цветной бульвар D . 5 KV . 12", - "gold": [ - { - "start": 20, - "end": 90, - "text": "АБХАЗИЯ , GRODNENSKAYA OBLAST G . GRODNO Цветной бульвар D . 5 KV . 12" - } - ], - "predicted": [ - { - "start": 69, - "end": 82, - "text": "бульвар D . 5", - "confidence": 0.44, - "signals": [ - "street_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 1982, - "text": "Переехали на Комсомольский проспект , дом 34 кв . 118", - "gold": [ - { - "start": 13, - "end": 53, - "text": "Комсомольский проспект , дом 34 кв . 118" - } - ], - "predicted": [ - { - "start": 0, - "end": 53, - "text": "Переехали на Комсомольский проспект , дом 34 кв . 118", - "confidence": 0.72, - "signals": [ - "street_marker", - "house_marker", - "unit_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2069, - "text": "Автосервис КАПИТАН-АВТО на метро Коптево Вольск , КРОНШТАДТСКИЙ БУЛЬВАР , 35б Округ САО , Люблино Головинский ( доехать , узнать ) ( ? ? ? )", - "gold": [ - { - "start": 27, - "end": 109, - "text": "метро Коптево Вольск , КРОНШТАДТСКИЙ БУЛЬВАР , 35б Округ САО , Люблино Головинский" - } - ], - "predicted": [ - { - "start": 50, - "end": 77, - "text": "КРОНШТАДТСКИЙ БУЛЬВАР , 35б", - "confidence": 0.44, - "signals": [ - "street_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2074, - "text": "Никитский бул . , 12", - "gold": [ - { - "start": 0, - "end": 20, - "text": "Никитский бул . , 12" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 2078, - "text": "живу г . Северодвинск ул . блока д . 65 не далеко от центра", - "gold": [ - { - "start": 9, - "end": 39, - "text": "Северодвинск ул . блока д . 65" - } - ], - "predicted": [ - { - "start": 22, - "end": 39, - "text": "ул . блока д . 65", - "confidence": 0.67, - "signals": [ - "street_marker", - "house_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2104, - "text": "Магазин на ПР . ТИМИРЯЗЕВСКАЯ , дом 110 , кв . 219 , заказы на s . komarov@bmstu . ru .", - "gold": [ - { - "start": 11, - "end": 50, - "text": "ПР . ТИМИРЯЗЕВСКАЯ , дом 110 , кв . 219" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 2108, - "text": "Лев — Северодвинск ул . Толстого , дом 96", - "gold": [ - { - "start": 6, - "end": 41, - "text": "Северодвинск ул . Толстого , дом 96" - } - ], - "predicted": [ - { - "start": 19, - "end": 41, - "text": "ул . Толстого , дом 96", - "confidence": 0.72, - "signals": [ - "street_marker", - "house_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2113, - "text": " Сербия Ленинградская обл . Всеволожский район наб . Фонтанки д . 30 ", - "gold": [ - { - "start": 21, - "end": 168, - "text": "Сербия Ленинградская обл . Всеволожский район наб . Фонтанки д . 30" - } - ], - "predicted": [ - { - "start": 129, - "end": 168, - "text": "наб . Фонтанки д . 30", - "confidence": 0.67, - "signals": [ - "street_marker", - "house_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2115, - "text": "КОМПАНИЯ ООО « RUTUBE » АДРЕС ГОРОД НОВОШАХТИНСК УЛИЦА ДАНИЛОВСКАЯ ДОМ 10 КОРПУС 1", - "gold": [ - { - "start": 36, - "end": 82, - "text": "НОВОШАХТИНСК УЛИЦА ДАНИЛОВСКАЯ ДОМ 10 КОРПУС 1" - } - ], - "predicted": [ - { - "start": 30, - "end": 82, - "text": "ГОРОД НОВОШАХТИНСК УЛИЦА ДАНИЛОВСКАЯ ДОМ 10 КОРПУС 1", - "confidence": 0.99, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2142, - "text": "Адрес офиса: пр . Обручева , д . 1 . Email: tarasov . ivan@rambler . ru .", - "gold": [ - { - "start": 13, - "end": 34, - "text": "пр . Обручева , д . 1" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 2185, - "text": "Библиотека в выхинО-ЖуЛЕБИНО районе , пр . Электрозаводская , д . 58 , кв . 198 , email vitali1975@rambler . ru .", - "gold": [ - { - "start": 13, - "end": 79, - "text": "выхинО-ЖуЛЕБИНО районе , пр . Электрозаводская , д . 58 , кв . 198" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 2216, - "text": "Адрес доставки: Ейск ул . Вавилова 56 , кв . 3", - "gold": [ - { - "start": 16, - "end": 46, - "text": "Ейск ул . Вавилова 56 , кв . 3" - } - ], - "predicted": [ - { - "start": 21, - "end": 46, - "text": "ул . Вавилова 56 , кв . 3", - "confidence": 0.67, - "signals": [ - "address_cue", - "street_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2229, - "text": "К проекту присоединился и кронштадт . У Морского собора по адресу Якорная площадь , дом 1 , оборудовали специальный павильон , в котором показывали карты островов Финского залива . Но на одном из стендов адрес значился : “ Якроная пл . 1 , Kronshtadt City” , и организаторы уверяли , что такие ошибки демонстрируют живую природу города .", - "gold": [ - { - "start": 66, - "end": 89, - "text": "Якорная площадь , дом 1" - }, - { - "start": 223, - "end": 250, - "text": "Якроная пл . 1 , Kronshtadt" - } - ], - "predicted": [ - { - "start": 66, - "end": 89, - "text": "Якорная площадь , дом 1", - "confidence": 0.9, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "parsed_street", - "parsed_house" - ] - }, - { - "start": 210, - "end": 237, - "text": "значился : “ Якроная пл . 1", - "confidence": 0.62, - "signals": [ - "address_cue", - "street_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text", - "span_includes_context" - ] - }, - { - "source_row": 2255, - "text": "Производственная компания « Альфа » приглашает к сотрудничеству дилеров . Мы предлагаем выгодные условия партнерства и поддержку на всех этапах . Подробности по телефону или по электронной почте . Адрес для корреспонденции : MOSKVA , ул . Центральная , д . 1 . Наш инн 7707083893 .", - "gold": [ - { - "start": 225, - "end": 258, - "text": "MOSKVA , ул . Центральная , д . 1" - } - ], - "predicted": [ - { - "start": 203, - "end": 258, - "text": "для корреспонденции : MOSKVA , ул . Центральная , д . 1", - "confidence": 0.85, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2269, - "text": "Поставщик : ООО \" Лучшее Решение \" , ИНН : 7707083893 , юридический адрес : г . Бузулук , ул . Примерная , д . 1 . Предлагаем к поставке партию строительных материалов согласно приложенной спецификации . Готовы предоставить все необходимые документы .", - "gold": [ - { - "start": 80, - "end": 112, - "text": "Бузулук , ул . Примерная , д . 1" - } - ], - "predicted": [ - { - "start": 76, - "end": 112, - "text": "г . Бузулук , ул . Примерная , д . 1", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2285, - "text": "Поставщик : ООО « Альфа » ( ИНН 7707083893 ) , юридический адрес : 105082 , г . Каменск-Уральский , ул . Бакунинская , д . 5 . Договор № 123 / 2023 от 15 . 03 . 2023 .", - "gold": [ - { - "start": 80, - "end": 124, - "text": "Каменск-Уральский , ул . Бакунинская , д . 5" - } - ], - "predicted": [ - { - "start": 67, - "end": 124, - "text": "105082 , г . Каменск-Уральский , ул . Бакунинская , д . 5", - "confidence": 0.99, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "postal_code", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2316, - "text": "Самовывоз: Димитровград , ул . Малышева , д . 51 , +7 ( 343 ) 200-10-20", - "gold": [ - { - "start": 11, - "end": 48, - "text": "Димитровград , ул . Малышева , д . 51" - } - ], - "predicted": [ - { - "start": 26, - "end": 48, - "text": "ул . Малышева , д . 51", - "confidence": 0.67, - "signals": [ - "street_marker", - "house_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2329, - "text": "Адрес доставки: АЛАТЫРЬ ул . Светланская 15 , офис 207", - "gold": [ - { - "start": 16, - "end": 54, - "text": "АЛАТЫРЬ ул . Светланская 15 , офис 207" - } - ], - "predicted": [ - { - "start": 24, - "end": 54, - "text": "ул . Светланская 15 , офис 207", - "confidence": 0.67, - "signals": [ - "address_cue", - "street_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2332, - "text": "Отделение в посёлке Малаховка , Быковское шоссе д . 9", - "gold": [ - { - "start": 20, - "end": 53, - "text": "Малаховка , Быковское шоссе д . 9" - } - ], - "predicted": [ - { - "start": 32, - "end": 53, - "text": "Быковское шоссе д . 9", - "confidence": 0.67, - "signals": [ - "street_marker", - "house_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2336, - "text": "+7 985 469-24-69 , Зеленодольск , московское шоссе , д . 15 Прайс-лист на https://admin . arxiv . org/products . Connected via 192 . 168 . 248 . 29 .", - "gold": [ - { - "start": 19, - "end": 59, - "text": "Зеленодольск , московское шоссе , д . 15" - } - ], - "predicted": [ - { - "start": 34, - "end": 59, - "text": "московское шоссе , д . 15", - "confidence": 0.67, - "signals": [ - "street_marker", - "house_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2351, - "text": "MOY ADRES PROPISKI: ЧЕРНОГОРИЯ , ANDIZHANSKAYA OBLAST G . ANDIZHAN ul . navroz D . 10 KV . 6", - "gold": [ - { - "start": 20, - "end": 92, - "text": "ЧЕРНОГОРИЯ , ANDIZHANSKAYA OBLAST G . ANDIZHAN ul . navroz D . 10 KV . 6" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 2370, - "text": "В связи с ненадлежащим исполнением условий договора № 123 от 15 . 01 . 2023 , прошу Вас в срок до 01 . 03 . 2024 направить в наш адрес письменные объяснения . Адрес для направления документов : 101000 , Рыбинск , ул . Примерная , д . 1 , офис 5 . Реквизиты контрагента : ООО \" СтройДом \" , ИНН 7707083893 , КПП 667890123 .", - "gold": [ - { - "start": 203, - "end": 244, - "text": "Рыбинск , ул . Примерная , д . 1 , офис 5" - } - ], - "predicted": [ - { - "start": 165, - "end": 244, - "text": "для направления документов : 101000 , Рыбинск , ул . Примерная , д . 1 , офис 5", - "confidence": 0.98, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "postal_code", - "unit_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2372, - "text": "сервак 23 . 233 . 99 . 78:22 лежит опять . Заявитель: Ingram Клим Макарович , г . Бузулук , бул . Цветной бульвар , д . 98 , кв . 120 , подробнее на https://ranepa . ru/help .", - "gold": [ - { - "start": 82, - "end": 133, - "text": "Бузулук , бул . Цветной бульвар , д . 98 , кв . 120" - } - ], - "predicted": [ - { - "start": 98, - "end": 133, - "text": "Цветной бульвар , д . 98 , кв . 120", - "confidence": 0.72, - "signals": [ - "street_marker", - "house_marker", - "unit_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2380, - "text": "MOY ADRES PROPISKI: НЕПАЛ , CHUYSKAYA OBLAST G . BISHKEK академика королёва D . 10 KV . 15", - "gold": [ - { - "start": 20, - "end": 90, - "text": "НЕПАЛ , CHUYSKAYA OBLAST G . BISHKEK академика королёва D . 10 KV . 15" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 2382, - "text": "Вниманию ООО \" СтройКомПлюс \" , расположенного по адресу : г . Тверь , ул . Строителей , д . 15 , ИНН 7707083893 , сообщаем о нарушении сроков поставки строительных материалов . В соответствии с пунктом 3 . 2 Договора № 145 от 10 . 01 . 2023 , поставка должна была быть осуществлена до 01 . 03 . 2023 . На сегодняшний день материалы не поступили , что создает существенные препятствия для выполнения наших обязательств перед заказчиком .", - "gold": [ - { - "start": 63, - "end": 95, - "text": "Тверь , ул . Строителей , д . 15" - } - ], - "predicted": [ - { - "start": 59, - "end": 112, - "text": "г . Тверь , ул . Строителей , д . 15 , ИНН 7707083893", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2389, - "text": "Зарегистрирован: г . Северодвинск , калмыкии обл . , ул . толстого , д . 103", - "gold": [ - { - "start": 21, - "end": 76, - "text": "Северодвинск , калмыкии обл . , ул . толстого , д . 103" - } - ], - "predicted": [ - { - "start": 17, - "end": 76, - "text": "г . Северодвинск , калмыкии обл . , ул . толстого , д . 103", - "confidence": 0.73, - "signals": [ - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2407, - "text": "При заключении кредитного договора № 456789012345 , прошу указать следующие данные : Шувалов Платон Артёмович , ИНН 500100732259 , дата рождения 01 . 01 . 1990 . Проживает по адресу : г . Алатырь , ул . Ленина , д . 1 , кв . 1 .", - "gold": [ - { - "start": 188, - "end": 226, - "text": "Алатырь , ул . Ленина , д . 1 , кв . 1" - } - ], - "predicted": [ - { - "start": 184, - "end": 226, - "text": "г . Алатырь , ул . Ленина , д . 1 , кв . 1", - "confidence": 0.96, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "unit_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2411, - "text": "Локации включали не только улицы и проспекты , но и метки : “центр , около ТЦ Gulliver” , “рядом с метро Дворец Украина” , ул Обручева , 16 , Люблинская , 10 , ул Усачёва , 44А , УЛ БАУМАНСКАЯ , 34 , УЛ ЯРЦЕВСКАЯ , 12 . Иногда встречались устаревшие адреса , записанные как “ ул . Ленина , 5 , Киев ” или “г . ейск , центр , напротив театра” .", - "gold": [ - { - "start": 123, - "end": 217, - "text": "ул Обручева , 16 , Люблинская , 10 , ул Усачёва , 44А , УЛ БАУМАНСКАЯ , 34 , УЛ ЯРЦЕВСКАЯ , 12" - }, - { - "start": 276, - "end": 298, - "text": "ул . Ленина , 5 , Киев" - } - ], - "predicted": [ - { - "start": 160, - "end": 176, - "text": "ул Усачёва , 44А", - "confidence": 0.44, - "signals": [ - "street_marker", - "parsed_street", - "parsed_house" - ] - }, - { - "start": 179, - "end": 197, - "text": "УЛ БАУМАНСКАЯ , 34", - "confidence": 0.44, - "signals": [ - "street_marker", - "parsed_street", - "parsed_house" - ] - }, - { - "start": 200, - "end": 217, - "text": "УЛ ЯРЦЕВСКАЯ , 12", - "confidence": 0.44, - "signals": [ - "street_marker", - "parsed_street", - "parsed_house" - ] - }, - { - "start": 276, - "end": 291, - "text": "ул . Ленина , 5", - "confidence": 0.62, - "signals": [ - "address_cue", - "street_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text", - "spurious_address" - ] - }, - { - "source_row": 2425, - "text": "Бампер передний в цвет Ford Focus 3 ВОЛЬСК , ул . Плеханова , 10Ас3", - "gold": [ - { - "start": 36, - "end": 67, - "text": "ВОЛЬСК , ул . Плеханова , 10Ас3" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 2451, - "text": "К заявке на участие в тендере прилагаем следующую информацию : Общество с ограниченной ответственностью \" Альфа \" , ИНН 7707083893 , юридический адрес : г . Нижнем Новгород , ул . Ленина , д . 1 . Готовы предоставить дополнительные документы по запросу .", - "gold": [ - { - "start": 157, - "end": 194, - "text": "Нижнем Новгород , ул . Ленина , д . 1" - } - ], - "predicted": [ - { - "start": 153, - "end": 194, - "text": "г . Нижнем Новгород , ул . Ленина , д . 1", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2466, - "text": "Адрес доставки: Усолье-Сибирское ул . социалистическая 7 , кв . 12", - "gold": [ - { - "start": 16, - "end": 66, - "text": "Усолье-Сибирское ул . социалистическая 7 , кв . 12" - } - ], - "predicted": [ - { - "start": 33, - "end": 66, - "text": "ул . социалистическая 7 , кв . 12", - "confidence": 0.67, - "signals": [ - "address_cue", - "street_marker", - "unit_marker", - "parsed_street", - "parsed_house" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2514, - "text": "Бузулук , ул . Кольцовская , д . 9 , +7 ( 951 ) 270 63 31", - "gold": [ - { - "start": 0, - "end": 34, - "text": "Бузулук , ул . Кольцовская , д . 9" - } - ], - "predicted": [ - { - "start": 10, - "end": 34, - "text": "ул . Кольцовская , д . 9", - "confidence": 0.67, - "signals": [ - "street_marker", - "house_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2531, - "text": "MOY ADRES PROPISKI: КАТАР , KIEVSKAYA OBLAST G . ГЛАЗОВ UL . KRESHCHATIK D . 10 KV . 36", - "gold": [ - { - "start": 20, - "end": 87, - "text": "КАТАР , KIEVSKAYA OBLAST G . ГЛАЗОВ UL . KRESHCHATIK D . 10 KV . 36" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 2589, - "text": "src= 208 . 222 . 5 . 31:53 dst= 10 . 0 . 0 . 1 . В ромнах , на улице Гетьмана Мазепи 26 , международным письмам прописывают адрес иначе : Hetman Mazepa street , Romny , ГРЕНАДА , но индекс иногда теряется .", - "gold": [ - { - "start": 51, - "end": 87, - "text": "ромнах , на улице Гетьмана Мазепи 26" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 2600, - "text": "8 919 431-42-62 , адрес: пр . ферганская , д . 130 , кв . 220 .", - "gold": [ - { - "start": 25, - "end": 61, - "text": "пр . ферганская , д . 130 , кв . 220" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 2648, - "text": "City maps change faster than memories . A couple once met every evening at “Old Lantern Café , 52A Howard St . , Crestville , ” but the address updated to 52 Howard Street , and then the café moved entirely to 18 Seaview Drive with a bright phone number +353 85 224 9087 printed on menus . The original lot now displays a grocery’s poorly translated name “Lanterno Shop Market , ” leaving locals nostalgic for the old wooden chairs and the window facing the coordinates 53 . 3498 , -6 . 2603 which used to align perfectly with the sunset . When I visited on 07/14/2022 , the new waitress said the café is still the heart of the city but the heart beats in different ribs .", - "gold": [ - { - "start": 95, - "end": 123, - "text": "52A Howard St . , Crestville" - }, - { - "start": 155, - "end": 171, - "text": "52 Howard Street" - }, - { - "start": 210, - "end": 226, - "text": "18 Seaview Drive" - } - ], - "predicted": [], - "failure_reasons": [ - "missed_address" - ] - }, - { - "source_row": 2698, - "text": "Гражданин Тихонов Лев Савельевич , 15 . 05 . 1985 г . р . , ИНН 500100732259 , проживающий по адресу : г . Romny , ул . Ленина , д . 5 , кв . 10 . Прошу предоставить справку о состоянии лицевого счета .", - "gold": [ - { - "start": 107, - "end": 144, - "text": "Romny , ул . Ленина , д . 5 , кв . 10" - } - ], - "predicted": [ - { - "start": 103, - "end": 144, - "text": "г . Romny , ул . Ленина , д . 5 , кв . 10", - "confidence": 0.96, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "unit_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2707, - "text": "Автосервис : Алатырь , Ул . Дубнинская , д . 83 , стр . 5 ( 03 . 06 с 10 : 00 ) - 12000 руб работа под ключ + стоимость деталей ( ~ 15000 - 16000 руб )", - "gold": [ - { - "start": 13, - "end": 57, - "text": "Алатырь , Ул . Дубнинская , д . 83 , стр . 5" - } - ], - "predicted": [ - { - "start": 23, - "end": 57, - "text": "Ул . Дубнинская , д . 83 , стр . 5", - "confidence": 0.72, - "signals": [ - "street_marker", - "house_marker", - "unit_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_drops_gold_text" - ] - }, - { - "source_row": 2764, - "text": "Центр оказывает посильную помощь семьям , воспитывающим несовершеннолетних детей , инвалидам , одиноким пожилым гражданам . На сайте Вы найдете информацию о реализации различных программ , работе клубов , режиме работы учреждения , деятельности структурных подразделений , оказываемых социальных услугах . Вы можете обратиться к специалистам учреждения через интернет , отправить письмо на нашу электронную почту belkids @ beladm . ru . Важная информация для семей с детьми до 1 , 5 лет Уважаемые родители ! Информируем вас о возможности получения необходимых детских вещей во временное пользование в рамках социального проек . . . . . . ели для близнецов )  Прогулочная коляска  Стул для кормления  Автолюлька для новорожденного  Автокресло для детей от 6 месяцев Кто может воспользоваться услугой :  Семьи , где родители учатся очно  Многодетные семьи  Неполные семьи и одинокие мамы  Семьи с детьми - инвалидами  Малообеспеченные семьи и семьи в трудной жизненной ситуации Условия предоставления :  Срок пользования согласовывается индивидуально  Все предметы предоставляются временно Где получить помощь :  Отделение « Семейный МФЦ » МБУ « Комплексный центр социального обслуживания населения города Белгорода »  Адрес : г . Белгород , ул . Королева , д . 9  Телефон : + 7 ( 4722 ) 55 - 15 - 75 , + 7 ( 4722 ) 52 - 58 - 13  Режим работы : пн - пт с 9 : 00 до 18 : 00 ( перерыв 13 : 00 - 14 : 00 ) Дополнительные услуги центра  Бесплатное юридическое сопровождение  Психологическая поддержка  Педагогическая помощь  Социальное консультирование  Помощь в получении мер поддержки  Организация досуга На территории Белгородской области запущена работа мобильного приложения « Куратор семьи » , предназначенного для семей участников СВО и для семей граждан пострадавших в результате обстрелов со стороны ВСУ . Не упустите возможность получить необходимую поддержку для вашей семьи ! Важно : все услуги предоставляются бесплатно в режиме « одного окна » . напиши выступление для родительского собрания кратко", - "gold": [ - { - "start": 1242, - "end": 1274, - "text": "Белгород , ул . Королева , д . 9" - } - ], - "predicted": [ - { - "start": 1238, - "end": 1413, - "text": "г . Белгород , ул . Королева , д . 9  Телефон : + 7 ( 4722 ) 55 - 15 - 75 , + 7 ( 4722 ) 52 - 58 - 13  Режим работы : пн - пт с 9 : 00 до 18 : 00 ( перерыв 13 : 00 - 14 : 00", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - }, - { - "source_row": 2815, - "text": "Реквизиты для оплаты : ООО « Альфа » , ИНН : 7707083893 . Адрес : г . New Springs , ул . Ленина , д . 1 .", - "gold": [ - { - "start": 70, - "end": 103, - "text": "New Springs , ул . Ленина , д . 1" - } - ], - "predicted": [ - { - "start": 66, - "end": 103, - "text": "г . New Springs , ул . Ленина , д . 1", - "confidence": 0.91, - "signals": [ - "address_cue", - "street_marker", - "house_marker", - "location_marker", - "parsed_street", - "parsed_house", - "unparsed_text" - ] - } - ], - "failure_reasons": [ - "span_includes_context" - ] - } - ], - "source": { - "dataset": "redmadrobot-rnd/pii_benchmark", - "revision": "f77ea831274daf980cc45c61a93c226be9d978d6", - "sha256": "6bf544a380a3ee5bec94b946124bea3afaecce49e734679ad0f0c0e7c12977bb" - } -} diff --git a/evaluation/redmadrobot_report.json b/evaluation/redmadrobot_report.json deleted file mode 100644 index 7b8052a..0000000 --- a/evaluation/redmadrobot_report.json +++ /dev/null @@ -1,1528 +0,0 @@ -{ - "scope": "independent, untuned external evaluation on address snippets from the RedMadRobot Russian PII NER benchmark", - "source": { - "repository": "redmadrobot-rnd/pii_benchmark", - "license": "MIT", - "revision": "f77ea831274daf980cc45c61a93c226be9d978d6", - "sha256": "6bf544a380a3ee5bec94b946124bea3afaecce49e734679ad0f0c0e7c12977bb", - "url": "https://huggingface.co/datasets/redmadrobot-rnd/pii_benchmark/resolve/f77ea831274daf980cc45c61a93c226be9d978d6/test.csv", - "limitations": "production-log-shaped and manually annotated, with real personal values replaced; includes synthetic document-style examples and hard negatives" - }, - "windowing": { - "max_non_location_tokens_between_spans": 3, - "country_is_context_only": true - }, - "source_rows": 493, - "address_snippets": 578, - "matching": "one-to-one same-label span overlap; address windows are oracle-cropped from the benchmark's BIO annotations", - "metric_definitions": { - "span_overlap_micro": "micro precision, recall, and F1 for one-to-one same-label spans with any character overlap", - "exact_span_recall": "exact-boundary same-label matches divided by gold span count", - "fields": "per-field span-overlap precision, recall, and F1" - }, - "span_overlap_micro": { - "tp": 581, - "fp": 387, - "fn": 429, - "support": 1010, - "precision": 0.600207, - "recall": 0.575248, - "f1": 0.587462 - }, - "micro": { - "tp": 581, - "fp": 387, - "fn": 429, - "support": 1010, - "precision": 0.600207, - "recall": 0.575248, - "f1": 0.587462 - }, - "macro_field_f1": 0.592198, - "overlap_matches": 581, - "exact_span_matches": 288, - "exact_span_recall": 0.285149, - "fields": { - "REGION": { - "tp": 66, - "fp": 10, - "fn": 110, - "support": 176, - "precision": 0.868421, - "recall": 0.375, - "f1": 0.52381 - }, - "DISTRICT": { - "tp": 47, - "fp": 3, - "fn": 114, - "support": 161, - "precision": 0.94, - "recall": 0.291925, - "f1": 0.445498 - }, - "CITY": { - "tp": 195, - "fp": 118, - "fn": 144, - "support": 339, - "precision": 0.623003, - "recall": 0.575221, - "f1": 0.59816 - }, - "STREET": { - "tp": 139, - "fp": 256, - "fn": 32, - "support": 171, - "precision": 0.351899, - "recall": 0.812865, - "f1": 0.491166 - }, - "HOUSE": { - "tp": 134, - "fp": 0, - "fn": 29, - "support": 163, - "precision": 1.0, - "recall": 0.822086, - "f1": 0.902357 - } - }, - "failure_sample": [ - { - "source_row": 1, - "text": "Курганской области", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 18 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 18 - } - ] - }, - { - "source_row": 8, - "text": "Ненецком крае в Забайкальском крае", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 13 - }, - { - "label": "REGION", - "start": 16, - "end": 34 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 14, - "end": 15 - } - ] - }, - { - "source_row": 13, - "text": "Курганской области и Адыгее", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 18 - }, - { - "label": "REGION", - "start": 21, - "end": 27 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 19, - "end": 20 - }, - { - "label": "STREET", - "start": 0, - "end": 10 - } - ] - }, - { - "source_row": 23, - "text": "Преображенское район , Алатырь", - "gold": [ - { - "label": "DISTRICT", - "start": 0, - "end": 20 - }, - { - "label": "CITY", - "start": 23, - "end": 30 - } - ], - "predicted": [ - { - "label": "DISTRICT", - "start": 0, - "end": 14 - } - ] - }, - { - "source_row": 25, - "text": "Димитровград", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 12 - } - ], - "predicted": [] - }, - { - "source_row": 30, - "text": "Торжка", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 6 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 6 - } - ] - }, - { - "source_row": 37, - "text": "hIgh mEadow", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 11 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 4 - }, - { - "label": "STREET", - "start": 5, - "end": 11 - } - ] - }, - { - "source_row": 37, - "text": "Hig Medow , ” “ HiGH MedDOw", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 9 - }, - { - "label": "CITY", - "start": 16, - "end": 27 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 9 - } - ] - }, - { - "source_row": 37, - "text": "ВОЛОГОДСКАЯ", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 11 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 11 - } - ] - }, - { - "source_row": 37, - "text": "Mountain Rd", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 11 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 8 - }, - { - "label": "STREET", - "start": 9, - "end": 11 - } - ] - }, - { - "source_row": 48, - "text": "ТУГУРО-ЧУМИКАНСКОМУ району Хабаровского края", - "gold": [ - { - "label": "DISTRICT", - "start": 0, - "end": 26 - }, - { - "label": "REGION", - "start": 27, - "end": 44 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 6 - }, - { - "label": "STREET", - "start": 27, - "end": 44 - } - ] - }, - { - "source_row": 54, - "text": "ул . Крымский Вал , 10 , Бузулук", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 17 - }, - { - "label": "HOUSE", - "start": 20, - "end": 22 - }, - { - "label": "CITY", - "start": 25, - "end": 32 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 17 - } - ] - }, - { - "source_row": 91, - "text": "Тамбовская области", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 18 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 18 - } - ] - }, - { - "source_row": 96, - "text": "выборгском районе", - "gold": [ - { - "label": "DISTRICT", - "start": 0, - "end": 17 - } - ], - "predicted": [] - }, - { - "source_row": 101, - "text": "Лондоне", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 7 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 7 - } - ] - }, - { - "source_row": 143, - "text": "Новгородской области", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 20 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 20 - } - ] - }, - { - "source_row": 151, - "text": "пр . бауманская , д . 71", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 15 - }, - { - "label": "HOUSE", - "start": 18, - "end": 24 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 22, - "end": 24 - }, - { - "label": "REGION", - "start": 5, - "end": 15 - }, - { - "label": "STREET", - "start": 18, - "end": 19 - } - ] - }, - { - "source_row": 156, - "text": "Псковская области", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 17 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 10, - "end": 17 - }, - { - "label": "REGION", - "start": 0, - "end": 9 - } - ] - }, - { - "source_row": 156, - "text": "Магаданская области", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 19 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 19 - } - ] - }, - { - "source_row": 161, - "text": "Польша , district= Центральный округ", - "gold": [ - { - "label": "DISTRICT", - "start": 19, - "end": 36 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 6 - }, - { - "label": "STREET", - "start": 9, - "end": 36 - } - ] - }, - { - "source_row": 163, - "text": "Подольск Курортном район", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 8 - }, - { - "label": "DISTRICT", - "start": 9, - "end": 24 - } - ], - "predicted": [ - { - "label": "DISTRICT", - "start": 0, - "end": 18 - } - ] - }, - { - "source_row": 164, - "text": "зЕлЕНоДолЬСК находится в Сахалинская крае", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 12 - }, - { - "label": "REGION", - "start": 25, - "end": 41 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 12 - }, - { - "label": "REGION", - "start": 13, - "end": 22 - }, - { - "label": "STREET", - "start": 25, - "end": 41 - } - ] - }, - { - "source_row": 169, - "text": "люберцы", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 7 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 7 - } - ] - }, - { - "source_row": 185, - "text": "Новосибирску", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 12 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 12 - } - ] - }, - { - "source_row": 186, - "text": "Троицкий административный округ Новошахтинск", - "gold": [ - { - "label": "DISTRICT", - "start": 0, - "end": 31 - }, - { - "label": "CITY", - "start": 32, - "end": 44 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 32, - "end": 44 - }, - { - "label": "STREET", - "start": 0, - "end": 25 - } - ] - }, - { - "source_row": 186, - "text": "Бабушкинский , Бирюлёво , Вешняки", - "gold": [ - { - "label": "DISTRICT", - "start": 0, - "end": 12 - }, - { - "label": "DISTRICT", - "start": 15, - "end": 23 - }, - { - "label": "DISTRICT", - "start": 26, - "end": 33 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 15, - "end": 23 - }, - { - "label": "STREET", - "start": 0, - "end": 12 - } - ] - }, - { - "source_row": 193, - "text": "Калуге", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 6 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 6 - } - ] - }, - { - "source_row": 209, - "text": "замБИЯ адыгее , НЕВСКОМ район , Златоуст", - "gold": [ - { - "label": "REGION", - "start": 7, - "end": 13 - }, - { - "label": "DISTRICT", - "start": 16, - "end": 29 - }, - { - "label": "CITY", - "start": 32, - "end": 40 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 7, - "end": 13 - }, - { - "label": "DISTRICT", - "start": 16, - "end": 23 - }, - { - "label": "STREET", - "start": 0, - "end": 6 - } - ] - }, - { - "source_row": 218, - "text": "3-а , Каширское ш . , Вольск , Глазов , АМУРСКАЯ ОБЛ", - "gold": [ - { - "label": "HOUSE", - "start": 0, - "end": 3 - }, - { - "label": "STREET", - "start": 6, - "end": 17 - }, - { - "label": "CITY", - "start": 22, - "end": 28 - }, - { - "label": "CITY", - "start": 31, - "end": 37 - }, - { - "label": "REGION", - "start": 40, - "end": 52 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 22, - "end": 28 - }, - { - "label": "REGION", - "start": 40, - "end": 48 - }, - { - "label": "STREET", - "start": 6, - "end": 17 - } - ] - }, - { - "source_row": 220, - "text": "площади Восстания", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 17 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 7 - }, - { - "label": "STREET", - "start": 8, - "end": 17 - } - ] - }, - { - "source_row": 226, - "text": "Чапаевск , Ямайка", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 8 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 8 - }, - { - "label": "STREET", - "start": 11, - "end": 17 - } - ] - }, - { - "source_row": 230, - "text": "пр-т Ленина", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 11 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 3, - "end": 4 - }, - { - "label": "STREET", - "start": 5, - "end": 11 - } - ] - }, - { - "source_row": 230, - "text": "стр . 7А", - "gold": [ - { - "label": "HOUSE", - "start": 0, - "end": 8 - } - ], - "predicted": [ - { - "label": "HOUSE", - "start": 6, - "end": 8 - }, - { - "label": "STREET", - "start": 0, - "end": 3 - } - ] - }, - { - "source_row": 246, - "text": "АБХАЗИЯ , almatinskaya OBLAST G . TALDYKORGAN ul . abaya D . 10 KV . 4", - "gold": [ - { - "label": "REGION", - "start": 10, - "end": 29 - }, - { - "label": "CITY", - "start": 34, - "end": 45 - }, - { - "label": "STREET", - "start": 46, - "end": 56 - }, - { - "label": "HOUSE", - "start": 57, - "end": 70 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 7 - }, - { - "label": "HOUSE", - "start": 69, - "end": 70 - }, - { - "label": "STREET", - "start": 64, - "end": 66 - } - ] - }, - { - "source_row": 259, - "text": "Чебоксарах", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 10 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 10 - } - ] - }, - { - "source_row": 262, - "text": "ROSSIYA , тАмБовсКАя OBLAST G . MYTISHCHI Стромынка D . 5 KV . 12", - "gold": [ - { - "label": "REGION", - "start": 10, - "end": 27 - }, - { - "label": "CITY", - "start": 32, - "end": 41 - }, - { - "label": "STREET", - "start": 42, - "end": 51 - }, - { - "label": "HOUSE", - "start": 52, - "end": 65 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 7 - }, - { - "label": "HOUSE", - "start": 63, - "end": 65 - }, - { - "label": "REGION", - "start": 10, - "end": 20 - }, - { - "label": "STREET", - "start": 58, - "end": 60 - } - ] - }, - { - "source_row": 267, - "text": "Костромская области:", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 20 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 12, - "end": 19 - }, - { - "label": "REGION", - "start": 0, - "end": 11 - } - ] - }, - { - "source_row": 271, - "text": "Усолье-Сибирское", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 16 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 6 - }, - { - "label": "STREET", - "start": 7, - "end": 16 - } - ] - }, - { - "source_row": 281, - "text": "Чукотский автономный округ", - "gold": [ - { - "label": "DISTRICT", - "start": 0, - "end": 26 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 26 - } - ] - }, - { - "source_row": 293, - "text": "Невинномысск г . УСОЛЬЕ-СИБИРСКОЕ , УЛ . ДУБИНИНСКАЯ , 70 стр . 1", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 12 - }, - { - "label": "CITY", - "start": 17, - "end": 33 - }, - { - "label": "STREET", - "start": 36, - "end": 52 - }, - { - "label": "HOUSE", - "start": 55, - "end": 65 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 17, - "end": 33 - }, - { - "label": "HOUSE", - "start": 64, - "end": 65 - }, - { - "label": "STREET", - "start": 36, - "end": 52 - } - ] - }, - { - "source_row": 298, - "text": "вологодская области", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 19 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 19 - } - ] - }, - { - "source_row": 307, - "text": "Ненецком крае", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 13 - } - ], - "predicted": [] - }, - { - "source_row": 312, - "text": "Сахалин", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 7 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 7 - } - ] - }, - { - "source_row": 326, - "text": "Ардатова до Нижнего Ломова , от Чкаловска до Лукоянова , от Красных Прудов до границы Сергачского района", - "gold": [ - { - "label": "CITY", - "start": 0, - "end": 8 - }, - { - "label": "CITY", - "start": 12, - "end": 26 - }, - { - "label": "CITY", - "start": 32, - "end": 41 - }, - { - "label": "CITY", - "start": 45, - "end": 54 - }, - { - "label": "CITY", - "start": 60, - "end": 74 - }, - { - "label": "DISTRICT", - "start": 86, - "end": 104 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 12, - "end": 19 - }, - { - "label": "STREET", - "start": 0, - "end": 8 - } - ] - }, - { - "source_row": 330, - "text": "Латинский квартал", - "gold": [ - { - "label": "DISTRICT", - "start": 0, - "end": 17 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 17 - } - ] - }, - { - "source_row": 333, - "text": "Ульяновская области", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 19 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 12, - "end": 19 - }, - { - "label": "REGION", - "start": 0, - "end": 11 - } - ] - }, - { - "source_row": 339, - "text": "округу Барнстейбл", - "gold": [ - { - "label": "DISTRICT", - "start": 0, - "end": 17 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 17 - } - ] - }, - { - "source_row": 350, - "text": "пр . Хавская , д . 149 , кв . 230", - "gold": [ - { - "label": "STREET", - "start": 0, - "end": 12 - }, - { - "label": "HOUSE", - "start": 15, - "end": 33 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 0, - "end": 2 - }, - { - "label": "HOUSE", - "start": 30, - "end": 33 - }, - { - "label": "REGION", - "start": 5, - "end": 12 - }, - { - "label": "STREET", - "start": 25, - "end": 27 - } - ] - }, - { - "source_row": 357, - "text": "Адмиралтейском район в БРАТСК", - "gold": [ - { - "label": "DISTRICT", - "start": 0, - "end": 20 - }, - { - "label": "CITY", - "start": 23, - "end": 29 - } - ], - "predicted": [ - { - "label": "CITY", - "start": 21, - "end": 22 - }, - { - "label": "DISTRICT", - "start": 0, - "end": 14 - }, - { - "label": "STREET", - "start": 23, - "end": 29 - } - ] - }, - { - "source_row": 364, - "text": "Алтай", - "gold": [ - { - "label": "REGION", - "start": 0, - "end": 5 - } - ], - "predicted": [ - { - "label": "STREET", - "start": 0, - "end": 5 - } - ] - } - ], - "slices": { - "multi_field": { - "source_rows": 206, - "address_snippets": 217, - "matching": "one-to-one same-label span overlap; address windows are oracle-cropped from the benchmark's BIO annotations", - "micro": { - "tp": 419, - "fp": 130, - "fn": 170, - "support": 589, - "precision": 0.763206, - "recall": 0.711375, - "f1": 0.73638 - }, - "macro_field_f1": 0.7125, - "overlap_matches": 419, - "exact_span_matches": 212, - "exact_span_recall": 0.359932, - "fields": { - "REGION": { - "tp": 28, - "fp": 9, - "fn": 32, - "support": 60, - "precision": 0.756757, - "recall": 0.466667, - "f1": 0.57732 - }, - "DISTRICT": { - "tp": 36, - "fp": 2, - "fn": 30, - "support": 66, - "precision": 0.947368, - "recall": 0.545455, - "f1": 0.692308 - }, - "CITY": { - "tp": 104, - "fp": 61, - "fn": 51, - "support": 155, - "precision": 0.630303, - "recall": 0.670968, - "f1": 0.65 - }, - "STREET": { - "tp": 125, - "fp": 58, - "fn": 30, - "support": 155, - "precision": 0.68306, - "recall": 0.806452, - "f1": 0.739645 - }, - "HOUSE": { - "tp": 126, - "fp": 0, - "fn": 27, - "support": 153, - "precision": 1.0, - "recall": 0.823529, - "f1": 0.903226 - } - } - }, - "street_and_house": { - "source_rows": 135, - "address_snippets": 144, - "matching": "one-to-one same-label span overlap; address windows are oracle-cropped from the benchmark's BIO annotations", - "micro": { - "tp": 326, - "fp": 73, - "fn": 102, - "support": 428, - "precision": 0.817043, - "recall": 0.761682, - "f1": 0.788392 - }, - "macro_field_f1": 0.620307, - "overlap_matches": 326, - "exact_span_matches": 179, - "exact_span_recall": 0.418224, - "fields": { - "REGION": { - "tp": 9, - "fp": 7, - "fn": 9, - "support": 18, - "precision": 0.5625, - "recall": 0.5, - "f1": 0.529412 - }, - "DISTRICT": { - "tp": 1, - "fp": 2, - "fn": 7, - "support": 8, - "precision": 0.333333, - "recall": 0.125, - "f1": 0.181818 - }, - "CITY": { - "tp": 71, - "fp": 42, - "fn": 30, - "support": 101, - "precision": 0.628319, - "recall": 0.70297, - "f1": 0.663551 - }, - "STREET": { - "tp": 119, - "fp": 22, - "fn": 29, - "support": 148, - "precision": 0.843972, - "recall": 0.804054, - "f1": 0.823529 - }, - "HOUSE": { - "tp": 126, - "fp": 0, - "fn": 27, - "support": 153, - "precision": 1.0, - "recall": 0.823529, - "f1": 0.903226 - } - } - }, - "administrative_only": { - "source_rows": 334, - "address_snippets": 403, - "matching": "one-to-one same-label span overlap; address windows are oracle-cropped from the benchmark's BIO annotations", - "micro": { - "tp": 221, - "fp": 298, - "fn": 319, - "support": 540, - "precision": 0.425819, - "recall": 0.409259, - "f1": 0.417375 - }, - "macro_field_f1": 0.513519, - "overlap_matches": 221, - "exact_span_matches": 98, - "exact_span_recall": 0.181481, - "fields": { - "REGION": { - "tp": 56, - "fp": 3, - "fn": 101, - "support": 157, - "precision": 0.949153, - "recall": 0.356688, - "f1": 0.518519 - }, - "DISTRICT": { - "tp": 44, - "fp": 1, - "fn": 106, - "support": 150, - "precision": 0.977778, - "recall": 0.293333, - "f1": 0.451282 - }, - "CITY": { - "tp": 121, - "fp": 70, - "fn": 112, - "support": 233, - "precision": 0.633508, - "recall": 0.519313, - "f1": 0.570755 - }, - "STREET": { - "tp": 0, - "fp": 224, - "fn": 0, - "support": 0, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "HOUSE": { - "tp": 0, - "fp": 0, - "fn": 0, - "support": 0, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - } - } - } - } -} diff --git a/evaluation/release_gates.json b/evaluation/release_gates.json deleted file mode 100644 index 705e6a6..0000000 --- a/evaluation/release_gates.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "2.0.0a1 legacy-reference regression gate", - "scope": "Extraction-only rows from the legacy Good worksheet; not an independently reviewed nationwide benchmark.", - "minimum_rows": 500, - "minimum_metrics": { - "exact_address_rate": 0.8, - "no_unparsed_rate": 0.75, - "micro.f1": 0.955, - "fields.postal_code.f1": 0.999, - "fields.region.f1": 0.9, - "fields.city.f1": 0.985, - "fields.street.f1": 0.955, - "fields.street_type.f1": 0.9, - "fields.house_num.f1": 0.95, - "fields.corpus.f1": 0.95, - "fields.structure.f1": 0.96, - "fields.apartment.f1": 0.85 - } -} diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index c22a19b..0000000 --- a/examples/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# Examples - -All examples assume v2 is installed from the repository checkout: - -```bash -python -m pip install . -``` - -The core package remains offline and dependency-free. Examples that add a web -framework or contact a customer system say so explicitly. - -## Single address - -```bash -python examples/basic.py -python examples/basic.py "г. Москва, ул. Тверская, д.4, кв.12" -``` - -The program prints the full JSON-compatible result followed by its review -decision. - -## Address span in a message - -```bash -python examples/detect_in_message.py -``` - -The detector returns ordered half-open spans into the original message and a -`ParsedAddress` for each span. It deliberately requires strong address evidence -and does not treat every place name or number as an address. - -## Streaming ETL - -Each input line is treated as one raw address. Each output line is one JSON -object, so memory use does not grow with the file: - -```bash -printf '%s\n' \ - 'Ополченская 5-30' \ - 'Самара Авроры 7 12' | - python examples/jsonl_etl.py > parsed.jsonl -``` - -Blank lines are retained as empty parse results. Add a filter in the calling -pipeline if blank lines should instead be rejected. - -## FastAPI wrapper - -FastAPI and Uvicorn are optional application dependencies, not package runtime -dependencies: - -```bash -python -m pip install fastapi uvicorn -uvicorn examples.fastapi_app:app --reload -``` - -Then: - -```bash -curl -sS http://127.0.0.1:8000/parse \ - -H 'content-type: application/json' \ - -d '{"address":"СПб Невский проспект 10 корп 2 кв 15"}' -``` - -Add the authentication, request-size limits, rate limits, observability, and -deployment controls required by your environment before exposing this service. - -## Customer-managed FIAS/GAR resolver - -First inspect the proposed generic request without making a network call: - -```bash -python examples/fias_gar_http.py "Ополченская 5-30" -``` - -To send it to an endpoint you control: - -```bash -python examples/fias_gar_http.py \ - "Ополченская 5-30" \ - --resolver-url https://resolver.internal.example/v1/candidates \ - --send -``` - -The example defines an illustrative HTTP contract; FIAS/GAR does not impose -that contract. Adapt field names, authentication, candidate ranking, and -registry freshness policy to your system. `--send` is explicit because the -parser itself must never hide a network call. diff --git a/examples/basic.py b/examples/basic.py deleted file mode 100644 index 1653ed6..0000000 --- a/examples/basic.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Parse one address and make review routing explicit.""" - -from __future__ import annotations - -import argparse -import json - -from address_normalizer import ParsedAddress, parse - - -def needs_review(result: ParsedAddress) -> bool: - """Use structural evidence, not an uncalibrated global score threshold.""" - - required_fields_missing = result.street is None or result.house_num is None - unresolved_evidence = bool( - result.warnings or result.alternatives or result.unparsed - ) - return required_fields_missing or unresolved_evidence - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("address", nargs="?", default="Ополченская 5-30") - args = parser.parse_args() - - result = parse(args.address) - print(json.dumps(result.as_dict(), ensure_ascii=False, indent=2)) - print(f"needs_review={str(needs_review(result)).lower()}") - - -if __name__ == "__main__": - main() diff --git a/examples/detect_in_message.py b/examples/detect_in_message.py deleted file mode 100644 index 8d0105a..0000000 --- a/examples/detect_in_message.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Detect address spans in a free-form message without a registry lookup.""" - -from __future__ import annotations - -import json - -from address_normalizer import detect_addresses - - -MESSAGE = ( - "Курьер приедет по адресу: Москва, ул. Тверская, " - "д. 13, кв. 4. Позвоните заранее." -) - - -def main() -> None: - for detected in detect_addresses(MESSAGE): - assert MESSAGE[detected.start : detected.end] == detected.text - print(json.dumps(detected.as_dict(), ensure_ascii=False, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/examples/fastapi_app.py b/examples/fastapi_app.py deleted file mode 100644 index a058370..0000000 --- a/examples/fastapi_app.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Optional FastAPI wrapper; FastAPI is not a package runtime dependency.""" - -from __future__ import annotations - -from address_normalizer import ParsedAddressDict, parse -from fastapi import FastAPI -from pydantic import BaseModel - - -class ParseRequest(BaseModel): - address: str - - -app = FastAPI(title="address-normalizer example", version="1") - - -@app.post("/parse") -def parse_address(request: ParseRequest) -> ParsedAddressDict: - """Extract fields; callers must still review or resolve the result.""" - - return parse(request.address).as_dict() diff --git a/examples/fias_gar_http.py b/examples/fias_gar_http.py deleted file mode 100644 index 3610fc7..0000000 --- a/examples/fias_gar_http.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Build or send a request to a customer-managed FIAS/GAR resolver. - -The package itself never performs this request. This illustrative contract must -be adapted to the resolver, authentication, and registry version you operate. -""" - -from __future__ import annotations - -import argparse -import json -from typing import Any -from urllib.request import Request, urlopen - -from address_normalizer import ParsedAddress, parse - - -FIELD_NAMES = ( - "postal_code", - "region", - "district", - "city", - "settlement", - "street", - "street_type", - "house_num", - "corpus", - "structure", - "apartment", -) - - -def resolver_payload(result: ParsedAddress) -> dict[str, Any]: - """Map selected values and ambiguity evidence to a generic resolver input.""" - - components = { - name: part.value - for name in FIELD_NAMES - if (part := getattr(result, name)) is not None - } - return { - "raw": result.raw, - "components": components, - "warnings": list(result.warnings), - "alternatives": [ - alternative.as_dict() for alternative in result.alternatives - ], - } - - -def send_json(url: str, payload: dict[str, Any]) -> Any: - """POST JSON using only the standard library.""" - - body = json.dumps(payload, ensure_ascii=False).encode("utf-8") - request = Request( - url, - data=body, - headers={"content-type": "application/json; charset=utf-8"}, - method="POST", - ) - with urlopen(request, timeout=10) as response: - return json.load(response) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("address") - parser.add_argument( - "--resolver-url", - default="http://127.0.0.1:8080/v1/candidates", - help="customer-managed endpoint; used only together with --send", - ) - parser.add_argument( - "--send", - action="store_true", - help="perform the network request; default is a local dry run", - ) - args = parser.parse_args() - - payload = resolver_payload(parse(args.address)) - print(json.dumps({"request": payload}, ensure_ascii=False, indent=2)) - - if args.send: - response = send_json(args.resolver_url, payload) - print(json.dumps({"response": response}, ensure_ascii=False, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/examples/jsonl_etl.py b/examples/jsonl_etl.py deleted file mode 100644 index 0d0a61c..0000000 --- a/examples/jsonl_etl.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Convert newline-delimited raw addresses to newline-delimited parse results.""" - -from __future__ import annotations - -import json -import sys - -from address_normalizer import parse_iter - - -def main() -> None: - addresses = (line.rstrip("\r\n") for line in sys.stdin) - for source_line, result in enumerate(parse_iter(addresses), start=1): - record = { - "source_line": source_line, - "address": result.raw, - "parsed_address": result.as_dict(), - } - print(json.dumps(record, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/parsing.py b/parsing.py deleted file mode 100644 index ee46772..0000000 --- a/parsing.py +++ /dev/null @@ -1,283 +0,0 @@ -import re -import itertools -import operator - -import pandas as pd -from elasticsearch import Elasticsearch - -def inverdic(dic): - resdic = {} - for key, value in dic.items(): - for index in value: - if type(value) == set or type(value) == list: - if index in resdic.keys(): - if isinstance(resdic[index], set): - resdic[index].add(key) - elif isinstance(resdic[index], list): - resdic[index].append(key) - else: - resdic[index] = [resdic[index]] - resdic[index].append(key) - else: - resdic[index] = key - elif type(value) == dict: - resdic.update(inverdic(value)) - return resdic - - -""" -Словарь, который используется для детекции номера дома, корпуса и т.д. -""" -sep_house_signs = { - 'дом': {'д', 'дом'}, - 'владение': {'владение', 'вл'}, - 'корпус': {'к', 'корп', 'копр', 'кор', 'корпус'}, - 'строение': {'с', 'стр', 'строен', 'строение'}, - 'квартира': {'кв', 'квартира'}, - 'помещение': {'пом', 'помещение'}, - 'комната': {"ком", 'комн', 'комната'}, - 'кабинет': {"кабинет", "каб", "к-т", "каб-т"}, - 'офис': {'оф', 'офис'}, - 'литера': set("абвежз"), - 'прочее': {'литер', 'литера', 'лит'}, - 'дробь': {'/', '-'} -} - -house_signs_inv = inverdic(sep_house_signs) - - -def boost_keyword(dic): - replaces_lowered = {} - for key, value in replaces.items(): - if isinstance(key, tuple): - new_key = "(" + ' OR '.join(key) + ')^' + str(1 / len(key)) - replaces_lowered[new_key] = value - else: - replaces_lowered[key] = value - return replaces_lowered - - -''' -Этот словарь приводит все типы адресных объектов к стандартному виду (к тому что в ФИАС) -''' -replaces = { - 'обл': {"область", "обл", "обл-ть"}, - 'респ': {"республика", 'респ'}, - 'край': {'край'}, - 'г': {'г', 'гор', 'город'}, - ('ао', 'а.окр'): {'автономный округ', "автономный", 'аокр', 'а.окр'}, - ('а.обл', 'аобл'): {'автономная область', 'авт.обл', 'аобл', 'а обл', 'аобл'}, - - ('аллея', 'ал'): {'аллея', 'а', 'ал'}, - 'б-р': {'б-р', 'бульвар'}, - 'наб': {'наб', 'набережная'}, - 'пер': {'пер', 'переулок'}, - ('площадь', 'пл'): {'пл', "площадь"}, - ('проспект', 'пр-кт'): {"проспект", "пр", "пр-кт", "просп", 'пр-т'}, - "пр-д": {"проезд", "пр-д", "прд"}, - "ул": {"улица", "ул", "у", 'ул-ца'}, - - 'р-н': {'район', "р", "р-н"}, - 'п': {'поселок', 'посёлок', "пос"}, - 'пгт': {'поселок городского типа', 'посёлок городского типа', 'пос. гор. типа', 'пос.гор.типа', 'пос гор типа'}, - - 'г': {'г', 'гор', 'город'}, - 'с': {'с', 'село', 'сел'}, - 'д': {'д', 'дер', 'деревня', 'д-ня'}, - 'с/п': {"сельский поселок", "сельский посёлок", "сп", 'сельское поселение', 'сельпо', 'сп', 'сел.п.', } -} -replaces_inv = inverdic(boost_keyword(replaces)) - - -def del_sp_char(string): - ''' - Стоплист. Удаляет из строки символы переноса строки, но словарь можно дополнить при надобности - ''' - for stopword in {r'\n', r'\r', '\\', '(', ')', ':'}: - string = string.replace(stopword, ' ') - return re.sub(r"[\d]+", ' \g<0> ', string) - - -def preprocess(string): - ''' - Отделяет всё что можно друг от друга чтобы облегчить токенизацию - ''' - string = del_sp_char(string) - # Превращает "2c3" в "2 c 3" и "2-3" в "2 - 3" - string = re.sub(r"[\d]+|[\W]+", ' \g<0> ', string) - - string = string.replace(',', ', ') # то же самое, только с запятыми - string = re.sub(r'\,|\.|\-|\'|\"|\(|\)', '', string) - string = re.sub(r' +', ' ', string) - return string - - -def multiple_replace(dict, text, compiled=False): - ''' - Преобразует словарь замен (dict) в паттерн замен для регулярок и тут же применяет его - ''' - if not compiled: - regex = re.compile(r"\b(%s)\b" % "|".join(map(re.escape, dict.keys()))) - else: - regex = compiled - - # For each match, look-up corresponding value in dictionary - return regex.sub(lambda mo: dict[mo.string[mo.start():mo.end()]], text) - - -def tokenize(string, comma=False): - ''' - Токенизатор. Раздвигает слипшиеся буквы и цифры вроде 2с3 или корп1 - Вход: строка - Выход: массив из слов (токены) - ''' - # string = re.sub(r"[\d]+", ' \g<0> ', string).lower() — это уже сделано при препроцессинге - string = string.lower() - if not comma: - return re.findall(r'[\d]+|[\w]+', string) - if comma: - return re.findall(r'[\d]+|[\w]+|\,', string) - - -def tokens_to_string(tokens, string): - ''' - Ищет токены в строке и возвращает их позицию начала - ''' - pattern = r".?.?".join(tokens) - found = re.search(pattern, string.lower()) - if found == None: - split = len(string) - else: - split = found.start() - return split - - -def extract_index(string, errors=False): # 100% works !!! - ''' - Извлекает индекс из строки - Вход: строка - Выход: адрес без индекса, индекс - ''' - index = re.findall(r'[^| |,][\d]{5}[ |$|, ]', string) - if len(index) > 1 and errors: - print("Два индекса в строке \"%s\" ?" % string) - - if index != []: - index = index[0] - string = string.replace(index, '').strip() - index = index.replace(',', '') - else: - index = None - return string, index - - -def clarify_address(tokens, types): - ''' - Разбивает строку с номером дома на ещё более точные части. - Возвращает номер дома, корпуса и строения - ''' - # add missing "number" type - for i, token in enumerate(tokens): - if token.isdigit(): - types[i] = 'число' - if i == 0: - types[i] = "дом" - elif types[i - 1] in sep_house_signs and types[i - 1] != 'литера' and types[i - 1] != 'дробь': - types[i] = types[i - 1] - elif i >= 2 and types[i - 1] == 'дробь': - types[i] = types[i - 2] - types[i - 1] = types[i - 2] - elif types[i] == 'литера' and i != 0: - tokens[i - 1] += tokens[i] - - # write this info somewhere - dic = {} - for token, typ in zip(tokens, types): - if token not in house_signs_inv and typ != 'препинания': - if typ not in dic: - dic[typ] = token - - # rename - dic['Дом'] = dic.get('дом', '') - if len(dic.get('корпус', '')) > len(dic.get('строение', '')): - dic['Корпус/строение'] = dic['корпус'] - elif dic.get('строение', False): - dic['Корпус/строение'] = dic['строение'] - return dic - - -def extract_house_tokens(tokens): - ''' - находит последовательность номеров дома/корпуса/строения среди токенов и возвращает их - ''' - a = lambda x: "число" if x.isdigit() and len(x) < 6 else "препинания" if x == ',' else "не распознано" - types = [house_signs_inv.get(x, a(x)) for x in tokens] - types_bin = [0 if x == 'не распознано' else 1 for x in types] - array = list((list(y) for (x, y) in itertools.groupby((enumerate(types_bin)), operator.itemgetter(1)) if x == 1)) - if len(array) == 0: - return [], [] - longest_seq = max(reversed(array), key=len) - return [tokens[i] for (i, _) in longest_seq], [types[i] for (i, _) in longest_seq] - - -def extract_house(string): # from 2.0 - ''' - Обёртка для процедуры извлечения номера дома/корпуса от оставшейся строки - Вход: адрес(строка) - Выход: адрес без номеров дома/корпуса, номера дома/корпуса строкой - ''' - tokens = tokenize(string, comma=True) - house_tokens, house_types = extract_house_tokens(tokens) - - split = tokens_to_string(house_tokens, string) - - house = clarify_address(house_tokens, house_types) - address = string[:split] - # house = string[split:] - return address, house - - -stopwords = { - 'российская': '', - 'федерация': '', - 'орел': 'орёл', - 'мо': 'московская обл', - 'большой': "(б OR большой)", - 'большая': "(б OR большая)", - 'малый': "(м OR малый)", - 'малая': "(м OR малая)", - 'средний': '(ср OR с OR средний)', - 'средняя': '(ср OR с OR средняя)', - 'нижний': '(н OR нижний)', - 'б': "(б OR большая OR большой)", - 'с': '(ср OR с OR средняя OR средний)', - 'ср': '(ср OR с OR средняя OR средний)', - 'м': "(м OR малый OR малая)", - 'н': '(н OR нижний)', - '/': '' -} - - -def optimize_for_search(string): - ''' - вводит небольшие изменения в строку поиска для более точного поиска - ''' - string = string.replace('ё', 'е') - string = multiple_replace(stopwords, string.lower()) - string = multiple_replace(replaces_inv, string) - # string = re.sub(r"[а-яА-Я]{4,}", '\g<0>~^2', string) - return string - - -housenum_replaces = { - '/': '\/' -} - - -def optimize_housenum(string): - ''' - Пока что делает escape для "/" - ''' - # creates escape characters for elasticsearch - string = multiple_replace(housenum_replaces, string) - return '"' + string + '"' diff --git a/pyproject.toml b/pyproject.toml index 91024dd..e935795 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,4 +64,4 @@ warn_unreachable = true [tool.pytest.ini_options] addopts = "-q" -testpaths = ["tests_v2"] +testpaths = ["tests"] diff --git a/ref/references.xlsx b/ref/references.xlsx deleted file mode 100644 index 07e5e2c..0000000 Binary files a/ref/references.xlsx and /dev/null differ diff --git a/release-policy.toml b/release-policy.toml deleted file mode 100644 index fc4b195..0000000 --- a/release-policy.toml +++ /dev/null @@ -1,16 +0,0 @@ -schema_version = 1 - -[publication] -# Maintainer decisions and their repository-local evidence. Publication requires -# both statuses to remain "approved". -license_status = "approved" -license_evidence = ["LICENSE", "LICENSING.md"] -model_provenance_status = "approved" -model_provenance_evidence = [ - "LICENSING.md", - "ref/references.xlsx", - "evaluation/legacy_reference_500.jsonl", - "training/README.md", - "training/model_evaluation.json", - "src/address_normalizer/data/model.json", -] diff --git a/requirements-evaluation.txt b/requirements-evaluation.txt deleted file mode 100644 index b2536a2..0000000 --- a/requirements-evaluation.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Runtime installation remains dependency-free. This extra is only needed to -# prepare the large external Parquet benchmark. -pyarrow==25.0.0 -pymongo==4.17.0 diff --git a/requirements-legacy.txt b/requirements-legacy.txt deleted file mode 100644 index 6b58039..0000000 --- a/requirements-legacy.txt +++ /dev/null @@ -1,4 +0,0 @@ -elasticsearch -pandas -simpledbf -tqdm diff --git a/scripts/build_reproducibly.py b/scripts/build_reproducibly.py deleted file mode 100644 index dc8dfc4..0000000 --- a/scripts/build_reproducibly.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -"""Build twice with a fixed epoch and keep only byte-identical artifacts.""" - -from __future__ import annotations - -import argparse -import copy -import gzip -from hashlib import sha256 -import os -from pathlib import Path -import subprocess -import sys -import tarfile -import tempfile - - -DEFAULT_SOURCE_DATE_EPOCH = "1704067200" # 2024-01-01T00:00:00Z - - -def canonicalize_sdist(path: Path, epoch: int) -> None: - """Remove ambient filesystem and gzip metadata from a setuptools sdist.""" - temporary = path.with_name(f".{path.name}.canonical") - try: - with tarfile.open(path, mode="r:gz") as source: - members = source.getmembers() - with temporary.open("wb") as raw_output: - with gzip.GzipFile( - filename="", - mode="wb", - fileobj=raw_output, - compresslevel=9, - mtime=epoch, - ) as compressed: - with tarfile.open( - fileobj=compressed, - mode="w", - format=tarfile.PAX_FORMAT, - ) as target: - for original in sorted(members, key=lambda member: member.name): - member = copy.copy(original) - member.uid = 0 - member.gid = 0 - member.uname = "" - member.gname = "" - member.mtime = epoch - member.mode = 0o755 if member.isdir() else 0o644 - member.pax_headers = {} - file_data = source.extractfile(original) if original.isfile() else None - target.addfile(member, file_data) - os.replace(temporary, path) - finally: - temporary.unlink(missing_ok=True) - - -def artifact_bytes(directory: Path) -> dict[str, bytes]: - artifacts = { - path.name: path.read_bytes() - for pattern in ("*.whl", "*.tar.gz") - for path in directory.glob(pattern) - } - if len([name for name in artifacts if name.endswith(".whl")]) != 1: - raise SystemExit("reproducible build failed: expected exactly one wheel") - if len([name for name in artifacts if name.endswith(".tar.gz")]) != 1: - raise SystemExit("reproducible build failed: expected exactly one sdist") - return artifacts - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--output", type=Path, default=Path("dist")) - parser.add_argument( - "--source-date-epoch", - default=os.environ.get("SOURCE_DATE_EPOCH", DEFAULT_SOURCE_DATE_EPOCH), - ) - args = parser.parse_args() - if not args.source_date_epoch.isdigit(): - raise SystemExit("reproducible build failed: SOURCE_DATE_EPOCH must be an integer") - if args.output.exists() and any(args.output.iterdir()): - raise SystemExit(f"reproducible build failed: output is not empty: {args.output}") - - environment = os.environ.copy() - environment["SOURCE_DATE_EPOCH"] = args.source_date_epoch - with tempfile.TemporaryDirectory(prefix="address-normalizer-build-a-") as first_dir: - with tempfile.TemporaryDirectory(prefix="address-normalizer-build-b-") as second_dir: - first = Path(first_dir) - second = Path(second_dir) - for output in (first, second): - subprocess.run( - [sys.executable, "-m", "build", "--outdir", str(output)], - check=True, - env=environment, - ) - sdist = next(output.glob("*.tar.gz")) - canonicalize_sdist(sdist, int(args.source_date_epoch)) - first_artifacts = artifact_bytes(first) - second_artifacts = artifact_bytes(second) - if first_artifacts != second_artifacts: - details = [] - for name in sorted(first_artifacts.keys() | second_artifacts.keys()): - first_hash = ( - sha256(first_artifacts[name]).hexdigest() - if name in first_artifacts - else "missing" - ) - second_hash = ( - sha256(second_artifacts[name]).hexdigest() - if name in second_artifacts - else "missing" - ) - details.append(f"{name}: first={first_hash}, second={second_hash}") - raise SystemExit( - "reproducible build failed: artifacts are not byte-identical; " - + "; ".join(details) - ) - - args.output.mkdir(parents=True, exist_ok=True) - for name, content in first_artifacts.items(): - target = args.output / name - target.write_bytes(content) - print(f"{name}: {len(content)} bytes sha256={sha256(content).hexdigest()}") - - print(f"reproducible build: OK (SOURCE_DATE_EPOCH={args.source_date_epoch})") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/check_artifacts.py b/scripts/check_artifacts.py deleted file mode 100644 index 375a803..0000000 --- a/scripts/check_artifacts.py +++ /dev/null @@ -1,434 +0,0 @@ -#!/usr/bin/env python3 -"""Inspect built distributions and write a small provenance manifest.""" - -from __future__ import annotations - -import argparse -import base64 -import csv -from email.parser import BytesParser -from hashlib import sha256 -import io -import json -import os -from pathlib import Path, PurePosixPath -import platform -import re -import subprocess -import tarfile -import tomllib -import zipfile - - -ROOT = Path(__file__).resolve().parents[1] -PACKAGE_ROOT = ROOT / "src" / "address_normalizer" -MODEL_PATH = PACKAGE_ROOT / "data" / "model.json" -WHEEL_MAX_BYTES = 256 * 1024 -SDIST_MAX_BYTES = 256 * 1024 -MODEL_MAX_BYTES = 64 * 1024 - -FORBIDDEN_PARTS = { - ".cache", - ".git", - ".github", - ".ipynb_checkpoints", - "__pycache__", - "evaluation", - "ref", - "tests", - "tests_v2", - "training", -} -FORBIDDEN_SUFFIXES = { - ".ipynb", - ".jsonl", - ".pem", - ".pyc", - ".pyo", - ".sqlite", - ".xlsx", -} - - -def fail(message: str) -> None: - raise SystemExit(f"artifact check failed: {message}") - - -def package_version() -> str: - init_text = (PACKAGE_ROOT / "__init__.py").read_text(encoding="utf-8") - match = re.search( - r'^__version__\s*=\s*["\'](?P[^"\']+)["\']\s*$', - init_text, - flags=re.MULTILINE, - ) - if match is None: - fail("address_normalizer.__version__ must be a string literal") - return match.group("version") - - -def expected_package_files() -> set[str]: - expected: set[str] = set() - for path in PACKAGE_ROOT.rglob("*"): - if not path.is_file() or "__pycache__" in path.parts: - continue - relative = path.relative_to(PACKAGE_ROOT).as_posix() - if path.suffix == ".py" or relative in {"py.typed", "data/model.json"}: - expected.add(f"address_normalizer/{relative}") - continue - fail(f"unexpected source-package file {relative}") - return expected - - -def project_metadata() -> dict[str, object]: - return tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))[ - "project" - ] - - -def configured_license_files() -> set[str]: - patterns = project_metadata().get("license-files", []) - if not isinstance(patterns, list): - fail("project.license-files must be a list when present") - return { - path.relative_to(ROOT).as_posix() - for pattern in patterns - if isinstance(pattern, str) - for path in ROOT.glob(pattern) - if path.is_file() - } - - -def expected_sdist_documentation() -> set[str]: - expected = { - "CHANGELOG.md", - "CONTRIBUTING.md", - "LICENSING.md", - "MANIFEST.in", - "README.md", - "README.ru.md", - "SECURITY.md", - "SUPPORT.md", - "pyproject.toml", - } - allowed_suffixes = {"docs": {".md"}, "examples": {".md", ".py"}} - for directory, suffixes in allowed_suffixes.items(): - for path in (ROOT / directory).rglob("*"): - if ( - not path.is_file() - or "__pycache__" in path.parts - or path.suffix in {".pyc", ".pyo"} - ): - continue - if path.suffix not in suffixes: - fail(f"unexpected {directory} artifact {path.relative_to(ROOT)}") - expected.add(path.relative_to(ROOT).as_posix()) - return expected - - -def safe_archive_name(name: str) -> PurePosixPath: - if "\\" in name: - fail(f"archive member uses a backslash: {name!r}") - path = PurePosixPath(name) - if path.is_absolute() or ".." in path.parts: - fail(f"unsafe archive member: {name!r}") - return path - - -def forbidden_member(name: str) -> bool: - path = safe_archive_name(name) - lower_parts = {part.lower() for part in path.parts} - return bool(lower_parts & FORBIDDEN_PARTS) or path.suffix.lower() in FORBIDDEN_SUFFIXES - - -def verify_record(archive: zipfile.ZipFile, record_name: str) -> None: - rows = list(csv.reader(io.StringIO(archive.read(record_name).decode("utf-8")))) - recorded = {row[0]: row[1:] for row in rows} - if set(recorded) != set(archive.namelist()): - fail("wheel RECORD does not enumerate every archive member exactly once") - - for name in archive.namelist(): - digest, size = recorded[name] - if name == record_name: - if digest or size: - fail("wheel RECORD must not hash itself") - continue - if not digest.startswith("sha256="): - fail(f"wheel RECORD lacks a SHA-256 digest for {name}") - expected_digest = digest.removeprefix("sha256=") - actual_digest = base64.urlsafe_b64encode( - sha256(archive.read(name)).digest() - ).rstrip(b"=").decode("ascii") - if expected_digest != actual_digest: - fail(f"wheel RECORD digest mismatch for {name}") - if size != str(len(archive.read(name))): - fail(f"wheel RECORD size mismatch for {name}") - - -def check_wheel(path: Path, expected_version: str) -> dict[str, object]: - if path.stat().st_size > WHEEL_MAX_BYTES: - fail(f"wheel is {path.stat().st_size} bytes; limit is {WHEEL_MAX_BYTES}") - - with zipfile.ZipFile(path) as archive: - names = archive.namelist() - if len(names) != len(set(names)): - fail("wheel contains duplicate member names") - for name in names: - safe_archive_name(name) - if forbidden_member(name): - fail(f"wheel contains forbidden member {name}") - - expected_files = expected_package_files() - package_files = {name for name in names if name.startswith("address_normalizer/")} - if package_files != expected_files: - missing = sorted(expected_files - package_files) - unexpected = sorted(package_files - expected_files) - fail(f"wheel package files differ; missing={missing}, unexpected={unexpected}") - - metadata_names = [name for name in names if name.endswith(".dist-info/METADATA")] - record_names = [name for name in names if name.endswith(".dist-info/RECORD")] - wheel_names = [name for name in names if name.endswith(".dist-info/WHEEL")] - if len(metadata_names) != 1 or len(record_names) != 1 or len(wheel_names) != 1: - fail("wheel must contain one METADATA, WHEEL, and RECORD file") - - metadata = BytesParser().parsebytes(archive.read(metadata_names[0])) - if metadata["Name"] != "address-normalizer": - fail(f"unexpected distribution name {metadata['Name']!r}") - if metadata["Version"] != expected_version: - fail( - f"metadata version {metadata['Version']!r} does not match " - f"source version {expected_version!r}" - ) - if metadata.get_all("Requires-Dist"): - fail(f"runtime dependencies found: {metadata.get_all('Requires-Dist')}") - if metadata["Requires-Python"] != ">=3.10": - fail(f"unexpected Requires-Python value {metadata['Requires-Python']!r}") - dist_info = metadata_names[0].rsplit("/", 1)[0] - expected_dist_info = f"address_normalizer-{expected_version}.dist-info" - if dist_info != expected_dist_info: - fail(f"unexpected dist-info directory {dist_info!r}") - - project = project_metadata() - expected_license = project.get("license") - actual_license = metadata["License-Expression"] - if expected_license != actual_license: - fail( - f"wheel license expression {actual_license!r} does not match " - f"pyproject value {expected_license!r}" - ) - license_files = configured_license_files() - metadata_license_files = set(metadata.get_all("License-File", [])) - if metadata_license_files != license_files: - fail( - f"wheel License-File metadata differs; expected={sorted(license_files)}, " - f"actual={sorted(metadata_license_files)}" - ) - - allowed_dist_info = { - f"{dist_info}/METADATA", - f"{dist_info}/RECORD", - f"{dist_info}/WHEEL", - f"{dist_info}/entry_points.txt", - f"{dist_info}/top_level.txt", - *(f"{dist_info}/licenses/{name}" for name in license_files), - } - allowed_names = expected_files | allowed_dist_info - if set(names) != allowed_names: - missing = sorted(allowed_names - set(names)) - unexpected = sorted(set(names) - allowed_names) - fail(f"wheel members differ; missing={missing}, unexpected={unexpected}") - - wheel_metadata = archive.read(wheel_names[0]).decode("utf-8") - if "Tag: py3-none-any" not in wheel_metadata: - fail("wheel is not tagged as platform-independent py3-none-any") - - model_bytes = archive.read("address_normalizer/data/model.json") - if model_bytes != MODEL_PATH.read_bytes(): - fail("wheel model differs from the source model") - if len(model_bytes) > MODEL_MAX_BYTES: - fail(f"model is {len(model_bytes)} bytes; limit is {MODEL_MAX_BYTES}") - try: - json.loads(model_bytes) - except (UnicodeDecodeError, json.JSONDecodeError) as error: - fail(f"bundled model is not valid UTF-8 JSON: {error}") - - verify_record(archive, record_names[0]) - - return { - "file": path.name, - "sha256": sha256(path.read_bytes()).hexdigest(), - "size": path.stat().st_size, - } - - -def check_sdist(path: Path, expected_version: str) -> dict[str, object]: - if path.stat().st_size > SDIST_MAX_BYTES: - fail(f"sdist is {path.stat().st_size} bytes; limit is {SDIST_MAX_BYTES}") - - expected_root = f"address_normalizer-{expected_version}" - with tarfile.open(path, mode="r:gz") as archive: - members = archive.getmembers() - names = [member.name for member in members] - if len(names) != len(set(names)): - fail("sdist contains duplicate member names") - for member in members: - member_path = safe_archive_name(member.name) - if not member_path.parts or member_path.parts[0] != expected_root: - fail(f"sdist member is outside {expected_root}: {member.name}") - if member.issym() or member.islnk() or member.isdev(): - fail(f"sdist contains a link or device: {member.name}") - if forbidden_member("/".join(member_path.parts[1:])): - fail(f"sdist contains forbidden member {member.name}") - - relative_files = { - "/".join(safe_archive_name(member.name).parts[1:]) - for member in members - if member.isfile() - } - expected_files = { - *expected_sdist_documentation(), - "PKG-INFO", - *(f"src/{name}" for name in expected_package_files()), - "setup.cfg", - "src/address_normalizer.egg-info/PKG-INFO", - "src/address_normalizer.egg-info/SOURCES.txt", - "src/address_normalizer.egg-info/dependency_links.txt", - "src/address_normalizer.egg-info/entry_points.txt", - "src/address_normalizer.egg-info/top_level.txt", - *configured_license_files(), - } - if relative_files != expected_files: - missing = sorted(expected_files - relative_files) - unexpected = sorted(relative_files - expected_files) - fail(f"sdist files differ; missing={missing}, unexpected={unexpected}") - - expected_directories = {""} - for name in expected_files: - parent = PurePosixPath(name).parent - while str(parent) != ".": - expected_directories.add(parent.as_posix()) - parent = parent.parent - relative_directories = { - "/".join(safe_archive_name(member.name).parts[1:]) - for member in members - if member.isdir() - } - if relative_directories != expected_directories: - missing = sorted(expected_directories - relative_directories) - unexpected = sorted(relative_directories - expected_directories) - fail(f"sdist directories differ; missing={missing}, unexpected={unexpected}") - - model_member = archive.extractfile( - f"{expected_root}/src/address_normalizer/data/model.json" - ) - if model_member is None or model_member.read() != MODEL_PATH.read_bytes(): - fail("sdist model differs from the source model") - - return { - "file": path.name, - "sha256": sha256(path.read_bytes()).hexdigest(), - "size": path.stat().st_size, - } - - -def git_details() -> tuple[str | None, bool | None]: - try: - revision = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - status = subprocess.run( - ["git", "status", "--porcelain"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ).stdout - except (FileNotFoundError, subprocess.CalledProcessError): - return None, None - return revision, bool(status) - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("dist", type=Path, help="directory containing one wheel and one sdist") - parser.add_argument( - "--expected-version", - help="fail unless built metadata matches this version as well as the source", - ) - parser.add_argument("--write-manifest", type=Path) - parser.add_argument( - "--verify-manifest", - type=Path, - help="verify artifacts and source provenance exactly match a prior manifest", - ) - args = parser.parse_args() - - wheels = sorted(args.dist.glob("*.whl")) - sdists = sorted(args.dist.glob("*.tar.gz")) - if len(wheels) != 1 or len(sdists) != 1: - fail( - f"expected exactly one wheel and one .tar.gz in {args.dist}; " - f"found {len(wheels)} wheel(s) and {len(sdists)} sdist(s)" - ) - - version = package_version() - if args.expected_version is not None and args.expected_version != version: - fail( - f"requested version {args.expected_version!r} does not match " - f"source version {version!r}" - ) - - pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) - if pyproject["project"].get("dependencies") != []: - fail("pyproject runtime dependencies must remain an explicit empty list") - if pyproject["build-system"]["requires"] != ["setuptools==80.9.0"]: - fail("build backend must stay exactly pinned for reproducible builds") - - artifacts = [ - check_wheel(wheels[0], version), - check_sdist(sdists[0], version), - ] - revision, dirty = git_details() - manifest = { - "schema_version": 1, - "distribution": "address-normalizer", - "version": version, - "source_revision": os.environ.get("GITHUB_SHA", revision), - "source_tree_dirty": dirty, - "build_python": platform.python_version(), - "build_backend": "setuptools==80.9.0", - "runtime_dependencies": [], - "model": { - "file": "src/address_normalizer/data/model.json", - "sha256": sha256(MODEL_PATH.read_bytes()).hexdigest(), - "size": MODEL_PATH.stat().st_size, - }, - "artifacts": artifacts, - } - if args.verify_manifest: - expected_manifest = json.loads(args.verify_manifest.read_text(encoding="utf-8")) - if expected_manifest != manifest: - fail( - f"current artifact provenance does not match {args.verify_manifest}; " - f"expected={json.dumps(expected_manifest, sort_keys=True)}, " - f"actual={json.dumps(manifest, sort_keys=True)}" - ) - if args.write_manifest: - args.write_manifest.parent.mkdir(parents=True, exist_ok=True) - args.write_manifest.write_text( - json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - - for artifact in artifacts: - print(f"{artifact['file']}: {artifact['size']} bytes sha256={artifact['sha256']}") - print(f"model.json: {MODEL_PATH.stat().st_size} bytes") - print("artifact policy: OK") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/check_license.py b/scripts/check_license.py deleted file mode 100644 index d0a7e20..0000000 --- a/scripts/check_license.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -"""Enforce the recorded license and model-provenance release state.""" - -from __future__ import annotations - -import argparse -from pathlib import Path -import tomllib - - -ROOT = Path(__file__).resolve().parents[1] -LICENSE_NAMES = ("LICENSE", "LICENSE.txt", "LICENSE.md", "COPYING", "COPYING.txt") -BLOCKER_TEXT = "No license currently applies to this repository" -POLICY_PATH = ROOT / "release-policy.toml" - - -def fail(message: str) -> None: - raise SystemExit(f"license gate failed: {message}") - - -def main() -> int: - parser = argparse.ArgumentParser() - mode = parser.add_mutually_exclusive_group(required=True) - mode.add_argument("--expect-blocked", action="store_true") - mode.add_argument("--require-publishable", action="store_true") - args = parser.parse_args() - - project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))[ - "project" - ] - policy = tomllib.loads(POLICY_PATH.read_text(encoding="utf-8")) - if policy.get("schema_version") != 1: - fail("release-policy.toml has an unsupported schema") - publication = policy.get("publication") - if not isinstance(publication, dict): - fail("release-policy.toml lacks a [publication] table") - license_status = publication.get("license_status") - model_status = publication.get("model_provenance_status") - license_expression = project.get("license") - license_patterns = project.get("license-files") - license_files = [ROOT / name for name in LICENSE_NAMES if (ROOT / name).is_file()] - blocker = ROOT / "LICENSING.md" - blocker_is_current = blocker.is_file() and BLOCKER_TEXT in blocker.read_text( - encoding="utf-8" - ) - - if args.expect_blocked: - if license_status != "blocked" or model_status != "blocked": - fail("both release-policy.toml publication statuses must remain blocked") - if license_expression or license_patterns or license_files: - fail("license metadata or a license file appeared; update the release policy deliberately") - if not blocker_is_current: - fail("LICENSING.md no longer records the known publication blocker") - print( - "publication status: BLOCKED by license and compact-model provenance; " - "package publication must remain disabled" - ) - return 0 - - if license_status != "approved": - fail("release-policy.toml license_status is not approved") - if model_status != "approved": - fail("release-policy.toml model_provenance_status is not approved") - for key in ("license_evidence", "model_provenance_evidence"): - evidence = publication.get(key) - if not isinstance(evidence, list) or not evidence: - fail(f"release-policy.toml {key} must list recorded evidence") - for item in evidence: - if not isinstance(item, str) or not (ROOT / item).is_file(): - fail(f"release-policy.toml {key} references a missing file: {item!r}") - if not isinstance(license_expression, str) or not license_expression.strip(): - fail("project.license must contain the maintainer-approved SPDX expression") - if not isinstance(license_patterns, list) or not license_patterns: - fail("project.license-files must identify the approved license file") - matched_files = [ - path - for pattern in license_patterns - for path in ROOT.glob(pattern) - if path.is_file() - ] - if not matched_files: - fail("project.license-files does not match a repository file") - if blocker_is_current: - fail("LICENSING.md still says that no license applies") - print(f"license status: publishable ({license_expression})") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/check_release_ref.py b/scripts/check_release_ref.py deleted file mode 100644 index 9b3a76e..0000000 --- a/scripts/check_release_ref.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -"""Require a release workflow to run from the immutable matching version tag.""" - -from __future__ import annotations - -import argparse -import os -from pathlib import Path -import re -import subprocess - - -ROOT = Path(__file__).resolve().parents[1] -RELEASE_VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:(?:a|b|rc)[0-9]+)?$") - - -def fail(message: str) -> None: - raise SystemExit(f"release ref check failed: {message}") - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--version", required=True) - args = parser.parse_args() - - if RELEASE_VERSION.fullmatch(args.version) is None: - fail(f"{args.version!r} is not an allowed alpha/beta/rc/stable version") - expected_tag = f"v{args.version}" - ref_type = os.environ.get("GITHUB_REF_TYPE") - ref_name = os.environ.get("GITHUB_REF_NAME") - revision = os.environ.get("GITHUB_SHA") - if ref_type != "tag" or ref_name != expected_tag: - fail( - f"workflow must be dispatched from tag {expected_tag!r}; " - f"received ref_type={ref_type!r}, ref_name={ref_name!r}" - ) - if not revision: - fail("GITHUB_SHA is missing") - - tagged_commit = subprocess.run( - ["git", "rev-parse", f"refs/tags/{expected_tag}^{{commit}}"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - checked_out_commit = subprocess.run( - ["git", "rev-parse", "HEAD^{commit}"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - if tagged_commit != revision or checked_out_commit != revision: - fail( - f"tag, GITHUB_SHA, and checkout differ: " - f"tag={tagged_commit}, github={revision}, checkout={checked_out_commit}" - ) - print(f"release ref: OK ({expected_tag} -> {revision})") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/smoke_installed.py b/scripts/smoke_installed.py deleted file mode 100644 index 9870def..0000000 --- a/scripts/smoke_installed.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -"""Smoke-test an installed wheel without source-tree or network access.""" - -from __future__ import annotations - -from contextlib import redirect_stdout -import importlib.metadata -import io -import json -from pathlib import Path -import sys - - -def deny_network(event: str, _args: tuple[object, ...]) -> None: - if event.startswith(("socket.", "urllib.", "http.client.")): - raise RuntimeError(f"unexpected network operation: {event}") - - -def main() -> int: - sys.addaudithook(deny_network) - - import address_normalizer - from address_normalizer import parse, parse_many - from address_normalizer.cli import main as cli_main - - package_path = Path(address_normalizer.__file__).resolve() - if "site-packages" not in package_path.parts: - raise AssertionError(f"not importing an installed wheel: {package_path}") - if importlib.metadata.requires("address-normalizer"): - raise AssertionError("the installed distribution has runtime dependencies") - if importlib.metadata.version("address-normalizer") != address_normalizer.__version__: - raise AssertionError("runtime and distribution versions differ") - - result = parse("г. Москва, ул. Тверская, д.4, кв.12") - assert result.house_num is not None and result.house_num.value == "4" - assert len(parse_many(["Ополченская 5-30", "Невский проспект 10"])) == 2 - - single_output = io.StringIO() - with redirect_stdout(single_output): - assert cli_main(["Москва", "Тверская", "1"]) == 0 - json.loads(single_output.getvalue()) - - old_stdin = sys.stdin - batch_output = io.StringIO() - try: - sys.stdin = io.StringIO("Ополченская 5-30\nНевский проспект 10\n") - with redirect_stdout(batch_output): - assert cli_main(["--jsonl"]) == 0 - finally: - sys.stdin = old_stdin - lines = batch_output.getvalue().splitlines() - assert len(lines) == 2 - for line in lines: - json.loads(line) - - print(f"installed smoke: OK ({package_path})") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/test_artifact_policy.py b/scripts/test_artifact_policy.py deleted file mode 100644 index b1c7cfa..0000000 --- a/scripts/test_artifact_policy.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -"""Focused negative tests for distribution allowlists.""" - -from __future__ import annotations - -import argparse -from pathlib import Path -import shutil -import tarfile -import tempfile -import zipfile - -import check_artifacts - - -def expect_failure(action: object, expected_text: str) -> None: - try: - action() # type: ignore[operator] - except SystemExit as error: - if expected_text not in str(error): - raise AssertionError(f"unexpected checker failure: {error}") from error - else: - raise AssertionError("malicious archive unexpectedly passed") - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("dist", type=Path) - args = parser.parse_args() - wheel = next(args.dist.glob("*.whl")) - sdist = next(args.dist.glob("*.tar.gz")) - version = check_artifacts.package_version() - - with tempfile.TemporaryDirectory(prefix="artifact-policy-test-") as temp_dir: - temp = Path(temp_dir) - bad_wheel = temp / wheel.name - shutil.copyfile(wheel, bad_wheel) - with zipfile.ZipFile(bad_wheel, mode="a") as archive: - archive.writestr("payload.sh", "#!/bin/sh\n") - expect_failure( - lambda: check_artifacts.check_wheel(bad_wheel, version), - "wheel members differ", - ) - - bad_sdist = temp / sdist.name - expected_root = f"address_normalizer-{version}" - with tarfile.open(sdist, mode="r:gz") as source: - with tarfile.open(bad_sdist, mode="w:gz") as target: - for member in source.getmembers(): - target.addfile(member, source.extractfile(member) if member.isfile() else None) - payload = b"raise RuntimeError('unexpected source payload')\n" - member = tarfile.TarInfo(f"{expected_root}/src/evil.py") - member.size = len(payload) - import io - - target.addfile(member, io.BytesIO(payload)) - expect_failure( - lambda: check_artifacts.check_sdist(bad_sdist, version), - "sdist files differ", - ) - - print("artifact policy negative tests: OK") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/address_normalizer/data/model.json b/src/address_normalizer/data/model.json index beb575e..d5e4592 100644 --- a/src/address_normalizer/data/model.json +++ b/src/address_normalizer/data/model.json @@ -1 +1 @@ -{"format":"address-normalizer-compact-sequence-v1","training":{"algorithm":"epoch-averaged structured perceptron","seed":2017,"epochs":10,"examples":359,"source":"source-verifiable legacy reference rows with deterministic marker-free views","dataset":"evaluation/legacy_reference_500.jsonl","dataset_sha256":"853916e36cfc5a52d06add0524ba64ca2c771073c1c4e98cd5645116e7f74588","split":"SHA-256 by canonical address group: 70/15/15"},"labels":["O","REGION","DISTRICT","CITY","SETTLEMENT","STREET"],"emissions":{"CITY\tbias":0.7,"CITY\tbos":1.1,"CITY\teos":0.9,"CITY\tkind=word":0.7,"CITY\tnext=б":-1.8,"CITY\tnext=большой":1.0,"CITY\tnext=буденного":1.0,"CITY\tnext=вал":-0.9,"CITY\tnext=видное":-1.0,"CITY\tnext=вокзальная":-1.0,"CITY\tnext=володарского":0.9,"CITY\tnext=воронина":1.0,"CITY\tnext=г":-3.0,"CITY\tnext=гагарина":1.9,"CITY\tnext=гоголя":1.0,"CITY\tnext=донского":-1.0,"CITY\tnext=й":-1.0,"CITY\tnext=калинина":-1.0,"CITY\tnext=каширское":-2.0,"CITY\tnext=кирова":1.0,"CITY\tnext=колпакова":2.0,"CITY\tnext=ленинградская":-1.5,"CITY\tnext=ленинский":-0.5,"CITY\tnext=ленинского":2.0,"CITY\tnext=можайское":1.0,"CITY\tnext=москва":1.2,"CITY\tnext=новгород":3.5,"CITY\tnext=октябрьский":1.0,"CITY\tnext=он":-1.0,"CITY\tnext=ополчения":-1.0,"CITY\tnext=переулок":-1.0,"CITY\tnext=петербург":2.9,"CITY\tnext=пр":-2.8,"CITY\tnext=правды":0.9,"CITY\tnext=сампсониевский":-1.0,"CITY\tnext=северный":2.8,"CITY\tnext=т":-1.0,"CITY\tnext=ш":-0.9,"CITY\tnext=этаж":-1.0,"CITY\tnext=я":-1.9,"CITY\tnext=ямского":1.0,"CITY\tnext_kind=word":-0.2,"CITY\tposition=first":1.1,"CITY\tposition=last":-1.2,"CITY\tposition=middle":0.8,"CITY\tprefix1=а":-1.0,"CITY\tprefix1=б":-0.9,"CITY\tprefix1=в":1.7,"CITY\tprefix1=г":-1.0,"CITY\tprefix1=д":-0.2,"CITY\tprefix1=е":2.5,"CITY\tprefix1=з":-1.0,"CITY\tprefix1=и":-1.0,"CITY\tprefix1=к":0.1,"CITY\tprefix1=л":-1.0,"CITY\tprefix1=м":4.2,"CITY\tprefix1=н":1.4,"CITY\tprefix1=о":-1.0,"CITY\tprefix1=п":-0.9,"CITY\tprefix1=р":-0.4,"CITY\tprefix1=с":1.1,"CITY\tprefix1=т":2.0,"CITY\tprefix1=у":-1.0,"CITY\tprefix1=ш":-1.0,"CITY\tprefix1=э":-0.9,"CITY\tprefix1=я":-1.0,"CITY\tprefix2=а":-1.0,"CITY\tprefix2=б":-1.0,"CITY\tprefix2=ба":2.0,"CITY\tprefix2=бо":-1.0,"CITY\tprefix2=бр":-0.9,"CITY\tprefix2=ва":-1.0,"CITY\tprefix2=ви":3.6,"CITY\tprefix2=во":-0.9,"CITY\tprefix2=г":-1.0,"CITY\tprefix2=д":-1.0,"CITY\tprefix2=дм":-1.0,"CITY\tprefix2=до":1.8,"CITY\tprefix2=ер":2.5,"CITY\tprefix2=зе":-1.0,"CITY\tprefix2=ис":-1.0,"CITY\tprefix2=ка":1.9,"CITY\tprefix2=ки":1.0,"CITY\tprefix2=ко":-1.8,"CITY\tprefix2=ку":-1.0,"CITY\tprefix2=ле":-1.0,"CITY\tprefix2=ми":-0.6,"CITY\tprefix2=мо":2.8,"CITY\tprefix2=мы":2.0,"CITY\tprefix2=на":-2.6,"CITY\tprefix2=ни":3.5,"CITY\tprefix2=но":0.5,"CITY\tprefix2=оч":-1.0,"CITY\tprefix2=п":-1.0,"CITY\tprefix2=пе":2.1,"CITY\tprefix2=по":1.0,"CITY\tprefix2=пр":-3.0,"CITY\tprefix2=р":-1.0,"CITY\tprefix2=ре":1.6,"CITY\tprefix2=ро":-1.0,"CITY\tprefix2=с":-0.9,"CITY\tprefix2=са":1.9,"CITY\tprefix2=см":1.0,"CITY\tprefix2=со":-0.9,"CITY\tprefix2=то":2.0,"CITY\tprefix2=ул":-1.0,"CITY\tprefix2=ш":-1.0,"CITY\tprefix2=эн":-0.9,"CITY\tprefix2=яр":-1.0,"CITY\tprev=боровский":-0.1,"CITY\tprev=г":0.7,"CITY\tprev=зеленый":-1.0,"CITY\tprev=калужская":1.0,"CITY\tprev=кировская":0.9,"CITY\tprev=ленинский":1.0,"CITY\tprev=марьиной":-1.0,"CITY\tprev=москва":-10.5,"CITY\tprev=московская":2.8,"CITY\tprev=нижегородская":1.9,"CITY\tprev=нижний":3.0,"CITY\tprev=ордынка":-1.0,"CITY\tprev=санкт":3.8,"CITY\tprev=смоленск":-1.0,"CITY\tprev=ямского":-0.9,"CITY\tprev_kind=word":-0.4,"CITY\tsuffix1=а":2.6,"CITY\tsuffix1=б":-1.0,"CITY\tsuffix1=в":2.6,"CITY\tsuffix1=г":2.8,"CITY\tsuffix1=д":2.0,"CITY\tsuffix1=е":-0.1,"CITY\tsuffix1=и":3.0,"CITY\tsuffix1=й":-2.4,"CITY\tsuffix1=к":2.2,"CITY\tsuffix1=л":-1.0,"CITY\tsuffix1=м":-1.8,"CITY\tsuffix1=о":0.2,"CITY\tsuffix1=п":-1.0,"CITY\tsuffix1=р":-2.0,"CITY\tsuffix1=с":-0.9,"CITY\tsuffix1=т":1.9,"CITY\tsuffix1=ш":-1.0,"CITY\tsuffix1=ы":-1.0,"CITY\tsuffix1=я":-4.4,"CITY\tsuffix2=а":-1.0,"CITY\tsuffix2=ая":-2.5,"CITY\tsuffix2=б":-1.0,"CITY\tsuffix2=ва":1.3,"CITY\tsuffix2=во":0.3,"CITY\tsuffix2=г":-1.0,"CITY\tsuffix2=га":2.8,"CITY\tsuffix2=го":-1.0,"CITY\tsuffix2=д":-1.0,"CITY\tsuffix2=ий":0.6,"CITY\tsuffix2=ия":-1.0,"CITY\tsuffix2=ка":-1.0,"CITY\tsuffix2=кт":1.9,"CITY\tsuffix2=ля":-0.9,"CITY\tsuffix2=на":-0.9,"CITY\tsuffix2=но":0.9,"CITY\tsuffix2=ов":2.6,"CITY\tsuffix2=од":3.0,"CITY\tsuffix2=ое":-0.1,"CITY\tsuffix2=ой":-1.0,"CITY\tsuffix2=ок":-1.7,"CITY\tsuffix2=ом":-1.8,"CITY\tsuffix2=п":-1.0,"CITY\tsuffix2=пр":-1.0,"CITY\tsuffix2=р":-1.0,"CITY\tsuffix2=ра":-0.6,"CITY\tsuffix2=рг":3.8,"CITY\tsuffix2=ры":-1.0,"CITY\tsuffix2=с":-0.9,"CITY\tsuffix2=ск":3.9,"CITY\tsuffix2=ти":2.0,"CITY\tsuffix2=ул":-1.0,"CITY\tsuffix2=ха":2.0,"CITY\tsuffix2=ш":-1.0,"CITY\tsuffix2=щи":1.0,"CITY\tsuffix2=ый":-2.0,"CITY\tsuffix3=а":-1.0,"CITY\tsuffix3=б":-1.0,"CITY\tsuffix3=г":-1.0,"CITY\tsuffix3=д":-1.0,"CITY\tsuffix3=ект":-1.0,"CITY\tsuffix3=ина":-0.9,"CITY\tsuffix3=ино":0.9,"CITY\tsuffix3=ира":-0.6,"CITY\tsuffix3=иха":2.0,"CITY\tsuffix3=ищи":2.0,"CITY\tsuffix3=кая":-2.5,"CITY\tsuffix3=ква":4.1,"CITY\tsuffix3=кий":-2.9,"CITY\tsuffix3=кое":-3.7,"CITY\tsuffix3=ком":-1.8,"CITY\tsuffix3=кры":-1.0,"CITY\tsuffix3=лок":-1.7,"CITY\tsuffix3=ний":3.5,"CITY\tsuffix3=нка":-1.0,"CITY\tsuffix3=нкт":2.9,"CITY\tsuffix3=ное":3.6,"CITY\tsuffix3=нск":1.0,"CITY\tsuffix3=ный":-2.0,"CITY\tsuffix3=ова":-2.8,"CITY\tsuffix3=ово":0.3,"CITY\tsuffix3=ого":-1.0,"CITY\tsuffix3=оля":-0.9,"CITY\tsuffix3=ощи":-1.0,"CITY\tsuffix3=п":-1.0,"CITY\tsuffix3=пр":-1.0,"CITY\tsuffix3=р":-1.0,"CITY\tsuffix3=рия":-1.0,"CITY\tsuffix3=ров":2.8,"CITY\tsuffix3=род":3.0,"CITY\tsuffix3=рск":1.0,"CITY\tsuffix3=с":-0.9,"CITY\tsuffix3=сов":-0.9,"CITY\tsuffix3=тов":0.7,"CITY\tsuffix3=тти":2.0,"CITY\tsuffix3=уга":2.8,"CITY\tsuffix3=ул":-1.0,"CITY\tsuffix3=ург":3.8,"CITY\tsuffix3=ш":-1.0,"CITY\tsuffix3=шой":-1.0,"CITY\tsuffix3=ьск":1.9,"CITY\tword=а":-1.0,"CITY\tword=б":-1.0,"CITY\tword=балашиха":2.0,"CITY\tword=большой":-1.0,"CITY\tword=брюсов":-0.9,"CITY\tword=вавилова":-1.0,"CITY\tword=видное":3.6,"CITY\tword=воронина":-0.9,"CITY\tword=г":-1.0,"CITY\tword=д":-1.0,"CITY\tword=дмитрия":-1.0,"CITY\tword=домодедово":1.8,"CITY\tword=ермолино":2.5,"CITY\tword=зеленый":-1.0,"CITY\tword=искры":-1.0,"CITY\tword=калуга":2.8,"CITY\tword=каширское":-0.9,"CITY\tword=киров":2.8,"CITY\tword=кирова":-1.8,"CITY\tword=ком":-1.8,"CITY\tword=кутузовский":-1.0,"CITY\tword=ленинский":-1.0,"CITY\tword=мира":-0.6,"CITY\tword=можайское":-0.8,"CITY\tword=москва":4.1,"CITY\tword=московская":-0.5,"CITY\tword=мытищи":2.0,"CITY\tword=народного":-1.0,"CITY\tword=нахабино":-1.6,"CITY\tword=нижний":3.5,"CITY\tword=новгород":3.0,"CITY\tword=ново":-1.5,"CITY\tword=новоивановское":-1.0,"CITY\tword=очаковское":-1.0,"CITY\tword=п":-1.0,"CITY\tword=переулок":-1.7,"CITY\tword=петербург":3.8,"CITY\tword=подольск":1.9,"CITY\tword=поля":-0.9,"CITY\tword=пр":-1.0,"CITY\tword=пречистенка":-1.0,"CITY\tword=проспект":-1.0,"CITY\tword=р":-1.0,"CITY\tword=реутов":1.6,"CITY\tword=рощи":-1.0,"CITY\tword=с":-0.9,"CITY\tword=садовническая":-1.0,"CITY\tword=санкт":2.9,"CITY\tword=смоленск":1.0,"CITY\tword=сокольнический":-0.9,"CITY\tword=тольятти":2.0,"CITY\tword=ул":-1.0,"CITY\tword=ш":-1.0,"CITY\tword=электрогорск":1.0,"CITY\tword=электродный":-1.0,"CITY\tword=энтузиастов":-0.9,"CITY\tword=ярославская":-1.0,"DISTRICT\tbias":-0.8,"DISTRICT\tbos":-0.6,"DISTRICT\teos":-1.0,"DISTRICT\tkind=word":-0.8,"DISTRICT\tnext=большой":-1.0,"DISTRICT\tnext=видное":1.0,"DISTRICT\tnext=ермолино":0.9,"DISTRICT\tnext=колпакова":-1.0,"DISTRICT\tnext=комсомола":-1.0,"DISTRICT\tnext=нахабино":1.0,"DISTRICT\tnext=новгород":-0.9,"DISTRICT\tnext=новоивановское":1.6,"DISTRICT\tnext=пр":-0.6,"DISTRICT\tnext=р":1.0,"DISTRICT\tnext=северный":-0.8,"DISTRICT\tnext_kind=word":0.2,"DISTRICT\tposition=first":-0.6,"DISTRICT\tposition=last":-1.0,"DISTRICT\tposition=middle":0.8,"DISTRICT\tprefix1=б":1.9,"DISTRICT\tprefix1=д":-0.8,"DISTRICT\tprefix1=к":1.0,"DISTRICT\tprefix1=л":-0.6,"DISTRICT\tprefix1=м":-1.0,"DISTRICT\tprefix1=н":-0.9,"DISTRICT\tprefix1=о":1.6,"DISTRICT\tprefix1=п":-1.0,"DISTRICT\tprefix1=с":-1.0,"DISTRICT\tprefix2=бо":1.9,"DISTRICT\tprefix2=до":-0.8,"DISTRICT\tprefix2=кр":1.0,"DISTRICT\tprefix2=ле":-0.6,"DISTRICT\tprefix2=мы":-1.0,"DISTRICT\tprefix2=ни":-0.9,"DISTRICT\tprefix2=од":1.6,"DISTRICT\tprefix2=пе":-1.0,"DISTRICT\tprefix2=са":-1.0,"DISTRICT\tprev=большой":-1.0,"DISTRICT\tprev=калужская":0.9,"DISTRICT\tprev=московская":1.8,"DISTRICT\tprev=нижегородская":-0.9,"DISTRICT\tprev=санкт":-1.0,"DISTRICT\tprev_kind=word":-0.2,"DISTRICT\tsuffix1=г":-1.0,"DISTRICT\tsuffix1=и":-1.0,"DISTRICT\tsuffix1=й":3.0,"DISTRICT\tsuffix1=о":-1.8,"DISTRICT\tsuffix2=во":-0.8,"DISTRICT\tsuffix2=го":-1.0,"DISTRICT\tsuffix2=ий":3.0,"DISTRICT\tsuffix2=рг":-1.0,"DISTRICT\tsuffix2=щи":-1.0,"DISTRICT\tsuffix3=ищи":-1.0,"DISTRICT\tsuffix3=кий":3.9,"DISTRICT\tsuffix3=ний":-0.9,"DISTRICT\tsuffix3=ово":-0.8,"DISTRICT\tsuffix3=ого":-1.0,"DISTRICT\tsuffix3=ург":-1.0,"DISTRICT\tword=боровский":1.9,"DISTRICT\tword=домодедово":-0.8,"DISTRICT\tword=красногорский":1.0,"DISTRICT\tword=ленинградский":-0.6,"DISTRICT\tword=ленинский":1.0,"DISTRICT\tword=ленинского":-1.0,"DISTRICT\tword=мытищи":-1.0,"DISTRICT\tword=нижний":-0.9,"DISTRICT\tword=одинцовский":1.6,"DISTRICT\tword=петербург":-1.0,"DISTRICT\tword=сампсониевский":-1.0,"O\tbias":1.6,"O\tbos":0.1,"O\teos":0.9,"O\tkind=word":1.6,"O\tnext=б":1.4,"O\tnext=балашиха":-0.2,"O\tnext=г":3.0,"O\tnext=гагарина":-0.9,"O\tnext=гоголя":-1.0,"O\tnext=домодедово":-0.5,"O\tnext=й":-1.0,"O\tnext=каширское":2.0,"O\tnext=киров":-0.7,"O\tnext=комсомола":-1.0,"O\tnext=ленинского":-1.0,"O\tnext=марьиной":1.0,"O\tnext=москва":4.3,"O\tnext=мытищи":-0.6,"O\tnext=нижний":-1.0,"O\tnext=новгород":-1.0,"O\tnext=он":1.0,"O\tnext=переулок":1.0,"O\tnext=петербург":-0.9,"O\tnext=подольск":-0.3,"O\tnext=пр":0.9,"O\tnext=правды":-0.9,"O\tnext=реутов":-0.4,"O\tnext=рощи":-1.0,"O\tnext=с":-2.4,"O\tnext=т":1.0,"O\tnext=электрогорск":-0.1,"O\tnext=этаж":1.0,"O\tnext=ямского":-1.0,"O\tnext_kind=word":0.7,"O\tposition=first":0.1,"O\tposition=last":1.1,"O\tposition=middle":0.4,"O\tprefix1=а":2.0,"O\tprefix1=б":0.2,"O\tprefix1=в":-1.7,"O\tprefix1=г":1.1,"O\tprefix1=д":1.0,"O\tprefix1=е":-0.9,"O\tprefix1=з":-0.8,"O\tprefix1=и":-1.0,"O\tprefix1=й":-1.0,"O\tprefix1=к":-1.1,"O\tprefix1=л":-1.0,"O\tprefix1=м":-0.6,"O\tprefix1=н":-4.0,"O\tprefix1=о":-2.0,"O\tprefix1=п":3.8,"O\tprefix1=р":0.7,"O\tprefix1=т":-1.9,"O\tprefix1=у":3.0,"O\tprefix1=ш":3.9,"O\tprefix1=э":1.0,"O\tprefix1=я":0.9,"O\tprefix2=а":2.7,"O\tprefix2=ав":-0.7,"O\tprefix2=бр":-1.0,"O\tprefix2=бу":1.2,"O\tprefix2=ва":-1.6,"O\tprefix2=ви":-1.0,"O\tprefix2=вл":0.9,"O\tprefix2=г":3.0,"O\tprefix2=га":-0.9,"O\tprefix2=гр":-1.0,"O\tprefix2=д":1.0,"O\tprefix2=ер":-0.9,"O\tprefix2=зе":-0.8,"O\tprefix2=ис":-1.0,"O\tprefix2=й":-1.0,"O\tprefix2=к":1.0,"O\tprefix2=ка":-2.8,"O\tprefix2=ки":-0.7,"O\tprefix2=ко":1.4,"O\tprefix2=ле":-1.0,"O\tprefix2=ма":-1.0,"O\tprefix2=ми":-1.0,"O\tprefix2=мо":1.4,"O\tprefix2=ни":-2.0,"O\tprefix2=но":-2.0,"O\tprefix2=оз":-1.0,"O\tprefix2=ок":-1.0,"O\tprefix2=он":1.0,"O\tprefix2=ор":-1.0,"O\tprefix2=п":1.9,"O\tprefix2=пе":0.7,"O\tprefix2=пл":1.0,"O\tprefix2=по":-1.0,"O\tprefix2=пр":1.2,"O\tprefix2=р":1.7,"O\tprefix2=ро":-1.0,"O\tprefix2=с":1.9,"O\tprefix2=са":-0.9,"O\tprefix2=см":-1.0,"O\tprefix2=т":1.0,"O\tprefix2=та":-0.9,"O\tprefix2=то":-2.0,"O\tprefix2=ул":3.0,"O\tprefix2=ш":1.9,"O\tprefix2=шо":2.0,"O\tprefix2=эн":-1.0,"O\tprefix2=эт":2.0,"O\tprefix2=я":0.9,"O\tprev=а":2.7,"O\tprev=боровский":0.1,"O\tprev=бульвар":0.9,"O\tprev=г":2.3,"O\tprev=ермолино":-0.9,"O\tprev=зеленый":1.0,"O\tprev=земляной":-1.0,"O\tprev=калуга":-0.8,"O\tprev=каширское":1.9,"O\tprev=ком":2.4,"O\tprev=ленинского":-1.0,"O\tprev=марьиной":-1.0,"O\tprev=москва":6.5,"O\tprev=нижегородская":-1.0,"O\tprev=нижний":-1.0,"O\tprev=новоивановское":-1.0,"O\tprev=ордынка":-2.4,"O\tprev=песчаный":-0.9,"O\tprev=поля":-1.8,"O\tprev=проспект":1.0,"O\tprev=р":1.0,"O\tprev=рощи":-1.0,"O\tprev=санкт":-2.0,"O\tprev=сокольнический":-0.6,"O\tprev=ш":-0.9,"O\tprev=ямского":-1.0,"O\tprev_kind=word":1.5,"O\tsuffix1=а":-3.3,"O\tsuffix1=в":-2.0,"O\tsuffix1=г":1.0,"O\tsuffix1=д":1.9,"O\tsuffix1=е":-0.9,"O\tsuffix1=ж":2.0,"O\tsuffix1=и":-2.0,"O\tsuffix1=й":-5.8,"O\tsuffix1=к":1.8,"O\tsuffix1=л":1.4,"O\tsuffix1=м":2.4,"O\tsuffix1=н":1.0,"O\tsuffix1=о":-1.9,"O\tsuffix1=п":1.9,"O\tsuffix1=р":4.0,"O\tsuffix1=с":1.9,"O\tsuffix1=т":1.1,"O\tsuffix1=у":-1.0,"O\tsuffix1=ш":1.9,"O\tsuffix1=щ":0.9,"O\tsuffix1=ы":-2.9,"O\tsuffix1=ь":1.0,"O\tsuffix1=я":-2.8,"O\tsuffix2=а":2.7,"O\tsuffix2=ад":0.9,"O\tsuffix2=аж":2.0,"O\tsuffix2=ал":-1.6,"O\tsuffix2=ар":1.2,"O\tsuffix2=ау":-1.0,"O\tsuffix2=ая":-2.7,"O\tsuffix2=ва":-0.2,"O\tsuffix2=г":3.0,"O\tsuffix2=го":-1.0,"O\tsuffix2=д":1.0,"O\tsuffix2=ды":-1.9,"O\tsuffix2=дь":1.0,"O\tsuffix2=ер":-0.9,"O\tsuffix2=ещ":0.9,"O\tsuffix2=зд":1.0,"O\tsuffix2=ий":-3.0,"O\tsuffix2=й":-1.0,"O\tsuffix2=к":1.0,"O\tsuffix2=ка":-1.9,"O\tsuffix2=кт":0.1,"O\tsuffix2=ла":-1.0,"O\tsuffix2=ля":-1.0,"O\tsuffix2=на":-1.9,"O\tsuffix2=но":-0.9,"O\tsuffix2=ов":-2.0,"O\tsuffix2=од":-1.0,"O\tsuffix2=ое":-2.9,"O\tsuffix2=ой":-1.0,"O\tsuffix2=ок":2.7,"O\tsuffix2=ом":2.4,"O\tsuffix2=он":1.0,"O\tsuffix2=п":1.9,"O\tsuffix2=пр":2.0,"O\tsuffix2=р":1.7,"O\tsuffix2=ра":-1.0,"O\tsuffix2=рг":-2.0,"O\tsuffix2=ры":-1.0,"O\tsuffix2=с":1.9,"O\tsuffix2=се":2.0,"O\tsuffix2=ск":-1.9,"O\tsuffix2=т":1.0,"O\tsuffix2=ти":-1.0,"O\tsuffix2=ул":3.0,"O\tsuffix2=ш":1.9,"O\tsuffix2=щи":-1.0,"O\tsuffix2=ый":-0.8,"O\tsuffix2=я":0.9,"O\tsuffix3=а":2.7,"O\tsuffix3=адь":1.0,"O\tsuffix3=вал":-1.6,"O\tsuffix3=вар":1.2,"O\tsuffix3=вды":-1.9,"O\tsuffix3=г":3.0,"O\tsuffix3=д":1.0,"O\tsuffix3=езд":1.0,"O\tsuffix3=ект":1.0,"O\tsuffix3=ина":-1.9,"O\tsuffix3=ино":-0.9,"O\tsuffix3=ира":-1.0,"O\tsuffix3=й":-1.0,"O\tsuffix3=к":1.0,"O\tsuffix3=кая":-2.0,"O\tsuffix3=ква":-0.2,"O\tsuffix3=кий":-2.0,"O\tsuffix3=кое":-1.9,"O\tsuffix3=ком":2.4,"O\tsuffix3=кры":-1.0,"O\tsuffix3=лад":0.9,"O\tsuffix3=лок":2.7,"O\tsuffix3=мау":-1.0,"O\tsuffix3=мещ":0.9,"O\tsuffix3=ная":-0.7,"O\tsuffix3=ний":-1.0,"O\tsuffix3=нка":-1.9,"O\tsuffix3=нкт":-0.9,"O\tsuffix3=ное":-1.0,"O\tsuffix3=ной":-1.0,"O\tsuffix3=нск":-1.0,"O\tsuffix3=ный":-0.8,"O\tsuffix3=ого":-1.0,"O\tsuffix3=ола":-1.0,"O\tsuffix3=оля":-1.0,"O\tsuffix3=он":1.0,"O\tsuffix3=ощи":-1.0,"O\tsuffix3=п":1.9,"O\tsuffix3=пр":2.0,"O\tsuffix3=р":1.7,"O\tsuffix3=род":-1.0,"O\tsuffix3=с":1.9,"O\tsuffix3=сов":-1.0,"O\tsuffix3=ссе":2.0,"O\tsuffix3=т":1.0,"O\tsuffix3=таж":2.0,"O\tsuffix3=тов":-1.0,"O\tsuffix3=тти":-1.0,"O\tsuffix3=ул":3.0,"O\tsuffix3=ург":-2.0,"O\tsuffix3=ш":1.9,"O\tsuffix3=ьер":-0.9,"O\tsuffix3=ьск":-0.9,"O\tsuffix3=я":0.9,"O\tword=а":2.7,"O\tword=авиамоторная":-0.7,"O\tword=брюсов":-1.0,"O\tword=бульвар":1.2,"O\tword=вал":-1.6,"O\tword=видное":-1.0,"O\tword=влад":0.9,"O\tword=г":3.0,"O\tword=гагарина":-0.9,"O\tword=гримау":-1.0,"O\tword=д":1.0,"O\tword=ермолино":-0.9,"O\tword=зеленый":-0.8,"O\tword=искры":-1.0,"O\tword=й":-1.0,"O\tword=к":1.0,"O\tword=калинина":-1.0,"O\tword=карьер":-0.9,"O\tword=каширское":-0.9,"O\tword=кировская":-0.7,"O\tword=ком":2.4,"O\tword=комсомола":-1.0,"O\tword=ленинского":-1.0,"O\tword=марьиной":-1.0,"O\tword=мира":-1.0,"O\tword=можайское":-1.0,"O\tword=москва":-0.2,"O\tword=московская":2.6,"O\tword=нижегородская":-1.0,"O\tword=нижний":-1.0,"O\tword=новгород":-1.0,"O\tword=новомарьинская":-1.0,"O\tword=озерковская":-1.0,"O\tword=октябрьский":-1.0,"O\tword=он":1.0,"O\tword=ордынка":-1.0,"O\tword=п":1.9,"O\tword=переулок":2.7,"O\tword=петербург":-2.0,"O\tword=площадь":1.0,"O\tword=подольск":-0.9,"O\tword=поля":-1.0,"O\tword=помещ":0.9,"O\tword=пр":2.0,"O\tword=правды":-1.9,"O\tword=пречистенка":-0.9,"O\tword=проезд":1.0,"O\tword=проспект":1.0,"O\tword=р":1.7,"O\tword=рощи":-1.0,"O\tword=с":1.9,"O\tword=санкт":-0.9,"O\tword=смоленск":-1.0,"O\tword=т":1.0,"O\tword=тамбовская":-0.9,"O\tword=товарищеский":-1.0,"O\tword=тольятти":-1.0,"O\tword=ул":3.0,"O\tword=ш":1.9,"O\tword=шоссе":2.0,"O\tword=энтузиастов":-1.0,"O\tword=этаж":2.0,"O\tword=я":0.9,"REGION\tbias":-1.1,"REGION\tbos":-0.5,"REGION\teos":-3.5,"REGION\tkind=word":-1.1,"REGION\tnext=балашиха":0.2,"REGION\tnext=боровский":0.9,"REGION\tnext=домодедово":0.5,"REGION\tnext=калуга":1.0,"REGION\tnext=киров":1.6,"REGION\tnext=красногорский":1.0,"REGION\tnext=ленинский":1.3,"REGION\tnext=можайское":-1.0,"REGION\tnext=москва":-5.5,"REGION\tnext=мытищи":1.6,"REGION\tnext=нижний":1.6,"REGION\tnext=новгород":-0.6,"REGION\tnext=одинцовский":1.6,"REGION\tnext=петербург":-1.0,"REGION\tnext=подольск":0.3,"REGION\tnext=реутов":0.4,"REGION\tnext=рощи":-1.0,"REGION\tnext=сенная":-1.3,"REGION\tnext=смоленск":0.6,"REGION\tnext=тольятти":0.7,"REGION\tnext=шоссе":-0.6,"REGION\tnext=электрогорск":0.1,"REGION\tnext_kind=word":2.4,"REGION\tposition=first":-0.5,"REGION\tposition=middle":-0.6,"REGION\tprefix1=в":-0.6,"REGION\tprefix1=д":-1.0,"REGION\tprefix1=к":1.9,"REGION\tprefix1=м":-1.4,"REGION\tprefix1=н":1.0,"REGION\tprefix1=с":-1.0,"REGION\tprefix2=вя":-0.6,"REGION\tprefix2=де":-1.0,"REGION\tprefix2=ка":1.3,"REGION\tprefix2=ки":0.6,"REGION\tprefix2=ма":-1.0,"REGION\tprefix2=мо":-0.4,"REGION\tprefix2=ни":1.0,"REGION\tprefix2=са":-0.3,"REGION\tprefix2=см":-0.7,"REGION\tprev=ул":-0.6,"REGION\tprev_kind=word":-0.6,"REGION\tsuffix1=а":-1.0,"REGION\tsuffix1=в":-1.0,"REGION\tsuffix1=е":-0.6,"REGION\tsuffix1=й":-1.6,"REGION\tsuffix1=т":-1.0,"REGION\tsuffix1=я":4.1,"REGION\tsuffix2=ая":4.1,"REGION\tsuffix2=ва":-1.0,"REGION\tsuffix2=ий":-0.6,"REGION\tsuffix2=кт":-1.0,"REGION\tsuffix2=ов":-1.0,"REGION\tsuffix2=ое":-0.6,"REGION\tsuffix2=ой":-1.0,"REGION\tsuffix3=кая":4.1,"REGION\tsuffix3=ква":-1.0,"REGION\tsuffix3=кое":-0.6,"REGION\tsuffix3=ний":-0.6,"REGION\tsuffix3=нкт":-1.0,"REGION\tsuffix3=ной":-1.0,"REGION\tsuffix3=ров":-1.0,"REGION\tword=вятская":-0.6,"REGION\tword=дербеневская":-1.0,"REGION\tword=калужская":1.9,"REGION\tword=каширское":-0.6,"REGION\tword=киров":-1.0,"REGION\tword=кировская":1.6,"REGION\tword=марьиной":-1.0,"REGION\tword=москва":-1.0,"REGION\tword=московская":0.6,"REGION\tword=нижегородская":1.6,"REGION\tword=нижний":-0.6,"REGION\tword=самарская":0.7,"REGION\tword=санкт":-1.0,"REGION\tword=смоленская":-0.7,"SETTLEMENT\tbias":-1.2,"SETTLEMENT\tbos":-0.4,"SETTLEMENT\teos":-0.3,"SETTLEMENT\tkind=word":-1.2,"SETTLEMENT\tnext=вокзальная":2.0,"SETTLEMENT\tnext=гагарина":-1.0,"SETTLEMENT\tnext=калинина":2.6,"SETTLEMENT\tnext=каширское":1.9,"SETTLEMENT\tnext=кирова":-1.0,"SETTLEMENT\tnext=ленинградская":-1.9,"SETTLEMENT\tnext=нижний":-0.6,"SETTLEMENT\tnext=пр":-0.9,"SETTLEMENT\tnext=сампсониевский":-1.0,"SETTLEMENT\tnext=ш":-1.0,"SETTLEMENT\tnext_kind=word":-0.9,"SETTLEMENT\tposition=first":-0.4,"SETTLEMENT\tposition=last":-1.4,"SETTLEMENT\tposition=middle":0.6,"SETTLEMENT\tprefix1=б":-1.0,"SETTLEMENT\tprefix1=в":-0.7,"SETTLEMENT\tprefix1=е":-1.0,"SETTLEMENT\tprefix1=к":-2.0,"SETTLEMENT\tprefix1=м":-1.0,"SETTLEMENT\tprefix1=н":3.1,"SETTLEMENT\tprefix1=р":-0.3,"SETTLEMENT\tprefix1=с":2.7,"SETTLEMENT\tprefix1=э":-1.0,"SETTLEMENT\tprefix2=бо":-1.0,"SETTLEMENT\tprefix2=ви":-0.7,"SETTLEMENT\tprefix2=ер":-1.0,"SETTLEMENT\tprefix2=ка":-1.0,"SETTLEMENT\tprefix2=ко":-1.0,"SETTLEMENT\tprefix2=мо":-1.0,"SETTLEMENT\tprefix2=на":2.1,"SETTLEMENT\tprefix2=ни":-0.6,"SETTLEMENT\tprefix2=но":1.6,"SETTLEMENT\tprefix2=ре":-0.3,"SETTLEMENT\tprefix2=се":2.7,"SETTLEMENT\tprefix2=эт":-1.0,"SETTLEMENT\tprev=балашиха":-0.4,"SETTLEMENT\tprev=домодедово":0.9,"SETTLEMENT\tprev=красногорский":1.0,"SETTLEMENT\tprev=москва":-1.0,"SETTLEMENT\tprev=нижний":-1.0,"SETTLEMENT\tprev=одинцовский":1.6,"SETTLEMENT\tprev=петербург":-1.0,"SETTLEMENT\tprev=смоленск":-0.9,"SETTLEMENT\tprev_kind=word":-0.8,"SETTLEMENT\tsuffix1=а":-1.0,"SETTLEMENT\tsuffix1=д":-1.0,"SETTLEMENT\tsuffix1=е":1.8,"SETTLEMENT\tsuffix1=ж":-1.0,"SETTLEMENT\tsuffix1=о":0.6,"SETTLEMENT\tsuffix1=я":-0.6,"SETTLEMENT\tsuffix2=аж":-1.0,"SETTLEMENT\tsuffix2=ая":-0.6,"SETTLEMENT\tsuffix2=ва":-1.0,"SETTLEMENT\tsuffix2=во":-1.9,"SETTLEMENT\tsuffix2=но":2.5,"SETTLEMENT\tsuffix2=од":-1.0,"SETTLEMENT\tsuffix2=ое":1.8,"SETTLEMENT\tsuffix2=ой":-1.0,"SETTLEMENT\tsuffix2=ый":1.0,"SETTLEMENT\tsuffix3=ино":2.5,"SETTLEMENT\tsuffix3=кая":-0.6,"SETTLEMENT\tsuffix3=кое":2.5,"SETTLEMENT\tsuffix3=ное":-0.7,"SETTLEMENT\tsuffix3=ный":1.0,"SETTLEMENT\tsuffix3=ова":-1.0,"SETTLEMENT\tsuffix3=ово":-1.9,"SETTLEMENT\tsuffix3=род":-1.0,"SETTLEMENT\tsuffix3=таж":-1.0,"SETTLEMENT\tsuffix3=шой":-1.0,"SETTLEMENT\tword=большой":-1.0,"SETTLEMENT\tword=видное":-0.7,"SETTLEMENT\tword=ермолино":-1.0,"SETTLEMENT\tword=каширское":-1.0,"SETTLEMENT\tword=колпакова":-1.0,"SETTLEMENT\tword=можайское":-1.0,"SETTLEMENT\tword=научный":-1.4,"SETTLEMENT\tword=нахабино":3.5,"SETTLEMENT\tword=нижегородская":-0.6,"SETTLEMENT\tword=новгород":-1.0,"SETTLEMENT\tword=ново":-1.9,"SETTLEMENT\tword=новоивановское":4.5,"SETTLEMENT\tword=революционный":-0.3,"SETTLEMENT\tword=северный":2.7,"SETTLEMENT\tword=этаж":-1.0,"STREET\tbias":0.8,"STREET\tbos":0.3,"STREET\teos":3.0,"STREET\tkind=word":0.8,"STREET\tnext=б":0.4,"STREET\tnext=боровский":-0.9,"STREET\tnext=буденного":-1.0,"STREET\tnext=вал":0.9,"STREET\tnext=вокзальная":-1.0,"STREET\tnext=володарского":-0.9,"STREET\tnext=воронина":-1.0,"STREET\tnext=донского":1.0,"STREET\tnext=ермолино":-0.9,"STREET\tnext=й":2.0,"STREET\tnext=калинина":-1.6,"STREET\tnext=калуга":-1.0,"STREET\tnext=каширское":-1.9,"STREET\tnext=киров":-0.9,"STREET\tnext=колпакова":-1.0,"STREET\tnext=комсомола":2.0,"STREET\tnext=красногорский":-1.0,"STREET\tnext=ленинградская":3.4,"STREET\tnext=ленинский":-0.8,"STREET\tnext=ленинского":-1.0,"STREET\tnext=марьиной":-1.0,"STREET\tnext=мытищи":-1.0,"STREET\tnext=нахабино":-1.0,"STREET\tnext=новгород":-1.0,"STREET\tnext=новоивановское":-1.6,"STREET\tnext=одинцовский":-1.6,"STREET\tnext=октябрьский":-1.0,"STREET\tnext=ополчения":1.0,"STREET\tnext=петербург":-1.0,"STREET\tnext=пр":3.4,"STREET\tnext=р":-1.0,"STREET\tnext=рощи":2.0,"STREET\tnext=с":2.4,"STREET\tnext=сампсониевский":2.0,"STREET\tnext=северный":-2.0,"STREET\tnext=сенная":1.3,"STREET\tnext=смоленск":-0.6,"STREET\tnext=тольятти":-0.7,"STREET\tnext=ш":1.9,"STREET\tnext=шоссе":0.6,"STREET\tnext=я":1.9,"STREET\tnext_kind=word":-2.2,"STREET\tposition=first":0.3,"STREET\tposition=last":2.5,"STREET\tposition=middle":-2.0,"STREET\tprefix1=а":-1.0,"STREET\tprefix1=б":-0.2,"STREET\tprefix1=в":1.3,"STREET\tprefix1=г":-0.1,"STREET\tprefix1=д":1.0,"STREET\tprefix1=е":-0.6,"STREET\tprefix1=з":1.8,"STREET\tprefix1=и":2.0,"STREET\tprefix1=й":1.0,"STREET\tprefix1=к":0.1,"STREET\tprefix1=л":2.6,"STREET\tprefix1=м":-0.2,"STREET\tprefix1=н":-0.6,"STREET\tprefix1=о":1.4,"STREET\tprefix1=п":-1.9,"STREET\tprefix1=с":-1.8,"STREET\tprefix1=т":-0.1,"STREET\tprefix1=у":-2.0,"STREET\tprefix1=ш":-2.9,"STREET\tprefix1=э":0.9,"STREET\tprefix1=я":0.1,"STREET\tprefix2=а":-1.7,"STREET\tprefix2=ав":0.7,"STREET\tprefix2=б":1.0,"STREET\tprefix2=ба":-2.0,"STREET\tprefix2=бо":0.1,"STREET\tprefix2=бр":1.9,"STREET\tprefix2=бу":-1.2,"STREET\tprefix2=ва":2.6,"STREET\tprefix2=ви":-1.9,"STREET\tprefix2=вл":-0.9,"STREET\tprefix2=во":0.9,"STREET\tprefix2=вя":0.6,"STREET\tprefix2=г":-2.0,"STREET\tprefix2=га":0.9,"STREET\tprefix2=гр":1.0,"STREET\tprefix2=де":1.0,"STREET\tprefix2=дм":1.0,"STREET\tprefix2=до":-1.0,"STREET\tprefix2=ер":-0.6,"STREET\tprefix2=зе":1.8,"STREET\tprefix2=ис":2.0,"STREET\tprefix2=й":1.0,"STREET\tprefix2=к":-1.0,"STREET\tprefix2=ка":0.6,"STREET\tprefix2=ки":-0.9,"STREET\tprefix2=ко":1.4,"STREET\tprefix2=кр":-1.0,"STREET\tprefix2=ку":1.0,"STREET\tprefix2=ле":2.6,"STREET\tprefix2=ма":2.0,"STREET\tprefix2=ми":1.6,"STREET\tprefix2=мо":-2.8,"STREET\tprefix2=мы":-1.0,"STREET\tprefix2=на":0.5,"STREET\tprefix2=ни":-1.0,"STREET\tprefix2=но":-0.1,"STREET\tprefix2=од":-1.6,"STREET\tprefix2=оз":1.0,"STREET\tprefix2=ок":1.0,"STREET\tprefix2=он":-1.0,"STREET\tprefix2=ор":1.0,"STREET\tprefix2=оч":1.0,"STREET\tprefix2=п":-0.9,"STREET\tprefix2=пе":-1.8,"STREET\tprefix2=пл":-1.0,"STREET\tprefix2=пр":1.8,"STREET\tprefix2=р":-0.7,"STREET\tprefix2=ре":-1.3,"STREET\tprefix2=ро":2.0,"STREET\tprefix2=с":-1.0,"STREET\tprefix2=са":0.3,"STREET\tprefix2=се":-2.7,"STREET\tprefix2=см":0.7,"STREET\tprefix2=со":0.9,"STREET\tprefix2=т":-1.0,"STREET\tprefix2=та":0.9,"STREET\tprefix2=ул":-2.0,"STREET\tprefix2=ш":-0.9,"STREET\tprefix2=шо":-2.0,"STREET\tprefix2=эн":1.9,"STREET\tprefix2=эт":-1.0,"STREET\tprefix2=я":-0.9,"STREET\tprefix2=яр":1.0,"STREET\tprev=а":-2.7,"STREET\tprev=балашиха":0.4,"STREET\tprev=большой":1.0,"STREET\tprev=бульвар":-0.9,"STREET\tprev=г":-3.0,"STREET\tprev=домодедово":-0.9,"STREET\tprev=ермолино":0.9,"STREET\tprev=земляной":1.0,"STREET\tprev=калуга":0.8,"STREET\tprev=калужская":-1.9,"STREET\tprev=каширское":-1.9,"STREET\tprev=кировская":-0.9,"STREET\tprev=ком":-2.4,"STREET\tprev=красногорский":-1.0,"STREET\tprev=ленинский":-1.0,"STREET\tprev=ленинского":1.0,"STREET\tprev=марьиной":2.0,"STREET\tprev=москва":5.0,"STREET\tprev=московская":-4.6,"STREET\tprev=нижний":-1.0,"STREET\tprev=новоивановское":1.0,"STREET\tprev=одинцовский":-1.6,"STREET\tprev=ордынка":3.4,"STREET\tprev=песчаный":0.9,"STREET\tprev=петербург":1.0,"STREET\tprev=поля":1.8,"STREET\tprev=проспект":-1.0,"STREET\tprev=р":-1.0,"STREET\tprev=рощи":1.0,"STREET\tprev=санкт":-0.8,"STREET\tprev=смоленск":1.9,"STREET\tprev=сокольнический":0.6,"STREET\tprev=ул":0.6,"STREET\tprev=ш":0.9,"STREET\tprev=ямского":1.9,"STREET\tprev_kind=word":0.5,"STREET\tsuffix1=а":2.7,"STREET\tsuffix1=б":1.0,"STREET\tsuffix1=в":0.4,"STREET\tsuffix1=г":-2.8,"STREET\tsuffix1=д":-2.9,"STREET\tsuffix1=е":-0.2,"STREET\tsuffix1=ж":-1.0,"STREET\tsuffix1=й":6.8,"STREET\tsuffix1=к":-4.0,"STREET\tsuffix1=л":-0.4,"STREET\tsuffix1=м":-0.6,"STREET\tsuffix1=н":-1.0,"STREET\tsuffix1=о":2.9,"STREET\tsuffix1=п":-0.9,"STREET\tsuffix1=р":-2.0,"STREET\tsuffix1=с":-1.0,"STREET\tsuffix1=т":-2.0,"STREET\tsuffix1=у":1.0,"STREET\tsuffix1=ш":-0.9,"STREET\tsuffix1=щ":-0.9,"STREET\tsuffix1=ы":3.9,"STREET\tsuffix1=ь":-1.0,"STREET\tsuffix1=я":3.7,"STREET\tsuffix2=а":-1.7,"STREET\tsuffix2=ад":-0.9,"STREET\tsuffix2=аж":-1.0,"STREET\tsuffix2=ал":1.6,"STREET\tsuffix2=ар":-1.2,"STREET\tsuffix2=ау":1.0,"STREET\tsuffix2=ая":1.7,"STREET\tsuffix2=б":1.0,"STREET\tsuffix2=ва":0.9,"STREET\tsuffix2=во":2.4,"STREET\tsuffix2=г":-2.0,"STREET\tsuffix2=га":-2.8,"STREET\tsuffix2=го":3.0,"STREET\tsuffix2=ды":1.9,"STREET\tsuffix2=дь":-1.0,"STREET\tsuffix2=ер":0.9,"STREET\tsuffix2=ещ":-0.9,"STREET\tsuffix2=зд":-1.0,"STREET\tsuffix2=ия":1.0,"STREET\tsuffix2=й":1.0,"STREET\tsuffix2=к":-1.0,"STREET\tsuffix2=ка":2.9,"STREET\tsuffix2=кт":-1.0,"STREET\tsuffix2=ла":1.0,"STREET\tsuffix2=ля":1.9,"STREET\tsuffix2=на":2.8,"STREET\tsuffix2=но":-2.5,"STREET\tsuffix2=ов":0.4,"STREET\tsuffix2=од":-1.0,"STREET\tsuffix2=ое":1.8,"STREET\tsuffix2=ой":4.0,"STREET\tsuffix2=ок":-1.0,"STREET\tsuffix2=ом":-0.6,"STREET\tsuffix2=он":-1.0,"STREET\tsuffix2=п":-0.9,"STREET\tsuffix2=пр":-1.0,"STREET\tsuffix2=р":-0.7,"STREET\tsuffix2=ра":1.6,"STREET\tsuffix2=рг":-0.8,"STREET\tsuffix2=ры":2.0,"STREET\tsuffix2=с":-1.0,"STREET\tsuffix2=се":-2.0,"STREET\tsuffix2=ск":-2.0,"STREET\tsuffix2=т":-1.0,"STREET\tsuffix2=ти":-1.0,"STREET\tsuffix2=ул":-2.0,"STREET\tsuffix2=ха":-2.0,"STREET\tsuffix2=ш":-0.9,"STREET\tsuffix2=щи":1.0,"STREET\tsuffix2=ый":1.8,"STREET\tsuffix2=я":-0.9,"STREET\tsuffix3=а":-1.7,"STREET\tsuffix3=адь":-1.0,"STREET\tsuffix3=б":1.0,"STREET\tsuffix3=вал":1.6,"STREET\tsuffix3=вар":-1.2,"STREET\tsuffix3=вды":1.9,"STREET\tsuffix3=г":-2.0,"STREET\tsuffix3=езд":-1.0,"STREET\tsuffix3=ина":2.8,"STREET\tsuffix3=ино":-2.5,"STREET\tsuffix3=ира":1.6,"STREET\tsuffix3=иха":-2.0,"STREET\tsuffix3=ищи":-1.0,"STREET\tsuffix3=й":1.0,"STREET\tsuffix3=к":-1.0,"STREET\tsuffix3=кая":1.0,"STREET\tsuffix3=ква":-2.9,"STREET\tsuffix3=кий":1.0,"STREET\tsuffix3=кое":3.7,"STREET\tsuffix3=ком":-0.6,"STREET\tsuffix3=кры":2.0,"STREET\tsuffix3=лад":-0.9,"STREET\tsuffix3=лок":-1.0,"STREET\tsuffix3=мау":1.0,"STREET\tsuffix3=мещ":-0.9,"STREET\tsuffix3=ная":0.7,"STREET\tsuffix3=ний":-1.0,"STREET\tsuffix3=нка":2.9,"STREET\tsuffix3=нкт":-1.0,"STREET\tsuffix3=ное":-1.9,"STREET\tsuffix3=ной":2.0,"STREET\tsuffix3=ный":1.8,"STREET\tsuffix3=ова":3.8,"STREET\tsuffix3=ово":2.4,"STREET\tsuffix3=ого":3.0,"STREET\tsuffix3=ола":1.0,"STREET\tsuffix3=оля":1.9,"STREET\tsuffix3=он":-1.0,"STREET\tsuffix3=ощи":2.0,"STREET\tsuffix3=п":-0.9,"STREET\tsuffix3=пр":-1.0,"STREET\tsuffix3=р":-0.7,"STREET\tsuffix3=рия":1.0,"STREET\tsuffix3=ров":-1.8,"STREET\tsuffix3=род":-1.0,"STREET\tsuffix3=рск":-1.0,"STREET\tsuffix3=с":-1.0,"STREET\tsuffix3=сов":1.9,"STREET\tsuffix3=ссе":-2.0,"STREET\tsuffix3=т":-1.0,"STREET\tsuffix3=таж":-1.0,"STREET\tsuffix3=тов":0.3,"STREET\tsuffix3=тти":-1.0,"STREET\tsuffix3=уга":-2.8,"STREET\tsuffix3=ул":-2.0,"STREET\tsuffix3=ург":-0.8,"STREET\tsuffix3=ш":-0.9,"STREET\tsuffix3=шой":2.0,"STREET\tsuffix3=ьер":0.9,"STREET\tsuffix3=ьск":-1.0,"STREET\tsuffix3=я":-0.9,"STREET\tword=а":-1.7,"STREET\tword=авиамоторная":0.7,"STREET\tword=б":1.0,"STREET\tword=балашиха":-2.0,"STREET\tword=большой":2.0,"STREET\tword=боровский":-1.9,"STREET\tword=брюсов":1.9,"STREET\tword=бульвар":-1.2,"STREET\tword=вавилова":1.0,"STREET\tword=вал":1.6,"STREET\tword=видное":-1.9,"STREET\tword=влад":-0.9,"STREET\tword=воронина":0.9,"STREET\tword=вятская":0.6,"STREET\tword=г":-2.0,"STREET\tword=гагарина":0.9,"STREET\tword=гримау":1.0,"STREET\tword=дербеневская":1.0,"STREET\tword=дмитрия":1.0,"STREET\tword=домодедово":-1.0,"STREET\tword=ермолино":-0.6,"STREET\tword=зеленый":1.8,"STREET\tword=искры":2.0,"STREET\tword=й":1.0,"STREET\tword=к":-1.0,"STREET\tword=калинина":1.0,"STREET\tword=калуга":-2.8,"STREET\tword=калужская":-1.9,"STREET\tword=карьер":0.9,"STREET\tword=каширское":3.4,"STREET\tword=киров":-1.8,"STREET\tword=кирова":1.8,"STREET\tword=кировская":-0.9,"STREET\tword=колпакова":1.0,"STREET\tword=ком":-0.6,"STREET\tword=комсомола":1.0,"STREET\tword=красногорский":-1.0,"STREET\tword=кутузовский":1.0,"STREET\tword=ленинградский":0.6,"STREET\tword=ленинского":2.0,"STREET\tword=марьиной":2.0,"STREET\tword=мира":1.6,"STREET\tword=можайское":2.8,"STREET\tword=москва":-2.9,"STREET\tword=московская":-2.7,"STREET\tword=мытищи":-1.0,"STREET\tword=народного":1.0,"STREET\tword=научный":1.4,"STREET\tword=нахабино":-1.9,"STREET\tword=нижний":-1.0,"STREET\tword=новгород":-1.0,"STREET\tword=ново":3.4,"STREET\tword=новоивановское":-3.5,"STREET\tword=новомарьинская":1.0,"STREET\tword=одинцовский":-1.6,"STREET\tword=озерковская":1.0,"STREET\tword=октябрьский":1.0,"STREET\tword=он":-1.0,"STREET\tword=ордынка":1.0,"STREET\tword=очаковское":1.0,"STREET\tword=п":-0.9,"STREET\tword=переулок":-1.0,"STREET\tword=петербург":-0.8,"STREET\tword=площадь":-1.0,"STREET\tword=подольск":-1.0,"STREET\tword=поля":1.9,"STREET\tword=помещ":-0.9,"STREET\tword=пр":-1.0,"STREET\tword=правды":1.9,"STREET\tword=пречистенка":1.9,"STREET\tword=проезд":-1.0,"STREET\tword=р":-0.7,"STREET\tword=революционный":0.3,"STREET\tword=реутов":-1.6,"STREET\tword=рощи":2.0,"STREET\tword=с":-1.0,"STREET\tword=садовническая":1.0,"STREET\tword=самарская":-0.7,"STREET\tword=сампсониевский":1.0,"STREET\tword=санкт":-1.0,"STREET\tword=северный":-2.7,"STREET\tword=смоленская":0.7,"STREET\tword=сокольнический":0.9,"STREET\tword=т":-1.0,"STREET\tword=тамбовская":0.9,"STREET\tword=товарищеский":1.0,"STREET\tword=тольятти":-1.0,"STREET\tword=ул":-2.0,"STREET\tword=ш":-0.9,"STREET\tword=шоссе":-2.0,"STREET\tword=электрогорск":-1.0,"STREET\tword=электродный":1.0,"STREET\tword=энтузиастов":1.9,"STREET\tword=этаж":-1.0,"STREET\tword=я":-0.9,"STREET\tword=ярославская":1.0},"transitions":{"\tCITY":1.1,"\tDISTRICT":-0.6,"\tO":0.1,"\tREGION":-0.5,"\tSETTLEMENT":-0.4,"\tSTREET":0.3,"CITY\t":0.9,"CITY\tCITY":-2.6,"CITY\tDISTRICT":-0.5,"CITY\tO":0.3,"CITY\tSETTLEMENT":0.4,"CITY\tSTREET":2.2,"DISTRICT\t":-1.0,"DISTRICT\tCITY":1.0,"DISTRICT\tO":-0.6,"DISTRICT\tSETTLEMENT":0.8,"DISTRICT\tSTREET":-1.0,"O\t":0.9,"O\tCITY":1.8,"O\tO":2.6,"O\tREGION":-0.6,"O\tSTREET":-3.1,"REGION\t":-3.5,"REGION\tCITY":4.2,"REGION\tDISTRICT":3.1,"REGION\tO":-0.6,"REGION\tSETTLEMENT":-1.0,"REGION\tSTREET":-3.3,"SETTLEMENT\t":-0.3,"SETTLEMENT\tCITY":-0.6,"SETTLEMENT\tDISTRICT":-1.0,"SETTLEMENT\tO":-1.9,"SETTLEMENT\tSTREET":2.6,"STREET\t":3.0,"STREET\tCITY":-4.2,"STREET\tDISTRICT":-1.8,"STREET\tO":1.7,"STREET\tSETTLEMENT":-1.0,"STREET\tSTREET":3.1}} \ No newline at end of file +{"format":"address-normalizer-compact-sequence-v1","training":{"algorithm":"epoch-averaged structured perceptron","seed":2017,"epochs":10,"examples":359,"source":"source-verifiable legacy reference rows with deterministic marker-free views","dataset":"benchmarks/legacy_500.jsonl","dataset_sha256":"853916e36cfc5a52d06add0524ba64ca2c771073c1c4e98cd5645116e7f74588","split":"SHA-256 by canonical address group: 70/15/15"},"labels":["O","REGION","DISTRICT","CITY","SETTLEMENT","STREET"],"emissions":{"CITY\tbias":0.7,"CITY\tbos":1.1,"CITY\teos":0.9,"CITY\tkind=word":0.7,"CITY\tnext=б":-1.8,"CITY\tnext=большой":1.0,"CITY\tnext=буденного":1.0,"CITY\tnext=вал":-0.9,"CITY\tnext=видное":-1.0,"CITY\tnext=вокзальная":-1.0,"CITY\tnext=володарского":0.9,"CITY\tnext=воронина":1.0,"CITY\tnext=г":-3.0,"CITY\tnext=гагарина":1.9,"CITY\tnext=гоголя":1.0,"CITY\tnext=донского":-1.0,"CITY\tnext=й":-1.0,"CITY\tnext=калинина":-1.0,"CITY\tnext=каширское":-2.0,"CITY\tnext=кирова":1.0,"CITY\tnext=колпакова":2.0,"CITY\tnext=ленинградская":-1.5,"CITY\tnext=ленинский":-0.5,"CITY\tnext=ленинского":2.0,"CITY\tnext=можайское":1.0,"CITY\tnext=москва":1.2,"CITY\tnext=новгород":3.5,"CITY\tnext=октябрьский":1.0,"CITY\tnext=он":-1.0,"CITY\tnext=ополчения":-1.0,"CITY\tnext=переулок":-1.0,"CITY\tnext=петербург":2.9,"CITY\tnext=пр":-2.8,"CITY\tnext=правды":0.9,"CITY\tnext=сампсониевский":-1.0,"CITY\tnext=северный":2.8,"CITY\tnext=т":-1.0,"CITY\tnext=ш":-0.9,"CITY\tnext=этаж":-1.0,"CITY\tnext=я":-1.9,"CITY\tnext=ямского":1.0,"CITY\tnext_kind=word":-0.2,"CITY\tposition=first":1.1,"CITY\tposition=last":-1.2,"CITY\tposition=middle":0.8,"CITY\tprefix1=а":-1.0,"CITY\tprefix1=б":-0.9,"CITY\tprefix1=в":1.7,"CITY\tprefix1=г":-1.0,"CITY\tprefix1=д":-0.2,"CITY\tprefix1=е":2.5,"CITY\tprefix1=з":-1.0,"CITY\tprefix1=и":-1.0,"CITY\tprefix1=к":0.1,"CITY\tprefix1=л":-1.0,"CITY\tprefix1=м":4.2,"CITY\tprefix1=н":1.4,"CITY\tprefix1=о":-1.0,"CITY\tprefix1=п":-0.9,"CITY\tprefix1=р":-0.4,"CITY\tprefix1=с":1.1,"CITY\tprefix1=т":2.0,"CITY\tprefix1=у":-1.0,"CITY\tprefix1=ш":-1.0,"CITY\tprefix1=э":-0.9,"CITY\tprefix1=я":-1.0,"CITY\tprefix2=а":-1.0,"CITY\tprefix2=б":-1.0,"CITY\tprefix2=ба":2.0,"CITY\tprefix2=бо":-1.0,"CITY\tprefix2=бр":-0.9,"CITY\tprefix2=ва":-1.0,"CITY\tprefix2=ви":3.6,"CITY\tprefix2=во":-0.9,"CITY\tprefix2=г":-1.0,"CITY\tprefix2=д":-1.0,"CITY\tprefix2=дм":-1.0,"CITY\tprefix2=до":1.8,"CITY\tprefix2=ер":2.5,"CITY\tprefix2=зе":-1.0,"CITY\tprefix2=ис":-1.0,"CITY\tprefix2=ка":1.9,"CITY\tprefix2=ки":1.0,"CITY\tprefix2=ко":-1.8,"CITY\tprefix2=ку":-1.0,"CITY\tprefix2=ле":-1.0,"CITY\tprefix2=ми":-0.6,"CITY\tprefix2=мо":2.8,"CITY\tprefix2=мы":2.0,"CITY\tprefix2=на":-2.6,"CITY\tprefix2=ни":3.5,"CITY\tprefix2=но":0.5,"CITY\tprefix2=оч":-1.0,"CITY\tprefix2=п":-1.0,"CITY\tprefix2=пе":2.1,"CITY\tprefix2=по":1.0,"CITY\tprefix2=пр":-3.0,"CITY\tprefix2=р":-1.0,"CITY\tprefix2=ре":1.6,"CITY\tprefix2=ро":-1.0,"CITY\tprefix2=с":-0.9,"CITY\tprefix2=са":1.9,"CITY\tprefix2=см":1.0,"CITY\tprefix2=со":-0.9,"CITY\tprefix2=то":2.0,"CITY\tprefix2=ул":-1.0,"CITY\tprefix2=ш":-1.0,"CITY\tprefix2=эн":-0.9,"CITY\tprefix2=яр":-1.0,"CITY\tprev=боровский":-0.1,"CITY\tprev=г":0.7,"CITY\tprev=зеленый":-1.0,"CITY\tprev=калужская":1.0,"CITY\tprev=кировская":0.9,"CITY\tprev=ленинский":1.0,"CITY\tprev=марьиной":-1.0,"CITY\tprev=москва":-10.5,"CITY\tprev=московская":2.8,"CITY\tprev=нижегородская":1.9,"CITY\tprev=нижний":3.0,"CITY\tprev=ордынка":-1.0,"CITY\tprev=санкт":3.8,"CITY\tprev=смоленск":-1.0,"CITY\tprev=ямского":-0.9,"CITY\tprev_kind=word":-0.4,"CITY\tsuffix1=а":2.6,"CITY\tsuffix1=б":-1.0,"CITY\tsuffix1=в":2.6,"CITY\tsuffix1=г":2.8,"CITY\tsuffix1=д":2.0,"CITY\tsuffix1=е":-0.1,"CITY\tsuffix1=и":3.0,"CITY\tsuffix1=й":-2.4,"CITY\tsuffix1=к":2.2,"CITY\tsuffix1=л":-1.0,"CITY\tsuffix1=м":-1.8,"CITY\tsuffix1=о":0.2,"CITY\tsuffix1=п":-1.0,"CITY\tsuffix1=р":-2.0,"CITY\tsuffix1=с":-0.9,"CITY\tsuffix1=т":1.9,"CITY\tsuffix1=ш":-1.0,"CITY\tsuffix1=ы":-1.0,"CITY\tsuffix1=я":-4.4,"CITY\tsuffix2=а":-1.0,"CITY\tsuffix2=ая":-2.5,"CITY\tsuffix2=б":-1.0,"CITY\tsuffix2=ва":1.3,"CITY\tsuffix2=во":0.3,"CITY\tsuffix2=г":-1.0,"CITY\tsuffix2=га":2.8,"CITY\tsuffix2=го":-1.0,"CITY\tsuffix2=д":-1.0,"CITY\tsuffix2=ий":0.6,"CITY\tsuffix2=ия":-1.0,"CITY\tsuffix2=ка":-1.0,"CITY\tsuffix2=кт":1.9,"CITY\tsuffix2=ля":-0.9,"CITY\tsuffix2=на":-0.9,"CITY\tsuffix2=но":0.9,"CITY\tsuffix2=ов":2.6,"CITY\tsuffix2=од":3.0,"CITY\tsuffix2=ое":-0.1,"CITY\tsuffix2=ой":-1.0,"CITY\tsuffix2=ок":-1.7,"CITY\tsuffix2=ом":-1.8,"CITY\tsuffix2=п":-1.0,"CITY\tsuffix2=пр":-1.0,"CITY\tsuffix2=р":-1.0,"CITY\tsuffix2=ра":-0.6,"CITY\tsuffix2=рг":3.8,"CITY\tsuffix2=ры":-1.0,"CITY\tsuffix2=с":-0.9,"CITY\tsuffix2=ск":3.9,"CITY\tsuffix2=ти":2.0,"CITY\tsuffix2=ул":-1.0,"CITY\tsuffix2=ха":2.0,"CITY\tsuffix2=ш":-1.0,"CITY\tsuffix2=щи":1.0,"CITY\tsuffix2=ый":-2.0,"CITY\tsuffix3=а":-1.0,"CITY\tsuffix3=б":-1.0,"CITY\tsuffix3=г":-1.0,"CITY\tsuffix3=д":-1.0,"CITY\tsuffix3=ект":-1.0,"CITY\tsuffix3=ина":-0.9,"CITY\tsuffix3=ино":0.9,"CITY\tsuffix3=ира":-0.6,"CITY\tsuffix3=иха":2.0,"CITY\tsuffix3=ищи":2.0,"CITY\tsuffix3=кая":-2.5,"CITY\tsuffix3=ква":4.1,"CITY\tsuffix3=кий":-2.9,"CITY\tsuffix3=кое":-3.7,"CITY\tsuffix3=ком":-1.8,"CITY\tsuffix3=кры":-1.0,"CITY\tsuffix3=лок":-1.7,"CITY\tsuffix3=ний":3.5,"CITY\tsuffix3=нка":-1.0,"CITY\tsuffix3=нкт":2.9,"CITY\tsuffix3=ное":3.6,"CITY\tsuffix3=нск":1.0,"CITY\tsuffix3=ный":-2.0,"CITY\tsuffix3=ова":-2.8,"CITY\tsuffix3=ово":0.3,"CITY\tsuffix3=ого":-1.0,"CITY\tsuffix3=оля":-0.9,"CITY\tsuffix3=ощи":-1.0,"CITY\tsuffix3=п":-1.0,"CITY\tsuffix3=пр":-1.0,"CITY\tsuffix3=р":-1.0,"CITY\tsuffix3=рия":-1.0,"CITY\tsuffix3=ров":2.8,"CITY\tsuffix3=род":3.0,"CITY\tsuffix3=рск":1.0,"CITY\tsuffix3=с":-0.9,"CITY\tsuffix3=сов":-0.9,"CITY\tsuffix3=тов":0.7,"CITY\tsuffix3=тти":2.0,"CITY\tsuffix3=уга":2.8,"CITY\tsuffix3=ул":-1.0,"CITY\tsuffix3=ург":3.8,"CITY\tsuffix3=ш":-1.0,"CITY\tsuffix3=шой":-1.0,"CITY\tsuffix3=ьск":1.9,"CITY\tword=а":-1.0,"CITY\tword=б":-1.0,"CITY\tword=балашиха":2.0,"CITY\tword=большой":-1.0,"CITY\tword=брюсов":-0.9,"CITY\tword=вавилова":-1.0,"CITY\tword=видное":3.6,"CITY\tword=воронина":-0.9,"CITY\tword=г":-1.0,"CITY\tword=д":-1.0,"CITY\tword=дмитрия":-1.0,"CITY\tword=домодедово":1.8,"CITY\tword=ермолино":2.5,"CITY\tword=зеленый":-1.0,"CITY\tword=искры":-1.0,"CITY\tword=калуга":2.8,"CITY\tword=каширское":-0.9,"CITY\tword=киров":2.8,"CITY\tword=кирова":-1.8,"CITY\tword=ком":-1.8,"CITY\tword=кутузовский":-1.0,"CITY\tword=ленинский":-1.0,"CITY\tword=мира":-0.6,"CITY\tword=можайское":-0.8,"CITY\tword=москва":4.1,"CITY\tword=московская":-0.5,"CITY\tword=мытищи":2.0,"CITY\tword=народного":-1.0,"CITY\tword=нахабино":-1.6,"CITY\tword=нижний":3.5,"CITY\tword=новгород":3.0,"CITY\tword=ново":-1.5,"CITY\tword=новоивановское":-1.0,"CITY\tword=очаковское":-1.0,"CITY\tword=п":-1.0,"CITY\tword=переулок":-1.7,"CITY\tword=петербург":3.8,"CITY\tword=подольск":1.9,"CITY\tword=поля":-0.9,"CITY\tword=пр":-1.0,"CITY\tword=пречистенка":-1.0,"CITY\tword=проспект":-1.0,"CITY\tword=р":-1.0,"CITY\tword=реутов":1.6,"CITY\tword=рощи":-1.0,"CITY\tword=с":-0.9,"CITY\tword=садовническая":-1.0,"CITY\tword=санкт":2.9,"CITY\tword=смоленск":1.0,"CITY\tword=сокольнический":-0.9,"CITY\tword=тольятти":2.0,"CITY\tword=ул":-1.0,"CITY\tword=ш":-1.0,"CITY\tword=электрогорск":1.0,"CITY\tword=электродный":-1.0,"CITY\tword=энтузиастов":-0.9,"CITY\tword=ярославская":-1.0,"DISTRICT\tbias":-0.8,"DISTRICT\tbos":-0.6,"DISTRICT\teos":-1.0,"DISTRICT\tkind=word":-0.8,"DISTRICT\tnext=большой":-1.0,"DISTRICT\tnext=видное":1.0,"DISTRICT\tnext=ермолино":0.9,"DISTRICT\tnext=колпакова":-1.0,"DISTRICT\tnext=комсомола":-1.0,"DISTRICT\tnext=нахабино":1.0,"DISTRICT\tnext=новгород":-0.9,"DISTRICT\tnext=новоивановское":1.6,"DISTRICT\tnext=пр":-0.6,"DISTRICT\tnext=р":1.0,"DISTRICT\tnext=северный":-0.8,"DISTRICT\tnext_kind=word":0.2,"DISTRICT\tposition=first":-0.6,"DISTRICT\tposition=last":-1.0,"DISTRICT\tposition=middle":0.8,"DISTRICT\tprefix1=б":1.9,"DISTRICT\tprefix1=д":-0.8,"DISTRICT\tprefix1=к":1.0,"DISTRICT\tprefix1=л":-0.6,"DISTRICT\tprefix1=м":-1.0,"DISTRICT\tprefix1=н":-0.9,"DISTRICT\tprefix1=о":1.6,"DISTRICT\tprefix1=п":-1.0,"DISTRICT\tprefix1=с":-1.0,"DISTRICT\tprefix2=бо":1.9,"DISTRICT\tprefix2=до":-0.8,"DISTRICT\tprefix2=кр":1.0,"DISTRICT\tprefix2=ле":-0.6,"DISTRICT\tprefix2=мы":-1.0,"DISTRICT\tprefix2=ни":-0.9,"DISTRICT\tprefix2=од":1.6,"DISTRICT\tprefix2=пе":-1.0,"DISTRICT\tprefix2=са":-1.0,"DISTRICT\tprev=большой":-1.0,"DISTRICT\tprev=калужская":0.9,"DISTRICT\tprev=московская":1.8,"DISTRICT\tprev=нижегородская":-0.9,"DISTRICT\tprev=санкт":-1.0,"DISTRICT\tprev_kind=word":-0.2,"DISTRICT\tsuffix1=г":-1.0,"DISTRICT\tsuffix1=и":-1.0,"DISTRICT\tsuffix1=й":3.0,"DISTRICT\tsuffix1=о":-1.8,"DISTRICT\tsuffix2=во":-0.8,"DISTRICT\tsuffix2=го":-1.0,"DISTRICT\tsuffix2=ий":3.0,"DISTRICT\tsuffix2=рг":-1.0,"DISTRICT\tsuffix2=щи":-1.0,"DISTRICT\tsuffix3=ищи":-1.0,"DISTRICT\tsuffix3=кий":3.9,"DISTRICT\tsuffix3=ний":-0.9,"DISTRICT\tsuffix3=ово":-0.8,"DISTRICT\tsuffix3=ого":-1.0,"DISTRICT\tsuffix3=ург":-1.0,"DISTRICT\tword=боровский":1.9,"DISTRICT\tword=домодедово":-0.8,"DISTRICT\tword=красногорский":1.0,"DISTRICT\tword=ленинградский":-0.6,"DISTRICT\tword=ленинский":1.0,"DISTRICT\tword=ленинского":-1.0,"DISTRICT\tword=мытищи":-1.0,"DISTRICT\tword=нижний":-0.9,"DISTRICT\tword=одинцовский":1.6,"DISTRICT\tword=петербург":-1.0,"DISTRICT\tword=сампсониевский":-1.0,"O\tbias":1.6,"O\tbos":0.1,"O\teos":0.9,"O\tkind=word":1.6,"O\tnext=б":1.4,"O\tnext=балашиха":-0.2,"O\tnext=г":3.0,"O\tnext=гагарина":-0.9,"O\tnext=гоголя":-1.0,"O\tnext=домодедово":-0.5,"O\tnext=й":-1.0,"O\tnext=каширское":2.0,"O\tnext=киров":-0.7,"O\tnext=комсомола":-1.0,"O\tnext=ленинского":-1.0,"O\tnext=марьиной":1.0,"O\tnext=москва":4.3,"O\tnext=мытищи":-0.6,"O\tnext=нижний":-1.0,"O\tnext=новгород":-1.0,"O\tnext=он":1.0,"O\tnext=переулок":1.0,"O\tnext=петербург":-0.9,"O\tnext=подольск":-0.3,"O\tnext=пр":0.9,"O\tnext=правды":-0.9,"O\tnext=реутов":-0.4,"O\tnext=рощи":-1.0,"O\tnext=с":-2.4,"O\tnext=т":1.0,"O\tnext=электрогорск":-0.1,"O\tnext=этаж":1.0,"O\tnext=ямского":-1.0,"O\tnext_kind=word":0.7,"O\tposition=first":0.1,"O\tposition=last":1.1,"O\tposition=middle":0.4,"O\tprefix1=а":2.0,"O\tprefix1=б":0.2,"O\tprefix1=в":-1.7,"O\tprefix1=г":1.1,"O\tprefix1=д":1.0,"O\tprefix1=е":-0.9,"O\tprefix1=з":-0.8,"O\tprefix1=и":-1.0,"O\tprefix1=й":-1.0,"O\tprefix1=к":-1.1,"O\tprefix1=л":-1.0,"O\tprefix1=м":-0.6,"O\tprefix1=н":-4.0,"O\tprefix1=о":-2.0,"O\tprefix1=п":3.8,"O\tprefix1=р":0.7,"O\tprefix1=т":-1.9,"O\tprefix1=у":3.0,"O\tprefix1=ш":3.9,"O\tprefix1=э":1.0,"O\tprefix1=я":0.9,"O\tprefix2=а":2.7,"O\tprefix2=ав":-0.7,"O\tprefix2=бр":-1.0,"O\tprefix2=бу":1.2,"O\tprefix2=ва":-1.6,"O\tprefix2=ви":-1.0,"O\tprefix2=вл":0.9,"O\tprefix2=г":3.0,"O\tprefix2=га":-0.9,"O\tprefix2=гр":-1.0,"O\tprefix2=д":1.0,"O\tprefix2=ер":-0.9,"O\tprefix2=зе":-0.8,"O\tprefix2=ис":-1.0,"O\tprefix2=й":-1.0,"O\tprefix2=к":1.0,"O\tprefix2=ка":-2.8,"O\tprefix2=ки":-0.7,"O\tprefix2=ко":1.4,"O\tprefix2=ле":-1.0,"O\tprefix2=ма":-1.0,"O\tprefix2=ми":-1.0,"O\tprefix2=мо":1.4,"O\tprefix2=ни":-2.0,"O\tprefix2=но":-2.0,"O\tprefix2=оз":-1.0,"O\tprefix2=ок":-1.0,"O\tprefix2=он":1.0,"O\tprefix2=ор":-1.0,"O\tprefix2=п":1.9,"O\tprefix2=пе":0.7,"O\tprefix2=пл":1.0,"O\tprefix2=по":-1.0,"O\tprefix2=пр":1.2,"O\tprefix2=р":1.7,"O\tprefix2=ро":-1.0,"O\tprefix2=с":1.9,"O\tprefix2=са":-0.9,"O\tprefix2=см":-1.0,"O\tprefix2=т":1.0,"O\tprefix2=та":-0.9,"O\tprefix2=то":-2.0,"O\tprefix2=ул":3.0,"O\tprefix2=ш":1.9,"O\tprefix2=шо":2.0,"O\tprefix2=эн":-1.0,"O\tprefix2=эт":2.0,"O\tprefix2=я":0.9,"O\tprev=а":2.7,"O\tprev=боровский":0.1,"O\tprev=бульвар":0.9,"O\tprev=г":2.3,"O\tprev=ермолино":-0.9,"O\tprev=зеленый":1.0,"O\tprev=земляной":-1.0,"O\tprev=калуга":-0.8,"O\tprev=каширское":1.9,"O\tprev=ком":2.4,"O\tprev=ленинского":-1.0,"O\tprev=марьиной":-1.0,"O\tprev=москва":6.5,"O\tprev=нижегородская":-1.0,"O\tprev=нижний":-1.0,"O\tprev=новоивановское":-1.0,"O\tprev=ордынка":-2.4,"O\tprev=песчаный":-0.9,"O\tprev=поля":-1.8,"O\tprev=проспект":1.0,"O\tprev=р":1.0,"O\tprev=рощи":-1.0,"O\tprev=санкт":-2.0,"O\tprev=сокольнический":-0.6,"O\tprev=ш":-0.9,"O\tprev=ямского":-1.0,"O\tprev_kind=word":1.5,"O\tsuffix1=а":-3.3,"O\tsuffix1=в":-2.0,"O\tsuffix1=г":1.0,"O\tsuffix1=д":1.9,"O\tsuffix1=е":-0.9,"O\tsuffix1=ж":2.0,"O\tsuffix1=и":-2.0,"O\tsuffix1=й":-5.8,"O\tsuffix1=к":1.8,"O\tsuffix1=л":1.4,"O\tsuffix1=м":2.4,"O\tsuffix1=н":1.0,"O\tsuffix1=о":-1.9,"O\tsuffix1=п":1.9,"O\tsuffix1=р":4.0,"O\tsuffix1=с":1.9,"O\tsuffix1=т":1.1,"O\tsuffix1=у":-1.0,"O\tsuffix1=ш":1.9,"O\tsuffix1=щ":0.9,"O\tsuffix1=ы":-2.9,"O\tsuffix1=ь":1.0,"O\tsuffix1=я":-2.8,"O\tsuffix2=а":2.7,"O\tsuffix2=ад":0.9,"O\tsuffix2=аж":2.0,"O\tsuffix2=ал":-1.6,"O\tsuffix2=ар":1.2,"O\tsuffix2=ау":-1.0,"O\tsuffix2=ая":-2.7,"O\tsuffix2=ва":-0.2,"O\tsuffix2=г":3.0,"O\tsuffix2=го":-1.0,"O\tsuffix2=д":1.0,"O\tsuffix2=ды":-1.9,"O\tsuffix2=дь":1.0,"O\tsuffix2=ер":-0.9,"O\tsuffix2=ещ":0.9,"O\tsuffix2=зд":1.0,"O\tsuffix2=ий":-3.0,"O\tsuffix2=й":-1.0,"O\tsuffix2=к":1.0,"O\tsuffix2=ка":-1.9,"O\tsuffix2=кт":0.1,"O\tsuffix2=ла":-1.0,"O\tsuffix2=ля":-1.0,"O\tsuffix2=на":-1.9,"O\tsuffix2=но":-0.9,"O\tsuffix2=ов":-2.0,"O\tsuffix2=од":-1.0,"O\tsuffix2=ое":-2.9,"O\tsuffix2=ой":-1.0,"O\tsuffix2=ок":2.7,"O\tsuffix2=ом":2.4,"O\tsuffix2=он":1.0,"O\tsuffix2=п":1.9,"O\tsuffix2=пр":2.0,"O\tsuffix2=р":1.7,"O\tsuffix2=ра":-1.0,"O\tsuffix2=рг":-2.0,"O\tsuffix2=ры":-1.0,"O\tsuffix2=с":1.9,"O\tsuffix2=се":2.0,"O\tsuffix2=ск":-1.9,"O\tsuffix2=т":1.0,"O\tsuffix2=ти":-1.0,"O\tsuffix2=ул":3.0,"O\tsuffix2=ш":1.9,"O\tsuffix2=щи":-1.0,"O\tsuffix2=ый":-0.8,"O\tsuffix2=я":0.9,"O\tsuffix3=а":2.7,"O\tsuffix3=адь":1.0,"O\tsuffix3=вал":-1.6,"O\tsuffix3=вар":1.2,"O\tsuffix3=вды":-1.9,"O\tsuffix3=г":3.0,"O\tsuffix3=д":1.0,"O\tsuffix3=езд":1.0,"O\tsuffix3=ект":1.0,"O\tsuffix3=ина":-1.9,"O\tsuffix3=ино":-0.9,"O\tsuffix3=ира":-1.0,"O\tsuffix3=й":-1.0,"O\tsuffix3=к":1.0,"O\tsuffix3=кая":-2.0,"O\tsuffix3=ква":-0.2,"O\tsuffix3=кий":-2.0,"O\tsuffix3=кое":-1.9,"O\tsuffix3=ком":2.4,"O\tsuffix3=кры":-1.0,"O\tsuffix3=лад":0.9,"O\tsuffix3=лок":2.7,"O\tsuffix3=мау":-1.0,"O\tsuffix3=мещ":0.9,"O\tsuffix3=ная":-0.7,"O\tsuffix3=ний":-1.0,"O\tsuffix3=нка":-1.9,"O\tsuffix3=нкт":-0.9,"O\tsuffix3=ное":-1.0,"O\tsuffix3=ной":-1.0,"O\tsuffix3=нск":-1.0,"O\tsuffix3=ный":-0.8,"O\tsuffix3=ого":-1.0,"O\tsuffix3=ола":-1.0,"O\tsuffix3=оля":-1.0,"O\tsuffix3=он":1.0,"O\tsuffix3=ощи":-1.0,"O\tsuffix3=п":1.9,"O\tsuffix3=пр":2.0,"O\tsuffix3=р":1.7,"O\tsuffix3=род":-1.0,"O\tsuffix3=с":1.9,"O\tsuffix3=сов":-1.0,"O\tsuffix3=ссе":2.0,"O\tsuffix3=т":1.0,"O\tsuffix3=таж":2.0,"O\tsuffix3=тов":-1.0,"O\tsuffix3=тти":-1.0,"O\tsuffix3=ул":3.0,"O\tsuffix3=ург":-2.0,"O\tsuffix3=ш":1.9,"O\tsuffix3=ьер":-0.9,"O\tsuffix3=ьск":-0.9,"O\tsuffix3=я":0.9,"O\tword=а":2.7,"O\tword=авиамоторная":-0.7,"O\tword=брюсов":-1.0,"O\tword=бульвар":1.2,"O\tword=вал":-1.6,"O\tword=видное":-1.0,"O\tword=влад":0.9,"O\tword=г":3.0,"O\tword=гагарина":-0.9,"O\tword=гримау":-1.0,"O\tword=д":1.0,"O\tword=ермолино":-0.9,"O\tword=зеленый":-0.8,"O\tword=искры":-1.0,"O\tword=й":-1.0,"O\tword=к":1.0,"O\tword=калинина":-1.0,"O\tword=карьер":-0.9,"O\tword=каширское":-0.9,"O\tword=кировская":-0.7,"O\tword=ком":2.4,"O\tword=комсомола":-1.0,"O\tword=ленинского":-1.0,"O\tword=марьиной":-1.0,"O\tword=мира":-1.0,"O\tword=можайское":-1.0,"O\tword=москва":-0.2,"O\tword=московская":2.6,"O\tword=нижегородская":-1.0,"O\tword=нижний":-1.0,"O\tword=новгород":-1.0,"O\tword=новомарьинская":-1.0,"O\tword=озерковская":-1.0,"O\tword=октябрьский":-1.0,"O\tword=он":1.0,"O\tword=ордынка":-1.0,"O\tword=п":1.9,"O\tword=переулок":2.7,"O\tword=петербург":-2.0,"O\tword=площадь":1.0,"O\tword=подольск":-0.9,"O\tword=поля":-1.0,"O\tword=помещ":0.9,"O\tword=пр":2.0,"O\tword=правды":-1.9,"O\tword=пречистенка":-0.9,"O\tword=проезд":1.0,"O\tword=проспект":1.0,"O\tword=р":1.7,"O\tword=рощи":-1.0,"O\tword=с":1.9,"O\tword=санкт":-0.9,"O\tword=смоленск":-1.0,"O\tword=т":1.0,"O\tword=тамбовская":-0.9,"O\tword=товарищеский":-1.0,"O\tword=тольятти":-1.0,"O\tword=ул":3.0,"O\tword=ш":1.9,"O\tword=шоссе":2.0,"O\tword=энтузиастов":-1.0,"O\tword=этаж":2.0,"O\tword=я":0.9,"REGION\tbias":-1.1,"REGION\tbos":-0.5,"REGION\teos":-3.5,"REGION\tkind=word":-1.1,"REGION\tnext=балашиха":0.2,"REGION\tnext=боровский":0.9,"REGION\tnext=домодедово":0.5,"REGION\tnext=калуга":1.0,"REGION\tnext=киров":1.6,"REGION\tnext=красногорский":1.0,"REGION\tnext=ленинский":1.3,"REGION\tnext=можайское":-1.0,"REGION\tnext=москва":-5.5,"REGION\tnext=мытищи":1.6,"REGION\tnext=нижний":1.6,"REGION\tnext=новгород":-0.6,"REGION\tnext=одинцовский":1.6,"REGION\tnext=петербург":-1.0,"REGION\tnext=подольск":0.3,"REGION\tnext=реутов":0.4,"REGION\tnext=рощи":-1.0,"REGION\tnext=сенная":-1.3,"REGION\tnext=смоленск":0.6,"REGION\tnext=тольятти":0.7,"REGION\tnext=шоссе":-0.6,"REGION\tnext=электрогорск":0.1,"REGION\tnext_kind=word":2.4,"REGION\tposition=first":-0.5,"REGION\tposition=middle":-0.6,"REGION\tprefix1=в":-0.6,"REGION\tprefix1=д":-1.0,"REGION\tprefix1=к":1.9,"REGION\tprefix1=м":-1.4,"REGION\tprefix1=н":1.0,"REGION\tprefix1=с":-1.0,"REGION\tprefix2=вя":-0.6,"REGION\tprefix2=де":-1.0,"REGION\tprefix2=ка":1.3,"REGION\tprefix2=ки":0.6,"REGION\tprefix2=ма":-1.0,"REGION\tprefix2=мо":-0.4,"REGION\tprefix2=ни":1.0,"REGION\tprefix2=са":-0.3,"REGION\tprefix2=см":-0.7,"REGION\tprev=ул":-0.6,"REGION\tprev_kind=word":-0.6,"REGION\tsuffix1=а":-1.0,"REGION\tsuffix1=в":-1.0,"REGION\tsuffix1=е":-0.6,"REGION\tsuffix1=й":-1.6,"REGION\tsuffix1=т":-1.0,"REGION\tsuffix1=я":4.1,"REGION\tsuffix2=ая":4.1,"REGION\tsuffix2=ва":-1.0,"REGION\tsuffix2=ий":-0.6,"REGION\tsuffix2=кт":-1.0,"REGION\tsuffix2=ов":-1.0,"REGION\tsuffix2=ое":-0.6,"REGION\tsuffix2=ой":-1.0,"REGION\tsuffix3=кая":4.1,"REGION\tsuffix3=ква":-1.0,"REGION\tsuffix3=кое":-0.6,"REGION\tsuffix3=ний":-0.6,"REGION\tsuffix3=нкт":-1.0,"REGION\tsuffix3=ной":-1.0,"REGION\tsuffix3=ров":-1.0,"REGION\tword=вятская":-0.6,"REGION\tword=дербеневская":-1.0,"REGION\tword=калужская":1.9,"REGION\tword=каширское":-0.6,"REGION\tword=киров":-1.0,"REGION\tword=кировская":1.6,"REGION\tword=марьиной":-1.0,"REGION\tword=москва":-1.0,"REGION\tword=московская":0.6,"REGION\tword=нижегородская":1.6,"REGION\tword=нижний":-0.6,"REGION\tword=самарская":0.7,"REGION\tword=санкт":-1.0,"REGION\tword=смоленская":-0.7,"SETTLEMENT\tbias":-1.2,"SETTLEMENT\tbos":-0.4,"SETTLEMENT\teos":-0.3,"SETTLEMENT\tkind=word":-1.2,"SETTLEMENT\tnext=вокзальная":2.0,"SETTLEMENT\tnext=гагарина":-1.0,"SETTLEMENT\tnext=калинина":2.6,"SETTLEMENT\tnext=каширское":1.9,"SETTLEMENT\tnext=кирова":-1.0,"SETTLEMENT\tnext=ленинградская":-1.9,"SETTLEMENT\tnext=нижний":-0.6,"SETTLEMENT\tnext=пр":-0.9,"SETTLEMENT\tnext=сампсониевский":-1.0,"SETTLEMENT\tnext=ш":-1.0,"SETTLEMENT\tnext_kind=word":-0.9,"SETTLEMENT\tposition=first":-0.4,"SETTLEMENT\tposition=last":-1.4,"SETTLEMENT\tposition=middle":0.6,"SETTLEMENT\tprefix1=б":-1.0,"SETTLEMENT\tprefix1=в":-0.7,"SETTLEMENT\tprefix1=е":-1.0,"SETTLEMENT\tprefix1=к":-2.0,"SETTLEMENT\tprefix1=м":-1.0,"SETTLEMENT\tprefix1=н":3.1,"SETTLEMENT\tprefix1=р":-0.3,"SETTLEMENT\tprefix1=с":2.7,"SETTLEMENT\tprefix1=э":-1.0,"SETTLEMENT\tprefix2=бо":-1.0,"SETTLEMENT\tprefix2=ви":-0.7,"SETTLEMENT\tprefix2=ер":-1.0,"SETTLEMENT\tprefix2=ка":-1.0,"SETTLEMENT\tprefix2=ко":-1.0,"SETTLEMENT\tprefix2=мо":-1.0,"SETTLEMENT\tprefix2=на":2.1,"SETTLEMENT\tprefix2=ни":-0.6,"SETTLEMENT\tprefix2=но":1.6,"SETTLEMENT\tprefix2=ре":-0.3,"SETTLEMENT\tprefix2=се":2.7,"SETTLEMENT\tprefix2=эт":-1.0,"SETTLEMENT\tprev=балашиха":-0.4,"SETTLEMENT\tprev=домодедово":0.9,"SETTLEMENT\tprev=красногорский":1.0,"SETTLEMENT\tprev=москва":-1.0,"SETTLEMENT\tprev=нижний":-1.0,"SETTLEMENT\tprev=одинцовский":1.6,"SETTLEMENT\tprev=петербург":-1.0,"SETTLEMENT\tprev=смоленск":-0.9,"SETTLEMENT\tprev_kind=word":-0.8,"SETTLEMENT\tsuffix1=а":-1.0,"SETTLEMENT\tsuffix1=д":-1.0,"SETTLEMENT\tsuffix1=е":1.8,"SETTLEMENT\tsuffix1=ж":-1.0,"SETTLEMENT\tsuffix1=о":0.6,"SETTLEMENT\tsuffix1=я":-0.6,"SETTLEMENT\tsuffix2=аж":-1.0,"SETTLEMENT\tsuffix2=ая":-0.6,"SETTLEMENT\tsuffix2=ва":-1.0,"SETTLEMENT\tsuffix2=во":-1.9,"SETTLEMENT\tsuffix2=но":2.5,"SETTLEMENT\tsuffix2=од":-1.0,"SETTLEMENT\tsuffix2=ое":1.8,"SETTLEMENT\tsuffix2=ой":-1.0,"SETTLEMENT\tsuffix2=ый":1.0,"SETTLEMENT\tsuffix3=ино":2.5,"SETTLEMENT\tsuffix3=кая":-0.6,"SETTLEMENT\tsuffix3=кое":2.5,"SETTLEMENT\tsuffix3=ное":-0.7,"SETTLEMENT\tsuffix3=ный":1.0,"SETTLEMENT\tsuffix3=ова":-1.0,"SETTLEMENT\tsuffix3=ово":-1.9,"SETTLEMENT\tsuffix3=род":-1.0,"SETTLEMENT\tsuffix3=таж":-1.0,"SETTLEMENT\tsuffix3=шой":-1.0,"SETTLEMENT\tword=большой":-1.0,"SETTLEMENT\tword=видное":-0.7,"SETTLEMENT\tword=ермолино":-1.0,"SETTLEMENT\tword=каширское":-1.0,"SETTLEMENT\tword=колпакова":-1.0,"SETTLEMENT\tword=можайское":-1.0,"SETTLEMENT\tword=научный":-1.4,"SETTLEMENT\tword=нахабино":3.5,"SETTLEMENT\tword=нижегородская":-0.6,"SETTLEMENT\tword=новгород":-1.0,"SETTLEMENT\tword=ново":-1.9,"SETTLEMENT\tword=новоивановское":4.5,"SETTLEMENT\tword=революционный":-0.3,"SETTLEMENT\tword=северный":2.7,"SETTLEMENT\tword=этаж":-1.0,"STREET\tbias":0.8,"STREET\tbos":0.3,"STREET\teos":3.0,"STREET\tkind=word":0.8,"STREET\tnext=б":0.4,"STREET\tnext=боровский":-0.9,"STREET\tnext=буденного":-1.0,"STREET\tnext=вал":0.9,"STREET\tnext=вокзальная":-1.0,"STREET\tnext=володарского":-0.9,"STREET\tnext=воронина":-1.0,"STREET\tnext=донского":1.0,"STREET\tnext=ермолино":-0.9,"STREET\tnext=й":2.0,"STREET\tnext=калинина":-1.6,"STREET\tnext=калуга":-1.0,"STREET\tnext=каширское":-1.9,"STREET\tnext=киров":-0.9,"STREET\tnext=колпакова":-1.0,"STREET\tnext=комсомола":2.0,"STREET\tnext=красногорский":-1.0,"STREET\tnext=ленинградская":3.4,"STREET\tnext=ленинский":-0.8,"STREET\tnext=ленинского":-1.0,"STREET\tnext=марьиной":-1.0,"STREET\tnext=мытищи":-1.0,"STREET\tnext=нахабино":-1.0,"STREET\tnext=новгород":-1.0,"STREET\tnext=новоивановское":-1.6,"STREET\tnext=одинцовский":-1.6,"STREET\tnext=октябрьский":-1.0,"STREET\tnext=ополчения":1.0,"STREET\tnext=петербург":-1.0,"STREET\tnext=пр":3.4,"STREET\tnext=р":-1.0,"STREET\tnext=рощи":2.0,"STREET\tnext=с":2.4,"STREET\tnext=сампсониевский":2.0,"STREET\tnext=северный":-2.0,"STREET\tnext=сенная":1.3,"STREET\tnext=смоленск":-0.6,"STREET\tnext=тольятти":-0.7,"STREET\tnext=ш":1.9,"STREET\tnext=шоссе":0.6,"STREET\tnext=я":1.9,"STREET\tnext_kind=word":-2.2,"STREET\tposition=first":0.3,"STREET\tposition=last":2.5,"STREET\tposition=middle":-2.0,"STREET\tprefix1=а":-1.0,"STREET\tprefix1=б":-0.2,"STREET\tprefix1=в":1.3,"STREET\tprefix1=г":-0.1,"STREET\tprefix1=д":1.0,"STREET\tprefix1=е":-0.6,"STREET\tprefix1=з":1.8,"STREET\tprefix1=и":2.0,"STREET\tprefix1=й":1.0,"STREET\tprefix1=к":0.1,"STREET\tprefix1=л":2.6,"STREET\tprefix1=м":-0.2,"STREET\tprefix1=н":-0.6,"STREET\tprefix1=о":1.4,"STREET\tprefix1=п":-1.9,"STREET\tprefix1=с":-1.8,"STREET\tprefix1=т":-0.1,"STREET\tprefix1=у":-2.0,"STREET\tprefix1=ш":-2.9,"STREET\tprefix1=э":0.9,"STREET\tprefix1=я":0.1,"STREET\tprefix2=а":-1.7,"STREET\tprefix2=ав":0.7,"STREET\tprefix2=б":1.0,"STREET\tprefix2=ба":-2.0,"STREET\tprefix2=бо":0.1,"STREET\tprefix2=бр":1.9,"STREET\tprefix2=бу":-1.2,"STREET\tprefix2=ва":2.6,"STREET\tprefix2=ви":-1.9,"STREET\tprefix2=вл":-0.9,"STREET\tprefix2=во":0.9,"STREET\tprefix2=вя":0.6,"STREET\tprefix2=г":-2.0,"STREET\tprefix2=га":0.9,"STREET\tprefix2=гр":1.0,"STREET\tprefix2=де":1.0,"STREET\tprefix2=дм":1.0,"STREET\tprefix2=до":-1.0,"STREET\tprefix2=ер":-0.6,"STREET\tprefix2=зе":1.8,"STREET\tprefix2=ис":2.0,"STREET\tprefix2=й":1.0,"STREET\tprefix2=к":-1.0,"STREET\tprefix2=ка":0.6,"STREET\tprefix2=ки":-0.9,"STREET\tprefix2=ко":1.4,"STREET\tprefix2=кр":-1.0,"STREET\tprefix2=ку":1.0,"STREET\tprefix2=ле":2.6,"STREET\tprefix2=ма":2.0,"STREET\tprefix2=ми":1.6,"STREET\tprefix2=мо":-2.8,"STREET\tprefix2=мы":-1.0,"STREET\tprefix2=на":0.5,"STREET\tprefix2=ни":-1.0,"STREET\tprefix2=но":-0.1,"STREET\tprefix2=од":-1.6,"STREET\tprefix2=оз":1.0,"STREET\tprefix2=ок":1.0,"STREET\tprefix2=он":-1.0,"STREET\tprefix2=ор":1.0,"STREET\tprefix2=оч":1.0,"STREET\tprefix2=п":-0.9,"STREET\tprefix2=пе":-1.8,"STREET\tprefix2=пл":-1.0,"STREET\tprefix2=пр":1.8,"STREET\tprefix2=р":-0.7,"STREET\tprefix2=ре":-1.3,"STREET\tprefix2=ро":2.0,"STREET\tprefix2=с":-1.0,"STREET\tprefix2=са":0.3,"STREET\tprefix2=се":-2.7,"STREET\tprefix2=см":0.7,"STREET\tprefix2=со":0.9,"STREET\tprefix2=т":-1.0,"STREET\tprefix2=та":0.9,"STREET\tprefix2=ул":-2.0,"STREET\tprefix2=ш":-0.9,"STREET\tprefix2=шо":-2.0,"STREET\tprefix2=эн":1.9,"STREET\tprefix2=эт":-1.0,"STREET\tprefix2=я":-0.9,"STREET\tprefix2=яр":1.0,"STREET\tprev=а":-2.7,"STREET\tprev=балашиха":0.4,"STREET\tprev=большой":1.0,"STREET\tprev=бульвар":-0.9,"STREET\tprev=г":-3.0,"STREET\tprev=домодедово":-0.9,"STREET\tprev=ермолино":0.9,"STREET\tprev=земляной":1.0,"STREET\tprev=калуга":0.8,"STREET\tprev=калужская":-1.9,"STREET\tprev=каширское":-1.9,"STREET\tprev=кировская":-0.9,"STREET\tprev=ком":-2.4,"STREET\tprev=красногорский":-1.0,"STREET\tprev=ленинский":-1.0,"STREET\tprev=ленинского":1.0,"STREET\tprev=марьиной":2.0,"STREET\tprev=москва":5.0,"STREET\tprev=московская":-4.6,"STREET\tprev=нижний":-1.0,"STREET\tprev=новоивановское":1.0,"STREET\tprev=одинцовский":-1.6,"STREET\tprev=ордынка":3.4,"STREET\tprev=песчаный":0.9,"STREET\tprev=петербург":1.0,"STREET\tprev=поля":1.8,"STREET\tprev=проспект":-1.0,"STREET\tprev=р":-1.0,"STREET\tprev=рощи":1.0,"STREET\tprev=санкт":-0.8,"STREET\tprev=смоленск":1.9,"STREET\tprev=сокольнический":0.6,"STREET\tprev=ул":0.6,"STREET\tprev=ш":0.9,"STREET\tprev=ямского":1.9,"STREET\tprev_kind=word":0.5,"STREET\tsuffix1=а":2.7,"STREET\tsuffix1=б":1.0,"STREET\tsuffix1=в":0.4,"STREET\tsuffix1=г":-2.8,"STREET\tsuffix1=д":-2.9,"STREET\tsuffix1=е":-0.2,"STREET\tsuffix1=ж":-1.0,"STREET\tsuffix1=й":6.8,"STREET\tsuffix1=к":-4.0,"STREET\tsuffix1=л":-0.4,"STREET\tsuffix1=м":-0.6,"STREET\tsuffix1=н":-1.0,"STREET\tsuffix1=о":2.9,"STREET\tsuffix1=п":-0.9,"STREET\tsuffix1=р":-2.0,"STREET\tsuffix1=с":-1.0,"STREET\tsuffix1=т":-2.0,"STREET\tsuffix1=у":1.0,"STREET\tsuffix1=ш":-0.9,"STREET\tsuffix1=щ":-0.9,"STREET\tsuffix1=ы":3.9,"STREET\tsuffix1=ь":-1.0,"STREET\tsuffix1=я":3.7,"STREET\tsuffix2=а":-1.7,"STREET\tsuffix2=ад":-0.9,"STREET\tsuffix2=аж":-1.0,"STREET\tsuffix2=ал":1.6,"STREET\tsuffix2=ар":-1.2,"STREET\tsuffix2=ау":1.0,"STREET\tsuffix2=ая":1.7,"STREET\tsuffix2=б":1.0,"STREET\tsuffix2=ва":0.9,"STREET\tsuffix2=во":2.4,"STREET\tsuffix2=г":-2.0,"STREET\tsuffix2=га":-2.8,"STREET\tsuffix2=го":3.0,"STREET\tsuffix2=ды":1.9,"STREET\tsuffix2=дь":-1.0,"STREET\tsuffix2=ер":0.9,"STREET\tsuffix2=ещ":-0.9,"STREET\tsuffix2=зд":-1.0,"STREET\tsuffix2=ия":1.0,"STREET\tsuffix2=й":1.0,"STREET\tsuffix2=к":-1.0,"STREET\tsuffix2=ка":2.9,"STREET\tsuffix2=кт":-1.0,"STREET\tsuffix2=ла":1.0,"STREET\tsuffix2=ля":1.9,"STREET\tsuffix2=на":2.8,"STREET\tsuffix2=но":-2.5,"STREET\tsuffix2=ов":0.4,"STREET\tsuffix2=од":-1.0,"STREET\tsuffix2=ое":1.8,"STREET\tsuffix2=ой":4.0,"STREET\tsuffix2=ок":-1.0,"STREET\tsuffix2=ом":-0.6,"STREET\tsuffix2=он":-1.0,"STREET\tsuffix2=п":-0.9,"STREET\tsuffix2=пр":-1.0,"STREET\tsuffix2=р":-0.7,"STREET\tsuffix2=ра":1.6,"STREET\tsuffix2=рг":-0.8,"STREET\tsuffix2=ры":2.0,"STREET\tsuffix2=с":-1.0,"STREET\tsuffix2=се":-2.0,"STREET\tsuffix2=ск":-2.0,"STREET\tsuffix2=т":-1.0,"STREET\tsuffix2=ти":-1.0,"STREET\tsuffix2=ул":-2.0,"STREET\tsuffix2=ха":-2.0,"STREET\tsuffix2=ш":-0.9,"STREET\tsuffix2=щи":1.0,"STREET\tsuffix2=ый":1.8,"STREET\tsuffix2=я":-0.9,"STREET\tsuffix3=а":-1.7,"STREET\tsuffix3=адь":-1.0,"STREET\tsuffix3=б":1.0,"STREET\tsuffix3=вал":1.6,"STREET\tsuffix3=вар":-1.2,"STREET\tsuffix3=вды":1.9,"STREET\tsuffix3=г":-2.0,"STREET\tsuffix3=езд":-1.0,"STREET\tsuffix3=ина":2.8,"STREET\tsuffix3=ино":-2.5,"STREET\tsuffix3=ира":1.6,"STREET\tsuffix3=иха":-2.0,"STREET\tsuffix3=ищи":-1.0,"STREET\tsuffix3=й":1.0,"STREET\tsuffix3=к":-1.0,"STREET\tsuffix3=кая":1.0,"STREET\tsuffix3=ква":-2.9,"STREET\tsuffix3=кий":1.0,"STREET\tsuffix3=кое":3.7,"STREET\tsuffix3=ком":-0.6,"STREET\tsuffix3=кры":2.0,"STREET\tsuffix3=лад":-0.9,"STREET\tsuffix3=лок":-1.0,"STREET\tsuffix3=мау":1.0,"STREET\tsuffix3=мещ":-0.9,"STREET\tsuffix3=ная":0.7,"STREET\tsuffix3=ний":-1.0,"STREET\tsuffix3=нка":2.9,"STREET\tsuffix3=нкт":-1.0,"STREET\tsuffix3=ное":-1.9,"STREET\tsuffix3=ной":2.0,"STREET\tsuffix3=ный":1.8,"STREET\tsuffix3=ова":3.8,"STREET\tsuffix3=ово":2.4,"STREET\tsuffix3=ого":3.0,"STREET\tsuffix3=ола":1.0,"STREET\tsuffix3=оля":1.9,"STREET\tsuffix3=он":-1.0,"STREET\tsuffix3=ощи":2.0,"STREET\tsuffix3=п":-0.9,"STREET\tsuffix3=пр":-1.0,"STREET\tsuffix3=р":-0.7,"STREET\tsuffix3=рия":1.0,"STREET\tsuffix3=ров":-1.8,"STREET\tsuffix3=род":-1.0,"STREET\tsuffix3=рск":-1.0,"STREET\tsuffix3=с":-1.0,"STREET\tsuffix3=сов":1.9,"STREET\tsuffix3=ссе":-2.0,"STREET\tsuffix3=т":-1.0,"STREET\tsuffix3=таж":-1.0,"STREET\tsuffix3=тов":0.3,"STREET\tsuffix3=тти":-1.0,"STREET\tsuffix3=уга":-2.8,"STREET\tsuffix3=ул":-2.0,"STREET\tsuffix3=ург":-0.8,"STREET\tsuffix3=ш":-0.9,"STREET\tsuffix3=шой":2.0,"STREET\tsuffix3=ьер":0.9,"STREET\tsuffix3=ьск":-1.0,"STREET\tsuffix3=я":-0.9,"STREET\tword=а":-1.7,"STREET\tword=авиамоторная":0.7,"STREET\tword=б":1.0,"STREET\tword=балашиха":-2.0,"STREET\tword=большой":2.0,"STREET\tword=боровский":-1.9,"STREET\tword=брюсов":1.9,"STREET\tword=бульвар":-1.2,"STREET\tword=вавилова":1.0,"STREET\tword=вал":1.6,"STREET\tword=видное":-1.9,"STREET\tword=влад":-0.9,"STREET\tword=воронина":0.9,"STREET\tword=вятская":0.6,"STREET\tword=г":-2.0,"STREET\tword=гагарина":0.9,"STREET\tword=гримау":1.0,"STREET\tword=дербеневская":1.0,"STREET\tword=дмитрия":1.0,"STREET\tword=домодедово":-1.0,"STREET\tword=ермолино":-0.6,"STREET\tword=зеленый":1.8,"STREET\tword=искры":2.0,"STREET\tword=й":1.0,"STREET\tword=к":-1.0,"STREET\tword=калинина":1.0,"STREET\tword=калуга":-2.8,"STREET\tword=калужская":-1.9,"STREET\tword=карьер":0.9,"STREET\tword=каширское":3.4,"STREET\tword=киров":-1.8,"STREET\tword=кирова":1.8,"STREET\tword=кировская":-0.9,"STREET\tword=колпакова":1.0,"STREET\tword=ком":-0.6,"STREET\tword=комсомола":1.0,"STREET\tword=красногорский":-1.0,"STREET\tword=кутузовский":1.0,"STREET\tword=ленинградский":0.6,"STREET\tword=ленинского":2.0,"STREET\tword=марьиной":2.0,"STREET\tword=мира":1.6,"STREET\tword=можайское":2.8,"STREET\tword=москва":-2.9,"STREET\tword=московская":-2.7,"STREET\tword=мытищи":-1.0,"STREET\tword=народного":1.0,"STREET\tword=научный":1.4,"STREET\tword=нахабино":-1.9,"STREET\tword=нижний":-1.0,"STREET\tword=новгород":-1.0,"STREET\tword=ново":3.4,"STREET\tword=новоивановское":-3.5,"STREET\tword=новомарьинская":1.0,"STREET\tword=одинцовский":-1.6,"STREET\tword=озерковская":1.0,"STREET\tword=октябрьский":1.0,"STREET\tword=он":-1.0,"STREET\tword=ордынка":1.0,"STREET\tword=очаковское":1.0,"STREET\tword=п":-0.9,"STREET\tword=переулок":-1.0,"STREET\tword=петербург":-0.8,"STREET\tword=площадь":-1.0,"STREET\tword=подольск":-1.0,"STREET\tword=поля":1.9,"STREET\tword=помещ":-0.9,"STREET\tword=пр":-1.0,"STREET\tword=правды":1.9,"STREET\tword=пречистенка":1.9,"STREET\tword=проезд":-1.0,"STREET\tword=р":-0.7,"STREET\tword=революционный":0.3,"STREET\tword=реутов":-1.6,"STREET\tword=рощи":2.0,"STREET\tword=с":-1.0,"STREET\tword=садовническая":1.0,"STREET\tword=самарская":-0.7,"STREET\tword=сампсониевский":1.0,"STREET\tword=санкт":-1.0,"STREET\tword=северный":-2.7,"STREET\tword=смоленская":0.7,"STREET\tword=сокольнический":0.9,"STREET\tword=т":-1.0,"STREET\tword=тамбовская":0.9,"STREET\tword=товарищеский":1.0,"STREET\tword=тольятти":-1.0,"STREET\tword=ул":-2.0,"STREET\tword=ш":-0.9,"STREET\tword=шоссе":-2.0,"STREET\tword=электрогорск":-1.0,"STREET\tword=электродный":1.0,"STREET\tword=энтузиастов":1.9,"STREET\tword=этаж":-1.0,"STREET\tword=я":-0.9,"STREET\tword=ярославская":1.0},"transitions":{"\tCITY":1.1,"\tDISTRICT":-0.6,"\tO":0.1,"\tREGION":-0.5,"\tSETTLEMENT":-0.4,"\tSTREET":0.3,"CITY\t":0.9,"CITY\tCITY":-2.6,"CITY\tDISTRICT":-0.5,"CITY\tO":0.3,"CITY\tSETTLEMENT":0.4,"CITY\tSTREET":2.2,"DISTRICT\t":-1.0,"DISTRICT\tCITY":1.0,"DISTRICT\tO":-0.6,"DISTRICT\tSETTLEMENT":0.8,"DISTRICT\tSTREET":-1.0,"O\t":0.9,"O\tCITY":1.8,"O\tO":2.6,"O\tREGION":-0.6,"O\tSTREET":-3.1,"REGION\t":-3.5,"REGION\tCITY":4.2,"REGION\tDISTRICT":3.1,"REGION\tO":-0.6,"REGION\tSETTLEMENT":-1.0,"REGION\tSTREET":-3.3,"SETTLEMENT\t":-0.3,"SETTLEMENT\tCITY":-0.6,"SETTLEMENT\tDISTRICT":-1.0,"SETTLEMENT\tO":-1.9,"SETTLEMENT\tSTREET":2.6,"STREET\t":3.0,"STREET\tCITY":-4.2,"STREET\tDISTRICT":-1.8,"STREET\tO":1.7,"STREET\tSETTLEMENT":-1.0,"STREET\tSTREET":3.1}} \ No newline at end of file diff --git a/tests.py b/tests.py deleted file mode 100644 index d3a41a2..0000000 --- a/tests.py +++ /dev/null @@ -1,83 +0,0 @@ -import pandas as pd -from api import get_addr - -dadata_LUT = {'original': 'Исходный адрес', - 'fullname': 'Адрес', - 'index': 'Индекс', - 'postalcode': 'Индекс', - 'country': 'Страна', - 'region_type': 'Тип региона', - 'region': 'Регион', - 'area_type': 'Тип района', - 'area': 'Район', - 'city_type': 'Тип города', - 'city': 'Город', - '65_type': 'Тип н/п', - '65': 'Н/п', - '???': 'Адм. округ', - 'town': 'Н/п', - 'town_type': 'Тип н/п', - 'district_type': 'Тип района', - 'district': 'Район города', - 'street_type': 'Тип улицы', - 'street': 'Улица', - 'house_type': 'Тип дома', - # 'housenum':'Дом', - # 'build_type':'Тип корпуса/строения', - # 'buildnum':'Корпус/строение', - # 'struc_type':'Тип корпуса/строения', - # 'strucnum':'Корпус/строение', - 'flat_type': 'Тип квартиры', - 'flat_num': 'Номер Квартиры', - 'houseid': "Код ФИАС" - } - - -def score(ref, orig_col="Исходный адрес", func=get_addr, - cols_to_score=["Регион", "Город", "Н/п", "Район", "Улица", "Дом", "Корпус/строение"]): - df_1 = func(ref[orig_col]).rename(index=str, columns=dadata_LUT) - if 'Индекс' in cols_to_score: - ref["Индекс"] = ref["Индекс"].fillna('9999999').astype(int).astype(str).replace('9999999', '') - ref, df_1 = ref.fillna(''), df_1.fillna('') - df_1 = df_1.to_dict(orient='records') - N = ref.shape[0] - ref = ref.to_dict(orient='records') - n = 0 - correct = 0 - df = [] - if len(df_1) != len(ref): - return "не совпадают размеры таблицы" - for i, row in enumerate(ref): - for key, value in row.items(): - if key in cols_to_score: # ["Регион", "Индекс", "Район", "Город", "Н/п", "Улица", "Дом", "Корпус/строение"]: - n += 1 - if str(value) == str(df_1[i][key]): - correct += 1 - else: - df.append(df_1[i]) - print("\n{0:03.1f}% correct fields".format(correct / n * 100)) - df = pd.DataFrame(df).drop_duplicates() - print("\n{0:03.1f}% correct lines".format((N - df.shape[0]) / N * 100)) - return df - - -def score_by_id(ref, orig_col="Исходный адрес", func=get_addr): - ref['Тип корпуса/строения'].astype(str) - ref = ref[[orig_col, 'Код ФИАС']][ - (ref['Уровень по ФИАС'] == '8: дом') & (ref['Тип корпуса/строения'].astype(str) != 'nan')] - predicted_df = func(ref[orig_col]) - true_id = ref['Код ФИАС'] - predicted_id = predicted_df['houseid'] - correct = 0 - incorrect = 0 - for true, predicted in zip(true_id, predicted_id): - if true == predicted: - correct += 1 - else: - incorrect += 1 - accuracy = correct / (correct + incorrect) * 100 - print() - print("Accuracy: {0:03.3f}%".format(accuracy)) - - -ref = pd.read_excel('ref/references.xlsx') diff --git a/tests_v2/test_adversarial.py b/tests/test_adversarial.py similarity index 100% rename from tests_v2/test_adversarial.py rename to tests/test_adversarial.py diff --git a/tests_v2/test_api.py b/tests/test_api.py similarity index 100% rename from tests_v2/test_api.py rename to tests/test_api.py diff --git a/tests_v2/test_detection.py b/tests/test_detection.py similarity index 61% rename from tests_v2/test_detection.py rename to tests/test_detection.py index 1729ef1..a0fec88 100644 --- a/tests_v2/test_detection.py +++ b/tests/test_detection.py @@ -1,26 +1,11 @@ from __future__ import annotations -import importlib.util import json -from pathlib import Path import pytest from address_normalizer import DetectedAddress, detect_addresses - -ROOT = Path(__file__).parents[1] - - -def _load_evaluator(): - path = ROOT / "evaluation/evaluate_detection.py" - spec = importlib.util.spec_from_file_location("evaluate_detection", path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - def test_detects_exact_span_and_keeps_component_offsets_relative(): message = ( "Курьер приедет по адресу: Москва, ул. Тверская, " @@ -86,41 +71,3 @@ def test_conservative_policy_rejects_weak_address_evidence(message): def test_detection_rejects_non_string_input(): with pytest.raises(TypeError, match="text must be a string"): detect_addresses(None) - - -def test_detection_reference_has_detailed_scenario_columns_and_passes(): - evaluator = _load_evaluator() - rows = evaluator.load_rows(ROOT / "evaluation/detection_reference.jsonl") - - assert len(rows) == 30 - assert all( - { - "id", - "message", - "expected", - "scenario_family", - "context_style", - "address_style", - "boundary_style", - "polarity", - "ambiguity", - "notes", - } - <= set(row) - for row in rows - ) - report = evaluator.evaluate(rows) - assert report["exact_span_micro"]["f1"] == 1.0 - assert report["negative_message_specificity"] == 1.0 - assert report["failure_sample"] == [] - - -def test_committed_detection_report_matches_current_result(): - evaluator = _load_evaluator() - rows = evaluator.load_rows(ROOT / "evaluation/detection_reference.jsonl") - actual = evaluator.evaluate(rows) - committed = json.loads( - (ROOT / "evaluation/detection_report.json").read_text(encoding="utf-8") - ) - - assert actual == committed diff --git a/tests_v2/test_model.py b/tests/test_model.py similarity index 89% rename from tests_v2/test_model.py rename to tests/test_model.py index 1e255cd..fae4a3c 100644 --- a/tests_v2/test_model.py +++ b/tests/test_model.py @@ -9,7 +9,7 @@ from address_normalizer import parse from address_normalizer.tagger import CompactSequenceTagger from address_normalizer.tokenizer import tokenize -from training.real_corpus import ( +from tools.model_data import ( build_examples, corpus_summary, load_reference_rows, @@ -38,13 +38,13 @@ def test_bundled_model_was_trained_on_grouped_real_examples(): ) training = payload["training"] assert training["algorithm"] == "epoch-averaged structured perceptron" - assert training["dataset"] == "evaluation/legacy_reference_500.jsonl" + assert training["dataset"] == "benchmarks/legacy_500.jsonl" assert training["examples"] >= 300 def test_real_corpus_has_disjoint_nonempty_splits(): examples = build_examples( - load_reference_rows(ROOT / "evaluation/legacy_reference_500.jsonl") + load_reference_rows(ROOT / "benchmarks/legacy_500.jsonl") ) summary = corpus_summary(examples) assert summary["leaking_groups"] == [] diff --git a/tests_v2/test_runtime_contract.py b/tests/test_runtime_contract.py similarity index 100% rename from tests_v2/test_runtime_contract.py rename to tests/test_runtime_contract.py diff --git a/tests_v2/test_spans.py b/tests/test_spans.py similarity index 100% rename from tests_v2/test_spans.py rename to tests/test_spans.py diff --git a/tests_v2/test_datamos_evaluation.py b/tests_v2/test_datamos_evaluation.py deleted file mode 100644 index 1e64403..0000000 --- a/tests_v2/test_datamos_evaluation.py +++ /dev/null @@ -1,63 +0,0 @@ -from pathlib import Path -import sys - -ROOT = Path(__file__).parents[1] -sys.path.insert(0, str(ROOT / "evaluation")) - -from datamos_data import ( - expected_components, - group_id_and_split, - rejection_reason, -) -from evaluate_datamos import _summary, score - - -def _row() -> dict: - return { - "OnTerritoryOfMoscow": "да", - "ADR_TYPE": "Официальный", - "SOSTAD": "Зарегистрирован в АР", - "STATUS": "Внесён в ГКН", - "SIMPLE_ADDRESS": "Косинская улица, дом 26А", - "ADDRESS": "город Москва, Косинская улица, дом 26А", - "P7": "Косинская улица", - "L1_VALUE": "26А", - "L2_VALUE": "", - "L3_VALUE": "", - "N_FIAS": "235212A3-01E8-4CC3-87D5-59F00C83898A", - } - - -def test_datamos_high_confidence_filter_and_grouping(): - row = _row() - assert rejection_reason(row) is None - assert expected_components(row) == { - "street": "Косинская улица", - "house_num": "26А", - "corpus": None, - "structure": None, - } - group_id, split = group_id_and_split(row) - assert len(group_id) == 32 - assert split in {"train", "validation", "test"} - row["STATUS"] = "Аннулирован в ГКН" - assert rejection_reason(row) == "not_in_gkn" - - -def test_exact_moscow_component_evaluation(): - row = { - "source_row": 0, - "fias_id": "235212a3-01e8-4cc3-87d5-59f00c83898a", - "tier": "house_only", - "raw": "Косинская улица, дом 26А", - "expected": expected_components(_row()), - } - report = score([row]) - assert report["micro"]["f1"] == 1.0 - assert report["exact_address_rate"] == 1.0 - assert report["exact_component_value_micro"] == report["micro"] - assert report["metric_definitions"]["fields"].startswith("per-field") - summary = _summary(report) - assert summary["micro"] == report["exact_component_value_micro"] - assert "metric_definitions" not in summary - assert "exact_component_value_micro" not in summary diff --git a/tests_v2/test_deepparse_evaluation.py b/tests_v2/test_deepparse_evaluation.py deleted file mode 100644 index 02d93d6..0000000 --- a/tests_v2/test_deepparse_evaluation.py +++ /dev/null @@ -1,119 +0,0 @@ -from pathlib import Path -import sys - -ROOT = Path(__file__).parents[1] -sys.path.insert(0, str(ROOT / "evaluation")) - -from deepparse_data import ( - expected_components, - group_id_and_split, - mapped_labels, - normalized_address_id, - quality_tier, -) -from evaluate_deepparse import _score, _summary, gold_spans, predicted_spans -from address_normalizer import parse - - -def test_deepparse_tags_map_to_package_fields(): - tags = ( - "Country", - "Municipality", - "StreetName", - "StreetName", - "StreetNumber", - "Unit", - ) - labels = mapped_labels(tags) - assert labels == ( - "O", - "CITY", - "STREET", - "STREET", - "HOUSE", - "APARTMENT", - ) - assert expected_components( - ("Россия", "Самара", "ул", "Авроры", "7", "12"), - labels, - ) == { - "postal_code": None, - "region": None, - "district": None, - "city": "Самара", - "street": "ул Авроры", - "house_num": "7", - "apartment": "12", - } - assert quality_tier(tags) == "street_house_unit" - - -def test_format_variants_share_a_split_and_exact_text_ids_do_not(): - tokens = ("Самара", "ул", "Авроры", "7", "12") - tags = ( - "Municipality", - "StreetName", - "StreetName", - "StreetNumber", - "Unit", - ) - group_a = group_id_and_split(tokens, tags) - group_b = group_id_and_split(tokens[:-1], tags[:-1]) - assert group_a == group_b - assert normalized_address_id("Самара ул Авроры 7") == ( - normalized_address_id(" самара УЛ авроры 7 ") - ) - assert normalized_address_id("Самара ул Авроры 8") != ( - normalized_address_id("Самара ул Авроры 7") - ) - - -def test_gold_and_predicted_spans_align_on_a_conventional_address(): - row = { - "raw": "Россия Самара ул Авроры 7 12", - "tokens": ["Россия", "Самара", "ул", "Авроры", "7", "12"], - "labels": ["O", "CITY", "STREET", "STREET", "HOUSE", "APARTMENT"], - } - assert [ - (span.label, row["raw"][span.start : span.end]) - for span in gold_spans(row) - ] == [ - ("CITY", "Самара"), - ("STREET", "ул Авроры"), - ("HOUSE", "7"), - ("APARTMENT", "12"), - ] - result = parse(row["raw"]) - assert { - (span.label, row["raw"][span.start : span.end]) - for span in predicted_spans(result) - } >= { - ("CITY", "Самара"), - ("STREET", "ул Авроры"), - ("HOUSE", "7"), - ("APARTMENT", "12"), - } - - row.update( - { - "source_row": 1, - "example_id": "example", - "tier": "street_house_unit", - } - ) - report = _score([row]) - assert report["character_micro"]["f1"] == 1.0 - assert report["token_micro"]["f1"] == 1.0 - assert report["exact_token_sequence_rate"] == 1.0 - assert report["span_overlap_micro"] == report["micro"] - assert report["character_overlap_micro"] == report["character_micro"] - assert report["token_label_micro"] == report["token_micro"] - assert report["metric_definitions"]["fields"].startswith("per-field") - assert report["metric_definitions"]["character_fields"].startswith("per-field") - assert report["metric_definitions"]["token_fields"].startswith("per-field") - summary = _summary(report) - assert summary["micro"] == report["span_overlap_micro"] - assert "metric_definitions" not in summary - assert "span_overlap_micro" not in summary - assert "character_overlap_micro" not in summary - assert "token_label_micro" not in summary diff --git a/tests_v2/test_evaluation.py b/tests_v2/test_evaluation.py deleted file mode 100644 index 71c0690..0000000 --- a/tests_v2/test_evaluation.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -import subprocess -import sys - -import pytest - - -ROOT = Path(__file__).parents[1] - - -def test_legacy_reference_is_unique_and_fixed_size(): - rows = [ - json.loads(line) - for line in (ROOT / "evaluation/legacy_reference_500.jsonl") - .read_text(encoding="utf-8") - .splitlines() - if line.strip() - ] - assert len(rows) == 500 - assert len({row["raw"] for row in rows}) == 500 - assert all(row["expected"].get("street") for row in rows) - assert all(row["expected"].get("house_num") for row in rows) - - -def test_legacy_release_gate_passes(): - completed = subprocess.run( - [ - sys.executable, - str(ROOT / "evaluation/evaluate.py"), - "--data", - str(ROOT / "evaluation/legacy_reference_500.jsonl"), - "--gates", - str(ROOT / "evaluation/release_gates.json"), - ], - check=False, - capture_output=True, - text=True, - ) - assert completed.returncode == 0, completed.stdout + completed.stderr - report = json.loads(completed.stdout) - assert report["release_gate_passed"] is True - assert report["exact_component_value_micro"] == report["micro"] - assert "exact component values" in ( - report["metric_definitions"]["exact_component_value_micro"] - ) - committed = json.loads( - (ROOT / "evaluation/legacy_reference_500_report.json").read_text( - encoding="utf-8" - ) - ) - assert committed == report - - -@pytest.mark.parametrize( - ("filename", "aliases"), - [ - ("redmadrobot_report.json", {"span_overlap_micro": "micro"}), - ( - "deepparse_report.json", - { - "span_overlap_micro": "micro", - "character_overlap_micro": "character_micro", - "token_label_micro": "token_micro", - }, - ), - ("datamos_report.json", {"exact_component_value_micro": "micro"}), - ], -) -def test_committed_reports_use_explicit_metric_names(filename, aliases): - report = json.loads((ROOT / "evaluation" / filename).read_text(encoding="utf-8")) - - assert report["metric_definitions"] - for explicit_name, compatibility_name in aliases.items(): - assert report[explicit_name] == report[compatibility_name] diff --git a/tests_v2/test_external_evaluation.py b/tests_v2/test_external_evaluation.py deleted file mode 100644 index fe39b9d..0000000 --- a/tests_v2/test_external_evaluation.py +++ /dev/null @@ -1,117 +0,0 @@ -import json -from pathlib import Path -import sys - -ROOT = Path(__file__).parents[1] -sys.path.insert(0, str(ROOT)) - -from address_normalizer import parse -from evaluation.evaluate_redmadrobot import ( - AddressSnippet, - _gold_spans, - _reconstruct, - _without_failures, - evaluate, -) -from evaluation.evaluate_redmadrobot_detection import ( - Message as DetectionMessage, - Span as DetectionSpan, - evaluate as evaluate_detection, -) - - -def _snippet( - tokens: tuple[str, ...], - tags: tuple[str, ...], -) -> AddressSnippet: - text, offsets = _reconstruct(tokens) - return AddressSnippet( - source_row=0, - text=text, - tokens=tokens, - tags=tags, - offsets=offsets, - ) - - -def test_gold_bio_spans_are_reconstructed_from_tokens(): - snippet = _snippet( - ("ул", ".", "Ополченская", ",", "дом", "5"), - ( - "B-STREET", - "I-STREET", - "I-STREET", - "O", - "B-HOUSE", - "I-HOUSE", - ), - ) - assert [ - (span.label, snippet.text[span.start : span.end]) - for span in _gold_spans(snippet) - ] == [ - ("STREET", "ул . Ополченская"), - ("HOUSE", "дом 5"), - ] - - -def test_external_span_evaluation_accepts_overlapping_component_values(): - snippet = _snippet( - ("ул", ".", "Ополченская", ",", "дом", "5"), - ( - "B-STREET", - "I-STREET", - "I-STREET", - "O", - "B-HOUSE", - "I-HOUSE", - ), - ) - report = evaluate([snippet], parse) - assert report["micro"]["support"] == 2 - assert report["micro"]["tp"] == 2 - assert report["micro"]["f1"] == 1.0 - assert report["span_overlap_micro"] == report["micro"] - assert report["metric_definitions"]["fields"].startswith("per-field") - summary = _without_failures(report) - assert summary["micro"] == report["span_overlap_micro"] - assert "metric_definitions" not in summary - assert "span_overlap_micro" not in summary - - -def test_complete_message_detection_is_scored_without_oracle_cropping(): - text = "Текст до ул. Мира, д. 2 после" - expected_text = "ул. Мира, д. 2" - start = text.index(expected_text) - report = evaluate_detection( - [ - DetectionMessage( - source_row=1, - text=text, - gold=(DetectionSpan(start, start + len(expected_text)),), - ), - DetectionMessage( - source_row=2, - text="Встреча завтра в 18:30", - gold=(), - ), - ] - ) - - assert report["span_overlap_micro"]["f1"] == 1.0 - assert report["exact_span_micro"]["f1"] == 1.0 - assert report["negative_message_specificity"] == 1.0 - assert report["failure_case_count"] == 0 - - -def test_committed_message_detection_report_keeps_all_failure_cases(): - report = json.loads( - ( - ROOT / "evaluation/redmadrobot_detection_report.json" - ).read_text(encoding="utf-8") - ) - - assert report["source"]["revision"] - assert report["metric_definitions"]["span_overlap_micro"] - assert report["failure_case_count"] == len(report["failure_cases"]) - assert report["failure_case_count"] == 107 diff --git a/tests_v2/test_failure_analysis.py b/tests_v2/test_failure_analysis.py deleted file mode 100644 index ba20d74..0000000 --- a/tests_v2/test_failure_analysis.py +++ /dev/null @@ -1,85 +0,0 @@ -from __future__ import annotations - -import csv -import importlib.util -import json -from pathlib import Path - - -ROOT = Path(__file__).parents[1] - - -def _load_analyzer(): - path = ROOT / "evaluation/analyze_failures.py" - spec = importlib.util.spec_from_file_location("analyze_failures", path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def test_diagnostic_table_covers_every_reference_row_with_narrow_columns(): - with ( - ROOT / "evaluation/legacy_reference_500_diagnostics.csv" - ).open(encoding="utf-8", newline="") as source: - rows = list(csv.DictReader(source)) - - assert len(rows) == 500 - assert len({row["id"] for row in rows}) == 500 - required = { - "exact_address", - "triage_priority", - "diagnosis_status", - "mismatch_fields", - "missing_fields", - "extra_fields", - "wrong_value_fields", - "failure_types", - "primary_likely_cause", - "likely_causes", - "failure_summary", - "warnings", - "unparsed_spans", - "scenario_tags", - "street_marker_style", - "has_compact_punctuation", - "has_compound_number", - "has_unmarked_numeric_sequence", - "expected_street", - "actual_street", - "status_street", - "expected_house_num", - "actual_house_num", - "status_house_num", - } - assert required <= set(rows[0]) - assert sum(row["exact_address"] == "false" for row in rows) == 98 - assert all( - row["diagnosis_status"] == "heuristic_needs_human_review" - for row in rows - if row["exact_address"] == "false" - ) - - -def test_failure_summary_matches_current_diagnostics(): - analyzer = _load_analyzer() - source_rows = analyzer._load_rows( - ROOT / "evaluation/legacy_reference_500.jsonl" - ) - diagnostics = [analyzer.diagnose_row(row) for row in source_rows] - with ( - ROOT / "evaluation/legacy_reference_500_diagnostics.csv" - ).open(encoding="utf-8", newline="") as source: - committed_diagnostics = list(csv.DictReader(source)) - actual = analyzer.summarize(diagnostics) - committed = json.loads( - ( - ROOT / "evaluation/legacy_reference_500_failure_summary.json" - ).read_text(encoding="utf-8") - ) - - assert diagnostics == committed_diagnostics - assert actual == committed - assert actual["rows"] == 500 - assert actual["failed_rows"] == 98 - assert len(actual["representative_failure_sample"]) == 10 diff --git a/evaluation/evaluate.py b/tools/benchmark.py similarity index 86% rename from evaluation/evaluate.py rename to tools/benchmark.py index bfe339e..7309452 100644 --- a/evaluation/evaluate.py +++ b/tools/benchmark.py @@ -9,6 +9,10 @@ import sys from typing import Any, Callable, Iterable + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + from address_normalizer import parse from address_normalizer.types import ParsedAddress @@ -219,22 +223,51 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--data", type=Path, - default=Path(__file__).with_name("legacy_reference_500.jsonl"), + default=ROOT / "benchmarks/legacy_500.jsonl", + ) + parser.add_argument( + "--check", + action="store_true", + help="fail unless the release regression thresholds pass", ) - parser.add_argument("--gates", type=Path) parser.add_argument("--output", type=Path) args = parser.parse_args(argv) report = score_rows(_load_jsonl(args.data)) - if args.gates: - gates = json.loads(args.gates.read_text(encoding="utf-8")) + if args.check: + gates = { + "minimum_rows": 500, + "minimum_metrics": { + "exact_address_rate": 0.80, + "no_unparsed_rate": 0.75, + "exact_component_value_micro.f1": 0.95, + }, + } report["gates"] = _check_gates(report, gates) report["release_gate_passed"] = all( outcome["passed"] for outcome in report["gates"] ) rendered = json.dumps(report, ensure_ascii=False, indent=2) - print(rendered) + if args.check: + print( + json.dumps( + { + "rows": report["rows"], + "exact_address_rate": report["exact_address_rate"], + "no_unparsed_rate": report["no_unparsed_rate"], + "exact_component_value_micro": report[ + "exact_component_value_micro" + ], + "gates": report["gates"], + "release_gate_passed": report["release_gate_passed"], + }, + ensure_ascii=False, + indent=2, + ) + ) + else: + print(rendered) if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(f"{rendered}\n", encoding="utf-8") diff --git a/training/real_corpus.py b/tools/model_data.py similarity index 100% rename from training/real_corpus.py rename to tools/model_data.py diff --git a/training/baselines/synthetic_model.json b/tools/synthetic_model.json similarity index 100% rename from training/baselines/synthetic_model.json rename to tools/synthetic_model.json diff --git a/training/train_compact_tagger.py b/tools/train_model.py similarity index 95% rename from training/train_compact_tagger.py rename to tools/train_model.py index b64807f..3b9653c 100644 --- a/training/train_compact_tagger.py +++ b/tools/train_model.py @@ -23,8 +23,8 @@ START, token_features, ) -from evaluation.evaluate import score_rows -from training.real_corpus import ( +from benchmark import score_rows +from model_data import ( SequenceExample, build_examples, corpus_summary, @@ -222,7 +222,7 @@ def _model_payload( "source-verifiable legacy reference rows with deterministic " "marker-free views" ), - "dataset": "evaluation/legacy_reference_500.jsonl", + "dataset": "benchmarks/legacy_500.jsonl", "dataset_sha256": dataset_sha256, "split": "SHA-256 by canonical address group: 70/15/15", }, @@ -344,8 +344,7 @@ def train( report = { "scope": ( - "group-disjoint real-address model evaluation; source and " - "derived-model provenance are recorded in LICENSING.md" + "group-disjoint real-address model evaluation" ), "dataset": dataset_name, "dataset_sha256": dataset_sha256, @@ -380,7 +379,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--data", type=Path, - default=ROOT / "evaluation/legacy_reference_500.jsonl", + default=ROOT / "benchmarks/legacy_500.jsonl", ) parser.add_argument( "--output", @@ -390,12 +389,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--report", type=Path, - default=ROOT / "training/model_evaluation.json", ) parser.add_argument( "--baseline-model", type=Path, - default=ROOT / "training/baselines/synthetic_model.json", + default=ROOT / "tools/synthetic_model.json", ) parser.add_argument("--seed", type=int, default=2017) parser.add_argument("--epoch-grid", default="5,10,20,40,80") @@ -419,17 +417,18 @@ def main(argv: list[str] | None = None) -> int: json.dumps(payload, ensure_ascii=False, separators=(",", ":")), encoding="utf-8", ) - args.report.parent.mkdir(parents=True, exist_ok=True) - args.report.write_text( - f"{json.dumps(report, ensure_ascii=False, indent=2)}\n", - encoding="utf-8", - ) + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text( + f"{json.dumps(report, ensure_ascii=False, indent=2)}\n", + encoding="utf-8", + ) print( json.dumps( { "output": str(args.output), "bytes": args.output.stat().st_size, - "report": str(args.report), + "report": str(args.report) if args.report else None, "selected_epochs": report["selected_epochs"], "test_sequence": { name: { diff --git a/training/README.md b/training/README.md deleted file mode 100644 index b0279d8..0000000 --- a/training/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# Training the compact tagger - -The runtime model is a sparse linear-chain sequence tagger with Viterbi -inference. Training uses an epoch-averaged structured perceptron. Training and -inference require only the Python standard library. - -The real-address corpus is derived from the 500 source-verifiable rows in -`evaluation/legacy_reference_500.jsonl`: - -1. the deterministic parser records the exact residual word tokens that reach - the model; -2. source-verifiable address fields are aligned to those token offsets; -3. marker-free views are derived from the same real component names; -4. examples are grouped by canonical administrative/street identity; -5. SHA-256 assigns whole groups to train, validation, or test (70/15/15); -6. epoch count is chosen on validation only; -7. the final candidate is evaluated once on the untouched test groups. - -Regenerate the model and committed evaluation report: - -```bash -python training/train_compact_tagger.py -``` - -Verify the bundled artifact against fixed test gates: - -```bash -python training/evaluate_compact_tagger.py -``` - -The first real model is 37 KB. Compared with the preserved 20 KB synthetic -starter on the group-disjoint sequence test: - -| Metric | Synthetic starter | Real model | -| --- | ---: | ---: | -| Token accuracy | 71.3% | 96.5% | -| Complete sequence accuracy | 61.4% | 94.3% | -| Micro entity F1 | 78.1% | 96.9% | - -On the 21 corresponding end-to-end holdout rows, micro field F1 improves from -87.4% to 91.3%. The test split contains no independently measured -`DISTRICT` or `SETTLEMENT` tokens, so those classes must not be claimed as -validated by this result. See `model_evaluation.json` for the exact split, -tuning runs, class coverage, failures, and end-to-end comparison. - -`training/baselines/synthetic_model.json` preserves the pre-real-data model so -that regeneration remains reproducible and comparisons do not silently change -after the bundled model is replaced. - -## Provenance - -The maintainer authorized redistribution of the historical source workbook, -its committed 500-row derivative, and the compact model under GPL-3.0-only. -The decision, source commit, deterministic generation path, and external-data -boundary are recorded in `LICENSING.md`. - -The external preparation tools now expose 5,681,842 Deepparse training rows and -276,368 historical Moscow-registry training rows in group-disjoint splits. -They are not used by the current 37 KB model. Before training on them, define a -sampling policy so repeated clean formatting does not overwhelm the smaller -noisy-input corpus, keep the committed test groups sealed, and record the -derived-model rights for CC BY 4.0 and the Moscow source terms. - -For a production training release: - -1. independently review at least 300 aligned rows; -2. add substantially more district and settlement examples; -3. preserve canonical-address grouping across every split; -4. reserve a final dataset not used for feature or rule changes; -5. report confidence intervals and field metrics by region and source system; -6. document the right to redistribute examples and the derived model. diff --git a/training/__init__.py b/training/__init__.py deleted file mode 100644 index 2caf800..0000000 --- a/training/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Reproducible training and evaluation helpers.""" diff --git a/training/evaluate_compact_tagger.py b/training/evaluate_compact_tagger.py deleted file mode 100644 index 2d0e81f..0000000 --- a/training/evaluate_compact_tagger.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Evaluate the bundled model on the untouched canonical-group test split.""" - -from __future__ import annotations - -import json -from pathlib import Path -import sys - - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) -sys.path.insert(0, str(ROOT)) - -from address_normalizer.tagger import CompactSequenceTagger -from training.real_corpus import ( - build_examples, - corpus_summary, - load_reference_rows, -) -from training.train_compact_tagger import score_sequences - - -MINIMUMS = { - "token_accuracy": 0.95, - "sequence_accuracy": 0.90, - "macro_entity_f1": 0.95, - "micro_entity_f1": 0.95, -} -MAXIMUM_MODEL_BYTES = 250_000 - - -def main() -> int: - data_path = ROOT / "evaluation/legacy_reference_500.jsonl" - model_path = ROOT / "src/address_normalizer/data/model.json" - examples = build_examples(load_reference_rows(data_path)) - test_examples = [example for example in examples if example.split == "test"] - summary = corpus_summary(examples) - metrics = score_sequences( - CompactSequenceTagger.from_package(), - test_examples, - ) - supported_labels = [ - label - for label, values in metrics["labels"].items() - if label != "O" and values["support"] - ] - unsupported_labels = [ - label - for label, values in metrics["labels"].items() - if label != "O" and not values["support"] - ] - gates = [ - { - "metric": name, - "actual": metrics[name], - "minimum": minimum, - "passed": metrics[name] >= minimum, - } - for name, minimum in MINIMUMS.items() - ] - gates.extend( - ( - { - "metric": "model_bytes", - "actual": model_path.stat().st_size, - "maximum": MAXIMUM_MODEL_BYTES, - "passed": model_path.stat().st_size <= MAXIMUM_MODEL_BYTES, - }, - { - "metric": "leaking_groups", - "actual": len(summary["leaking_groups"]), - "maximum": 0, - "passed": not summary["leaking_groups"], - }, - ) - ) - report = { - "scope": ( - "untuned test split grouped by canonical address; source and " - "derived-model provenance are recorded in LICENSING.md" - ), - "test_groups": summary["splits"]["test"]["groups"], - "test_examples": len(test_examples), - "supported_test_labels": supported_labels, - "unsupported_test_labels": unsupported_labels, - "metrics": metrics, - "gates": gates, - "passed": all(gate["passed"] for gate in gates), - } - print(json.dumps(report, ensure_ascii=False, indent=2)) - return 0 if report["passed"] else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/training/model_evaluation.json b/training/model_evaluation.json deleted file mode 100644 index 4610490..0000000 --- a/training/model_evaluation.json +++ /dev/null @@ -1,1316 +0,0 @@ -{ - "scope": "group-disjoint real-address model evaluation; source and derived-model provenance are recorded in LICENSING.md", - "dataset": "evaluation/legacy_reference_500.jsonl", - "dataset_sha256": "853916e36cfc5a52d06add0524ba64ca2c771073c1c4e98cd5645116e7f74588", - "corpus": { - "examples": 429, - "groups": 108, - "splits": { - "train": { - "examples": 301, - "groups": 77, - "tokens": 498, - "positive_sequences": 272, - "views": { - "hierarchy_without_markers": 74, - "locality_and_street": 11, - "locality_only": 58, - "observed_residual": 83, - "street_only": 75 - }, - "labels": { - "CITY": 189, - "DISTRICT": 3, - "O": 77, - "REGION": 15, - "SETTLEMENT": 6, - "STREET": 208 - } - }, - "validation": { - "examples": 58, - "groups": 14, - "tokens": 101, - "positive_sequences": 50, - "views": { - "hierarchy_without_markers": 14, - "locality_and_street": 3, - "locality_only": 13, - "observed_residual": 16, - "street_only": 12 - }, - "labels": { - "CITY": 29, - "DISTRICT": 2, - "O": 16, - "REGION": 4, - "SETTLEMENT": 3, - "STREET": 47 - } - }, - "test": { - "examples": 70, - "groups": 17, - "tokens": 115, - "positive_sequences": 62, - "views": { - "hierarchy_without_markers": 17, - "locality_and_street": 2, - "locality_only": 16, - "observed_residual": 21, - "street_only": 14 - }, - "labels": { - "CITY": 45, - "O": 20, - "REGION": 3, - "STREET": 47 - } - } - }, - "leaking_groups": [] - }, - "tuning": [ - { - "epochs": 5, - "model_bytes": 32646, - "validation": { - "examples": 58, - "tokens": 101, - "token_accuracy": 0.821782, - "sequence_accuracy": 0.775862, - "macro_entity_f1": 0.423816, - "micro_entity_f1": 0.811765, - "labels": { - "O": { - "tp": 14, - "fp": 2, - "fn": 2, - "support": 16, - "precision": 0.875, - "recall": 0.875, - "f1": 0.875 - }, - "REGION": { - "tp": 1, - "fp": 0, - "fn": 3, - "support": 4, - "precision": 1.0, - "recall": 0.25, - "f1": 0.4 - }, - "DISTRICT": { - "tp": 0, - "fp": 0, - "fn": 2, - "support": 2, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "CITY": { - "tp": 24, - "fp": 1, - "fn": 5, - "support": 29, - "precision": 0.96, - "recall": 0.827586, - "f1": 0.888889 - }, - "SETTLEMENT": { - "tp": 0, - "fp": 0, - "fn": 3, - "support": 3, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "STREET": { - "tp": 44, - "fp": 15, - "fn": 3, - "support": 47, - "precision": 0.745763, - "recall": 0.93617, - "f1": 0.830189 - } - } - } - }, - { - "epochs": 10, - "model_bytes": 33434, - "validation": { - "examples": 58, - "tokens": 101, - "token_accuracy": 0.821782, - "sequence_accuracy": 0.741379, - "macro_entity_f1": 0.541778, - "micro_entity_f1": 0.826347, - "labels": { - "O": { - "tp": 14, - "fp": 5, - "fn": 2, - "support": 16, - "precision": 0.736842, - "recall": 0.875, - "f1": 0.8 - }, - "REGION": { - "tp": 4, - "fp": 0, - "fn": 0, - "support": 4, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "DISTRICT": { - "tp": 0, - "fp": 0, - "fn": 2, - "support": 2, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "CITY": { - "tp": 24, - "fp": 1, - "fn": 5, - "support": 29, - "precision": 0.96, - "recall": 0.827586, - "f1": 0.888889 - }, - "SETTLEMENT": { - "tp": 0, - "fp": 0, - "fn": 3, - "support": 3, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "STREET": { - "tp": 41, - "fp": 12, - "fn": 6, - "support": 47, - "precision": 0.773585, - "recall": 0.87234, - "f1": 0.82 - } - } - } - }, - { - "epochs": 20, - "model_bytes": 33971, - "validation": { - "examples": 58, - "tokens": 101, - "token_accuracy": 0.811881, - "sequence_accuracy": 0.724138, - "macro_entity_f1": 0.535961, - "micro_entity_f1": 0.814371, - "labels": { - "O": { - "tp": 14, - "fp": 5, - "fn": 2, - "support": 16, - "precision": 0.736842, - "recall": 0.875, - "f1": 0.8 - }, - "REGION": { - "tp": 4, - "fp": 0, - "fn": 0, - "support": 4, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "DISTRICT": { - "tp": 0, - "fp": 0, - "fn": 2, - "support": 2, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "CITY": { - "tp": 23, - "fp": 1, - "fn": 6, - "support": 29, - "precision": 0.958333, - "recall": 0.793103, - "f1": 0.867925 - }, - "SETTLEMENT": { - "tp": 0, - "fp": 0, - "fn": 3, - "support": 3, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "STREET": { - "tp": 41, - "fp": 13, - "fn": 6, - "support": 47, - "precision": 0.759259, - "recall": 0.87234, - "f1": 0.811881 - } - } - } - }, - { - "epochs": 40, - "model_bytes": 34672, - "validation": { - "examples": 58, - "tokens": 101, - "token_accuracy": 0.821782, - "sequence_accuracy": 0.741379, - "macro_entity_f1": 0.541629, - "micro_entity_f1": 0.826347, - "labels": { - "O": { - "tp": 14, - "fp": 5, - "fn": 2, - "support": 16, - "precision": 0.736842, - "recall": 0.875, - "f1": 0.8 - }, - "REGION": { - "tp": 4, - "fp": 0, - "fn": 0, - "support": 4, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "DISTRICT": { - "tp": 0, - "fp": 0, - "fn": 2, - "support": 2, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "CITY": { - "tp": 23, - "fp": 0, - "fn": 6, - "support": 29, - "precision": 1.0, - "recall": 0.793103, - "f1": 0.884615 - }, - "SETTLEMENT": { - "tp": 0, - "fp": 0, - "fn": 3, - "support": 3, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "STREET": { - "tp": 42, - "fp": 13, - "fn": 5, - "support": 47, - "precision": 0.763636, - "recall": 0.893617, - "f1": 0.823529 - } - } - } - }, - { - "epochs": 80, - "model_bytes": 35456, - "validation": { - "examples": 58, - "tokens": 101, - "token_accuracy": 0.80198, - "sequence_accuracy": 0.706897, - "macro_entity_f1": 0.511363, - "micro_entity_f1": 0.802395, - "labels": { - "O": { - "tp": 14, - "fp": 5, - "fn": 2, - "support": 16, - "precision": 0.736842, - "recall": 0.875, - "f1": 0.8 - }, - "REGION": { - "tp": 4, - "fp": 1, - "fn": 0, - "support": 4, - "precision": 0.8, - "recall": 1.0, - "f1": 0.888889 - }, - "DISTRICT": { - "tp": 0, - "fp": 0, - "fn": 2, - "support": 2, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "CITY": { - "tp": 23, - "fp": 1, - "fn": 6, - "support": 29, - "precision": 0.958333, - "recall": 0.793103, - "f1": 0.867925 - }, - "SETTLEMENT": { - "tp": 0, - "fp": 0, - "fn": 3, - "support": 3, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "STREET": { - "tp": 40, - "fp": 13, - "fn": 7, - "support": 47, - "precision": 0.754717, - "recall": 0.851064, - "f1": 0.8 - } - } - } - } - ], - "selected_epochs": 10, - "test_sequence": { - "synthetic_starter": { - "examples": 70, - "tokens": 115, - "token_accuracy": 0.713043, - "sequence_accuracy": 0.614286, - "macro_entity_f1": 0.808777, - "micro_entity_f1": 0.780952, - "labels": { - "O": { - "tp": 0, - "fp": 0, - "fn": 20, - "support": 20, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "REGION": { - "tp": 3, - "fp": 1, - "fn": 0, - "support": 3, - "precision": 0.75, - "recall": 1.0, - "f1": 0.857143 - }, - "DISTRICT": { - "tp": 0, - "fp": 2, - "fn": 0, - "support": 0, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "CITY": { - "tp": 45, - "fp": 23, - "fn": 0, - "support": 45, - "precision": 0.661765, - "recall": 1.0, - "f1": 0.79646 - }, - "SETTLEMENT": { - "tp": 0, - "fp": 0, - "fn": 0, - "support": 0, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "STREET": { - "tp": 34, - "fp": 7, - "fn": 13, - "support": 47, - "precision": 0.829268, - "recall": 0.723404, - "f1": 0.772727 - } - }, - "failure_sample": [ - { - "id": "legacy-good-0761:street_only", - "view": "street_only", - "tokens": [ - "Варшавское" - ], - "expected": [ - "STREET" - ], - "actual": [ - "CITY" - ] - }, - { - "id": "legacy-good-0563", - "view": "observed_residual", - "tokens": [ - "ком" - ], - "expected": [ - "O" - ], - "actual": [ - "STREET" - ] - }, - { - "id": "legacy-good-0585", - "view": "observed_residual", - "tokens": [ - "Москва", - "Москва", - "г" - ], - "expected": [ - "CITY", - "O", - "O" - ], - "actual": [ - "CITY", - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0763", - "view": "observed_residual", - "tokens": [ - "Москва", - "ул" - ], - "expected": [ - "CITY", - "O" - ], - "actual": [ - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0763:street_only", - "view": "street_only", - "tokens": [ - "Ленинский" - ], - "expected": [ - "STREET" - ], - "actual": [ - "DISTRICT" - ] - }, - { - "id": "legacy-good-0160", - "view": "observed_residual", - "tokens": [ - "шоссе" - ], - "expected": [ - "O" - ], - "actual": [ - "CITY" - ] - }, - { - "id": "legacy-good-0753", - "view": "observed_residual", - "tokens": [ - "уп", - "Адмирала", - "Макарова" - ], - "expected": [ - "O", - "STREET", - "STREET" - ], - "actual": [ - "CITY", - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0753:hierarchy_without_markers", - "view": "hierarchy_without_markers", - "tokens": [ - "Москва", - "Адмирала", - "Макарова" - ], - "expected": [ - "CITY", - "STREET", - "STREET" - ], - "actual": [ - "CITY", - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0753:street_only", - "view": "street_only", - "tokens": [ - "Адмирала", - "Макарова" - ], - "expected": [ - "STREET", - "STREET" - ], - "actual": [ - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0647", - "view": "observed_residual", - "tokens": [ - "ул" - ], - "expected": [ - "O" - ], - "actual": [ - "CITY" - ] - }, - { - "id": "legacy-good-0335", - "view": "observed_residual", - "tokens": [ - "г", - "Москва" - ], - "expected": [ - "O", - "CITY" - ], - "actual": [ - "CITY", - "CITY" - ] - }, - { - "id": "legacy-good-0947", - "view": "observed_residual", - "tokens": [ - "Соколово", - "Мещерская" - ], - "expected": [ - "STREET", - "STREET" - ], - "actual": [ - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0947:hierarchy_without_markers", - "view": "hierarchy_without_markers", - "tokens": [ - "Москва", - "Соколово", - "Мещерская" - ], - "expected": [ - "CITY", - "STREET", - "STREET" - ], - "actual": [ - "CITY", - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0353", - "view": "observed_residual", - "tokens": [ - "Москва", - "г" - ], - "expected": [ - "CITY", - "O" - ], - "actual": [ - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0353:street_only", - "view": "street_only", - "tokens": [ - "Николоямская" - ], - "expected": [ - "STREET" - ], - "actual": [ - "REGION" - ] - }, - { - "id": "legacy-good-0764", - "view": "observed_residual", - "tokens": [ - "Московская", - "Подольск", - "г" - ], - "expected": [ - "REGION", - "CITY", - "O" - ], - "actual": [ - "REGION", - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0358", - "view": "observed_residual", - "tokens": [ - "ул" - ], - "expected": [ - "O" - ], - "actual": [ - "CITY" - ] - }, - { - "id": "legacy-good-0391", - "view": "observed_residual", - "tokens": [ - "Москва", - "Москва", - "Академика", - "Королева" - ], - "expected": [ - "CITY", - "O", - "STREET", - "STREET" - ], - "actual": [ - "CITY", - "CITY", - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0391:hierarchy_without_markers", - "view": "hierarchy_without_markers", - "tokens": [ - "Москва", - "Академика", - "Королева" - ], - "expected": [ - "CITY", - "STREET", - "STREET" - ], - "actual": [ - "CITY", - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0391:street_only", - "view": "street_only", - "tokens": [ - "Академика", - "Королева" - ], - "expected": [ - "STREET", - "STREET" - ], - "actual": [ - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0796", - "view": "observed_residual", - "tokens": [ - "г", - "ул", - "д" - ], - "expected": [ - "O", - "O", - "O" - ], - "actual": [ - "CITY", - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0349", - "view": "observed_residual", - "tokens": [ - "этаж" - ], - "expected": [ - "O" - ], - "actual": [ - "CITY" - ] - }, - { - "id": "legacy-good-0349:street_only", - "view": "street_only", - "tokens": [ - "Космонавтов" - ], - "expected": [ - "STREET" - ], - "actual": [ - "CITY" - ] - }, - { - "id": "legacy-good-0814", - "view": "observed_residual", - "tokens": [ - "г" - ], - "expected": [ - "O" - ], - "actual": [ - "CITY" - ] - }, - { - "id": "legacy-good-0807", - "view": "observed_residual", - "tokens": [ - "Кронверкский", - "пр" - ], - "expected": [ - "STREET", - "O" - ], - "actual": [ - "DISTRICT", - "STREET" - ] - }, - { - "id": "legacy-good-0049", - "view": "observed_residual", - "tokens": [ - "Проспект" - ], - "expected": [ - "O" - ], - "actual": [ - "CITY" - ] - }, - { - "id": "legacy-good-0379", - "view": "observed_residual", - "tokens": [ - "Москва", - "Москва" - ], - "expected": [ - "CITY", - "O" - ], - "actual": [ - "CITY", - "CITY" - ] - } - ] - }, - "real_model": { - "examples": 70, - "tokens": 115, - "token_accuracy": 0.965217, - "sequence_accuracy": 0.942857, - "macro_entity_f1": 0.978495, - "micro_entity_f1": 0.96875, - "labels": { - "O": { - "tp": 18, - "fp": 0, - "fn": 2, - "support": 20, - "precision": 1.0, - "recall": 0.9, - "f1": 0.947368 - }, - "REGION": { - "tp": 3, - "fp": 0, - "fn": 0, - "support": 3, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "DISTRICT": { - "tp": 0, - "fp": 0, - "fn": 0, - "support": 0, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "CITY": { - "tp": 45, - "fp": 3, - "fn": 0, - "support": 45, - "precision": 0.9375, - "recall": 1.0, - "f1": 0.967742 - }, - "SETTLEMENT": { - "tp": 0, - "fp": 0, - "fn": 0, - "support": 0, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "STREET": { - "tp": 45, - "fp": 1, - "fn": 2, - "support": 47, - "precision": 0.978261, - "recall": 0.957447, - "f1": 0.967742 - } - }, - "failure_sample": [ - { - "id": "legacy-good-0753", - "view": "observed_residual", - "tokens": [ - "уп", - "Адмирала", - "Макарова" - ], - "expected": [ - "O", - "STREET", - "STREET" - ], - "actual": [ - "O", - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0753:street_only", - "view": "street_only", - "tokens": [ - "Адмирала", - "Макарова" - ], - "expected": [ - "STREET", - "STREET" - ], - "actual": [ - "CITY", - "STREET" - ] - }, - { - "id": "legacy-good-0391", - "view": "observed_residual", - "tokens": [ - "Москва", - "Москва", - "Академика", - "Королева" - ], - "expected": [ - "CITY", - "O", - "STREET", - "STREET" - ], - "actual": [ - "CITY", - "CITY", - "STREET", - "STREET" - ] - }, - { - "id": "legacy-good-0379", - "view": "observed_residual", - "tokens": [ - "Москва", - "Москва" - ], - "expected": [ - "CITY", - "O" - ], - "actual": [ - "CITY", - "STREET" - ] - } - ] - } - }, - "test_end_to_end": { - "rows": 21, - "synthetic_starter": { - "rows": 21, - "review_statuses": { - "legacy_reference_not_independently_rereviewed": 21 - }, - "metric_definitions": { - "exact_address_rate": "fraction of rows where every public component value matches", - "no_unparsed_rate": "fraction of rows with no residual word or number spans", - "exact_component_value_micro": "micro precision, recall, and F1 over case-insensitive exact component values after whitespace and ё/е folding" - }, - "exact_address_rate": 0.428571, - "no_unparsed_rate": 0.285714, - "exact_component_value_micro": { - "tp": 101, - "fp": 11, - "fn": 18, - "precision": 0.901786, - "recall": 0.848739, - "f1": 0.874459 - }, - "micro": { - "tp": 101, - "fp": 11, - "fn": 18, - "precision": 0.901786, - "recall": 0.848739, - "f1": 0.874459 - }, - "fields": { - "postal_code": { - "tp": 21, - "fp": 0, - "fn": 0, - "support": 21, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "region": { - "tp": 2, - "fp": 0, - "fn": 0, - "support": 2, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "district": { - "tp": 0, - "fp": 1, - "fn": 0, - "support": 0, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "city": { - "tp": 18, - "fp": 3, - "fn": 3, - "support": 21, - "precision": 0.857143, - "recall": 0.857143, - "f1": 0.857143 - }, - "settlement": { - "tp": 0, - "fp": 0, - "fn": 0, - "support": 0, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "street": { - "tp": 17, - "fp": 4, - "fn": 4, - "support": 21, - "precision": 0.809524, - "recall": 0.809524, - "f1": 0.809524 - }, - "street_type": { - "tp": 13, - "fp": 2, - "fn": 8, - "support": 21, - "precision": 0.866667, - "recall": 0.619048, - "f1": 0.722222 - }, - "house_num": { - "tp": 20, - "fp": 1, - "fn": 1, - "support": 21, - "precision": 0.952381, - "recall": 0.952381, - "f1": 0.952381 - }, - "corpus": { - "tp": 2, - "fp": 0, - "fn": 0, - "support": 2, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "structure": { - "tp": 5, - "fp": 0, - "fn": 0, - "support": 5, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "apartment": { - "tp": 3, - "fp": 0, - "fn": 2, - "support": 5, - "precision": 1.0, - "recall": 0.6, - "f1": 0.75 - } - } - }, - "real_model": { - "rows": 21, - "review_statuses": { - "legacy_reference_not_independently_rereviewed": 21 - }, - "metric_definitions": { - "exact_address_rate": "fraction of rows where every public component value matches", - "no_unparsed_rate": "fraction of rows with no residual word or number spans", - "exact_component_value_micro": "micro precision, recall, and F1 over case-insensitive exact component values after whitespace and ё/е folding" - }, - "exact_address_rate": 0.52381, - "no_unparsed_rate": 0.238095, - "exact_component_value_micro": { - "tp": 105, - "fp": 6, - "fn": 14, - "precision": 0.945946, - "recall": 0.882353, - "f1": 0.913043 - }, - "micro": { - "tp": 105, - "fp": 6, - "fn": 14, - "precision": 0.945946, - "recall": 0.882353, - "f1": 0.913043 - }, - "fields": { - "postal_code": { - "tp": 21, - "fp": 0, - "fn": 0, - "support": 21, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "region": { - "tp": 2, - "fp": 0, - "fn": 0, - "support": 2, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "district": { - "tp": 0, - "fp": 0, - "fn": 0, - "support": 0, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "city": { - "tp": 20, - "fp": 1, - "fn": 1, - "support": 21, - "precision": 0.952381, - "recall": 0.952381, - "f1": 0.952381 - }, - "settlement": { - "tp": 0, - "fp": 0, - "fn": 0, - "support": 0, - "precision": 0.0, - "recall": 0.0, - "f1": 0.0 - }, - "street": { - "tp": 19, - "fp": 2, - "fn": 2, - "support": 21, - "precision": 0.904762, - "recall": 0.904762, - "f1": 0.904762 - }, - "street_type": { - "tp": 13, - "fp": 2, - "fn": 8, - "support": 21, - "precision": 0.866667, - "recall": 0.619048, - "f1": 0.722222 - }, - "house_num": { - "tp": 20, - "fp": 1, - "fn": 1, - "support": 21, - "precision": 0.952381, - "recall": 0.952381, - "f1": 0.952381 - }, - "corpus": { - "tp": 2, - "fp": 0, - "fn": 0, - "support": 2, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "structure": { - "tp": 5, - "fp": 0, - "fn": 0, - "support": 5, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 - }, - "apartment": { - "tp": 3, - "fp": 0, - "fn": 2, - "support": 5, - "precision": 1.0, - "recall": 0.6, - "f1": 0.75 - } - } - } - }, - "model_bytes": { - "synthetic_starter": 20106, - "real_model": 37130 - } -} diff --git a/upload_fias.py b/upload_fias.py deleted file mode 100644 index f576613..0000000 --- a/upload_fias.py +++ /dev/null @@ -1,197 +0,0 @@ -# DISCLAIMER -# Я настоятельно рекомендую запускать этот скрипт построчно из Jupyter notebook. -# 1. Он очень долгий. Перегон dfb в csv занимает где-то 20 минут, загрузка названий улиц ещё 2, а номера домов это ещё часов на 8-10 -# 2. Требует очень много памяти и можно не заметить как на машине она закончится. Таблицы с csv занимают около 50Гб, в elastic это может весить ещё около 100гб. Удаляйте dfb после того как получили csv файлы. - - -import os -import csv -import glob -import shutil -import argparse - -import pandas as pd -from elasticsearch import Elasticsearch, helpers -from simpledbf import Dbf5 - -pd.options.display.max_columns = None - -es = Elasticsearch() - -def load_elastic(fn, index, doc_type, encoding='cp866', es=es): - ''' - Этот метод загружает указанный файлик в elastic. - ''' - with open(fn, encoding=encoding) as f: - reader = csv.DictReader(f) - helpers.bulk(es, reader, index=index, doc_type=doc_type, raise_on_error=False, stats_only=True) - print('done') - - -# # Оптимизация под полнотекстовый поиск -def full_address(GUID): - answer = es.search(index='fias', doc_type='address', body= - { - "size": 1, - "query": { - "bool": { - "must": [ - {"match": { - "AOGUID": GUID}}, - {"match": { - "ACTSTATUS": 1}} - ] - } - } - }) - entry = answer["hits"]["hits"][0]["_source"] - string = entry['SHORTNAME'] + " " + entry['OFFNAME'] - if len(entry['PARENTGUID']) > 5: - string = full_address(entry['PARENTGUID']) + ', ' + string - return string - - -def full_address_sep(GUID, _leaf=True): - address = {} - address['fullname'] = '' - answer = es.search(index='fias', doc_type='address', body= - { - "size": 1, - "query": { - "bool": { - "must": [ - {"match": { - "AOGUID": GUID}}, - {"match": { - "ACTSTATUS": 1}} - ] - } - } - }) - - try: - entry = answer["hits"]["hits"][0]["_source"] - level = entry["AOLEVEL"] - LUT = { - '1': 'region', - '2': 'aregion', - '3': 'area', - '4': 'city', - '5': 'district', - '6': 'town', - '7': 'street', - '8': 'building', - '9': 'placement', - '65': 'planning', - '75': 'land', - '90': 'additional', - '91': 'nestreet' - } - if _leaf: - address['guid'] = GUID - address['aolevel'] = entry['AOLEVEL'] - address[LUT.get(level, level)] = entry['OFFNAME'] - address[LUT.get(level, level) + "_type"] = entry['SHORTNAME'] - address['fullname'] = entry['SHORTNAME'] + " " + entry['OFFNAME'] - if len(entry['PARENTGUID']) > 5: - nest = full_address_sep(entry['PARENTGUID'], _leaf=False) - string = nest['fullname'] + ', ' + address['fullname'] - address.update(nest) - address['fullname'] = string - except Exception: - print("failed get address") - print(answer) - - return address - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser() - parser.add_argument('--fiasdir', default='') - parser.add_argument('--remove', default=False, action='store_true') - parser.add_argument('--dont-remove', dest='remove', action='store_false') - args = parser.parse_args() - fias_dir = os.path.join(args.fiasdir, 'fias_dbf/'); - fias_csv_dir = os.path.dirname(args.fiasdir + 'fias_csv/') - - files = glob.glob(os.path.join(fias_dir, 'ADDR*'), recursive=True) - - # # Для начала преобразуем всё в csv - - os.makedirs('fias_csv', exist_ok=True) - - files = glob.glob(os.path.join(fias_dir, 'ADDR*'), recursive=True) - for i, f in enumerate(files): - if f[-3:].lower() == 'dbf': - print('processing {0} of {1}. Filename: {2} '.format(i + 1, len(files), f), end='\r') - dbf = Dbf5(f, codec='cp866') - dbf.to_csv('fias_csv/ADDROBJ.csv') - if args.remove: - os.remove(f) # delete to save memory - - -# files = ['ESTSTAT.DBF', 'FLATTYPE.DBF', 'HSTSTAT.DBF', 'INTVSTAT.DBF', 'NDOCTYPE.DBF', -# 'OPERSTAT.DBF', 'ROOMTYPE.DBF', 'SOCRBASE.DBF', 'STRSTAT.DBF'] -# for i, f in enumerate(files): -# if f[-3:].lower() == 'dbf': -# print('processing {0} of {1}. Filename: {2} '.format(i + 1, len(files), f), end='\r') -# dbf = Dbf5(os.path.join(fias_dir, f), codec='cp866') -# dbf.to_csv('fias_csv/{0}.csv'.format(f[:-4])) - - files = glob.glob(os.path.join(fias_dir, 'HOUSE*'), recursive=True) - for i, f in enumerate(files): - if f[-3:].lower() == 'dbf': - print('processing {0} of {1}. Filename: {2} '.format(i + 1, len(files), f), end='\r') - dbf = Dbf5(f, codec='cp866') - dbf.to_csv('fias_csv/HOUSE.csv') - if args.remove: - os.remove(f) # delete to save memory - -# files = glob.glob(os.path.join(fias_dir, 'ROOM*'), recursive=True) -# for i, f in enumerate(files): -# if f[-3:].lower() == 'dbf': -# print('processing {0} of {1}. Filename: {2} '.format(i + 1, len(files), f), end='\r') -# dbf = Dbf5(f, codec='cp866') -# dbf.to_csv('fias_csv/ROOM.csv') - -# files = glob.glob(os.path.join(fias_dir, 'STEAD*'), recursive=True) -# for i, f in enumerate(files): -# if f[-3:].lower() == 'dbf': -# print('processing {0} of {1}. Filename: {2} '.format(i + 1, len(files), f), end='\r') -# dbf = Dbf5(f, codec='cp866') -# dbf.to_csv('fias_csv/STEAD.csv') - - # # Теперь надо всё это закинуть в Elastic - # Да, это не самый оптимальный путь (можно миновать csv). Но это уже как есть - - - # Загрузка самой главной таблицы - # На первых порах её нам хватит. Остальные загружаются при надобности - # Занимает 2 часа - # FIXME - # load_elastic(os.path.join(fias_csv_dir, 'ADDROBJ.csv'), 'fias', 'address') - - # Загрузка в полнотекстовый поиск, где есть и адрес и город и индекс - df_addr = pd.read_csv(os.path.join(fias_csv_dir, 'ADDROBJ.csv'), encoding='cp866', dtype=str, error_bad_lines=False) - # здесь могут быть ошибки парсинга на некоторых полях. - # Их можно просто пропустить а потом попытаться исправить самостоятельно. - start = 0 - finish = None - i = start - for _, value in df_addr[["AOGUID"]][df_addr['ACTSTATUS'] == '1'][start:finish].iterrows(): - if i % 50 == 0: - print(i, end="\r") - - full_addr = full_address_sep(value["AOGUID"]) - # print(full_addr) - es.index(index="fias_full_text", id=value["AOGUID"], doc_type='address', body=full_addr) - i += 1 - - # На данном этапе в elastic должна быть таблица fias_full_text. Далее мы её будем максимально активно использовать - - # Можно ставить на ночь. Это очень долго: 18Гб таблица весит - load_elastic(os.path.join(fias_csv_dir, 'HOUSE.csv'), 'fias_houses', 'home') - - # Удаляем все csv-таблицы, они теперь есть в elastic - shutil.rmtree("fias_csv")